diff --git a/.github/workflows/bootstrap_script.yml b/.github/workflows/bootstrap_script.yml new file mode 100644 index 0000000000..f119c4a0d4 --- /dev/null +++ b/.github/workflows/bootstrap_script.yml @@ -0,0 +1,123 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: test EasyBuild bootstrap script +on: [push, pull_request] +jobs: + setup: + runs-on: ubuntu-latest + outputs: + lmod7: Lmod-7.8.22 + lmod8: Lmod-8.4.27 + modulesTcl: modules-tcl-1.147 + modules3: modules-3.2.10 + modules4: modules-4.1.4 + steps: + - run: "true" + build: + needs: setup + runs-on: ubuntu-18.04 + strategy: + matrix: + # Don't run for Python 3.8, 3.9 , people should just use `pip install easybuild` + python: [2.7, 3.6, 3.7] + modules_tool: + # use variables defined by 'setup' job above, see also + # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context + - ${{needs.setup.outputs.lmod7}} + - ${{needs.setup.outputs.lmod8}} + module_syntax: [Lua] + lc_all: [""] + include: + # also test with module tools other than Lmod (only Tcl syntax) + - modules_tool: ${{needs.setup.outputs.modulesTcl}} + module_syntax: Tcl + python: 2.7 + - modules_tool: ${{needs.setup.outputs.modulesTcl}} + module_syntax: Tcl + python: 3.6 + - modules_tool: ${{needs.setup.outputs.modules3}} + module_syntax: Tcl + python: 2.7 + - modules_tool: ${{needs.setup.outputs.modules3}} + module_syntax: Tcl + python: 3.6 + - modules_tool: ${{needs.setup.outputs.modules4}} + module_syntax: Tcl + python: 2.7 + - modules_tool: ${{needs.setup.outputs.modules4}} + module_syntax: Tcl + python: 3.6 + # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set + # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) + - python: 3.6 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + lc_all: C + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: install OS & Python packages + run: | + # disable apt-get update, we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + # sudo apt-get update + # for modules tool + sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi + + - name: install modules tool + run: | + # avoid downloading modules tool sources into easybuild-framework dir + cd $HOME + export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh + # install Lmod + source $INSTALL_DEP ${{matrix.modules_tool}} $HOME + # changes in environment are not passed to other steps, so need to create files... + echo $MOD_INIT > mod_init + echo $PATH > path + if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi + + - name: test bootstrap script + run: | + # (re)initialize environment for modules tool + if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi + source $(cat $HOME/mod_init); type module + # also pick up changes to $PATH set by sourcing $HOME/mod_init + export PATH=$(cat $HOME/path) + + # define $EASYBUILD_MODULES_TOOL only for oldest module tools + # (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool) + if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then + export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl + elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then + export EASYBUILD_MODULES_TOOL=EnvironmentModulesC + fi + + # version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents + EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') + EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') + EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" + EB_BOOTSTRAP_EXPECTED="20210715.01 0ffdc17ed7eacf78369c9cd6743728f36e61bb8bf5c1bdc1e23cf2040b1ce301" + test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) + + # test bootstrap script + export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap + export EASYBUILD_BOOTSTRAP_DEPRECATED=1 + python easybuild/scripts/bootstrap_eb.py $PREFIX + unset EASYBUILD_BOOTSTRAP_DEPRECATED + # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module + unset PYTHONPATH + # simple sanity check on bootstrapped EasyBuild module + module use $PREFIX/modules/all + module load EasyBuild + eb --version diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml new file mode 100644 index 0000000000..d3bc6e05fb --- /dev/null +++ b/.github/workflows/container_tests.yml @@ -0,0 +1,95 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for container support +on: [push, pull_request] +jobs: + build: + # stick to Ubuntu 18.04, where we can still easily install yum via 'apt-get install' + runs-on: ubuntu-18.04 + strategy: + matrix: + python: [2.7, 3.6] + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: install OS & Python packages + run: | + # ensure package list is up to date to avoid 404's for old packages + sudo apt-get update -yqq + # for building Singularity images + sudo apt-get install rpm + sudo apt-get install yum + # for modules tool + sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi + + - name: install Lmod + run: | + # avoid downloading modules tool sources into easybuild-framework dir + cd $HOME + export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh + # install Lmod + source $INSTALL_DEP Lmod-8.4.27 $HOME + # changes in environment are not passed to other steps, so need to create files... + echo $MOD_INIT > mod_init + echo $PATH > path + if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi + + # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 + - name: install Singularity + run: | + # install alien, which can be used to convert RPMs to Debian packages + sudo apt-get install alien + alien --version + # determine latest version of Singularity available in EPEL, and download RPM + singularity_rpm=$(curl -sL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*singularity/singularity/g' | sed 's/rpm.*/rpm/g') + curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/${singularity_rpm} + # convert Singularity RPM to Debian package, and install it + sudo alien -d ${singularity_rpm} + sudo apt install ./singularity*.deb + singularity --version + + - name: install sources + run: | + # install from source distribution tarball, to test release as published on PyPI + python setup.py sdist + ls dist + export PREFIX=/tmp/$USER/$GITHUB_SHA + pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz + + - name: run test + run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME + # initialize environment for modules tool + if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi + source $(cat $HOME/mod_init); type module + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); + # also pick up changes to $PATH set by sourcing $MOD_INIT + export PREFIX=/tmp/$USER/$GITHUB_SHA + export PATH=$PREFIX/bin:$(cat $HOME/path) + export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH + eb --version + # create $HOME/.rpmmacros, see also https://github.com/apptainer/singularity/issues/241 + echo '%_var /var' > $HOME/.rpmmacros + echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros + # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild; + # see https://docs.easybuild.io/en/latest/Containers.html + curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb + export EASYBUILD_CONTAINERPATH=$PWD + export EASYBUILD_CONTAINER_CONFIG='bootstrap=yum,osversion=7' + eb bzip2-1.0.8.eb --containerize --experimental --container-build-image + singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1) + singularity exec bzip2-1.0.8.sif bzip2 --help diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml new file mode 100644 index 0000000000..d86a49aa0b --- /dev/null +++ b/.github/workflows/eb_command.yml @@ -0,0 +1,88 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for the 'eb' command +on: [push, pull_request] +jobs: + test-eb: + runs-on: ubuntu-18.04 + strategy: + matrix: + python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10'] + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: install OS & Python packages + run: | + # check Python version + python -V + # update to latest pip, check version + pip install --upgrade pip + pip --version + # install packages required for modules tool + sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi + + - name: install modules tool + run: | + # avoid downloading modules tool sources into easybuild-framework dir + cd $HOME + export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh + # install Lmod + source $INSTALL_DEP Lmod-8.4.26 $HOME + # changes in environment are not passed to other steps, so need to create files... + echo $MOD_INIT > mod_init + echo $PATH > path + if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi + + - name: install EasyBuild framework + run: | + # install from source distribution tarball, to test release as published on PyPI + python setup.py sdist + ls dist + export PREFIX=/tmp/$USER/$GITHUB_SHA + pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + + - name: run tests for 'eb' command + env: + EB_VERBOSE: 1 + run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME + # initialize environment for modules tool + if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi + source $(cat $HOME/mod_init); type module + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); + # also pick up changes to $PATH set by sourcing $MOD_INIT + export PREFIX=/tmp/$USER/$GITHUB_SHA + export PATH=$PREFIX/bin:$(cat $HOME/path) + export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH + # run --version, capture (verbose) output + eb --version | tee eb_version.out 2>&1 + # determine active Python version + pymajver=$(python -c 'import sys; print(sys.version_info[0])') + pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))') + # check patterns in verbose output + for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do + echo "Looking for pattern \"${pattern}\" in eb_version.out..." + grep "$pattern" eb_version.out + done + # also check when specifying Python command via $EB_PYTHON + for eb_python in "python${pymajver}" "python${pymajminver}"; do + export EB_PYTHON="${eb_python}" + eb --version | tee eb_version.out 2>&1 + for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do + echo "Looking for pattern \"${pattern}\" in eb_version.out..." + grep "$pattern" eb_version.out + done + done diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 55ada2170c..becc9cc1cd 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -3,13 +3,17 @@ on: [push, pull_request] jobs: python-linting: runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10'] + steps: - uses: actions/checkout@v2 - name: set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} - name: install Python packages run: | @@ -18,4 +22,10 @@ jobs: - name: Run flake8 to verify PEP8-compliance of Python code run: | - flake8 + # don't check py2vs3/py3.py when testing with Python 2, and vice versa + if [[ "${{ matrix.python-version }}" =~ "2." ]]; then + py_excl=py3 + else + py_excl=py2 + fi + flake8 --exclude ./easybuild/tools/py2vs3/${py_excl}.py diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 326810b96d..41ea5139c2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -2,55 +2,78 @@ name: EasyBuild framework unit tests on: [push, pull_request] jobs: + setup: + runs-on: ubuntu-latest + outputs: + lmod7: Lmod-7.8.22 + lmod8: Lmod-8.4.27 + modulesTcl: modules-tcl-1.147 + modules3: modules-3.2.10 + modules4: modules-4.1.4 + steps: + - run: "true" build: + needs: setup runs-on: ubuntu-18.04 strategy: matrix: - python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] - modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] + python: [2.7, 3.6] + modules_tool: + # use variables defined by 'setup' job above, see also + # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context + - ${{needs.setup.outputs.lmod7}} + - ${{needs.setup.outputs.lmod8}} + - ${{needs.setup.outputs.modulesTcl}} + - ${{needs.setup.outputs.modules3}} + - ${{needs.setup.outputs.modules4}} module_syntax: [Lua, Tcl] - # exclude some configuration for non-Lmod modules tool: - # - don't test with Lua module syntax (only supported in Lmod) - # - exclude Python 3.x versions other than 3.6, to limit test configurations + lc_all: [""] + # don't test with Lua module syntax (only supported in Lmod) exclude: - - modules_tool: modules-tcl-1.147 + - modules_tool: ${{needs.setup.outputs.modulesTcl}} + module_syntax: Lua + - modules_tool: ${{needs.setup.outputs.modules3}} + module_syntax: Lua + - modules_tool: ${{needs.setup.outputs.modules4}} + module_syntax: Lua + include: + # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) + - python: 3.5 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + - python: 3.5 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl + - python: 3.7 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + - python: 3.7 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl + - python: 3.8 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua - - modules_tool: modules-3.2.10 + - python: 3.8 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl + - python: 3.9 + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua - - modules_tool: modules-4.1.4 + - python: 3.9 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl + - python: '3.10' + modules_tool: ${{needs.setup.outputs.lmod8}} module_syntax: Lua - - modules_tool: modules-tcl-1.147 - python: 3.5 - - modules_tool: modules-tcl-1.147 - python: 3.7 - - modules_tool: modules-tcl-1.147 - python: 3.8 - - modules_tool: modules-tcl-1.147 - python: 3.9 - - modules_tool: modules-3.2.10 - python: 3.5 - - modules_tool: modules-3.2.10 - python: 3.7 - - modules_tool: modules-3.2.10 - python: 3.8 - - modules_tool: modules-3.2.10 - python: 3.9 - - modules_tool: modules-4.1.4 - python: 3.5 - - modules_tool: modules-4.1.4 - python: 3.7 - - modules_tool: modules-4.1.4 - python: 3.8 - - modules_tool: modules-4.1.4 - python: 3.9 - - modules_tool: Lmod-7.8.22 - python: 3.5 - - modules_tool: Lmod-7.8.22 - python: 3.7 - - modules_tool: Lmod-7.8.22 - python: 3.8 - - modules_tool: Lmod-7.8.22 - python: 3.9 + - python: '3.10' + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Tcl + # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set + # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) + - python: 3.6 + modules_tool: ${{needs.setup.outputs.lmod8}} + module_syntax: Lua + lc_all: C fail-fast: false steps: - uses: actions/checkout@v2 @@ -69,7 +92,10 @@ jobs: # for modules tool sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 - sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi # for GitPython, python-hglib sudo apt-get install git mercurial # dep for GC3Pie @@ -89,12 +115,21 @@ jobs: # see https://github.com//easybuild-framework/settings/secrets GITHUB_TOKEN: ${{secrets.TEST_GITHUB_TOKEN}} run: | - if [ ! -z $GITHUB_TOKEN ]; then - if [ "x${{matrix.python}}" == 'x2.6' ]; + # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, + # and only when testing with Lua as module syntax, + # to avoid hitting GitHub rate limit; + # tests that require a GitHub token are skipped automatically when no GitHub token is available + if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.modules_syntax}}" == 'Lua' ]]; then + if [ ! -z $GITHUB_TOKEN ]; then + if [ "x${{matrix.python}}" == 'x2.6' ]; then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; - else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; - fi; - python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; + fi; + python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + fi + echo "GitHub token installed!" + else + echo "Installation of GitHub token skipped!" fi - name: install modules tool @@ -129,6 +164,7 @@ jobs: EB_VERBOSE: 1 EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} + LC_ALL: ${{matrix.lc_all}} run: | # run tests *outside* of checked out easybuild-framework directory, # to ensure we're testing installed version (see previous step) @@ -164,40 +200,7 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes|Blowfish" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) - - - name: test bootstrap script - run: | - # (re)initialize environment for modules tool - if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi - source $(cat $HOME/mod_init); type module - # also pick up changes to $PATH set by sourcing $HOME/mod_init - export PATH=$(cat $HOME/path) - - # define $EASYBUILD_MODULES_TOOL only for oldest module tools - # (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool) - if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl - elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesC - fi - - # version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) - - # test bootstrap script - export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap - python easybuild/scripts/bootstrap_eb.py $PREFIX - # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module - unset PYTHONPATH - # simple sanity check on bootstrapped EasyBuild module - module use $PREFIX/modules/all - module load EasyBuild - eb --version diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 10cf597707..27fde3d069 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,346 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. + +v4.5.5 (June 8th 2022) +---------------------- + +update/bugfix release + +- various enhancements, including: + - add toolchain definitions for nvompi (NVHPC + OpenMPI) (#3969), nvpsmpi (NVHPC + ParaStationMPI) (#3970), gfbf (GCC, FlexiBLAS, FFTW) (#4006) + - add support for FFTW.MPI toolchain component ($FFT*DIR variables) (#4012) + - add support for customizing EasyBuild command used in jobs via --job-eb-cmd (#4016) +- various bug fixes, including: + - fix copying of easyconfig file with --copy-ec without --rebuild if module is already installed (#3993) + - ignore deprecation warnings regarding cryptography in Python 3.6 + Blowfish with Python 3.10 in test suite output (#3999) + - fix typo in debug log message in easyblock.py (#4000) + - fix printing of list of attempted download URLs when url-encoded characters are used in URL (#4005) + - set $FFT(W)_LIB_DIR to imkl-FFTW's lib path in build environment if usempi toolchain option is enabled (#4011) + - correctly identify Apple Silicon M1 as Arm 64-bit by also considering arm64 next to aarch64 (#4014) + - fix 'eb --show-system-info' on Apple M1 system (#4015) +- other changes: + - change 'eb' command to import easybuild.framework to check if EasyBuild framework is available (#3995, #3998) + + +v4.5.4 (March 31st 2022) +------------------------ + +update/bugfix release + +- various enhancements, including: + - warn about potentially missing patches in --new-pr (#3759, #3966) + - add support for 'clone_into' field in git_config source spec to specify different name for top-level directory (#3949) + - add bash completion for easyconfigs from local dir but not robot search path (#3953) + - add a 'sync pr' message when the PR has a mergeable state but is showing a failed status for the test suite on the last commit (#3967) + - add gmpit toolchain definition (GCC + MPItrampoline) (#3971) + - use 'zypper search -i' to check whether specified OS dependency is installed on openSUSE + make sure that rpm is considered for checking OS dependencies on RHEL8 (#3973) + - add support for post-install patches (#3974) + - add support for 'download_instructions' easyconfig parameter key to specify some download or installation steps for user in case of complicated way of obtaining needed files (#3976, #3981) + - also try collecting AMD GPU info (via rocm-smi) for --show-system-info (#3978, #3982) +- various bug fixes, including: + - ensure --review-pr can find dependencies included in PR (#3979) + - run 'apt-get update' in GitHub Actions workflow for container tests to update container package list before installing anything (#3985) +- other changes: + - enable code linting check for all supported Python versions (#3725) + - update copyright lines for 2022 (#3986) + + +v4.5.3 (February 11th 2022) +--------------------------- + +update/bugfix release + +- various enhancements, including: + - also check for git in --check-eb-deps (#3954) + - add end2end test for 'eb --containerize' (#3958) +- various bug fixes, including: + - take into account that patch files can also be zipped when checking filename extension for patches (#3936) + - initialize BACKUP_PKG_URL with empty string in install_eb_dep.sh script (#3939) + - fix get_os_name and get_os_version to avoid reporting UNKNOWN in output of eb --show-system-info (#3942) + - consistently use actions/setup-python@v2 in CI workflows (#3944) + - switch to using pip3 for installing EasyBuild in container recipes generated by EasyBuild (#3945) + - specify easybuild.io as EasyBuild homepage in setup.py (#3947) + - avoid crash in get_os_version on modern SLES-based OSs (#3955) +- other changes: + - indicate compatibility with Python 3.10 in setup.py (#3940) + + +v4.5.2 (January 24th 2022) +-------------------------- + +update/bugfix release + +- various enhancements, including: + - automatically prepend value for env-for-shebang configuration setting with sysroot (#3919) + - enhance update_build_option to return original value + add support for update_build_options (#3923) + - fix compatibility with Python 3.10 (#3926, #3933) + - also take /etc/os-release into account in get_os_name and get_os_version (#3930) + - enhance get_cpu_architecture and get_cpu_family to be aware of RISC-V (#3931) +- various bug fixes, including: + - relax pattern checks in test_toy_exts_parallel (#3921) + - use sources.easybuild.io as backup URL for downloading Environment Modules sources to run test suite (#3928) + - make intelfftw toolchain component aware of GCCcore (#3929) +- other changes: + - deprecate use of (actual) patch files that don't have a filename ending with .patch (#3920) + - remove version restriction for keyring in requirements.txt to test with latest version (#3925) + - simplify requirements.txt by removing Python 2.6 support (#3927) + - only run GitHub tests when testing with Lua module syntax, to avoid hitting GitHub rate limit when running tests (#3938) + + +v4.5.1 (December 13th 2021) +--------------------------- + +update/bugfix release + +- various enhancements, including: + - also dump environment to reprod directory (#3374) + - determine which extensions can be skipped in parallel (if --parallel-extensions-install is enabled) (#3890) + - fall back to sequential installation for extensions with unknown dependencies when using --parallel-extensions-install (#3906) + - allow oversubscription in sanity check for OpenMPI-based toolchains (#3909) +- various bug fixes, including: + - don't try to ensure absolute path for path part of repositorypath (#3893, #3899) + - fix typo in EULA agreement error message (#3894) + - only remove lock if it was created in the same EasyBuild session (not if it existed already) (#3889) + - introduce EasyBlock.post_init method to correctly define builddir variable when build-in-installdir mode is enabled in easyconfig or easyblock (#3900) + - also show download progress bar when using --inject-checksums (#3905) + - pick up custom extract_cmd specified for extension (#3907) + - make test_run_cmd_async more robust against fluke failures (#3908) + + +v4.5.0 (October 29th 2021) +-------------------------- + +feature release + +- various enhancements, including: + - add --review-pr-max and --review-pr-filter options to limit easyconfigs used by --review-pr + retain order of easyconfigs being compared with (#3754) + - use Rich (if available) to show progress bars when installing easyconfigs (#3823, #3826, #3833, #3835, #3844, #3864, #3867, #3882) + - see also https://docs.easybuild.io/en/latest/Progress_bars.html + - add support for checking required/optional EasyBuild dependencies via 'eb --check-eb-deps' (#3829) + - add support for collecting GPU info (via nvidia-smi), and include it in --show-system-info and test report (#3851) + - added support for --insecure-download configuration option (#3859) + - make intelfftw toolchain component aware of imkl-FFTW module (#3861) + - make sure the contrib/hooks tree is included in the source tarball for easybuild-framework (#3862) + - add check_async_cmd function to facilitate checking on asynchronously running commands (#3865, #3881) + - add initial/experimental support for installing extensions in parallel (#3667, #3878) + - see also https://docs.easybuild.io/en/latest/Installing_extensions_in_parallel.html + - filter out duplicate paths added to module files, and print warning when they occur (#3770, #3874) +- various bug fixes, including: + - ensure that path configuration options have absolute path values (#3832) + - fix broken test by taking into account changed error raised by Python 3.9.7 when copying directory via shutil.copyfile (#3840) + - ensure newer location of CUDA stubs is taken into account by RPATH filter (#3850) + - replace 'which' by 'command -v' in 'eb' wrapper script to avoid dependency on 'which' command (#3852) + - refactor EasyBlock to decouple collecting of information on extension source/patch files from downloading them (#3860) + - in this context, the EasyBlock.fetch_extension_sources method was deprecated, and replaced by EasyBlock.collect_exts_file_info + - fix library paths to add to $LDFLAGS for intel-compilers toolchain component (#3866) + - remove '--depth 1' from git clone when 'commit' specified (#3871, #3872) + - make sure correct include directory is used for FlexiBLAS toolchain component (#3875) + - explictly disable rebase when pulling develop branch to create a merge commit, since not specifying how to reconcile divergent branches is an error with Git 2.33.1 and newer (#3879) + - tweak test_http_header_fields_urlpat to download from sources.easybuild.io rather than https://ftp.gnu.org (#3883) +- other changes: + - change copy_file function to raise an error when trying to copy non-existing file (#3836, #3855, #3877) + - only print the hook messages if EasyBuild is running in debug mode (#3843) + - deprecate old toolchain versions (pre-2019a common toolchains) (#3876, #3884) + - see also https://docs.easybuild.io/en/latest/Deprecated-easyconfigs.html#deprecated-toolchains + + +v4.4.2 (September 7th 2021) +--------------------------- + +update/bugfix release + +- various enhancements, including: + - add per-extension timing in output produced by eb command (#3734) + - add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI + CUDA) (#3736) + - include list of missing libraries in warning about missing FFTW libraries in imkl toolchain component (#3776) + - check for recursive symlinks by default before copying a folder (#3784) + - add --filter-ecs configuration option to filter out easyconfigs from set of easyconfigs to install (#3796) + - check type of source_tmpl value for extensions, ensure it's a string value (not a list) (#3799) + - also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) (#3800) + - report use of --ignore-test-failure in success message in output (#3806) + - add get_cuda_cc_template_value method to EasyConfig class (#3807) + - add support for fix_bash_shebang_for (#3808) + - pick up $MODULES_CMD to facilitate using Environment Modules 4.x as modules tool (#3816) + - use more sensible branch name for creating easyblocks PR with --new-pr (#3817) +- various bug fixes, including: + - remove Python 2.6 from list of supported Python versions in setup.py (#3767) + - don't add directory that doesn't include any files to $PATH or $LD_LIBRARY_PATH (#3769) + - make logdir writable also when --stop/--fetch is used and --read-only-installdir is enabled (#3771) + - fix forgotten renaming of 'l' to 'char' __init__.py that is created for included Python modules (#3773) + - fix verify_imports by deleting all imported modules before re-importing them one by one (#3780) + - fix ignore_test_failure not set for Extension instances (#3782) + - update iompi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iompi 2020.12) (#3785) + - don't parse patch files as easyconfigs when searching for where patch file is used (#3786) + - make sure git clone with a tag argument actually downloads a tag (#3795) + - fix CI by excluding GC3Pie 2.6.7 (which is broken with Python 2) and improve error reporting for option parsing (#3798) + - correctly resolve templates for patches in extensions when uploading to GitHub (#3805) + - add --easystack to ignored options when submitting job (#3813) +- other changes: + - speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802) + - speed up set_parallel method in EasyBlock class (#3812) + + +v4.4.1 (July 6th 2021) +---------------------- + +update/bugfix release + +- various enhancements, including: + - enhance detection of patch files supplied to 'eb' command with better error messages (#3709) + - add per-step timing information (#3716) + - add module-write hook (#3728) + - add ignore-test-failure configuration option to ignore failing test step (#3732) + - add toolchain definition for nvompic (NVHPC + OpenMPI) (#3735) + - warn about generic milestone in --review-pr and --merge-pr (#3751) +- various bug fixes, including: + - don't override COMPILER_MODULE_NAME, inherited from Ffmpi, in Fujitsu toolchain definition (#3721) + - avoid overwritting pr_nr in post_pr_test_report for reports with --include-easyblocks-from-pr (#3724, #3726) + - fix crash in get_config_dict when copying modules that were imported in easyconfig file (like 'import os') (#3729) + - parse C standard flags in CFLAGS for Fujitsu compiler (#3731) + - fix error message for --use-ccache (#3733) + - error out when passing a list to run_cmd* to avoid running wrong command unintended, unless shell=True is used (#3737) + - minor fixes to output of test reports when using multiple PRs (#3741) + - fix location for modules installed with intel-compilers toolchain in HierarchicalMNS by always checking toolchain compiler name against template map (#3745) + - avoid checking msg attribute of GitCommandError (#3756) + - consider sources provided as dict in EasyBlock.check_checksums_for (#3758) + - don't make changes to software installation directory when using --sanity-check-only (#3761) + - honor specified easyblock for extensions (#3762) +- other changes: + - make sure that tests requiring a github token have 'github' in the test name so that they can be easily filtered (#3730) + - deprecate EasyBuild bootstrap script (#3742) + - use temporary download folder in test_http_header_fields_urlpat (#3755) + + +v4.4.0 (June 2nd 2021) +---------------------- + +feature release + +- various enhancements, including: + - enhance apply_regex_substitutions to allow specifying action to take in case there are no matches (#3440) + - performance improvements for easyconfig parsing and eb startup (#3555) + - add support for downloading easyconfigs from multiple PRs with --from-pr (#3605, #3707, #3708) + - add support for prepending custom library paths in RPATH section via --rpath-override-dirs (#3650) + - allow amending easyconfig parameters which are not the default (#3651) + - update HierarchicalMNS for Intel OneAPI compilers (#3653) + - add support for --sanity-check-only (#3655) + - add support for running commands asynchronously via run_cmd + complete_cmd functions (#3656) + - add support for using oneAPI versions of 'intel' toolchain components (#3665) + - add toolchain definition for gofbf (foss with FlexiBLAS rather than OpenBLAS) (#3666) + - add support for using ARCH constant in easyconfig files (#3670) + - honor keyboard interrupt in 'eb' command (#3674) + - add toolchain definition for Fujitsu toolchain (#3677, #3704, #3712, #3713, #3714, #3717) + - extend sanity check step to check whether specific libraries are not linked into installed binaries/libraries (#3678) + - via banned-linked-shared-libs and required-linked-shared-libs EasyBuild configuration options + - via banned_linked_shared_libs and required_linked_shared_libs methods in toolchain support + - via banned_linked_shared_libs and required_linked_shared_libs methods in easyblock + - via banned_linked_shared_libs and required_linked_shared_libs easyconfig parameters + - add locate_solib function to locate Linux shared libraries (#3682) + - add system agnostic function to locate shared libraries (#3683) + - add update_build_option function to update specific build options after initializing the EasyBuild configuration (#3684) + - allow opting out of recursively unloaded of modules via recursive_module_unload easyconfig parameter (#3689) + - check for correct version values when parsing easystack file (#3693) + - run post-install commands specified for a specific extension (#3696) + - add support for --skip-extensions (#3702) + - include PR title in output produced by --merge-pr (#3706) +- various bug fixes, including: + - avoid metadata greedy behaviour when probing for external module metadata (mostly relevant for integration with Cray Programming Environment) (#3559) + - catch problems early on if --github-user is not specified for --new-pr & co (#3644) + - re-enable write permissions when installing with read-only-installdir (#3649) + - also run sanity check for extensions when using --module-only (#3655) + - improve logging when failing to load class from exts_classmap (#3657) + - fix use of --module-only on existing installations without write permissions (#3659) + - correct help text for subdir-user-modules (#3660) + - avoid picking up easyblocks outside of sandbox in framework tests (#3680) + - use unload/load in ModuleGeneratorLua.swap_module, since 'swap' is not supported by Lmod (#3685) + - update HierarchicalMNS to also return 'Toolchain//' as $MODULEPATH extension for cpe* Cray toolchains (#3686) + - make EasyConfigParser.get_config_dict return a copy rather than a reference (#3692) + - make sure that $TAPE is unset when using piped tar (#3698) + - fix extending message for changed files in new_pr_from_branch (#3699) + - enhance sched_getaffinity function to avoid early crash when counting available cores on systems with more than 1024 cores (#3701) + - correctly strip extension from filename in extract_cmd and back_up_file functions (#3705) +- other changes: + - deprecate adding a non-existing path to $MODULEPATH (#3637) + - bump cryptography requirement from 3.2.1 to 3.3.2 (#3643, #3648) + - test bootstrap script in separate workflow, and limit test configurations a bit (#3646) + - update setup.py to indicate compatibility with Python 3.8 and 3.9 (#3647) + - replace log_error parameter of which() by on_error (#3661, #3664) + - don't skip sanity check for --module-only --rebuild (#3645) + - move disable_templating function into the EasyConfig class (#3668) + - pin GitPython version for Python<3.6, don't test bootstrap script against Python 3.8/3.9 (#3675) + - tweak foss toolchain definition to switch from OpenBLAS to FlexiBLAS in foss/2021a (#3679) + - suggest missing SSH key when not able to read from remote repository in --check-github (#3681) + - drop support for Python 2.6 (#3715) + + +v4.3.4 (April 9th 2021) +----------------------- + +update/bugfix release + +- various enhancements, including: + - add support for filtering dependencies by using False as version (#3506) + - add create_unused_dir function to create a directory which does not yet exist (#3551) + - avoid running expensive 'module use' and 'module unuse' commands when using Lmod as modules tool, update $MODULEPATH directly instead (#3557, #3633) + - create CUDA cache (for JIT compiled PTX code) in build dir instead of $HOME (#3569) + - add "Citing" section to module files (#3596) + - add support for using fallback 'arch=*' key in dependency version specified as arch->version mapping (#3600) + - also check for pending change requests and mergeable_state in check_pr_eligible_to_merge (#3604) + - ignore undismissed 'changes requested' review if there is an 'approved' review by the same user (#3607, #3608) + - sort output of 'eb --search' in natural order (respecting numbers) (#3609) + - enhance 'eb' command to ensure that easybuild.main can be imported before settling on python* command to use (#3610) + - add --env-for-shebang configuration option to define the env command to use for shebangs (#3613) + - add templates for architecture independent Python wheels (#3618) + - mention easyblocks PR in gist when uploading test report for it + fix clean_gists.py script (#3622) + - also accept regular expression value for --accept-eula-for (#3630) + - update validate_github_token function to accept GitHub token in new format (#3632) +- various bug fixes, including: + - fix $BLAS_LIB_MT for OpenBLAS, ensure -lpthread is included (#3584) + - use '--opt=val' for passing settings from config file to option parser to avoid error for values starting with '-' or '--' (#3594) + - avoid raised exception when getting output from interactive command in run_cmd_qa (#3599) + - add option to write file from file-like object and use in download_file (#3614) + - make sure that path to eb is always found by tests (#3617) +- other changes: + - add pick_default_branch function to clean up duplicate code in tools/github.py (#3592) + - refactor the CI configuration to use inclusion instead of exclusion (#3616) + - use develop branch when testing push access in --check-github (#3629) + - deprecate --accept-eula, rename to --accept-eula-for (#3630) + + +v4.3.3 (February 23rd 2021) +--------------------------- + +update/bugfix release + +- various enhancements, including: + - advise PR labels in --review-pr and add support for --add-pr-labels (#3177) + - add support for using customized HTTP headers in download_file (#3472, #3583) + - also take toolchain dependencies into account when defining template values (#3541, #3560) + - add support for --accept-eula configuration option + 'accept_eula' easyconfig parameter (#3535, #3536, #3546) + - detect 'SYSTEM' toolchain as special case in easystack files (#3543) + - enhance extract_cmd function to use 'cp -a' for shell scripts (.sh) (#3545) + - allow use of alternate envvar(s) to $HOME for user modules (#3558) + - use https://sources.easybuild.io as fallback source URL (#3572, #3576) + - add toolchain definition for iibff toolchain (#3574) + - add %(cuda_cc_space_sep)s and %(cuda_cc_semicolon_sep)s templates (#3578) + - add support for intel-compiler toolchain (>= 2021.x versions, oneAPI) (#3581, #3582) +- various bug fixes, including: + - add --init and --recursive options to 'git submodule update' command that is used when creating source tarball for specific commit (#3537) + - filter out duplicate paths in RPATH wrapper script (#3538) + - don't clean up imported modules after verifying imports of included Python modules (#3544) + - avoid no-op changes to $LD_* environment variables in ModulesTool (#3553) + - fix UTF-8 encoding errors when running EasyBuild with Python 3.0.x-3.6.x (#3565) + - create lib64 symlink as a relative symlink (#3566) + - don't reuse variable name in the loop to fix adding extra compiler flags via toolchainopts (#3571) + - symlink 'lib' to 'lib64' if it doesn't exist (#3580) + - include %(mpi_cmd_prefix)s and %(cuda_*)s templates in output of --avail-easyconfig-templates (#3586) +- other changes: + - rename EasyBlock._skip_step to EasyBlock.skip_step, to make it part of the public API (#3561) + - make symlinking of posix_c.so to posix.so in test suite configuration conditional (#3570) + - use 'main' rather than 'master' branch in GitHub integration functionality (#3589) + + v4.3.2 (December 10th 2020) --------------------------- diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py index 1e0bbd8fcd..602694e1a7 100644 --- a/contrib/hooks/hpc2n_hooks.py +++ b/contrib/hooks/hpc2n_hooks.py @@ -162,21 +162,19 @@ def pre_module_hook(self, *args, **kwargs): self.log.info("[pre-module hook] Set I_MPI_PMI_LIBRARY in impi module") # Must be done this way, updating self.cfg['modextravars'] # directly doesn't work due to templating. - en_templ = self.cfg.enable_templating - self.cfg.enable_templating = False - shlib_ext = get_shared_lib_ext() - pmix_root = get_software_root('PMIx') - if pmix_root: - mpi_type = 'pmix_v3' - self.cfg['modextravars'].update({ - 'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext) - }) - self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) - # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. - self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) - else: - self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"}) - self.cfg.enable_templating = en_templ + with self.cfg.disable_templating(): + shlib_ext = get_shared_lib_ext() + pmix_root = get_software_root('PMIx') + if pmix_root: + mpi_type = 'pmix_v3' + self.cfg['modextravars'].update({ + 'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext) + }) + self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) + # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. + self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) + else: + self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"}) if self.name == 'OpenBLAS': self.log.info("[pre-module hook] Set OMP_NUM_THREADS=1 in OpenBLAS module") @@ -197,18 +195,14 @@ def pre_module_hook(self, *args, **kwargs): self.log.info("[pre-module hook] Set SLURM_MPI_TYPE=%s in OpenMPI module" % mpi_type) # Must be done this way, updating self.cfg['modextravars'] # directly doesn't work due to templating. - en_templ = self.cfg.enable_templating - self.cfg.enable_templating = False - self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) - # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. - self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) - self.cfg.enable_templating = en_templ + with self.cfg.disable_templating(): + self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) + # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. + self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) if self.name == 'PMIx': # This is a, hopefully, temporary workaround for https://github.com/pmix/pmix/issues/1114 if LooseVersion(self.version) > LooseVersion('2') and LooseVersion(self.version) < LooseVersion('3'): self.log.info("[pre-module hook] Set PMIX_MCA_gds=^ds21 in PMIx module") - en_templ = self.cfg.enable_templating - self.cfg.enable_templating = False - self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'}) - self.cfg.enable_templating = en_templ + with self.cfg.disable_templating(): + self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'}) diff --git a/easybuild/__init__.py b/easybuild/__init__.py index c1b9a77dde..109445bb2f 100644 --- a/easybuild/__init__.py +++ b/easybuild/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/exceptions.py b/easybuild/base/exceptions.py index c7a8690d1b..528b4cb0ce 100644 --- a/easybuild/base/exceptions.py +++ b/easybuild/base/exceptions.py @@ -1,5 +1,5 @@ # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index e95fb0e89f..140346e2a7 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -580,6 +580,8 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_ 'maxBytes': max_bytes, 'backupCount': backup_count, } + if sys.version_info[0] >= 3: + handleropts['encoding'] = 'utf-8' # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed directory = os.path.dirname(filename) if not os.path.exists(directory): diff --git a/easybuild/base/frozendict.py b/easybuild/base/frozendict.py index b7546b0b9d..6bfe91a82b 100644 --- a/easybuild/base/frozendict.py +++ b/easybuild/base/frozendict.py @@ -21,10 +21,10 @@ It can be used as a drop-in replacement for dictionaries where immutability is desired. """ import operator -from collections import Mapping from functools import reduce from easybuild.base import fancylogger +from easybuild.tools.py2vs3 import Mapping # minor adjustments: diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 79fa8a85f8..c587f4fdd0 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,13 +40,30 @@ from functools import reduce from optparse import Option, OptionGroup, OptionParser, OptionValueError, Values from optparse import SUPPRESS_HELP as nohelp # supported in optparse of python v2.4 -from optparse import gettext as _gettext # this is gettext.gettext normally from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels from easybuild.base.optcomplete import autocomplete, CompleterOption from easybuild.tools.py2vs3 import StringIO, configparser, string_type from easybuild.tools.utilities import mk_rst_table, nub, shell_quote +try: + import gettext + eb_translation = None + + def get_translation(): + global eb_translation + if not eb_translation: + # Finding a translation is expensive, so do only once + domain = gettext.textdomain() + eb_translation = gettext.translation(domain, gettext.bindtextdomain(domain), fallback=True) + return eb_translation + + def _gettext(message): + return get_translation().gettext(message) +except ImportError: + def _gettext(message): + return message + HELP_OUTPUT_FORMATS = ['', 'rst', 'short', 'config'] @@ -1376,8 +1393,7 @@ def parseconfigfiles(self): configfile_values[opt_dest] = newval else: configfile_cmdline_dest.append(opt_dest) - configfile_cmdline.append("--%s" % opt_name) - configfile_cmdline.append(val) + configfile_cmdline.append("--%s=%s" % (opt_name, val)) # reparse self.log.debug('parseconfigfiles: going to parse options through cmdline %s' % configfile_cmdline) diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index f0c172d9f3..263e4405cb 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -594,9 +594,8 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete if isinstance(debugfn, logging.Logger): debugfn.debug(txt) else: - f = open(debugfn, 'a') - f.write(txt) - f.close() + with open(debugfn, 'a') as fh: + fh.write(txt) # Exit with error code (we do not let the caller continue on purpose, this # is a run for completions only.) diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index 0df4c537ad..7f44c318cc 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -1,5 +1,5 @@ # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,6 +35,7 @@ import pprint import re import sys +from contextlib import contextmanager try: from cStringIO import StringIO # Python 2 @@ -185,6 +186,26 @@ def get_stderr(self): """Return output captured from stderr until now.""" return sys.stderr.getvalue() + @contextmanager + def mocked_stdout_stderr(self, mock_stdout=True, mock_stderr=True): + """Context manager to mock stdout and stderr""" + if mock_stdout: + self.mock_stdout(True) + if mock_stderr: + self.mock_stderr(True) + try: + if mock_stdout and mock_stderr: + yield sys.stdout, sys.stderr + elif mock_stdout: + yield sys.stdout + else: + yield sys.stderr + finally: + if mock_stdout: + self.mock_stdout(False) + if mock_stderr: + self.mock_stderr(False) + def tearDown(self): """Cleanup after running a test.""" self.mock_stdout(False) diff --git a/easybuild/framework/__init__.py b/easybuild/framework/__init__.py index c9304064b4..b2f628b039 100644 --- a/easybuild/framework/__init__.py +++ b/easybuild/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index acc2b64bfc..46764b1626 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -52,6 +52,7 @@ from distutils.version import LooseVersion import easybuild.tools.environment as env +import easybuild.tools.toolchain as toolchain from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import ITERATE_OPTIONS, EasyConfig, ActiveMNS, get_easyblock_class @@ -59,50 +60,58 @@ from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH -from easybuild.framework.easyconfig.tools import get_paths_for +from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict -from easybuild.framework.extension import resolve_exts_filter_template +from easybuild.framework.extension import Extension, resolve_exts_filter_template from easybuild.tools import config, run from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning +from easybuild.tools.config import DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 -from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name -from easybuild.tools.filetools import compute_checksum, copy_file, check_lock, create_lock, derive_alt_pypi_url -from easybuild.tools.filetools import diff_files, dir_contains_files, download_file, encode_class_name, extract_file +from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock +from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info +from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file +from easybuild.tools.filetools import encode_class_name, extract_file from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP -from easybuild.tools.hooks import load_hooks, run_hook -from easybuild.tools.run import run_cmd +from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook +from easybuild.tools.run import check_async_cmd, run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name +from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS +from easybuild.tools.output import show_progress_bars, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository -from easybuild.tools.systemtools import det_parallelism, use_group -from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, quote_str +from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_shared_lib_ext, use_group +from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION +EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' + +DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') + MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] # string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url) PYPI_PKG_URL_PATTERN = 'pypi.python.org/packages/source/' -# Directory name in which to store reproducability files +# Directory name in which to store reproducibility files REPROD = 'reprod' _log = fancylogger.getLogger('easyblock') @@ -139,7 +148,7 @@ def __init__(self, ec): # keep track of original working directory, so we can go back there self.orig_workdir = os.getcwd() - # list of pre- and post-step hooks + # dict of all hooks (mapping of name to function) self.hooks = load_hooks(build_option('hooks')) # list of patch/source files, along with checksums @@ -207,6 +216,10 @@ def __init__(self, ec): self.postmsg = '' # allow a post message to be set, which can be shown as last output self.current_step = None + # Create empty progress bar + self.progress_bar = None + self.pbar_task = None + # list of loaded modules self.loaded_modules = [] @@ -264,6 +277,16 @@ def __init__(self, ec): self.log.info("Init completed for application name %s version %s" % (self.name, self.version)) + def post_init(self): + """ + Run post-initialization tasks. + """ + if self.build_in_installdir: + # self.builddir is set by self.gen_builddir(), + # but needs to be correct if the build is performed in the installation directory + self.log.info("Changing build dir to %s", self.installdir) + self.builddir = self.installdir + # INIT/CLOSE LOG def _init_log(self): """ @@ -341,14 +364,14 @@ def get_checksum_for(self, checksums, filename=None, index=None): else: raise EasyBuildError("Invalid type for checksums (%s), should be list, tuple or None.", type(checksums)) - def fetch_source(self, source, checksum=None, extension=False): + def fetch_source(self, source, checksum=None, extension=False, download_instructions=None): """ Get a specific source (tarball, iso, url) Will be tested for existence or can be located :param source: source to be found (single dictionary in 'sources' list, or filename) :param checksum: checksum corresponding to source - :param extension: flag if being called from fetch_extension_sources() + :param extension: flag if being called from collect_exts_file_info() """ filename, download_filename, extract_cmd, source_urls, git_config = None, None, None, None, None @@ -377,7 +400,8 @@ def fetch_source(self, source, checksum=None, extension=False): # check if the sources can be located force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] path = self.obtain_file(filename, extension=extension, download_filename=download_filename, - force_download=force_download, urls=source_urls, git_config=git_config) + force_download=force_download, urls=source_urls, git_config=git_config, + download_instructions=download_instructions) if path is None: raise EasyBuildError('No file found for source %s', filename) @@ -431,58 +455,29 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): Add a list of patches. All patches will be checked if a file exists (or can be located) """ + post_install_patches = [] if patch_specs is None: - patch_specs = self.cfg['patches'] + # if no patch_specs are specified, use all pre-install and post-install patches + post_install_patches = self.cfg['postinstallpatches'] + patch_specs = self.cfg['patches'] + post_install_patches patches = [] for index, patch_spec in enumerate(patch_specs): - # check if the patches can be located - copy_file = False - suff = None - level = None - if isinstance(patch_spec, (list, tuple)): - if not len(patch_spec) == 2: - raise EasyBuildError("Unknown patch specification '%s', only 2-element lists/tuples are supported!", - str(patch_spec)) - patch_file = patch_spec[0] - - # this *must* be of typ int, nothing else - # no 'isinstance(..., int)', since that would make True/False also acceptable - if isinstance(patch_spec[1], int): - level = patch_spec[1] - elif isinstance(patch_spec[1], string_type): - # non-patch files are assumed to be files to copy - if not patch_spec[0].endswith('.patch'): - copy_file = True - suff = patch_spec[1] - else: - raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element", - str(patch_spec)) - else: - patch_file = patch_spec + patch_info = create_patch_info(patch_spec) + patch_info['postinstall'] = patch_spec in post_install_patches force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES] - path = self.obtain_file(patch_file, extension=extension, force_download=force_download) + path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download) if path: - self.log.debug('File %s found for patch %s' % (path, patch_spec)) - patchspec = { - 'name': patch_file, - 'path': path, - 'checksum': self.get_checksum_for(checksums, index=index), - } - if suff: - if copy_file: - patchspec['copy'] = suff - else: - patchspec['sourcepath'] = suff - if level is not None: - patchspec['level'] = level + self.log.debug('File %s found for patch %s', path, patch_spec) + patch_info['path'] = path + patch_info['checksum'] = self.get_checksum_for(checksums, index=index) if extension: - patches.append(patchspec) + patches.append(patch_info) else: - self.patches.append(patchspec) + self.patches.append(patch_info) else: raise EasyBuildError('No file found for patch %s', patch_spec) @@ -490,25 +485,38 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): self.log.info("Fetched extension patches: %s", patches) return patches else: - self.log.info("Added patches: %s" % self.patches) + self.log.info("Added patches: %s", self.patches) def fetch_extension_sources(self, skip_checksums=False): """ - Find source file for extensions. + Fetch source and patch files for extensions (DEPRECATED, use collect_exts_file_info instead). + """ + depr_msg = "EasyBlock.fetch_extension_sources is deprecated, use EasyBlock.collect_exts_file_info instead" + self.log.deprecated(depr_msg, '5.0') + return self.collect_exts_file_info(fetch_files=True, verify_checksums=not skip_checksums) + + def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): + """ + Collect information on source and patch files for extensions. + + :param fetch_files: whether or not to fetch files (if False, path to files will be missing from info) + :param verify_checksums: whether or not to verify checksums + :return: list of dict values, one per extension, with information on source/patch files. """ exts_sources = [] exts_list = self.cfg.get_ref('exts_list') + if verify_checksums and not fetch_files: + raise EasyBuildError("Can't verify checksums for extension files if they are not being fetched") + if self.dry_run: self.dry_run_msg("\nList of sources/patches for extensions:") force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES] for ext in exts_list: - if (isinstance(ext, list) or isinstance(ext, tuple)) and ext: - + if isinstance(ext, (list, tuple)) and ext: # expected format: (name, version, options (dict)) - ext_name = ext[0] if len(ext) == 1: exts_sources.append({'name': ext_name}) @@ -533,6 +541,10 @@ def fetch_extension_sources(self, skip_checksums=False): 'options': ext_options, } + # if a particular easyblock is specified, make sure it's used + # (this is picked up by init_ext_instances) + ext_src['easyblock'] = ext_options.get('easyblock', None) + # construct dictionary with template values; # inherited from parent, except for name/version templates which are specific to this extension template_values = copy.deepcopy(self.cfg.template_values) @@ -544,6 +556,8 @@ def fetch_extension_sources(self, skip_checksums=False): source_urls = ext_options.get('source_urls', []) checksums = ext_options.get('checksums', []) + download_instructions = ext_options.get('download_instructions') + if ext_options.get('nosource', None): self.log.debug("No sources for extension %s, as indicated by 'nosource'", ext_name) @@ -574,26 +588,39 @@ def fetch_extension_sources(self, skip_checksums=False): if 'source_urls' not in source: source['source_urls'] = source_urls - src = self.fetch_source(source, checksums, extension=True) - - # copy 'path' entry to 'src' for use with extensions - ext_src.update({'src': src['path']}) + if fetch_files: + src = self.fetch_source(source, checksums, extension=True, + download_instructions=download_instructions) + ext_src.update({ + # keep track of custom extract command (if any) + 'extract_cmd': src['cmd'], + # copy 'path' entry to 'src' for use with extensions + 'src': src['path'], + }) else: # use default template for name of source file if none is specified default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values) # if no sources are specified via 'sources', fall back to 'source_tmpl' - src_fn = ext_options.get('source_tmpl', default_source_tmpl) - src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, - force_download=force_download) - if src_path: - ext_src.update({'src': src_path}) - else: - raise EasyBuildError("Source for extension %s not found.", ext) + src_fn = ext_options.get('source_tmpl') + if src_fn is None: + src_fn = default_source_tmpl + elif not isinstance(src_fn, string_type): + error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s" + raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn) + + if fetch_files: + src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, + force_download=force_download, + download_instructions=download_instructions) + if src_path: + ext_src.update({'src': src_path}) + else: + raise EasyBuildError("Source for extension %s not found.", ext) # verify checksum for extension sources - if 'src' in ext_src and not skip_checksums: + if verify_checksums and 'src' in ext_src: src_path = ext_src['src'] src_fn = os.path.basename(src_path) @@ -613,12 +640,17 @@ def fetch_extension_sources(self, skip_checksums=False): raise EasyBuildError('Checksum verification for extension source %s failed', src_fn) # locate extension patches (if any), and verify checksums - ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True) + ext_patches = ext_options.get('patches', []) + if fetch_files: + ext_patches = self.fetch_patches(patch_specs=ext_patches, extension=True) + else: + ext_patches = [create_patch_info(p) for p in ext_patches] + if ext_patches: self.log.debug('Found patches for extension %s: %s', ext_name, ext_patches) ext_src.update({'patches': ext_patches}) - if not skip_checksums: + if verify_checksums: for patch in ext_patches: patch = patch['path'] # report both MD5 and SHA256 checksums, @@ -655,7 +687,7 @@ def fetch_extension_sources(self, skip_checksums=False): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - git_config=None): + git_config=None, download_instructions=None): """ Locate the file with the given name - searches in different subdirectories of source path @@ -669,6 +701,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No """ srcpaths = source_paths() + update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename) + # should we download or just try and find it? if re.match(r"^(https?|ftp)://", filename): # URL detected, so let's try and download it @@ -756,7 +790,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No break # no need to try other source paths - targetdir = os.path.join(srcpaths[0], self.name.lower()[0], self.name) + name_letter = self.name.lower()[0] + targetdir = os.path.join(srcpaths[0], name_letter, self.name) if foundfile: if self.dry_run: @@ -772,6 +807,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) + # add https://sources.easybuild.io as fallback source URL + source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, self.name)) + mkdir(targetdir, parents=True) for url in source_urls: @@ -836,8 +874,21 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + error_msg = "Couldn't find file %s anywhere, " + if download_instructions is None: + download_instructions = self.cfg['download_instructions'] + if download_instructions is not None and download_instructions != "": + msg = "\nDownload instructions:\n\n" + download_instructions + '\n' + print_msg(msg, prefix=False, stderr=True) + error_msg += "please follow the download instructions above, and make the file available " + error_msg += "in the active source path (%s)" % ':'.join(source_paths()) + else: + # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf') + failedpaths_msg = ', '.join(failedpaths).replace('%', '%%') + error_msg += "and downloading it didn't work either... " + error_msg += "Paths attempted (in order): %s " % failedpaths_msg + + raise EasyBuildError(error_msg, filename) # # GETTER/SETTER UTILITY FUNCTIONS @@ -927,9 +978,6 @@ def make_builddir(self): raise EasyBuildError("self.builddir not set, make sure gen_builddir() is called first!") self.log.debug("Creating the build directory %s (cleanup: %s)", self.builddir, self.cfg['cleanupoldbuild']) else: - self.log.info("Changing build dir to %s" % self.installdir) - self.builddir = self.installdir - self.log.info("Overriding 'cleanupoldinstall' (to False), 'cleanupoldbuild' (to True) " "and 'keeppreviousinstall' because we're building in the installation directory.") # force cleanup before installation @@ -1027,6 +1075,27 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): mkdir(dir_name, parents=True) + def set_up_cuda_cache(self): + """Set up CUDA PTX cache.""" + + cuda_cache_maxsize = build_option('cuda_cache_maxsize') + if cuda_cache_maxsize is None: + cuda_cache_maxsize = 1 * 1024 # 1 GiB default value + else: + cuda_cache_maxsize = int(cuda_cache_maxsize) + + if cuda_cache_maxsize == 0: + self.log.info("Disabling CUDA PTX cache since cache size was set to zero") + env.setvar('CUDA_CACHE_DISABLE', '1') + else: + cuda_cache_dir = build_option('cuda_cache_dir') + if not cuda_cache_dir: + cuda_cache_dir = os.path.join(self.builddir, 'eb-cuda-cache') + self.log.info("Enabling CUDA PTX cache of size %s MiB at %s", cuda_cache_maxsize, cuda_cache_dir) + env.setvar('CUDA_CACHE_DISABLE', '0') + env.setvar('CUDA_CACHE_PATH', cuda_cache_dir) + env.setvar('CUDA_CACHE_MAXSIZE', str(cuda_cache_maxsize * 1024 * 1024)) + # # MODULE UTILITY FUNCTIONS # @@ -1345,10 +1414,16 @@ def make_module_extend_modpath(self): # add user-specific module path; use statement will be guarded so no need to create the directories user_modpath = build_option('subdir_user_modules') if user_modpath: + user_envvars = build_option('envvars_user_modules') or [DEFAULT_ENVVAR_USERS_MODULES] user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) - txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd('HOME'), - guarded=True, user_modpath=user_modpath) + for user_envvar in user_envvars: + self.log.debug("Requested environment variable $%s to host additional branch for modules", + user_envvar) + default_value = user_envvar + "_NOT_DEFINED" + getenv_txt = self.module_generator.getenv_cmd(user_envvar, default=default_value) + txt += self.module_generator.use(user_modpath_exts, prefix=getenv_txt, + guarded=True, user_modpath=user_modpath) else: self.log.debug("Not including module path extensions, as specified.") return txt @@ -1387,9 +1462,17 @@ def make_module_req(self): note += "for paths are skipped for the statements below due to dry run" lines.append(self.module_generator.comment(note)) - # for these environment variables, the corresponding subdirectory must include at least one file - keys_requiring_files = set(('PATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'CPATH', - 'CMAKE_PREFIX_PATH', 'CMAKE_LIBRARY_PATH')) + # For these environment variables, the corresponding directory must include at least one file. + # The values determine if detection is done recursively, i.e. if it accepts directories where files + # are only in subdirectories. + keys_requiring_files = { + 'PATH': False, + 'LD_LIBRARY_PATH': False, + 'LIBRARY_PATH': True, + 'CPATH': True, + 'CMAKE_PREFIX_PATH': True, + 'CMAKE_LIBRARY_PATH': True, + } for key, reqs in sorted(requirements.items()): if isinstance(reqs, string_type): @@ -1419,20 +1502,16 @@ def make_module_req(self): if fixed_paths != paths: self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", key, paths, fixed_paths) paths = fixed_paths - # remove duplicate paths - # don't use 'set' here, since order in 'paths' is important! - uniq_paths = [] - for path in paths: - if path not in uniq_paths: - uniq_paths.append(path) - paths = uniq_paths + # remove duplicate paths preserving order + paths = nub(paths) if key in keys_requiring_files: # only retain paths that contain at least one file - retained_paths = [ - path for path in paths - if os.path.isdir(os.path.join(self.installdir, path)) - and dir_contains_files(os.path.join(self.installdir, path)) - ] + recursive = keys_requiring_files[key] + retained_paths = [] + for pth in paths: + fullpath = os.path.join(self.installdir, pth) + if os.path.isdir(fullpath) and dir_contains_files(fullpath, recursive=recursive): + retained_paths.append(pth) if retained_paths != paths: self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s", key, paths, retained_paths) @@ -1468,13 +1547,14 @@ def make_module_req_guess(self): 'CMAKE_LIBRARY_PATH': ['lib64'], # lib and lib32 are searched through the above } - def load_module(self, mod_paths=None, purge=True, extra_modules=None): + def load_module(self, mod_paths=None, purge=True, extra_modules=None, verbose=True): """ Load module for this software package/version, after purging all currently loaded modules. :param mod_paths: list of (additional) module paths to take into account :param purge: boolean indicating whether or not to purge currently loaded modules first :param extra_modules: list of extra modules to load (these are loaded *before* loading the 'self' module) + :param verbose: print modules being loaded when trace mode is enabled """ # self.full_mod_name might not be set (e.g. during unit tests) if self.full_mod_name is not None: @@ -1496,6 +1576,9 @@ def load_module(self, mod_paths=None, purge=True, extra_modules=None): if self.mod_subdir and not self.toolchain.is_system_toolchain(): mods.insert(0, self.toolchain.det_short_module_name()) + if verbose: + trace_msg("loading modules: %s..." % ', '.join(mods)) + # pass initial environment, to use it for resetting the environment before loading the modules self.modules_tool.load(mods, mod_paths=all_mod_paths, purge=purge, init_env=self.initial_environ) @@ -1507,7 +1590,7 @@ def load_module(self, mod_paths=None, purge=True, extra_modules=None): else: self.log.warning("Not loading module, since self.full_mod_name is not set.") - def load_fake_module(self, purge=False, extra_modules=None): + def load_fake_module(self, purge=False, extra_modules=None, verbose=False): """ Create and load fake module. @@ -1522,7 +1605,7 @@ def load_fake_module(self, purge=False, extra_modules=None): # load fake module self.modules_tool.prepend_module_path(os.path.join(fake_mod_path, self.mod_subdir), priority=10000) - self.load_module(purge=purge, extra_modules=extra_modules) + self.load_module(purge=purge, extra_modules=extra_modules, verbose=verbose) return (fake_mod_path, env) @@ -1562,29 +1645,313 @@ def prepare_for_extensions(self): def skip_extensions(self): """ - Called when self.skip is True - - use this to detect existing extensions and to remove them from self.ext_instances - - based on initial R version + Skip already installed extensions, + by removing them from list of Extension instances to install (self.ext_instances). + + This is done in parallel when EasyBuild is configured to install extensions in parallel. """ + self.update_exts_progress_bar("skipping installed extensions") + # obtaining untemplated reference value is required here to support legacy string templates like name/version exts_filter = self.cfg.get_ref('exts_filter') if not exts_filter or len(exts_filter) == 0: raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") + if build_option('parallel_extensions_install'): + self.skip_extensions_parallel(exts_filter) + else: + self.skip_extensions_sequential(exts_filter) + + def skip_extensions_sequential(self, exts_filter): + """ + Skip already installed extensions (checking sequentially), + by removing them from list of Extension instances to install (self.ext_instances). + """ + print_msg("skipping installed extensions (sequentially)", log=self.log) + + exts_cnt = len(self.ext_instances) + res = [] - for ext_inst in self.ext_instances: + for idx, ext_inst in enumerate(self.ext_instances): cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) - (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) - self.log.info("exts_filter result %s %s", cmdstdouterr, ec) - if ec: + (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, + regexp=False, trace=False) + self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_inst.name, ec, out) + if ec == 0: + print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + else: self.log.info("Not skipping %s", ext_inst.name) - self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) res.append(ext_inst) - else: - print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + + self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt)) self.ext_instances = res + self.update_exts_progress_bar("already installed extensions filtered out", total=len(self.ext_instances)) + + def skip_extensions_parallel(self, exts_filter): + """ + Skip already installed extensions (checking in parallel), + by removing them from list of Extension instances to install (self.ext_instances). + """ + self.log.experimental("Skipping installed extensions in parallel") + print_msg("skipping installed extensions (in parallel)", log=self.log) + + async_cmd_info_cache = {} + running_checks_ids = [] + installed_exts_ids = [] + exts_queue = list(enumerate(self.ext_instances[:])) + checked_exts_cnt = 0 + exts_cnt = len(self.ext_instances) + + # asynchronously run checks to see whether extensions are already installed + while exts_queue or running_checks_ids: + + # first handle completed checks + for idx in running_checks_ids[:]: + ext_name = self.ext_instances[idx].name + # don't read any output, just check whether command completed + async_cmd_info = check_async_cmd(*async_cmd_info_cache[idx], output_read_size=0, fail_on_error=False) + if async_cmd_info['done']: + out, ec = async_cmd_info['output'], async_cmd_info['exit_code'] + self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_name, ec, out) + running_checks_ids.remove(idx) + if ec == 0: + print_msg("skipping extension %s" % ext_name, log=self.log) + installed_exts_ids.append(idx) + + checked_exts_cnt += 1 + exts_pbar_label = "skipping installed extensions " + exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) + self.update_exts_progress_bar(exts_pbar_label) + + # start additional checks asynchronously + while exts_queue and len(running_checks_ids) < self.cfg['parallel']: + idx, ext = exts_queue.pop(0) + cmd, stdin = resolve_exts_filter_template(exts_filter, ext) + async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, + regexp=False, trace=False, asynchronous=True) + running_checks_ids.append(idx) + + # compose new list of extensions, skip over the ones that are already installed; + # note: original order in extensions list should be preserved! + retained_ext_instances = [] + for idx, ext in enumerate(self.ext_instances): + if idx not in installed_exts_ids: + retained_ext_instances.append(ext) + self.log.info("Not skipping %s", ext.name) + + self.ext_instances = retained_ext_instances + + def install_extensions(self, install=True): + """ + Install extensions. + + :param install: actually install extensions, don't just prepare environment for installing + + """ + self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + + if build_option('parallel_extensions_install'): + self.log.experimental("installing extensions in parallel") + self.install_extensions_parallel(install=install) + else: + self.install_extensions_sequential(install=install) + + def install_extensions_sequential(self, install=True): + """ + Install extensions sequentially. + + :param install: actually install extensions, don't just prepare environment for installing + """ + self.log.info("Installing extensions sequentially...") + + exts_cnt = len(self.ext_instances) + + for idx, ext in enumerate(self.ext_instances): + + self.log.info("Starting extension %s", ext.name) + + # always go back to original work dir to avoid running stuff from a dir that no longer exists + change_dir(self.orig_workdir) + + progress_info = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt) + self.update_exts_progress_bar(progress_info) + + tup = (ext.name, ext.version or '', idx + 1, exts_cnt) + print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, log=self.log) + start_time = datetime.now() + + if self.dry_run: + tup = (ext.name, ext.version, ext.__class__.__name__) + msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup + self.dry_run_msg(msg) + + self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + + # prepare toolchain build environment, but only when not doing a dry run + # since in that case the build environment is the same as for the parent + if self.dry_run: + self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...") + else: + # don't reload modules for toolchain, there is no need since they will be loaded already; + # the (fake) module for the parent software gets loaded before installing extensions + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs) + + # real work + if install: + try: + ext.prerun() + with self.module_generator.start_module_creation(): + txt = ext.run() + if txt: + self.module_extra_extensions += txt + ext.postrun() + finally: + if not self.dry_run: + ext_duration = datetime.now() - start_time + if ext_duration.total_seconds() >= 1: + print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) + elif self.logdebug or build_option('trace'): + print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) + + self.update_exts_progress_bar(progress_info, progress_size=1) + + def install_extensions_parallel(self, install=True): + """ + Install extensions in parallel. + + :param install: actually install extensions, don't just prepare environment for installing + """ + self.log.info("Installing extensions in parallel...") + + running_exts = [] + installed_ext_names = [] + + all_ext_names = [x['name'] for x in self.exts_all] + self.log.debug("List of names of all extensions: %s", all_ext_names) + + # take into account that some extensions may be installed already + to_install_ext_names = [x.name for x in self.ext_instances] + installed_ext_names = [n for n in all_ext_names if n not in to_install_ext_names] + + exts_cnt = len(all_ext_names) + exts_queue = self.ext_instances[:] + + def update_exts_progress_bar_helper(running_exts, progress_size): + """Helper function to update extensions progress bar.""" + running_exts_cnt = len(running_exts) + if running_exts_cnt > 1: + progress_info = "Installing %d extensions" % running_exts_cnt + elif running_exts_cnt == 1: + progress_info = "Installing extension " + else: + progress_info = "Not installing extensions (yet)" + + if running_exts_cnt: + progress_info += " (%d/%d done): " % (len(installed_ext_names), exts_cnt) + progress_info += ', '.join(e.name for e in running_exts) + + self.update_exts_progress_bar(progress_info, progress_size=progress_size) + + while exts_queue or running_exts: + + # always go back to original work dir to avoid running stuff from a dir that no longer exists + change_dir(self.orig_workdir) + + # check for extension installations that have completed + if running_exts: + self.log.info("Checking for completed extension installations (%d running)...", len(running_exts)) + for ext in running_exts[:]: + if self.dry_run or ext.async_cmd_check(): + self.log.info("Installation of %s completed!", ext.name) + ext.postrun() + running_exts.remove(ext) + installed_ext_names.append(ext.name) + update_exts_progress_bar_helper(running_exts, 1) + else: + self.log.debug("Installation of %s is still running...", ext.name) + + # try to start as many extension installations as we can, taking into account number of available cores, + # but only consider first 100 extensions still in the queue + max_iter = min(100, len(exts_queue)) + + for _ in range(max_iter): + + if not (exts_queue and len(running_exts) < self.cfg['parallel']): + break + + # check whether extension at top of the queue is ready to install + ext = exts_queue.pop(0) + + required_deps = ext.required_deps + if required_deps is None: + pending_deps = None + self.log.info("Required dependencies for %s are unknown!", ext.name) + else: + self.log.info("Required dependencies for %s: %s", ext.name, ', '.join(required_deps)) + pending_deps = [x for x in required_deps if x not in installed_ext_names] + self.log.info("Missing required dependencies for %s: %s", ext.name, ', '.join(pending_deps)) + + # if required dependencies could not be determined, wait until all preceding extensions are installed + if pending_deps is None: + if running_exts: + # add extension back at top of the queue, + # since we need to preverse installation order of extensions; + # break out of for loop since there is no point to keep checking + # until running installations have been completed + exts_queue.insert(0, ext) + break + else: + pending_deps = [] + + if self.dry_run: + tup = (ext.name, ext.version, ext.__class__.__name__) + msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup + self.dry_run_msg(msg) + running_exts.append(ext) + + # if some of the required dependencies are not installed yet, requeue this extension + elif pending_deps: + + # make sure all required dependencies are actually going to be installed, + # to avoid getting stuck in an infinite loop! + missing_deps = [x for x in required_deps if x not in all_ext_names] + if missing_deps: + raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s", + ext.name, ', '.join(missing_deps)) + else: + self.log.info("Required dependencies missing for extension %s (%s), adding it back to queue...", + ext.name, ', '.join(pending_deps)) + # purposely adding extension back in the queue at Nth place rather than at the end, + # since we assume that the required dependencies will be installed soon... + exts_queue.insert(max_iter, ext) + + else: + tup = (ext.name, ext.version or '') + print_msg("starting installation of extension %s %s..." % tup, silent=self.silent, log=self.log) + + # don't reload modules for toolchain, there is no need since they will be loaded already; + # the (fake) module for the parent software gets loaded before installing extensions + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs) + if install: + ext.prerun() + ext.run_async() + running_exts.append(ext) + self.log.info("Started installation of extension %s in the background...", ext.name) + update_exts_progress_bar_helper(running_exts, 0) + + # print progress info after every iteration (unless that info is already shown via progress bar) + if not show_progress_bars(): + msg = "%d out of %d extensions installed (%d queued, %d running: %s)" + installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts) + if running_cnt <= 3: + running_ext_names = ', '.join(x.name for x in running_exts) + else: + running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." + print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) # # MISCELLANEOUS UTILITY FUNCTIONS @@ -1642,19 +2009,19 @@ def check_accepted_eula(self, name=None, more_info=None): if name is None: name = self.name - accepted_eulas = build_option('accept_eula') or [] - if self.cfg['accept_eula'] or name in accepted_eulas: + accepted_eulas = build_option('accept_eula_for') or [] + if self.cfg['accept_eula'] or name in accepted_eulas or any(re.match(x, name) for x in accepted_eulas): self.log.info("EULA for %s is accepted", name) else: error_lines = [ - "The End User License Argreement (EULA) for %(name)s is currently not accepted!", + "The End User License Agreement (EULA) for %(name)s is currently not accepted!", ] if more_info: error_lines.append("(see %s for more information)" % more_info) error_lines.extend([ "You should either:", - "- add --accept-eula=%(name)s to the 'eb' command;", + "- add --accept-eula-for=%(name)s to the 'eb' command;", "- update your EasyBuild configuration to always accept the EULA for %(name)s;", "- add 'accept_eula = True' to the easyconfig file you are using;", '', @@ -1670,59 +2037,50 @@ def handle_iterate_opts(self): self.iter_idx += 1 # disable templating in this function, since we're messing about with values in self.cfg - prev_enable_templating = self.cfg.enable_templating - self.cfg.enable_templating = False - - # start iterative mode (only need to do this once) - if self.iter_idx == 0: - self.cfg.start_iterating() - - # handle configure/build/install options that are specified as lists (+ perhaps builddependencies) - # set first element to be used, keep track of list in self.iter_opts - # only needs to be done during first iteration, since after that the options won't be lists anymore - if self.iter_idx == 0: - # keep track of list, supply first element as first option to handle - for opt in self.cfg.iterate_options: - self.iter_opts[opt] = self.cfg[opt] # copy - self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) - - if self.iter_opts: - print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent) - self.log.info("Current iteration index: %s", self.iter_idx) - - # pop first element from all iterative easyconfig parameters as next value to use - for opt in self.iter_opts: - if len(self.iter_opts[opt]) > self.iter_idx: - self.cfg[opt] = self.iter_opts[opt][self.iter_idx] - else: - self.cfg[opt] = '' # empty list => empty option as next value - self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) - - # re-generate template values, which may be affected by changed parameters we're iterating over - self.cfg.generate_template_values() + with self.cfg.disable_templating(): + + # start iterative mode (only need to do this once) + if self.iter_idx == 0: + self.cfg.start_iterating() + + # handle configure/build/install options that are specified as lists (+ perhaps builddependencies) + # set first element to be used, keep track of list in self.iter_opts + # only needs to be done during first iteration, since after that the options won't be lists anymore + if self.iter_idx == 0: + # keep track of list, supply first element as first option to handle + for opt in self.cfg.iterate_options: + self.iter_opts[opt] = self.cfg[opt] # copy + self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) + + if self.iter_opts: + print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent) + self.log.info("Current iteration index: %s", self.iter_idx) + + # pop first element from all iterative easyconfig parameters as next value to use + for opt in self.iter_opts: + if len(self.iter_opts[opt]) > self.iter_idx: + self.cfg[opt] = self.iter_opts[opt][self.iter_idx] + else: + self.cfg[opt] = '' # empty list => empty option as next value + self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) - # re-enable templating before self.cfg values are used - self.cfg.enable_templating = prev_enable_templating + # re-generate template values, which may be affected by changed parameters we're iterating over + self.cfg.generate_template_values() def post_iter_step(self): """Restore options that were iterated over""" # disable templating, since we're messing about with values in self.cfg - prev_enable_templating = self.cfg.enable_templating - self.cfg.enable_templating = False + with self.cfg.disable_templating(): + for opt in self.iter_opts: + self.cfg[opt] = self.iter_opts[opt] - for opt in self.iter_opts: - self.cfg[opt] = self.iter_opts[opt] + # also need to take into account extensions, since those were iterated over as well + for ext in self.ext_instances: + ext.cfg[opt] = self.iter_opts[opt] - # also need to take into account extensions, since those were iterated over as well - for ext in self.ext_instances: - ext.cfg[opt] = self.iter_opts[opt] + self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) - self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) - - self.cfg.stop_iterating() - - # re-enable templating before self.cfg values are used - self.cfg.enable_templating = prev_enable_templating + self.cfg.stop_iterating() def det_iter_cnt(self): """Determine iteration count based on configure/build/install options that may be lists.""" @@ -1745,18 +2103,19 @@ def set_parallel(self): """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds.""" # set level of parallelism for build par = build_option('parallel') - if self.cfg['parallel'] is not None: - if par is None: - par = self.cfg['parallel'] - self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) - else: - par = min(int(par), int(self.cfg['parallel'])) - self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) - else: + cfg_par = self.cfg['parallel'] + if cfg_par is None: self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + elif par is None: + par = cfg_par + self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) + else: + par = min(int(par), int(cfg_par)) + self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) - self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel']) - self.log.info("Setting parallelism: %s" % self.cfg['parallel']) + par = det_parallelism(par, maxpar=self.cfg['maxparallel']) + self.log.info("Setting parallelism: %s" % par) + self.cfg['parallel'] = par def remove_module_file(self): """Remove module file (if it exists), and check for ghost installation directory (and deal with it).""" @@ -1783,6 +2142,18 @@ def remove_module_file(self): self.log.info("Removing existing module file %s", self.mod_filepath) remove_file(self.mod_filepath) + def report_test_failure(self, msg_or_error): + """ + Report a failing test either via an exception or warning depending on ignore-test-failure + + :param msg_or_error: failure description (string value or an EasyBuildError instance) + """ + if build_option('ignore_test_failure'): + print_warning("Test failure ignored: " + str(msg_or_error), log=self.log) + else: + exception = msg_or_error if isinstance(msg_or_error, EasyBuildError) else EasyBuildError(msg_or_error) + raise exception + # # STEP FUNCTIONS # @@ -1856,6 +2227,8 @@ def fetch_step(self, skip_checksums=False): raise EasyBuildError("EasyBuild-version %s is newer than the currently running one. Aborting!", easybuild_version) + start_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, self.cfg.count_files()) + if self.dry_run: self.dry_run_msg("Available download URLs for sources/patches:") @@ -1879,7 +2252,7 @@ def fetch_step(self, skip_checksums=False): self.dry_run_msg("\nList of patches:") # fetch patches - if self.cfg['patches']: + if self.cfg['patches'] + self.cfg['postinstallpatches']: if isinstance(self.cfg['checksums'], (list, tuple)): # if checksums are provided as a list, first entries are assumed to be for sources patches_checksums = self.cfg['checksums'][len(self.cfg['sources']):] @@ -1917,7 +2290,7 @@ def fetch_step(self, skip_checksums=False): # fetch extensions if self.cfg.get_ref('exts_list'): - self.exts = self.fetch_extension_sources(skip_checksums=skip_checksums) + self.exts = self.collect_exts_file_info(fetch_files=True, verify_checksums=not skip_checksums) # create parent dirs in install and modules path already # this is required when building in parallel @@ -1938,6 +2311,8 @@ def fetch_step(self, skip_checksums=False): else: self.log.info("Skipped installation dirs check per user request") + stop_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL) + def checksum_step(self): """Verify checksum of sources and patches, if a checksum is available.""" for fil in self.src + self.patches: @@ -1981,7 +2356,12 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): for fn, checksum in zip(sources + patches, checksums): if isinstance(checksum, dict): - checksum = checksum.get(fn) + # sources entry may be a dictionary rather than just a string value with filename + if isinstance(fn, dict): + filename = fn['filename'] + else: + filename = fn + checksum = checksum.get(filename) # take into account that we may encounter a tuple of valid SHA256 checksums # (see https://github.com/easybuilders/easybuild-framework/pull/2958) @@ -2044,11 +2424,15 @@ def extract_step(self): else: raise EasyBuildError("Unpacking source %s failed", src['name']) - def patch_step(self, beginpath=None): + def patch_step(self, beginpath=None, patches=None): """ Apply the patches """ - for patch in self.patches: + if patches is None: + # if no patches are specified, use all non-post-install patches + patches = [p for p in self.patches if not p['postinstall']] + + for patch in patches: self.log.info("Applying patch %s" % patch['name']) trace_msg("applying patch %s" % patch['name']) @@ -2099,16 +2483,38 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): if not self.build_in_installdir: self.rpath_filter_dirs.append(self.builddir) + self.rpath_include_dirs = [] + + # If we have override directories for RPATH, insert them first. + # This means they override all other options (including the installation itself). + if build_option('rpath_override_dirs') is not None: + # make sure we have a list + rpath_overrides = build_option('rpath_override_dirs') + if isinstance(rpath_overrides, string_type): + rpath_override_dirs = rpath_overrides.split(':') + # Filter out any empty values + rpath_override_dirs = list(filter(None, rpath_override_dirs)) + _log.debug("Converted RPATH override directories ('%s') to a list of paths: %s" % (rpath_overrides, + rpath_override_dirs)) + for path in rpath_override_dirs: + if not os.path.isabs(path): + raise EasyBuildError( + "Path used in rpath_override_dirs is not an absolute path: %s", path) + else: + raise EasyBuildError("Value for rpath_override_dirs has invalid type (%s), should be string: %s", + type(rpath_overrides), rpath_overrides) + self.rpath_include_dirs.extend(rpath_override_dirs) + # always include '/lib', '/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64 # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object # see also https://linux.die.net/man/8/ld-linux; - self.rpath_include_dirs = [ + self.rpath_include_dirs.extend([ os.path.join(self.installdir, 'lib'), os.path.join(self.installdir, 'lib64'), '$ORIGIN', '$ORIGIN/../lib', '$ORIGIN/../lib64', - ] + ]) if self.iter_idx > 0: # reset toolchain for iterative runs before preparing it again @@ -2121,7 +2527,7 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): curr_modpaths = curr_module_paths() for init_modpath in init_modpaths: full_mod_path = os.path.join(self.installdir_mod, init_modpath) - if full_mod_path not in curr_modpaths: + if os.path.exists(full_mod_path) and full_mod_path not in curr_modpaths: self.modules_tool.prepend_module_path(full_mod_path) # prepare toolchain: load toolchain module and dependencies, set up build environment @@ -2150,6 +2556,10 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): self.log.info("Loading extra modules: %s", extra_modules) self.modules_tool.load(extra_modules) + # Setup CUDA cache if required. If we don't do this, CUDA will use the $HOME for its cache files + if get_software_root('CUDA') or get_software_root('CUDAcore'): + self.set_up_cuda_cache() + # guess directory to start configure/build/install process in, and move there if start_dir: self.guess_start_dir() @@ -2172,6 +2582,13 @@ def test_step(self): return out + def _test_step(self): + """Run the test_step and handles failures""" + try: + self.test_step() + except EasyBuildError as err: + self.report_test_failure(err) + def stage_install_step(self): """ Install in a stage directory before actual installation. @@ -2182,144 +2599,163 @@ def install_step(self): """Install built software (abstract method).""" raise NotImplementedError - def extensions_step(self, fetch=False, install=True): + def init_ext_instances(self): """ - After make install, run this. - - only if variable len(exts_list) > 0 - - optionally: load module that was just created using temp module file - - find source for extensions, in 'extensions' (and 'packages' for legacy reasons) - - run extra_extensions + Create class instances for all extensions. """ - if not self.cfg.get_ref('exts_list'): - self.log.debug("No extensions in exts_list") - return - - # load fake module - fake_mod_data = None - if install and not self.dry_run: - - # load modules for build dependencies as extra modules - build_dep_mods = [dep['short_mod_name'] for dep in self.cfg.dependencies(build_only=True)] - - fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods) - - self.prepare_for_extensions() - - if fetch: - self.exts = self.fetch_extension_sources() + exts_list = self.cfg.get_ref('exts_list') - self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping + # early exit if there are no extensions + if not exts_list: + return - # actually install extensions - self.log.debug("Installing extensions") - exts_defaultclass = self.cfg['exts_defaultclass'] + self.ext_instances = [] exts_classmap = self.cfg['exts_classmap'] - # we really need a default class - if not exts_defaultclass and fake_mod_data: - self.clean_up_fake_module(fake_mod_data) - raise EasyBuildError("ERROR: No default extension class set for %s", self.name) + # self.exts may already be populated at this point through collect_exts_file_info; + # if it's not, we do it lightweight here, by skipping fetching of the files; + # information on location of source/patch files will be lacking in that case (but that should be fine) + if exts_list and not self.exts: + self.exts = self.collect_exts_file_info(fetch_files=False, verify_checksums=False) # obtain name and module path for default extention class + exts_defaultclass = self.cfg['exts_defaultclass'] if isinstance(exts_defaultclass, string_type): # proper way: derive module path from specified class name default_class = exts_defaultclass default_class_modpath = get_module_path(default_class, generic=True) else: - raise EasyBuildError("Improper default extension class specification, should be string.") + error_msg = "Improper default extension class specification, should be string: %s (%s)" + raise EasyBuildError(error_msg, exts_defaultclass, type(exts_defaultclass)) - # get class instances for all extensions - self.ext_instances = [] - for ext in self.exts: - self.log.debug("Creating class instance for extension %s...", ext['name']) + exts_cnt = len(self.exts) + + self.update_exts_progress_bar("creating internal datastructures for extensions") + + for idx, ext in enumerate(self.exts): + ext_name = ext['name'] + self.log.debug("Creating class instance for extension %s...", ext_name) + + # if a specific easyblock is specified for this extension, honor it; + # just passing this to get_easyblock_class is sufficient + easyblock = ext.get('easyblock', None) + if easyblock: + class_name = easyblock + mod_path = get_module_path(class_name) + else: + class_name = encode_class_name(ext_name) + mod_path = get_module_path(class_name, generic=False) cls, inst = None, None - class_name = encode_class_name(ext['name']) - mod_path = get_module_path(class_name, generic=False) - # try instantiating extension-specific class + # try instantiating extension-specific class, or honor specified easyblock try: # no error when importing class fails, in case we run into an existing easyblock # with a similar name (e.g., Perl Extension 'GO' vs 'Go' for which 'EB_Go' is available) - cls = get_easyblock_class(None, name=ext['name'], error_on_failed_import=False, + cls = get_easyblock_class(easyblock, name=ext_name, error_on_failed_import=False, error_on_missing_easyblock=False) - self.log.debug("Obtained class %s for extension %s", cls, ext['name']) + + self.log.debug("Obtained class %s for extension %s", cls, ext_name) if cls is not None: + # make sure that this easyblock can be used to install extensions + if not issubclass(cls, Extension): + raise EasyBuildError("%s easyblock can not be used to install extensions!", cls.__name__) + inst = cls(self, ext) except (ImportError, NameError) as err: - self.log.debug("Failed to use extension-specific class for extension %s: %s", ext['name'], err) + self.log.debug("Failed to use extension-specific class for extension %s: %s", ext_name, err) # alternative attempt: use class specified in class map (if any) - if inst is None and ext['name'] in exts_classmap: - - class_name = exts_classmap[ext['name']] + if inst is None and ext_name in exts_classmap: + class_name = exts_classmap[ext_name] mod_path = get_module_path(class_name) try: cls = get_class_for(mod_path, class_name) + self.log.debug("Obtained class %s for extension %s from exts_classmap", cls, ext_name) inst = cls(self, ext) - except (ImportError, NameError) as err: - raise EasyBuildError("Failed to load specified class %s for extension %s: %s", - class_name, ext['name'], err) + except Exception as err: + raise EasyBuildError("Failed to load specified class %s (from %s) specified via exts_classmap " + "for extension %s: %s", + class_name, mod_path, ext_name, err) # fallback attempt: use default class if inst is None: try: cls = get_class_for(default_class_modpath, default_class) - self.log.debug("Obtained class %s for installing extension %s", cls, ext['name']) + self.log.debug("Obtained class %s for installing extension %s", cls, ext_name) inst = cls(self, ext) self.log.debug("Installing extension %s with default class %s (from %s)", - ext['name'], default_class, default_class_modpath) + ext_name, default_class, default_class_modpath) except (ImportError, NameError) as err: raise EasyBuildError("Also failed to use default class %s from %s for extension %s: %s, giving up", - default_class, default_class_modpath, ext['name'], err) + default_class, default_class_modpath, ext_name, err) else: - self.log.debug("Installing extension %s with class %s (from %s)", ext['name'], class_name, mod_path) + self.log.debug("Installing extension %s with class %s (from %s)", ext_name, class_name, mod_path) self.ext_instances.append(inst) + pbar_label = "creating internal datastructures for extensions " + pbar_label += "(%d/%d done)" % (idx + 1, exts_cnt) + self.update_exts_progress_bar(pbar_label) - if self.skip: - self.skip_extensions() + def update_exts_progress_bar(self, info, progress_size=0, total=None): + """ + Update extensions progress bar with specified info and amount of progress made + """ + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=info, progress_size=progress_size, total=total) - exts_cnt = len(self.ext_instances) - for idx, ext in enumerate(self.ext_instances): + def extensions_step(self, fetch=False, install=True): + """ + After make install, run this. + - only if variable len(exts_list) > 0 + - optionally: load module that was just created using temp module file + - find source for extensions, in 'extensions' (and 'packages' for legacy reasons) + - run extra_extensions + """ + if not self.cfg.get_ref('exts_list'): + self.log.debug("No extensions in exts_list") + return - self.log.debug("Starting extension %s" % ext.name) + # load fake module + fake_mod_data = None + if install and not self.dry_run: - # always go back to original work dir to avoid running stuff from a dir that no longer exists - change_dir(self.orig_workdir) + # load modules for build dependencies as extra modules + build_dep_mods = [dep['short_mod_name'] for dep in self.cfg.dependencies(build_only=True)] - tup = (ext.name, ext.version or '', idx + 1, exts_cnt) - print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) + fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods) - if self.dry_run: - tup = (ext.name, ext.version, cls.__name__) - msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup - self.dry_run_msg(msg) + start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg['exts_list'])) - self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + self.prepare_for_extensions() - # prepare toolchain build environment, but only when not doing a dry run - # since in that case the build environment is the same as for the parent - if self.dry_run: - self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...") - else: - # don't reload modules for toolchain, there is no need since they will be loaded already; - # the (fake) module for the parent software gets loaded before installing extensions - ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, - rpath_filter_dirs=self.rpath_filter_dirs) + if fetch: + self.update_exts_progress_bar("fetching extension sources/patches") + self.exts = self.collect_exts_file_info(fetch_files=True) - # real work - if install: - ext.prerun() - txt = ext.run() - if txt: - self.module_extra_extensions += txt - ext.postrun() + self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping + + # actually install extensions + if install: + self.log.info("Installing extensions") + + # we really need a default class + if not self.cfg['exts_defaultclass'] and fake_mod_data: + self.clean_up_fake_module(fake_mod_data) + raise EasyBuildError("ERROR: No default extension class set for %s", self.name) + + self.init_ext_instances() + + if self.skip: + self.skip_extensions() + + self.install_extensions(install=install) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: self.clean_up_fake_module(fake_mod_data) + stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) + def package_step(self): """Package installed software (e.g., into an RPM), if requested, using selected package tool.""" @@ -2347,14 +2783,24 @@ def package_step(self): def fix_shebang(self): """Fix shebang lines for specified files.""" - for lang in ['perl', 'python']: + + env_for_shebang = build_option('env_for_shebang') + sysroot = build_option('sysroot') + if sysroot and not env_for_shebang.startswith(sysroot): + env_for_shebang = os.path.join(sysroot, env_for_shebang.lstrip('/')) + if os.path.exists(env_for_shebang.split(' ')[0]): + self.log.info("Path to 'env' command to use in patched shebang lines: %s", env_for_shebang) + else: + raise EasyBuildError("Path to 'env' command to use in shebang lines does not exist: %s", env_for_shebang) + + for lang in ['bash', 'perl', 'python']: shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang) fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang] if fix_shebang_for: if isinstance(fix_shebang_for, string_type): fix_shebang_for = [fix_shebang_for] - shebang = '#!/usr/bin/env %s' % lang + shebang = '#!%s %s' % (env_for_shebang, lang) for glob_pattern in fix_shebang_for: paths = glob.glob(os.path.join(self.installdir, glob_pattern)) self.log.info("Fixing '%s' shebang to '%s' for files that match '%s': %s", @@ -2381,31 +2827,67 @@ def fix_shebang(self): contents = shebang + '\n' + contents write_file(path, contents) - def post_install_step(self): + def run_post_install_commands(self, commands=None): """ - Do some postprocessing - - run post install commands if any were specified + Run post install commands that are specified via 'postinstallcmds' easyconfig parameter. """ + if commands is None: + commands = self.cfg['postinstallcmds'] + + if commands: + self.log.debug("Specified post install commands: %s", commands) - if self.cfg['postinstallcmds'] is not None: # make sure we have a list of commands - if not isinstance(self.cfg['postinstallcmds'], (list, tuple)): - raise EasyBuildError("Invalid value for 'postinstallcmds', should be list or tuple of strings.") - for cmd in self.cfg['postinstallcmds']: + if not isinstance(commands, (list, tuple)): + error_msg = "Invalid value for 'postinstallcmds', should be list or tuple of strings: %s" + raise EasyBuildError(error_msg, commands) + + for cmd in commands: if not isinstance(cmd, string_type): raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) run_cmd(cmd, simple=True, log_ok=True, log_all=True) + def apply_post_install_patches(self, patches=None): + """ + Apply post-install patch files that are specified via the 'postinstallpatches' easyconfig parameter. + """ + if patches is None: + patches = [p for p in self.patches if p['postinstall']] + + self.log.debug("Post-install patches to apply: %s", patches) + if patches: + self.patch_step(beginpath=self.installdir, patches=patches) + + def post_install_step(self): + """ + Do some postprocessing + - run post install commands if any were specified + """ + + self.run_post_install_commands() + self.apply_post_install_patches() + self.fix_shebang() + + lib_dir = os.path.join(self.installdir, 'lib') + lib64_dir = os.path.join(self.installdir, 'lib64') + # GCC linker searches system /lib64 path before the $LIBRARY_PATH paths. # However for each in $LIBRARY_PATH (where is often /lib) it searches /../lib64 first. # So we create /lib64 as a symlink to /lib to make it prefer EB installed libraries. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/5776 if build_option('lib64_lib_symlink'): - lib_dir = os.path.join(self.installdir, 'lib') - lib64_dir = os.path.join(self.installdir, 'lib64') if os.path.exists(lib_dir) and not os.path.exists(lib64_dir): - symlink(lib_dir, lib64_dir) + # create *relative* 'lib64' symlink to 'lib'; + # see https://github.com/easybuilders/easybuild-framework/issues/3564 + symlink('lib', lib64_dir, use_abspath_source=False) + + # symlink lib to lib64, which is helpful on OpenSUSE; + # see https://github.com/easybuilders/easybuild-framework/issues/3549 + if build_option('lib_lib64_symlink'): + if os.path.exists(lib64_dir) and not os.path.exists(lib_dir): + # create *relative* 'lib' symlink to 'lib64'; + symlink('lib64', lib_dir, use_abspath_source=False) def sanity_check_step(self, *args, **kwargs): """ @@ -2462,6 +2944,8 @@ def _sanity_check_step_multi_deps(self, *args, **kwargs): def sanity_check_rpath(self, rpath_dirs=None): """Sanity check binaries/libraries w.r.t. RPATH linking.""" + self.log.info("Checking RPATH linkage for binaries/libraries...") + fails = [] # hard reset $LD_LIBRARY_PATH before running RPATH sanity check @@ -2474,10 +2958,15 @@ def sanity_check_rpath(self, rpath_dirs=None): readelf_rpath_regex = re.compile('(RPATH)', re.M) if rpath_dirs is None: - rpath_dirs = ['bin', 'lib', 'lib64'] - self.log.info("Using default subdirs for binaries/libraries to verify RPATH linking: %s", rpath_dirs) + rpath_dirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs() + + if not rpath_dirs: + rpath_dirs = DEFAULT_BIN_LIB_SUBDIRS + self.log.info("Using default subdirectories for binaries/libraries to verify RPATH linking: %s", + rpath_dirs) else: - self.log.info("Using specified subdirs for binaries/libraries to verify RPATH linking: %s", rpath_dirs) + self.log.info("Using specified subdirectories for binaries/libraries to verify RPATH linking: %s", + rpath_dirs) for dirpath in [os.path.join(self.installdir, d) for d in rpath_dirs]: if os.path.exists(dirpath): @@ -2488,7 +2977,9 @@ def sanity_check_rpath(self, rpath_dirs=None): out, ec = run_cmd("file %s" % path, simple=False, trace=False) if ec: - fails.append("Failed to run 'file %s': %s" % (path, out)) + fail_msg = "Failed to run 'file %s': %s" % (path, out) + self.log.warning(fail_msg) + fails.append(fail_msg) # only run ldd/readelf on dynamically linked executables/libraries # example output: @@ -2530,6 +3021,129 @@ def sanity_check_rpath(self, rpath_dirs=None): return fails + def bin_lib_subdirs(self): + """ + List of subdirectories for binaries and libraries for this software installation. + This is used during the sanity check to check RPATH linking and banned/required linked shared libraries. + """ + return None + + def banned_linked_shared_libs(self): + """ + List of shared libraries which are not allowed to be linked in any installed binary/library. + Supported values are pure library names without 'lib' prefix or extension ('example'), + file names ('libexample.so'), and full paths ('/usr/lib64/libexample.so'). + """ + return [] + + def required_linked_shared_libs(self): + """ + List of shared libraries which must be linked in all installed binaries/libraries. + Supported values are pure library names without 'lib' prefix or extension ('example'), + file names ('libexample.so'), and full paths ('/usr/lib64/libexample.so'). + """ + return [] + + def sanity_check_linked_shared_libs(self, subdirs=None): + """ + Check whether specific shared libraries are (not) linked into installed binaries/libraries. + """ + self.log.info("Checking for banned/required linked shared libraries...") + + # list of libraries that can *not* be linked in any installed binary/library + banned_libs = build_option('banned_linked_shared_libs') or [] + banned_libs.extend(self.toolchain.banned_linked_shared_libs()) + banned_libs.extend(self.banned_linked_shared_libs()) + banned_libs.extend(self.cfg['banned_linked_shared_libs']) + + # list of libraries that *must* be linked in every installed binary/library + required_libs = build_option('required_linked_shared_libs') or [] + required_libs.extend(self.toolchain.required_linked_shared_libs()) + required_libs.extend(self.required_linked_shared_libs()) + required_libs.extend(self.cfg['required_linked_shared_libs']) + + # early return if there are no banned/required libraries + if not (banned_libs + required_libs): + self.log.info("No banned/required libraries specified") + return [] + else: + if banned_libs: + self.log.info("Banned libraries to check for: %s", ', '.join(banned_libs)) + if required_libs: + self.log.info("Required libraries to check for: %s", ', '.join(banned_libs)) + + shlib_ext = get_shared_lib_ext() + + # compose regular expressions for banned/required libraries + def regex_for_lib(lib): + """Compose regular expression for specified banned/required library.""" + # absolute path to library ('/usr/lib64/libexample.so') + if os.path.isabs(lib): + regex = re.compile(re.escape(lib)) + # full filename for library ('libexample.so') + elif lib.startswith('lib'): + regex = re.compile(r'(/|\s)' + re.escape(lib)) + # pure library name, without 'lib' prefix or extension ('example') + else: + regex = re.compile(r'(/|\s)lib%s\.%s' % (lib, shlib_ext)) + + return regex + + banned_lib_regexs = [regex_for_lib(x) for x in banned_libs] + if banned_lib_regexs: + self.log.debug("Regular expressions to check for banned libraries: %s", + '\n'.join("'%s'" % regex.pattern for regex in banned_lib_regexs)) + + required_lib_regexs = [regex_for_lib(x) for x in required_libs] + if required_lib_regexs: + self.log.debug("Regular expressions to check for required libraries: %s", + '\n'.join("'%s'" % regex.pattern for regex in required_lib_regexs)) + + if subdirs is None: + subdirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs() + + if subdirs: + self.log.info("Using specified subdirectories to check for banned/required linked shared libraries: %s", + subdirs) + else: + subdirs = DEFAULT_BIN_LIB_SUBDIRS + self.log.info("Using default subdirectories to check for banned/required linked shared libraries: %s", + subdirs) + + # filter to existing directories that are unique (after resolving symlinks) + dirpaths = [] + for subdir in subdirs: + dirpath = os.path.join(self.installdir, subdir) + if os.path.exists(dirpath) and os.path.isdir(dirpath): + dirpath = os.path.realpath(dirpath) + if dirpath not in dirpaths: + dirpaths.append(dirpath) + + failed_paths = [] + + for dirpath in dirpaths: + if os.path.exists(dirpath): + self.log.debug("Checking banned/required linked shared libraries in %s", dirpath) + + for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]: + self.log.debug("Checking banned/required linked shared libraries for %s", path) + + libs_check = check_linked_shared_libs(path, banned_patterns=banned_lib_regexs, + required_patterns=required_lib_regexs) + + # None indicates the path is not a dynamically linked binary or shared library, so ignore it + if libs_check is not None: + if libs_check: + self.log.debug("Check for banned/required linked shared libraries passed for %s", path) + else: + failed_paths.append(path) + + fail_msg = None + if failed_paths: + fail_msg = "Check for banned/required shared libraries failed for %s" % ', '.join(failed_paths) + + return fail_msg + def _sanity_check_step_common(self, custom_paths, custom_commands): """ Determine sanity check paths and commands to use. @@ -2669,14 +3283,26 @@ def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, ** else: self.dry_run_msg(" (none)") + self.sanity_check_linked_shared_libs() + if self.toolchain.use_rpath: self.sanity_check_rpath() else: - self.log.debug("Skiping RPATH sanity check") + self.log.debug("Skipping RPATH sanity check") def _sanity_check_step_extensions(self): """Sanity check on extensions (if any).""" failed_exts = [] + + if build_option('skip_extensions'): + self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...") + return + elif not self.ext_instances: + # class instances for extensions may not be initialized yet here, + # for example when using --module-only or --sanity-check-only + self.prepare_for_extensions() + self.init_ext_instances() + for ext in self.ext_instances: success, fail_msg = None, None res = ext.sanity_check_step() @@ -2768,12 +3394,16 @@ def xs2str(xs): fake_mod_data = None + # skip loading of fake module when using --sanity-check-only, load real module instead + if build_option('sanity_check_only') and not extension: + self.load_module(extra_modules=extra_modules) + # only load fake module for non-extensions, and not during dry run - if not (extension or self.dry_run): + elif not (extension or self.dry_run): try: # unload all loaded modules before loading fake module # this ensures that loading of dependencies is tested, and avoids conflicts with build dependencies - fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules) + fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules, verbose=True) except EasyBuildError as err: self.sanity_check_fail_msgs.append("loading fake module failed: %s" % err) self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) @@ -2781,7 +3411,12 @@ def xs2str(xs): if extra_modules: self.log.info("Loading extra modules for sanity check: %s", ', '.join(extra_modules)) - # chdir to installdir (better environment for running tests) + # allow oversubscription of P processes on C cores (P>C) for software installed on top of Open MPI; + # this is useful to avoid failing of sanity check commands that involve MPI + if self.toolchain.mpi_family() and self.toolchain.mpi_family() in toolchain.OPENMPI: + env.setvar('OMPI_MCA_rmaps_base_oversubscribe', '1') + + # change to install directory (better environment for running tests) if os.path.isdir(self.installdir): change_dir(self.installdir) @@ -2804,6 +3439,11 @@ def xs2str(xs): if not extension: self._sanity_check_step_extensions() + linked_shared_lib_fails = self.sanity_check_linked_shared_libs() + if linked_shared_lib_fails: + self.log.warning("Check for required/banned linked shared libraries failed!") + self.sanity_check_fail_msgs.append(linked_shared_lib_fails) + # cleanup if fake_mod_data: self.clean_up_fake_module(fake_mod_data) @@ -2814,7 +3454,7 @@ def xs2str(xs): self.log.warning("RPATH sanity check failed!") self.sanity_check_fail_msgs.extend(rpath_fails) else: - self.log.debug("Skiping RPATH sanity check") + self.log.debug("Skipping RPATH sanity check") # pass or fail if self.sanity_check_fail_msgs: @@ -2896,21 +3536,22 @@ def make_module_step(self, fake=False): else: trace_msg("generating module file @ %s" % self.mod_filepath) - txt = self.module_generator.MODULE_SHEBANG - if txt: - txt += '\n' + with self.module_generator.start_module_creation() as txt: + if self.modules_header: + txt += self.modules_header + '\n' - if self.modules_header: - txt += self.modules_header + '\n' + txt += self.make_module_description() + txt += self.make_module_group_check() + txt += self.make_module_deppaths() + txt += self.make_module_dep() + txt += self.make_module_extend_modpath() + txt += self.make_module_req() + txt += self.make_module_extra() + txt += self.make_module_footer() - txt += self.make_module_description() - txt += self.make_module_group_check() - txt += self.make_module_deppaths() - txt += self.make_module_dep() - txt += self.make_module_extend_modpath() - txt += self.make_module_req() - txt += self.make_module_extra() - txt += self.make_module_footer() + hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt]) + if hook_txt is not None: + txt = hook_txt if self.dry_run: # only report generating actual module file during dry run, don't mention temporary module files @@ -2943,7 +3584,13 @@ def make_module_step(self, fake=False): self.module_generator.create_symlinks(mod_symlink_paths, fake=fake) if ActiveMNS().mns.det_make_devel_module() and not fake and build_option('generate_devel_module'): - self.make_devel_module() + try: + self.make_devel_module() + except EasyBuildError as error: + if build_option('module_only'): + self.log.info("Using --module-only so can recover from error: %s", error) + else: + raise error else: self.log.info("Skipping devel module...") @@ -3044,18 +3691,22 @@ def update_config_template_run_step(self): self.cfg.template_values[name[0]] = str(getattr(self, name[0], None)) self.cfg.generate_template_values() - def _skip_step(self, step, skippable): + def skip_step(self, step, skippable): """Dedice whether or not to skip the specified step.""" - module_only = build_option('module_only') - force = build_option('force') or build_option('rebuild') skip = False + force = build_option('force') + module_only = build_option('module_only') + sanity_check_only = build_option('sanity_check_only') + skip_extensions = build_option('skip_extensions') + skip_test_step = build_option('skip_test_step') + skipsteps = self.cfg['skipsteps'] # under --skip, sanity check is not skipped cli_skip = self.skip and step != SANITYCHECK_STEP # skip step if specified as individual (skippable) step, or if --skip is used - if skippable and (cli_skip or step in self.cfg['skipsteps']): - self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, self.cfg['skipsteps']) + if skippable and (cli_skip or step in skipsteps): + self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, skipsteps) skip = True # skip step when only generating module file @@ -3070,9 +3721,23 @@ def _skip_step(self, step, skippable): self.log.info("Skipping %s step because of forced module-only mode", step) skip = True + elif sanity_check_only and step != SANITYCHECK_STEP: + self.log.info("Skipping %s step because of sanity-check-only mode", step) + skip = True + + elif skip_extensions and step == EXTENSIONS_STEP: + self.log.info("Skipping %s step as requested via skip-extensions", step) + skip = True + + elif skip_test_step and step == TEST_STEP: + self.log.info("Skipping %s step as requested via skip-test-step", step) + skip = True + else: - self.log.debug("Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s", - step, skippable, self.skip, self.cfg['skipsteps'], module_only, force) + msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, " + msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s)" + self.log.debug(msg, step, skippable, self.skip, skipsteps, module_only, force, + sanity_check_only, skip_extensions, skip_test_step) return skip @@ -3086,10 +3751,12 @@ def run_step(self, step, step_methods): run_hook(step, self.hooks, pre_step_hook=True, args=[self]) for step_method in step_methods: - self.log.info("Running method %s part of step %s" % (extract_method_name(step_method), step)) + # Remove leading underscore from e.g. "_test_step" + method_name = extract_method_name(step_method).lstrip('_') + self.log.info("Running method %s part of step %s", method_name, step) if self.dry_run: - self.dry_run_msg("[%s method]", step_method(self).__name__) + self.dry_run_msg("[%s method]", method_name) # if an known possible error occurs, just report it and continue try: @@ -3111,6 +3778,7 @@ def run_step(self, step, step_methods): run_hook(step, self.hooks, post_step_hook=True, args=[self]) if self.cfg['stop'] == step: + update_progress_bar(PROGRESS_BAR_EASYCONFIG) self.log.info("Stopping after %s step.", step) raise StopException(step) @@ -3162,7 +3830,7 @@ def install_step_spec(initial): prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False) configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True) build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True) - test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True) + test_step_spec = (TEST_STEP, 'testing', [lambda x: x._test_step], True) extensions_step_spec = (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step], False) # part 1: pre-iteration + first iteration @@ -3224,41 +3892,72 @@ def run_all_steps(self, run_test_cases): steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) + # figure out how many steps will actually be run (not be skipped) + step_cnt = 0 + for (step_name, _, _, _) in steps: + step_cnt += 1 + if self.cfg['stop'] == step_name: + break + + start_progress_bar(PROGRESS_BAR_EASYCONFIG, step_cnt, label="Installing %s" % self.full_mod_name) + print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) ignore_locks = build_option('ignore_locks') - if ignore_locks: - self.log.info("Ignoring locks...") - else: - lock_name = self.installdir.replace('/', '_') + lock_created = False + try: + if ignore_locks: + self.log.info("Ignoring locks...") + else: + lock_name = self.installdir.replace('/', '_') - # check if lock already exists; - # either aborts with an error or waits until it disappears (depends on --wait-on-lock) - check_lock(lock_name) + # check if lock already exists; + # either aborts with an error or waits until it disappears (depends on --wait-on-lock) + check_lock(lock_name) - # create lock to avoid that another installation running in parallel messes things up - create_lock(lock_name) + # create lock to avoid that another installation running in parallel messes things up + create_lock(lock_name) + lock_created = True - try: - for (step_name, descr, step_methods, skippable) in steps: - if self._skip_step(step_name, skippable): + # run post-initialization tasks first, before running any steps + self.post_init() + + for step_name, descr, step_methods, skippable in steps: + if self.skip_step(step_name, skippable): print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) else: + progress_label = "Installing %s: %s" % (self.full_mod_name, descr) + update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label, progress_size=0) + if self.dry_run: self.dry_run_msg("%s... [DRY RUN]\n", descr) else: print_msg("%s..." % descr, log=self.log, silent=self.silent) self.current_step = step_name - self.run_step(step_name, step_methods) + start_time = datetime.now() + try: + self.run_step(step_name, step_methods) + finally: + if not self.dry_run: + step_duration = datetime.now() - start_time + if step_duration.total_seconds() >= 1: + print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) + elif self.logdebug or build_option('trace'): + print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) + + update_progress_bar(PROGRESS_BAR_EASYCONFIG) except StopException: pass finally: - if not ignore_locks: + # remove lock, but only if it was created in this session (not if it was there already) + if lock_created: remove_lock(lock_name) + stop_progress_bar(PROGRESS_BAR_EASYCONFIG) + # return True for successfull build (or stopped build) return True @@ -3312,7 +4011,7 @@ def build_and_install_one(ecdict, init_env): # load easyblock easyblock = build_option('easyblock') if easyblock: - # set the value in the dict so this is included in the reproducability dump of the easyconfig + # set the value in the dict so this is included in the reproducibility dump of the easyconfig ecdict['ec']['easyblock'] = easyblock else: easyblock = fetch_parameters_from_easyconfig(rawtxt, ['easyblock'])[0] @@ -3336,10 +4035,6 @@ def build_and_install_one(ecdict, init_env): _log.debug("Skip set to %s" % skip) app.cfg['skip'] = skip - if build_option('skip_test_step'): - _log.debug('Adding test_step to skipped steps') - app.cfg.update('skipsteps', TEST_STEP, allow_duplicate=False) - # build easyconfig errormsg = '(no error)' # timing info @@ -3348,16 +4043,35 @@ def build_and_install_one(ecdict, init_env): run_test_cases = not build_option('skip_test_cases') and app.cfg['tests'] if not dry_run: - # create our reproducability files before carrying out the easyblock steps + # create our reproducibility files before carrying out the easyblock steps reprod_dir_root = os.path.dirname(app.logfile) reprod_dir = reproduce_build(app, reprod_dir_root) + if os.path.exists(app.installdir) and build_option('read_only_installdir') and ( + build_option('rebuild') or build_option('force')): + enabled_write_permissions = True + # re-enable write permissions so we can install additional modules + adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=True) + else: + enabled_write_permissions = False + result = app.run_all_steps(run_test_cases=run_test_cases) if not dry_run: - # also add any extension easyblocks used during the build for reproducability + # Copy over the build environment used during the configuraton + reprod_spec = os.path.join(reprod_dir, app.cfg.filename()) + try: + dump_env_easyblock(app, ec_path=reprod_spec, silent=True) + _log.debug("Created build environment dump for easyconfig %s", reprod_spec) + except EasyBuildError as err: + _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err) + + # also add any extension easyblocks used during the build for reproducibility if app.ext_instances: copy_easyblocks_for_reprod(app.ext_instances, reprod_dir) + # If not already done remove the granted write permissions if we did so + if enabled_write_permissions and os.lstat(app.installdir)[stat.ST_MODE] & stat.S_IWUSR: + adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=True) except EasyBuildError as err: first_n = 300 @@ -3374,6 +4088,21 @@ def build_and_install_one(ecdict, init_env): # successful (non-dry-run) build if result and not dry_run: + def ensure_writable_log_dir(log_dir): + """Make sure we can write into the log dir""" + if build_option('read_only_installdir'): + # temporarily re-enable write permissions for copying log/easyconfig to install dir + if os.path.exists(log_dir): + adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True) + else: + parent_dir = os.path.dirname(log_dir) + if os.path.exists(parent_dir): + adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False) + mkdir(log_dir, parents=True) + adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) + else: + mkdir(log_dir, parents=True) + adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True) if app.cfg['stop']: ended = 'STOPPED' @@ -3381,16 +4110,15 @@ def build_and_install_one(ecdict, init_env): new_log_dir = os.path.join(app.builddir, config.log_path(ec=app.cfg)) else: new_log_dir = os.path.dirname(app.logfile) + ensure_writable_log_dir(new_log_dir) + + # if we're only running the sanity check, we should not copy anything new to the installation directory + elif build_option('sanity_check_only'): + _log.info("Only running sanity check, so skipping build stats, easyconfigs archive, reprod files...") + else: new_log_dir = os.path.join(app.installdir, config.log_path(ec=app.cfg)) - if build_option('read_only_installdir'): - # temporarily re-enable write permissions for copying log/easyconfig to install dir - if os.path.exists(new_log_dir): - adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=False) - else: - adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=False) - mkdir(new_log_dir, parents=True) - adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=False) + ensure_writable_log_dir(new_log_dir) # collect build stats _log.info("Collecting build stats...") @@ -3398,14 +4126,20 @@ def build_and_install_one(ecdict, init_env): buildstats = get_build_stats(app, start_time, build_option('command_line')) _log.info("Build stats: %s" % buildstats) - # move the reproducability files to the final log directory - archive_reprod_dir = os.path.join(new_log_dir, REPROD) - if os.path.exists(archive_reprod_dir): - backup_dir = find_backup_name_candidate(archive_reprod_dir) - move_file(archive_reprod_dir, backup_dir) - _log.info("Existing reproducability directory %s backed up to %s", archive_reprod_dir, backup_dir) - move_file(reprod_dir, archive_reprod_dir) - _log.info("Wrote files for reproducability to %s", archive_reprod_dir) + try: + # move the reproducibility files to the final log directory + archive_reprod_dir = os.path.join(new_log_dir, REPROD) + if os.path.exists(archive_reprod_dir): + backup_dir = find_backup_name_candidate(archive_reprod_dir) + move_file(archive_reprod_dir, backup_dir) + _log.info("Existing reproducibility directory %s backed up to %s", archive_reprod_dir, backup_dir) + move_file(reprod_dir, archive_reprod_dir) + _log.info("Wrote files for reproducibility to %s", archive_reprod_dir) + except EasyBuildError as error: + if build_option('module_only'): + _log.info("Using --module-only so can recover from error: %s", error) + else: + raise error try: # upload easyconfig (and patch files) to central repository @@ -3424,23 +4158,35 @@ def build_and_install_one(ecdict, init_env): # cleanup logs app.close_log() - log_fn = os.path.basename(get_log_filename(app.name, app.version)) - application_log = os.path.join(new_log_dir, log_fn) - move_logs(app.logfile, application_log) - newspec = os.path.join(new_log_dir, app.cfg.filename()) - copy_file(spec, newspec) - _log.debug("Copied easyconfig file %s to %s", spec, newspec) + if build_option('sanity_check_only'): + _log.info("Only running sanity check, so not copying anything to software install directory...") + else: + log_fn = os.path.basename(get_log_filename(app.name, app.version)) + try: + application_log = os.path.join(new_log_dir, log_fn) + move_logs(app.logfile, application_log) - # copy patches - for patch in app.patches: - target = os.path.join(new_log_dir, os.path.basename(patch['path'])) - copy_file(patch['path'], target) - _log.debug("Copied patch %s to %s", patch['path'], target) + newspec = os.path.join(new_log_dir, app.cfg.filename()) + copy_file(spec, newspec) + _log.debug("Copied easyconfig file %s to %s", spec, newspec) - if build_option('read_only_installdir'): - # take away user write permissions (again) - adjust_permissions(new_log_dir, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=True) + # copy patches + for patch in app.patches: + target = os.path.join(new_log_dir, os.path.basename(patch['path'])) + copy_file(patch['path'], target) + _log.debug("Copied patch %s to %s", patch['path'], target) + + if build_option('read_only_installdir'): + # take away user write permissions (again) + perms = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH + adjust_permissions(new_log_dir, perms, add=False, recursive=True) + except EasyBuildError as error: + if build_option('module_only'): + application_log = None + _log.debug("Using --module-only so can recover from error: %s", error) + else: + raise error end_timestamp = datetime.now() @@ -3498,9 +4244,9 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): for easyblock_instance in easyblock_instances: for easyblock_class in inspect.getmro(type(easyblock_instance)): easyblock_path = inspect.getsourcefile(easyblock_class) - # if we reach EasyBlock or ExtensionEasyBlock class, we are done - # (ExtensionEasyblock is hardcoded to avoid a cyclical import) - if easyblock_class.__name__ in [EasyBlock.__name__, 'ExtensionEasyBlock']: + # if we reach EasyBlock, Extension or ExtensionEasyBlock class, we are done + # (Extension and ExtensionEasyblock are hardcoded to avoid a cyclical import) + if easyblock_class.__name__ in [EasyBlock.__name__, 'Extension', 'ExtensionEasyBlock']: break else: easyblock_paths.add(easyblock_path) @@ -3512,12 +4258,12 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): def reproduce_build(app, reprod_dir_root): """ - Create reproducability files (processed easyconfig and easyblocks used) from class instance + Create reproducibility files (processed easyconfig and easyblocks used) from class instance :param app: easyblock class instance :param reprod_dir_root: root directory in which to create the 'reprod' directory - :return reprod_dir: directory containing reproducability files + :return reprod_dir: directory containing reproducibility files """ ec_filename = app.cfg.filename() diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py index 3ae73bb0df..040a28347f 100644 --- a/easybuild/framework/easyconfig/__init__.py +++ b/easybuild/framework/easyconfig/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index 5ad1ccd904..f684985b21 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,8 @@ import platform from easybuild.base import fancylogger -from easybuild.tools.systemtools import get_os_name, get_os_type, get_os_version +from easybuild.tools.build_log import print_warning +from easybuild.tools.systemtools import KNOWN_ARCH_CONSTANTS, get_os_name, get_os_type, get_os_version _log = fancylogger.getLogger('easyconfig.constants', fname=False) @@ -42,8 +43,26 @@ EXTERNAL_MODULE_MARKER = 'EXTERNAL_MODULE' + +def _get_arch_constant(): + """ + Get value for ARCH constant. + """ + arch = platform.uname()[4] + + # macOS on Arm produces 'arm64' rather than 'aarch64' + if arch == 'arm64': + arch = 'aarch64' + + if arch not in KNOWN_ARCH_CONSTANTS: + print_warning("Using unknown value for ARCH constant: %s", arch) + + return arch + + # constants that can be used in easyconfig EASYCONFIG_CONSTANTS = { + 'ARCH': (_get_arch_constant(), "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"), 'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"), 'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"), 'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"), diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 99e3c8309f..91e0fe7fe4 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -82,17 +82,22 @@ 'toolchainopts': [None, 'Extra options for compilers', TOOLCHAIN], # BUILD easyconfig parameters + 'banned_linked_shared_libs': [[], "List of shared libraries (names, file names, or paths) which are not allowed " + "to be linked in any installed binary/library", BUILD], 'bitbucket_account': ['%(namelower)s', "Bitbucket account name to be used to resolve template values in source" " URLs", BUILD], 'buildopts': ['', 'Extra options passed to make step (default already has -j X)', BUILD], 'checksums': [[], "Checksums for sources and patches", BUILD], 'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD], 'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with (if supported)", BUILD], + 'download_instructions': ['', "Specify steps to aquire necessary file, if obtaining it is difficult", BUILD], 'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected " "based on the software name", BUILD], 'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD], 'enhance_sanity_check': [False, "Indicate that additional sanity check commands & paths should enhance " "the existin sanity check, not replace it", BUILD], + 'fix_bash_shebang_for': [None, "List of files for which Bash shebang should be fixed " + "to '#!/usr/bin/env bash' (glob patterns supported)", BUILD], 'fix_perl_shebang_for': [None, "List of files for which Perl shebang should be fixed " "to '#!/usr/bin/env perl' (glob patterns supported)", BUILD], 'fix_python_shebang_for': [None, "List of files for which Python shebang should be fixed " @@ -110,8 +115,13 @@ 'preinstallopts': ['', 'Extra prefix options for installation.', BUILD], 'pretestopts': ['', 'Extra prefix options for test.', BUILD], 'postinstallcmds': [[], 'Commands to run after the install step.', BUILD], + 'postinstallpatches': [[], 'Patch files to apply after running the install step.', BUILD], + 'required_linked_shared_libs': [[], "List of shared libraries (names, file names, or paths) which must be " + "linked in all installed binaries/libraries", BUILD], 'runtest': [None, ('Indicates if a test should be run after make; should specify argument ' 'after make (for e.g.,"test" for make test)'), BUILD], + 'bin_lib_subdirs': [[], "List of subdirectories for binaries and libraries, which is used during sanity check " + "to check RPATH linking and banned/required libraries", BUILD], 'sanity_check_commands': [[], ("format: [(name, options)] e.g. [('gzip','-h')]. " "Using a non-tuple is equivalent to (name, '-h')"), BUILD], 'sanity_check_paths': [{}, ("List of files and directories to check " @@ -190,10 +200,14 @@ 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], 'module_depends_on': [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' '(implies recursive unloading of modules).', MODULES], - 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], + 'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module " + "(True/False to hard enable/disable; None implies honoring " + "the --recursive-module-unload EasyBuild configuration setting", + MODULES], # MODULES documentation easyconfig parameters # (docurls is part of MANDATORY) + 'citing': [None, "Free-form text that describes how the software should be cited in publications", MODULES], 'docpaths': [None, "List of paths for documentation relative to installation directory", MODULES], 'examples': [None, "Free-form text with examples on using the software", MODULES], 'site_contacts': [None, "String/list of strings with site contacts for the software", MODULES], @@ -229,3 +243,8 @@ def get_easyconfig_parameter_default(param): else: _log.debug("Returning default value for easyconfig parameter %s: %s" % (param, DEFAULT_CONFIG[param][0])) return DEFAULT_CONFIG[param][0] + + +def is_easyconfig_parameter_default_value(param, value): + """Return True if the parameters is one of the default ones and the value equals its default value""" + return param in DEFAULT_CONFIG and get_easyconfig_parameter_default(param) == value diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 27cf331c41..e7abf1687d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -59,7 +59,7 @@ from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, template_constant_dict +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN @@ -220,7 +220,7 @@ def cache_aware_func(toolchain, incl_capabilities=False): return cache_aware_func -def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands, incl_capabilities=False): +def det_subtoolchain_version(current_tc, subtoolchain_names, optional_toolchains, cands, incl_capabilities=False): """ Returns unique version for subtoolchain, in tc dict. If there is no unique version: @@ -229,30 +229,46 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, optional toolchains or system toolchain without add_system_to_minimal_toolchains. * in all other cases, raises an exception. """ - uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name]) # init with "skipped" subtoolchain_version = None - # system toolchain: bottom of the hierarchy - if is_system_toolchain(subtoolchain_name): - add_system_to_minimal_toolchains = build_option('add_system_to_minimal_toolchains') - if not add_system_to_minimal_toolchains and build_option('add_dummy_to_minimal_toolchains'): - depr_msg = "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains" - _log.deprecated(depr_msg, '5.0') - add_system_to_minimal_toolchains = True - - if add_system_to_minimal_toolchains and not incl_capabilities: - subtoolchain_version = '' - elif len(uniq_subtc_versions) == 1: - subtoolchain_version = list(uniq_subtc_versions)[0] - elif len(uniq_subtc_versions) == 0: - if subtoolchain_name not in optional_toolchains: + # ensure we always have a tuple of alternative subtoolchain names, which makes things easier below + if isinstance(subtoolchain_names, string_type): + subtoolchain_names = (subtoolchain_names,) + + system_subtoolchain = False + + for subtoolchain_name in subtoolchain_names: + + uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name]) + + # system toolchain: bottom of the hierarchy + if is_system_toolchain(subtoolchain_name): + add_system_to_minimal_toolchains = build_option('add_system_to_minimal_toolchains') + if not add_system_to_minimal_toolchains and build_option('add_dummy_to_minimal_toolchains'): + depr_msg = "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains" + _log.deprecated(depr_msg, '5.0') + add_system_to_minimal_toolchains = True + + system_subtoolchain = True + + if add_system_to_minimal_toolchains and not incl_capabilities: + subtoolchain_version = '' + elif len(uniq_subtc_versions) == 1: + subtoolchain_version = list(uniq_subtc_versions)[0] + elif len(uniq_subtc_versions) > 1: + raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s", + subtoolchain_name, current_tc['name'], ', '.join(sorted(uniq_subtc_versions))) + + if subtoolchain_version is not None: + break + + if not system_subtoolchain and subtoolchain_version is None: + if not all(n in optional_toolchains for n in subtoolchain_names): + subtoolchain_names = ' or '.join(subtoolchain_names) # raise error if the subtoolchain considered now is not optional raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s", - subtoolchain_name, current_tc['name']) - else: - raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s", - subtoolchain_name, current_tc['name'], ', '.join(sorted(uniq_subtc_versions))) + subtoolchain_names, current_tc['name']) return subtoolchain_version @@ -356,11 +372,16 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): cands.append({'name': dep, 'version': current_tc_version}) # only retain candidates that match subtoolchain names - cands = [c for c in cands if c['name'] in subtoolchain_names] + cands = [c for c in cands if any(c['name'] == x or c['name'] in x for x in subtoolchain_names)] for subtoolchain_name in subtoolchain_names: subtoolchain_version = det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands, incl_capabilities=incl_capabilities) + + # narrow down alternative subtoolchain names to a single one, based on the selected version + if isinstance(subtoolchain_name, tuple): + subtoolchain_name = [cand['name'] for cand in cands if cand['version'] == subtoolchain_version][0] + # add to hierarchy and move to next if subtoolchain_version is not None and subtoolchain_name not in visited: tc = {'name': subtoolchain_name, 'version': subtoolchain_version} @@ -394,12 +415,9 @@ def disable_templating(ec): # Do what you want without templating # Templating set to previous value """ - old_enable_templating = ec.enable_templating - ec.enable_templating = False - try: - yield old_enable_templating - finally: - ec.enable_templating = old_enable_templating + _log.deprecated("disable_templating(ec) was replaced by ec.disable_templating()", '5.0') + with ec.disable_templating() as old_value: + yield old_value class EasyConfig(object): @@ -525,6 +543,29 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi self.software_license = None + @contextmanager + def disable_templating(self): + """Temporarily disable templating on the given EasyConfig + + Usage: + with ec.disable_templating(): + # Do what you want without templating + # Templating set to previous value + """ + old_enable_templating = self.enable_templating + self.enable_templating = False + try: + yield old_enable_templating + finally: + self.enable_templating = old_enable_templating + + def __str__(self): + """Return a string representation of this EasyConfig instance""" + if self.path: + return '%s EasyConfig @ %s' % (self.name, self.path) + else: + return 'Raw %s EasyConfig' % self.name + def filename(self): """Determine correct filename for this easyconfig file.""" @@ -634,7 +675,7 @@ def set_keys(self, params): """ # disable templating when setting easyconfig parameters # required to avoid problems with values that need more parsing to be done (e.g. dependencies) - with disable_templating(self): + with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config if key in self._config.keys(): @@ -686,7 +727,7 @@ def parse(self): # templating is disabled when parse_hook is called to allow for easy updating of mutable easyconfig parameters # (see also comment in resolve_template) - with disable_templating(self): + with self.disable_templating(): # if any lists of dependency versions are specified over which we should iterate, # deal with them now, before calling parse hook, parsing of dependencies & iterative easyconfig parameters self.handle_multi_deps() @@ -702,10 +743,13 @@ def parse(self): # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") - self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] - self['hiddendependencies'] = [ - self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies'] - ] + + def remove_false_versions(deps): + return [dep for dep in deps if not (isinstance(dep, dict) and dep['version'] is False)] + + self['dependencies'] = remove_false_versions(self._parse_dependency(dep) for dep in self['dependencies']) + self['hiddendependencies'] = remove_false_versions(self._parse_dependency(dep, hidden=True) for dep in + self['hiddendependencies']) # need to take into account that builddependencies may need to be iterated over, # i.e. when the value is a list of lists of tuples @@ -715,7 +759,7 @@ def parse(self): builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps] else: builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps] - self['builddependencies'] = builddeps + self['builddependencies'] = remove_false_versions(builddeps) # keep track of parsed multi deps, they'll come in handy during sanity check & module steps... self.multi_deps = self.get_parsed_multi_deps() @@ -729,6 +773,28 @@ def parse(self): # indicate that this is a parsed easyconfig self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"] + def count_files(self): + """ + Determine number of files (sources + patches) required for this easyconfig. + """ + cnt = len(self['sources']) + len(self['patches']) + + for ext in self['exts_list']: + if isinstance(ext, tuple) and len(ext) >= 3: + ext_opts = ext[2] + # check for 'sources' first, since that's also considered first by EasyBlock.fetch_extension_sources + if 'sources' in ext_opts: + cnt += len(ext_opts['sources']) + elif 'source_tmpl' in ext_opts: + cnt += 1 + else: + # assume there's always one source file; + # for extensions using PythonPackage, no 'source' or 'sources' may be specified + cnt += 1 + cnt += len(ext_opts.get('patches', [])) + + return cnt + def local_var_naming(self, local_var_naming_check): """Deal with local variables that do not follow the recommended naming scheme (if any).""" @@ -771,7 +837,10 @@ def check_deprecated(self, path): raise EasyBuildError("Wrong type for value of 'deprecated' easyconfig parameter: %s", type(deprecated)) if self.toolchain.is_deprecated(): - depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain']) + # allow use of deprecated toolchains when running unit tests, + # because test easyconfigs/modules often use old toolchain versions (and updating them is far from trivial) + if not build_option('unit_testing_mode'): + depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain']) if depr_msgs: depr_msg = ', '.join(depr_msgs) @@ -844,7 +913,7 @@ def validate_os_deps(self): raise EasyBuildError("Non-tuple value type for OS dependency specification: %s (type %s)", dep, type(dep)) - if not any([check_os_dependency(cand_dep) for cand_dep in dep]): + if not any(check_os_dependency(cand_dep) for cand_dep in dep): not_found.append(dep) if not_found: @@ -1145,7 +1214,7 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals :param backup: create backup of existing file before overwriting it """ # templated values should be dumped unresolved - with disable_templating(self): + with self.disable_templating(): # build dict of default values default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG]) default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options])) @@ -1289,12 +1358,19 @@ def mk_var_name_pair(var_name_pair, name): # if a version is already set in the available metadata, we retain it if 'version' not in existing_metadata: - res['version'] = [version] + # Use name of environment variable as value, not the current value of that environment variable. + # This is important in case the value of the environment variables changes by the time we really + # use it, for example by a loaded module being swapped with another version of that module. + # This is particularly important w.r.t. integration with the Cray Programming Environment, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/3559. + res['version'] = [version_var_name] self.log.info('setting external module %s version to be %s', mod_name, version) # if a prefix is already set in the available metadata, we retain it if 'prefix' not in existing_metadata: - res['prefix'] = prefix + # Use name of environment variable as value, not the current value of that environment variable. + # (see above for more info) + res['prefix'] = prefix_var_name self.log.info('setting external module %s prefix to be %s', mod_name, prefix_var_name) break @@ -1636,7 +1712,7 @@ def _generate_template_values(self, ignore=None): # step 1-3 work with easyconfig.templates constants # disable templating with creating dict with template values to avoid looping back to here via __getitem__ - with disable_templating(self): + with self.disable_templating(): if self.template_values is None: # if no template values are set yet, initiate with a minimal set of template values; # this is important for easyconfig that use %(version_minor)s to define 'toolchain', @@ -1649,7 +1725,7 @@ def _generate_template_values(self, ignore=None): # get updated set of template values, now with toolchain instance # (which is used to define the %(mpi_cmd_prefix)s template) - with disable_templating(self): + with self.disable_templating(): template_values = template_constant_dict(self, ignore=ignore, toolchain=toolchain) # update the template_values dict @@ -1693,7 +1769,7 @@ def get_ref(self, key): # see also comments in resolve_template # temporarily disable templating - with disable_templating(self): + with self.disable_templating(): ref = self[key] return ref @@ -1759,6 +1835,25 @@ def asdict(self): res[key] = value return res + def get_cuda_cc_template_value(self, key): + """ + Get template value based on --cuda-compute-capabilities EasyBuild configuration option + and cuda_compute_capabilities easyconfig parameter. + Returns user-friendly error message in case neither are defined, + or if an unknown key is used. + """ + if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC): + try: + return self.template_values[key] + except KeyError: + error_msg = "Template value '%s' is not defined!\n" + error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration " + error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." + raise EasyBuildError(error_msg, key) + else: + error_msg = "%s is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities" + raise EasyBuildError(error_msg, key) + def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix): """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters.""" @@ -1928,9 +2023,8 @@ def resolve_template(value, tmpl_dict): # self['x'] is a get, will return a reference to a templated version of self._config['x'] # and the ['y] = z part will be against this new reference # you will need to do - # self.enable_templating = False - # self['x']['y'] = z - # self.enable_templating = True + # with self.disable_templating(): + # self['x']['y'] = z # or (direct but evil) # self._config['x']['y'] = z # it can not be intercepted with __setitem__ because the set is done at a deeper level diff --git a/easybuild/framework/easyconfig/format/__init__.py b/easybuild/framework/easyconfig/format/__init__.py index 0b0de5ca72..125c7b73fd 100644 --- a/easybuild/framework/easyconfig/format/__init__.py +++ b/easybuild/framework/easyconfig/format/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/convert.py b/easybuild/framework/easyconfig/format/convert.py index 3765df9bf5..5ba1fa7f45 100644 --- a/easybuild/framework/easyconfig/format/convert.py +++ b/easybuild/framework/easyconfig/format/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 073cc1bdc5..c49e82fbe9 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -577,7 +577,7 @@ def get_version_toolchain(self, version=None, tcname=None, tcversion=None): self.log.debug("No toolchain version specified, using default %s" % tcversion) else: raise EasyBuildError("No toolchain version specified, no default found.") - elif any([tc.test(tcname, tcversion) for tc in tcs]): + elif any(tc.test(tcname, tcversion) for tc in tcs): self.log.debug("Toolchain '%s' version '%s' is supported in easyconfig" % (tcname, tcversion)) else: raise EasyBuildError("Toolchain '%s' version '%s' not supported in easyconfig (only %s)", diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 71d7b929ae..114cb800fd 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,6 +30,7 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +import copy import os import pprint import re @@ -129,7 +130,21 @@ def get_config_dict(self): if spec_tc_version is not None and not spec_tc_version == tc_version: raise EasyBuildError('Requested toolchain version %s not available, only %s', spec_tc_version, tc_version) - return cfg + # avoid passing anything by reference, so next time get_config_dict is called + # we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file; + # we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails... + cfg_copy = {} + for key in cfg: + # skip special variables like __builtins__, and imported modules (like 'os') + if key != '__builtins__' and "'module'" not in str(type(cfg[key])): + try: + cfg_copy[key] = copy.deepcopy(cfg[key]) + except Exception as err: + raise EasyBuildError("Failed to copy '%s' easyconfig parameter: %s" % (key, err)) + else: + self.log.debug("Not copying '%s' variable from parsed easyconfig", key) + + return cfg_copy def parse(self, txt): """ diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 5438c031ab..48361feec5 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/two.py b/easybuild/framework/easyconfig/format/two.py index 42d8f93bf7..0c1c3c6c20 100644 --- a/easybuild/framework/easyconfig/format/two.py +++ b/easybuild/framework/easyconfig/format/two.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index 5a643af823..8fc78e66a0 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index 18245de7ec..d87b16e64f 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,7 +29,7 @@ :author: Caroline De Brouwer (Ghent University) :author: Kenneth Hoste (Ghent University) """ - +import copy import os import platform from distutils.version import LooseVersion @@ -91,7 +91,9 @@ def get_config_dict(self): """ Return parsed easyconfig as a dictionary, based on specified arguments. """ - return self.parsed_yeb + # avoid passing anything by reference, so next time get_config_dict is called + # we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file + return copy.deepcopy(self.parsed_yeb) @only_if_module_is_available('yaml') def parse(self, txt): diff --git a/easybuild/framework/easyconfig/licenses.py b/easybuild/framework/easyconfig/licenses.py index 6c2ab7e70f..b7e28358af 100644 --- a/easybuild/framework/easyconfig/licenses.py +++ b/easybuild/framework/easyconfig/licenses.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index c5c9ff64af..f5ff4d9607 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py index 97578197b7..8bcf371d55 100644 --- a/easybuild/framework/easyconfig/style.py +++ b/easybuild/framework/easyconfig/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index a7915c09a3..d268536c63 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,7 +31,6 @@ :author: Fotis Georgatos (Uni.Lu, NTUA) :author: Kenneth Hoste (Ghent University) """ -import copy import re import platform @@ -46,7 +45,6 @@ # derived from easyconfig, but not from ._config directly TEMPLATE_NAMES_EASYCONFIG = [ - ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), ('module_name', "Module name"), ('nameletter', "First letter of software name"), ('toolchain_name', "Toolchain name"), @@ -80,11 +78,24 @@ TEMPLATE_SOFTWARE_VERSIONS = [ # software name, prefix for *ver and *shortver ('CUDA', 'cuda'), + ('CUDAcore', 'cuda'), ('Java', 'java'), ('Perl', 'perl'), ('Python', 'py'), ('R', 'r'), ] +# template values which are only generated dynamically +TEMPLATE_NAMES_DYNAMIC = [ + ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), + ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), + ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " + "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), + ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), + ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), + ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), + ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), +] + # constant templates that can be used in easyconfigs TEMPLATE_CONSTANTS = [ # source url constants @@ -143,6 +154,19 @@ ('SOURCE_%s' % suffix, '%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext), ('SOURCELOWER_%s' % suffix, '%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext), ] +for pyver in ('py2.py3', 'py2', 'py3'): + if pyver == 'py2.py3': + desc = 'Python 2 & Python 3' + name_infix = '' + else: + desc = 'Python ' + pyver[-1] + name_infix = pyver.upper() + '_' + TEMPLATE_CONSTANTS += [ + ('SOURCE_%sWHL' % name_infix, '%%(name)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package' % desc), + ('SOURCELOWER_%sWHL' % name_infix, '%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package with lowercase name' % desc), + ] # TODO derived config templates # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) ) @@ -217,10 +241,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) # step 2: define *ver and *shortver templates - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + if TEMPLATE_SOFTWARE_VERSIONS: - # copy to avoid changing original list below - deps = copy.copy(config.get('dependencies', [])) + name_to_prefix = dict((name.lower(), pref) for name, pref in TEMPLATE_SOFTWARE_VERSIONS) + deps = config.get('dependencies', []) # also consider build dependencies for *ver and *shortver templates; # we need to be a bit careful here, because for iterative installations @@ -232,15 +256,27 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # a cyclic import...); # we need to know to determine whether we're iterating over a list of build dependencies is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options') - if is_easyconfig: # if we're iterating over different lists of build dependencies, # only consider build dependencies when we're actually in iterative mode! if 'builddependencies' in config.iterate_options: if config.iterating: - deps += config.get('builddependencies', []) + build_deps = config.get('builddependencies') + else: + build_deps = None else: - deps += config.get('builddependencies', []) + build_deps = config.get('builddependencies') + if build_deps: + # Don't use += to avoid changing original list + deps = deps + build_deps + # include all toolchain deps (e.g. CUDAcore component in fosscuda); + # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain! + if config._toolchain is not None and config._toolchain.tcdeps: + # If we didn't create a new list above do it here + if build_deps: + deps.extend(config._toolchain.tcdeps) + else: + deps = deps + config._toolchain.tcdeps for dep in deps: if isinstance(dep, dict): @@ -262,15 +298,16 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) else: raise EasyBuildError("Unexpected type for dependency: %s", dep) - if isinstance(dep_name, string_type) and dep_name.lower() == name.lower() and dep_version: - dep_version = pick_dep_version(dep_version) - template_values['%sver' % pref] = dep_version - dep_version_parts = dep_version.split('.') - template_values['%smajver' % pref] = dep_version_parts[0] - if len(dep_version_parts) > 1: - template_values['%sminver' % pref] = dep_version_parts[1] - template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2]) - break + if isinstance(dep_name, string_type) and dep_version: + pref = name_to_prefix.get(dep_name.lower()) + if pref: + dep_version = pick_dep_version(dep_version) + template_values['%sver' % pref] = dep_version + dep_version_parts = dep_version.split('.') + template_values['%smajver' % pref] = dep_version_parts[0] + if len(dep_version_parts) > 1: + template_values['%sminver' % pref] = dep_version_parts[1] + template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2]) # step 3: add remaining from config for name in TEMPLATE_NAMES_CONFIG: @@ -294,6 +331,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) except Exception: _log.warning("Failed to get .lower() for name %s value %s (type %s)", name, value, type(value)) + # keep track of names of defined templates until now, + # so we can check whether names of additional dynamic template values are all known + common_template_names = set(template_values.keys()) + # step 5. add additional conditional templates if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'): try: @@ -310,10 +351,20 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) cuda_compute_capabilities = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities') if cuda_compute_capabilities: template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities) + template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) + template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) + unknown_names = [] + for key in template_values: + dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC) + if not (key in common_template_names or key in dynamic_template_names): + unknown_names.append(key) + if unknown_names: + raise EasyBuildError("One or more template values found with unknown name: %s", ','.join(unknown_names)) + return template_values diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 3e8cce5054..5d165c081b 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,6 +37,7 @@ :author: Ward Poelmans (Ghent University) """ import copy +import fnmatch import glob import os import re @@ -47,7 +48,8 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig -from easybuild.framework.easyconfig.easyconfig import create_paths, get_easyblock_class, process_easyconfig +from easybuild.framework.easyconfig.easyconfig import create_paths, det_file_info, get_easyblock_class +from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning @@ -55,7 +57,9 @@ from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files from easybuild.tools.filetools import read_file, resolve_path, which, write_file -from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_files_from_pr, download_repo +from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import det_pr_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data +from easybuild.tools.github import fetch_files_from_pr from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -304,7 +308,7 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): return paths -def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False): +def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=None): """Obtain alternative paths for easyconfig files.""" # paths where tweaked easyconfigs will be placed, easyconfigs listed on the command line take priority and will be @@ -315,12 +319,18 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False): tweaked_ecs_paths = (os.path.join(tmpdir, 'tweaked_easyconfigs'), os.path.join(tmpdir, 'tweaked_dep_easyconfigs')) - # path where files touched in PR will be downloaded to - pr_path = None - if from_pr: - pr_path = os.path.join(tmpdir, "files_pr%s" % from_pr) + # paths where files touched in PRs will be downloaded to, + # which are picked up via 'pr_paths' build option in fetch_files_from_pr + pr_paths = [] + if from_prs: + pr_paths = from_prs[:] + if review_pr and review_pr not in pr_paths: + pr_paths.append(review_pr) + + if pr_paths: + pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in pr_paths] - return tweaked_ecs_paths, pr_path + return tweaked_ecs_paths, pr_paths def det_easyconfig_paths(orig_paths): @@ -329,14 +339,22 @@ def det_easyconfig_paths(orig_paths): :param orig_paths: list of original easyconfig paths :return: list of paths to easyconfig files """ - from_pr = build_option('from_pr') + try: + from_prs = [int(x) for x in build_option('from_pr')] + except ValueError: + raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + robot_path = build_option('robot_path') # list of specified easyconfig files ec_files = orig_paths[:] - if from_pr is not None: - pr_files = fetch_easyconfigs_from_pr(from_pr) + if from_prs: + pr_files = [] + for pr in from_prs: + # path to where easyconfig files should be downloaded is determined via 'pr_paths' build option, + # which corresponds to the list of PR paths returned by alt_easyconfig_paths + pr_files.extend(fetch_easyconfigs_from_pr(pr)) if ec_files: # replace paths for specified easyconfigs that are touched in PR @@ -348,6 +366,10 @@ def det_easyconfig_paths(orig_paths): # if no easyconfigs are specified, use all the ones touched in the PR ec_files = [path for path in pr_files if path.endswith('.eb')] + filter_ecs = build_option('filter_ecs') + if filter_ecs: + ec_files = [ec for ec in ec_files + if not any(fnmatch.fnmatch(ec, filter_spec) for filter_spec in filter_ecs)] if ec_files and robot_path: ignore_subdirs = build_option('ignore_dirs') if not build_option('consider_archived_easyconfigs'): @@ -440,7 +462,7 @@ def find_related_easyconfigs(path, ec): toolchain_pattern = '' potential_paths = [glob.glob(ec_path) for ec_path in create_paths(path, name, '*')] - potential_paths = sum(potential_paths, []) # flatten + potential_paths = sorted(sum(potential_paths, []), reverse=True) # flatten _log.debug("found these potential paths: %s" % potential_paths) parsed_version = LooseVersion(version).version @@ -471,17 +493,22 @@ def find_related_easyconfigs(path, ec): else: _log.debug("No related easyconfigs in potential paths using '%s'" % regex) - return sorted(res) + return res -def review_pr(paths=None, pr=None, colored=True, branch='develop'): +def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False, max_ecs=None, filter_ecs=None): """ Print multi-diff overview between specified easyconfigs or PR and specified branch. :param pr: pull request number in easybuild-easyconfigs repo to review :param paths: path tuples (path, generated) of easyconfigs to review :param colored: boolean indicating whether a colored multi-diff should be generated :param branch: easybuild-easyconfigs branch to compare with + :param testing: whether to ignore PR labels (used in test_review_pr) """ + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO + if pr_target_repo != GITHUB_EASYCONFIGS_REPO: + raise EasyBuildError("Reviewing PRs for repositories other than easyconfigs hasn't been implemented yet") + tmpdir = tempfile.mkdtemp() download_repo_path = download_repo(branch=branch, path=tmpdir) @@ -504,13 +531,76 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop'): pr_msg = "new PR" _log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files)) if files: + if filter_ecs is not None: + files = [x for x in files if filter_ecs.search(x)] + if max_ecs is not None: + files = files[:max_ecs] lines.append(multidiff(ec['spec'], files, colored=colored)) else: lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])]) + if pr: + file_info = det_file_info(pr_files, download_repo_path) + + pr_target_account = build_option('pr_target_account') + github_user = build_option('github_user') + pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) + pr_labels = [label['name'] for label in pr_data['labels']] if not testing else [] + + expected_labels = det_pr_labels(file_info, pr_target_repo) + missing_labels = [label for label in expected_labels if label not in pr_labels] + + if missing_labels: + lines.extend(['', "This PR should be labelled with %s" % ', '.join(["'%s'" % ml for ml in missing_labels])]) + + if not pr_data['milestone']: + lines.extend(['', "This PR should be associated with a milestone"]) + elif '.x' in pr_data['milestone']['title']: + lines.extend(['', "This PR is associated with a generic '.x' milestone, " + "it should be associated to the next release milestone once merged"]) + return '\n'.join(lines) +def dump_env_easyblock(app, orig_env=None, ec_path=None, script_path=None, silent=False): + if orig_env is None: + orig_env = copy.deepcopy(os.environ) + if ec_path is None: + raise EasyBuildError("The path to the easyconfig relevant to this environment dump is required") + if script_path is None: + # Assume we are placing it alongside the easyconfig path + script_path = '%s.env' % os.path.splitext(ec_path)[0] + # Compose script + ecfile = os.path.basename(ec_path) + script_lines = [ + "#!/bin/bash", + "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), + "# usage: source %s" % os.path.basename(script_path), + ] + + script_lines.extend(['', "# toolchain & dependency modules"]) + if app.toolchain.modules: + script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules]) + else: + script_lines.append("# (no modules loaded)") + + script_lines.extend(['', "# build environment"]) + if app.toolchain.vars: + env_vars = sorted(app.toolchain.vars.items()) + script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars]) + else: + script_lines.append("# (no build environment defined)") + + write_file(script_path, '\n'.join(script_lines)) + msg = "Script to set up build environment for %s dumped to %s" % (ecfile, script_path) + if silent: + _log.info(msg) + else: + print_msg(msg, prefix=False) + + restore_env(orig_env) + + def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. @@ -545,31 +635,8 @@ def dump_env_script(easyconfigs): app.check_readiness_step() app.prepare_step(start_dir=False) - # compose script - ecfile = os.path.basename(ec.path) - script_lines = [ - "#!/bin/bash", - "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), - "# usage: source %s" % os.path.basename(script_path), - ] - - script_lines.extend(['', "# toolchain & dependency modules"]) - if app.toolchain.modules: - script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules]) - else: - script_lines.append("# (no modules loaded)") - - script_lines.extend(['', "# build environment"]) - if app.toolchain.vars: - env_vars = sorted(app.toolchain.vars.items()) - script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars]) - else: - script_lines.append("# (no build environment defined)") - - write_file(script_path, '\n'.join(script_lines)) - print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False) - - restore_env(orig_env) + # create the environment dump + dump_env_easyblock(app, orig_env=orig_env, ec_path=ec.path, script_path=script_path) def categorize_files_by_type(paths): @@ -592,6 +659,14 @@ def categorize_files_by_type(paths): # file must exist in order to check whether it's a patch file elif os.path.isfile(path) and is_patch_file(path): res['patch_files'].append(path) + elif path.endswith('.patch'): + if not os.path.exists(path): + raise EasyBuildError('File %s does not exist, did you mistype the path?', path) + elif not os.path.isfile(path): + raise EasyBuildError('File %s is expected to be a regular file, but is a folder instead', path) + else: + raise EasyBuildError('%s is not detected as a valid patch file. Please verify its contents!', + path) else: # anything else is considered to be an easyconfig file res['easyconfigs'].append(path) @@ -703,6 +778,9 @@ def avail_easyblocks(): def det_copy_ec_specs(orig_paths, from_pr): """Determine list of paths + target directory for --copy-ec.""" + if from_pr is not None and not isinstance(from_pr, list): + from_pr = [from_pr] + target_path, paths = None, [] # if only one argument is specified, use current directory as target directory @@ -724,8 +802,10 @@ def det_copy_ec_specs(orig_paths, from_pr): # to avoid potential trouble with already existing files in the working tmpdir # (note: we use a fixed subdirectory in the working tmpdir here rather than a unique random subdirectory, # to ensure that the caching for fetch_files_from_pr works across calls for the same PR) - tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_pr_%s' % from_pr) - pr_paths = fetch_files_from_pr(pr=from_pr, path=tmpdir) + tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_pr_%s' % '_'.join(str(pr) for pr in from_pr)) + pr_paths = [] + for pr in from_pr: + pr_paths.extend(fetch_files_from_pr(pr=pr, path=tmpdir)) # assume that files need to be copied to current working directory for now target_path = os.getcwd() diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 7d15be21b0..c3fdfa0904 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -45,7 +45,7 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS -from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default +from easybuild.framework.easyconfig.default import is_easyconfig_parameter_default_value from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION @@ -324,13 +324,31 @@ def __repr__(self): _log.debug("Overwriting %s with %s" % (key, fval)) ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) - elif get_easyconfig_parameter_default(key) != val: + elif not is_easyconfig_parameter_default_value(key, val): additions.append("%s = %s" % (key, val)) tweaks.pop(key) # add parameters or replace existing ones + special_values = { + # if the value is True/False/None then take that + 'True': True, + 'False': False, + 'None': None, + # if e.g. (literal) True is wanted, then it can be passed as "True"/'True' + "'True'": 'True', + '"True"': 'True', + "'False'": 'False', + '"False"': 'False', + "'None'": 'None', + '"None"': 'None', + } for (key, val) in tweaks.items(): + if isinstance(val, string_type) and val in special_values: + str_val = val + val = special_values[val] + else: + str_val = quote_str(val) regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P.*)$" % key, re.M) _log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern)) @@ -348,10 +366,10 @@ def __repr__(self): diff = res.group('val') != val if diff: - ectxt = regexp.sub("%s = %s" % (res.group('key'), quote_str(val)), ectxt) - _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val))) - elif get_easyconfig_parameter_default(key) != val: - additions.append("%s = %s" % (key, quote_str(val))) + ectxt = regexp.sub("%s = %s" % (res.group('key'), str_val), ectxt) + _log.info("Tweaked '%s' to '%s'" % (key, str_val)) + elif not is_easyconfig_parameter_default_value(key, val): + additions.append("%s = %s" % (key, str_val)) if additions: _log.info("Adding additional parameters to tweaked easyconfig file: %s" % additions) diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index 98737328d8..77495a1f31 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 8607734774..fe56253b3f 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Ghent University +# Copyright 2020-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,12 +27,13 @@ :author: Denis Kristak (Inuits) :author: Pavel Grochal (Inuits) +:author: Kenneth Hoste (HPC-UGent) """ - from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import only_if_module_is_available try: import yaml @@ -41,6 +42,25 @@ _log = fancylogger.getLogger('easystack', fname=False) +def check_value(value, context): + """ + Check whether specified value obtained from a YAML file in specified context represents is valid. + The value must be a string (not a float or an int). + """ + if not isinstance(value, string_type): + error_msg = '\n'.join([ + "Value %(value)s (of type %(type)s) obtained for %(context)s is not valid!", + "Make sure to wrap the value in single quotes (like '%(value)s') to avoid that it is interpreted " + "by the YAML parser as a non-string value.", + ]) + format_info = { + 'context': context, + 'type': type(value), + 'value': value, + } + raise EasyBuildError(error_msg % format_info) + + class EasyStack(object): """One class instance per easystack. General options + list of all SoftwareSpecs instances""" @@ -90,7 +110,12 @@ class EasyStackParser(object): def parse(filepath): """Parses YAML file and assigns obtained values to SW config instances as well as general config instance""" yaml_txt = read_file(filepath) - easystack_raw = yaml.safe_load(yaml_txt) + + try: + easystack_raw = yaml.safe_load(yaml_txt) + except yaml.YAMLError as err: + raise EasyBuildError("Failed to parse %s: %s" % (filepath, err)) + easystack = EasyStack() try: @@ -103,13 +128,13 @@ def parse(filepath): for name in software: # ensure we have a string value (YAML parser returns type = dict # if levels under the current attribute are present) - name = str(name) + check_value(name, "software name") try: toolchains = software[name]['toolchains'] except KeyError: raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath) for toolchain in toolchains: - toolchain = str(toolchain) + check_value(toolchain, "software %s" % name) if toolchain == 'SYSTEM': toolchain_name, toolchain_version = 'system', '' @@ -140,12 +165,13 @@ def parse(filepath): # Example of yaml structure: # ======================================================================== # versions: - # 2.25: - # 2.23: + # '2.25': + # '2.23': # versionsuffix: '-R-4.0.0' # ======================================================================== if isinstance(versions, dict): for version in versions: + check_value(version, "%s (with %s toolchain)" % (name, toolchain_name)) if versions[version] is not None: version_spec = versions[version] if 'versionsuffix' in version_spec: @@ -172,35 +198,25 @@ def parse(filepath): easystack.software_list.append(sw) continue - # is format read as a list of versions? - # ======================================================================== - # versions: - # [2.24, 2.51] - # ======================================================================== - elif isinstance(versions, list): - versions_list = versions + elif isinstance(versions, (list, tuple)): + pass - # format = multiple lines without ':' (read as a string)? - # ======================================================================== + # multiple lines without ':' is read as a single string; example: # versions: - # 2.24 - # 2.51 - # ======================================================================== - elif isinstance(versions, str): - versions_list = str(versions).split() + # '2.24' + # '2.51' + elif isinstance(versions, string_type): + versions = versions.split() - # format read as float (containing one version only)? - # ======================================================================== - # versions: - # 2.24 - # ======================================================================== - elif isinstance(versions, float): - versions_list = [str(versions)] + # single values like '2.24' should be wrapped in a list + else: + versions = [versions] - # if no version is a dictionary, versionsuffix isn't specified + # if version is not a dictionary, versionsuffix is not specified versionsuffix = '' - for version in versions_list: + for version in versions: + check_value(version, "%s (with %s toolchain)" % (name, toolchain_name)) sw = SoftwareSpecs( name=name, version=version, versionsuffix=versionsuffix, toolchain_name=toolchain_name, toolchain_version=toolchain_version) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 569a3bb414..badc42856c 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import check_async_cmd, run_cmd from easybuild.tools.py2vs3 import string_type @@ -105,19 +105,21 @@ def __init__(self, mself, ext, extra_params=None): name, version = self.ext['name'], self.ext.get('version', None) - # parent sanity check paths/commands are not relevant for extension + # parent sanity check paths/commands and postinstallcmds are not relevant for extension self.cfg['sanity_check_commands'] = [] self.cfg['sanity_check_paths'] = [] + self.cfg['postinstallcmds'] = [] # construct dict with template values that can be used self.cfg.template_values.update(template_constant_dict({'name': name, 'version': version})) # Add install/builddir templates with values from master. - for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - self.cfg.template_values[name[0]] = str(getattr(self.master, name[0], None)) + for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: + self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None)) # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) + self.src_extract_cmd = self.ext.get('extract_cmd', None) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) @@ -138,6 +140,12 @@ def __init__(self, mself, ext, extra_params=None): key, name, version, value) self.sanity_check_fail_msgs = [] + self.async_cmd_info = None + self.async_cmd_output = None + self.async_cmd_check_cnt = None + # initial read size should be relatively small, + # to avoid hanging for a long time until desired output is available in async_cmd_check + self.async_cmd_read_size = 1024 @property def name(self): @@ -159,17 +167,67 @@ def prerun(self): """ pass - def run(self): + def run(self, *args, **kwargs): """ - Actual installation of a extension. + Actual installation of an extension. """ pass + def run_async(self, *args, **kwargs): + """ + Asynchronous installation of an extension. + """ + raise NotImplementedError + def postrun(self): """ Stuff to do after installing a extension. """ - pass + self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) + + def async_cmd_start(self, cmd, inp=None): + """ + Start installation asynchronously using specified command. + """ + self.async_cmd_output = '' + self.async_cmd_check_cnt = 0 + self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True) + + def async_cmd_check(self): + """ + Check progress of installation command that was started asynchronously. + + :return: True if command completed, False otherwise + """ + if self.async_cmd_info is None: + raise EasyBuildError("No installation command running asynchronously for %s", self.name) + elif self.async_cmd_info is False: + self.log.info("No asynchronous command was started for extension %s", self.name) + return True + else: + self.log.debug("Checking on installation of extension %s...", self.name) + # use small read size, to avoid waiting for a long time until sufficient output is produced + res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size) + self.async_cmd_output += res['output'] + if res['done']: + self.log.info("Installation of extension %s completed!", self.name) + self.async_cmd_info = None + else: + self.async_cmd_check_cnt += 1 + self.log.debug("Installation of extension %s still running (checked %d times)", + self.name, self.async_cmd_check_cnt) + # increase read size after sufficient checks, + # to avoid that installation hangs due to output buffer filling up... + if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): + self.async_cmd_read_size *= 2 + + return res['done'] + + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + self.log.info("Don't know how to determine required dependencies for extension '%s'", self.name) + return None @property def toolchain(self): diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index c3b5c7a9fb..8037e78507 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). @@ -118,7 +118,7 @@ def run(self, unpack_src=False): if unpack_src: targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name)) self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options, - change_into_dir=False) + change_into_dir=False, cmd=self.src_extract_cmd) # setting start dir must be done from unpacked source directory for extension, # because start_dir value is usually a relative path (if it is set) diff --git a/easybuild/main.py b/easybuild/main.py index f13ed688b6..c43c0583df 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -55,24 +55,29 @@ from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak +from easybuild.tools.build_log import print_warning from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software from easybuild.tools.filetools import adjust_permissions, cleanup, copy_files, dump_index, load_index from easybuild.tools.filetools import locate_files, read_file, register_lock_cleanup_signal_handlers, write_file from easybuild.tools.github import check_github, close_pr, find_easybuild_easyconfig -from easybuild.tools.github import install_github_token, list_prs, merge_pr, new_branch_github, new_pr +from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_up_configuration, use_color +from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm +from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository +from easybuild.tools.systemtools import check_easybuild_deps from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state + _log = None @@ -110,8 +115,14 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) + start_progress_bar(STATUS_BAR, size=len(ecs)) + res = [] + ec_results = [] + failed_cnt = 0 + for ec in ecs: + ec_res = {} try: (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) @@ -124,6 +135,12 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): ec_res['err'] = err ec_res['traceback'] = traceback.format_exc() + if ec_res['success']: + ec_results.append(ec['full_mod_name'] + ' (' + colorize('OK', COLOR_GREEN) + ')') + else: + ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')') + failed_cnt += 1 + # keep track of success/total count if ec_res['success']: test_msg = "Successfully built %s" % ec['spec'] @@ -153,6 +170,19 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): res.append((ec, ec_res)) + if failed_cnt: + # if installations failed: indicate th + status_label = ' (%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED) + failed_ecs = [x for x in ec_results[::-1] if 'FAILED' in x] + ok_ecs = [x for x in ec_results[::-1] if x not in failed_ecs] + status_label += ', '.join(failed_ecs + ok_ecs) + else: + status_label = ': ' + ', '.join(ec_results[::-1]) + + update_progress_bar(STATUS_BAR, label=status_label) + + stop_progress_bar(STATUS_BAR) + return res @@ -207,7 +237,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): options, orig_paths = eb_go.options, eb_go.args global _log - (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings + (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, + from_pr_list, tweaked_ecs_paths) = cfg_settings # load hook implementations (if any) hooks = load_hooks(options.hooks) @@ -243,6 +274,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) + if options.check_eb_deps: + print_checks(check_easybuild_deps(modtool)) + # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() @@ -260,7 +294,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): merge_pr(options.merge_pr) elif options.review_pr: - print(review_pr(pr=options.review_pr, colored=use_color(options.color))) + print(review_pr(pr=options.review_pr, colored=use_color(options.color), testing=testing, + max_ecs=options.review_pr_max, filter_ecs=options.review_pr_filter)) + + elif options.add_pr_labels: + add_pr_labels(options.add_pr_labels) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' @@ -277,6 +315,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ + options.add_pr_labels, + options.check_eb_deps, options.check_github, options.create_index, options.install_github_token, @@ -299,6 +339,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) + if options.skip_test_step: + if options.ignore_test_failure: + raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. " + "Please use only one of them.") + else: + print_warning("Will not run the test step as requested via skip-test-step. " + "Consider using ignore-test-failure instead and verify the results afterwards") + # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: @@ -314,7 +362,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.copy_ec: # figure out list of files to copy + target location (taking into account --from-pr) - orig_paths, target_path = det_copy_ec_specs(orig_paths, options.from_pr) + orig_paths, target_path = det_copy_ec_specs(orig_paths, from_pr_list) categorized_paths = categorize_files_by_type(orig_paths) @@ -399,8 +447,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules + keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options or options.copy_ec + keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only + # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping - if not (forced or dry_run_mode or options.extended_dry_run or pr_options or options.inject_checksums): + if not keep_available_modules: retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: @@ -480,7 +531,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): dump_env_script(easyconfigs) elif options.inject_checksums: - inject_checksums(ordered_ecs, options.inject_checksums) + with rich_live_cm(): + inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] @@ -504,13 +556,18 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) + with rich_live_cm(): + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, + exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) - success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) + success_msg = "Build succeeded " + if build_option('ignore_test_failure'): + success_msg += "(with --ignore-test-failure) " + success_msg += "for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 1ccc317466..c1914cdf65 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -62,7 +62,7 @@ import urllib.request as std_urllib -EB_BOOTSTRAP_VERSION = '20210106.01' +EB_BOOTSTRAP_VERSION = '20210715.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False @@ -82,6 +82,9 @@ STAGE1_SUBDIR = 'eb_stage1' +# the EasyBuild bootstrap script is deprecated, and will only run if $EASYBUILD_BOOTSTRAP_DEPRECATED is defined +EASYBUILD_BOOTSTRAP_DEPRECATED = os.environ.pop('EASYBUILD_BOOTSTRAP_DEPRECATED', None) + # set print_debug to True for detailed progress info print_debug = os.environ.pop('EASYBUILD_BOOTSTRAP_DEBUG', False) @@ -854,6 +857,17 @@ def main(): self_txt = open(__file__).read() if IS_PY3: self_txt = self_txt.encode('utf-8') + + url = 'https://docs.easybuild.io/en/latest/Installation.html' + info("Use of the EasyBuild boostrap script is DEPRECATED (since June 2021).") + info("It is strongly recommended to use one of the installation methods outlined at %s instead!\n" % url) + if not EASYBUILD_BOOTSTRAP_DEPRECATED: + error("The EasyBuild bootstrap script will only run if $EASYBUILD_BOOTSTRAP_DEPRECATED is defined.") + else: + msg = "You have opted to continue with the EasyBuild bootstrap script by defining " + msg += "$EASYBUILD_BOOTSTRAP_DEPRECATED. Good luck!\n" + info(msg) + info("EasyBuild bootstrap script (version %s, MD5: %s)" % (EB_BOOTSTRAP_VERSION, md5(self_txt).hexdigest())) info("Found Python %s\n" % '; '.join(sys.version.split('\n'))) @@ -911,7 +925,7 @@ def main(): for path in orig_sys_path: include_path = True # exclude path if it's potentially an EasyBuild/VSC package, providing the 'easybuild'/'vsc' namespace, resp. - if any([os.path.exists(os.path.join(path, pkg, '__init__.py')) for pkg in ['easyblocks', 'easybuild', 'vsc']]): + if any(os.path.exists(os.path.join(path, pkg, '__init__.py')) for pkg in ['easyblocks', 'easybuild', 'vsc']): include_path = False # exclude any .egg paths if path.endswith('.egg'): diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index b5761713be..a290de3f42 100644 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -32,9 +32,10 @@ from easybuild.base.generaloption import simple_option from easybuild.base.rest import RestClient from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token from easybuild.tools.options import EasyBuildOptions +from easybuild.tools.py2vs3 import HTTPError, URLError HTTP_DELETE_OK = 204 @@ -49,6 +50,7 @@ def main(): 'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'), 'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'), 'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'), + 'dry-run': ("Only show which gists will be deleted but don't actually delete them", None, 'store_true', False), } go = simple_option(options) @@ -58,6 +60,7 @@ def main(): raise EasyBuildError("Please tell me what to do?") if go.options.github_user is None: + EasyBuildOptions.DEFAULT_LOGLEVEL = None # Don't overwrite log level eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[]) username = eb_go.options.github_user log.debug("Fetch github username from easybuild, found: %s", username) @@ -88,7 +91,8 @@ def main(): break log.info("Found %s gists", len(all_gists)) - regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P[0-9]+))?\)?$") + re_eb_gist = re.compile(r"(EasyBuild test report|EasyBuild log for failed build)(.*?)$") + re_pr_nr = re.compile(r"(EB )?PR #([0-9]+)") pr_cache = {} num_deleted = 0 @@ -96,43 +100,79 @@ def main(): for gist in all_gists: if not gist["description"]: continue - re_pr_num = regex.search(gist["description"]) - delete_gist = False - - if re_pr_num: - log.debug("Found a Easybuild gist (id=%s)", gist["id"]) - pr_num = re_pr_num.group("PR") - if go.options.all: - delete_gist = True - elif pr_num and go.options.closed_pr: - log.debug("Found Easybuild test report for PR #%s", pr_num) - - if pr_num not in pr_cache: - status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get() + + gist_match = re_eb_gist.search(gist["description"]) + + if not gist_match: + log.debug("Found a non-Easybuild gist (id=%s)", gist["id"]) + continue + + log.debug("Found an Easybuild gist (id=%s)", gist["id"]) + + pr_data = gist_match.group(2) + + pr_nrs_matches = re_pr_nr.findall(pr_data) + + if go.options.all: + delete_gist = True + elif not pr_nrs_matches: + log.debug("Found Easybuild test report without PR (id=%s).", gist["id"]) + delete_gist = go.options.orphans + elif go.options.closed_pr: + # All PRs must be closed + delete_gist = True + for pr_nr_match in pr_nrs_matches: + eb_str, pr_num = pr_nr_match + if eb_str or GITHUB_EASYBLOCKS_REPO in pr_data: + repo = GITHUB_EASYBLOCKS_REPO + else: + repo = GITHUB_EASYCONFIGS_REPO + + cache_key = "%s-%s" % (repo, pr_num) + + if cache_key not in pr_cache: + try: + status, pr = gh.repos[GITHUB_EB_MAIN][repo].pulls[pr_num].get() + except HTTPError as e: + status, pr = e.code, e.msg if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get pull-request #%s: error code %s, message = %s", pr_num, status, pr) - pr_cache[pr_num] = pr["state"] - - if pr_cache[pr_num] == "closed": - log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"]) - delete_gist = True - - elif not pr_num and go.options.orphans: - log.debug("Found Easybuild test report without PR (id=%s)", gist["id"]) - delete_gist = True + pr_cache[cache_key] = pr["state"] + + if pr_cache[cache_key] == "closed": + log.debug("Found report from closed %s PR #%s (id=%s)", repo, pr_num, gist["id"]) + elif delete_gist: + if len(pr_nrs_matches) > 1: + log.debug("Found at least 1 PR, that is not closed yet: %s/%s (id=%s)", + repo, pr_num, gist["id"]) + delete_gist = False + else: + delete_gist = True if delete_gist: - status, del_gist = gh.gists[gist["id"]].delete() + if go.options.dry_run: + log.info("DRY-RUN: Delete gist with id=%s", gist["id"]) + num_deleted += 1 + continue + try: + status, del_gist = gh.gists[gist["id"]].delete() + except HTTPError as e: + status, del_gist = e.code, e.msg + except URLError as e: + status, del_gist = None, e.reason if status != HTTP_DELETE_OK: - raise EasyBuildError("Unable to remove gist (id=%s): error code %s, message = %s", - gist["id"], status, del_gist) + log.warning("Unable to remove gist (id=%s): error code %s, message = %s", + gist["id"], status, del_gist) else: - log.info("Delete gist with id=%s", gist["id"]) + log.info("Deleted gist with id=%s", gist["id"]) num_deleted += 1 - log.info("Deleted %s gists", num_deleted) + if go.options.dry_run: + log.info("DRY-RUN: Would delete %s gists", num_deleted) + else: + log.info("Deleted %s gists", num_deleted) if __name__ == '__main__': diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py index 8c40e34d4f..5a02fde0b0 100644 --- a/easybuild/scripts/fix_docs.py +++ b/easybuild/scripts/fix_docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -55,15 +55,14 @@ # exclude self if os.path.basename(tmp) == os.path.basename(__file__): continue - with open(tmp) as f: + with open(tmp) as fh: temp = "tmp_file.py" - out = open(temp, 'w') - for line in f: - if "@author" in line: - out.write(re.sub(r"@author: (.*)", r":author: \1", line)) - elif "@param" in line: - out.write(re.sub(r"@param ([^:]*):", r":param \1:", line)) - else: - out.write(line) - out.close() + with open(temp, 'w') as out: + for line in fh: + if "@author" in line: + out.write(re.sub(r"@author: (.*)", r":author: \1", line)) + elif "@param" in line: + out.write(re.sub(r"@param ([^:]*):", r":param \1:", line)) + else: + out.write(line) os.rename(temp, tmp) diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh index 82c34b774d..0b6fb49d11 100755 --- a/easybuild/scripts/install_eb_dep.sh +++ b/easybuild/scripts/install_eb_dep.sh @@ -16,16 +16,18 @@ PKG_VERSION="${PKG##*-}" CONFIG_OPTIONS= PRECONFIG_CMD= -if [ "$PKG_NAME" == 'modules' ] && [ "$PKG_VERSION" == '3.2.10' ]; then - PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz" - BACKUP_PKG_URL="https://easybuilders.github.io/easybuild/files/${PKG}.tar.gz" - export PATH="$PREFIX/Modules/$PKG_VERSION/bin:$PATH" - export MOD_INIT="$PREFIX/Modules/$PKG_VERSION/init/bash" +BACKUP_PKG_URL= -elif [ "$PKG_NAME" == 'modules' ]; then +if [ "$PKG_NAME" == 'modules' ]; then PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz" - export PATH="$PREFIX/bin:$PATH" - export MOD_INIT="$PREFIX/init/bash" + BACKUP_PKG_URL="https://sources.easybuild.io/e/EnvironmentModules/${PKG}.tar.gz" + if [ "$PKG_VERSION" == '3.2.10' ]; then + export PATH="$PREFIX/Modules/$PKG_VERSION/bin:$PATH" + export MOD_INIT="$PREFIX/Modules/$PKG_VERSION/init/bash" + else + export PATH="$PREFIX/bin:$PATH" + export MOD_INIT="$PREFIX/init/bash" + fi elif [ "$PKG_NAME" == 'lua' ]; then PKG_URL="http://downloads.sourceforge.net/project/lmod/${PKG}.tar.gz" diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index 34cdbf8b0d..928a67d8be 100644 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -225,9 +225,8 @@ def make_module_extra(self): dirpath = os.path.dirname(easyblock_path) if not os.path.exists(dirpath): os.makedirs(dirpath) - f = open(easyblock_path, "w") - f.write(txt) - f.close() + with open(easyblock_path, "w") as fh: + fh.write(txt) except (IOError, OSError) as err: sys.stderr.write("ERROR! Writing template easyblock for %s to %s failed: %s" % (name, easyblock_path, err)) sys.exit(1) diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 23471f9344..9c4ab8b355 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 501e0aa047..c14e249076 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -1,6 +1,6 @@ #!/bin/bash ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/__init__.py b/easybuild/toolchains/__init__.py index ae86d1c599..842767b3d3 100644 --- a/easybuild/toolchains/__init__.py +++ b/easybuild/toolchains/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py index 6348249534..675c9eb5e8 100644 --- a/easybuild/toolchains/cgmpich.py +++ b/easybuild/toolchains/cgmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmpolf.py b/easybuild/toolchains/cgmpolf.py index fbd7d4fec9..ab7ac81b1d 100644 --- a/easybuild/toolchains/cgmpolf.py +++ b/easybuild/toolchains/cgmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py index 07e5c99925..9b73f92488 100644 --- a/easybuild/toolchains/cgmvapich2.py +++ b/easybuild/toolchains/cgmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvolf.py b/easybuild/toolchains/cgmvolf.py index 8d88de2133..321fbee7a6 100644 --- a/easybuild/toolchains/cgmvolf.py +++ b/easybuild/toolchains/cgmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py index 5f94167ba6..5c405c710c 100644 --- a/easybuild/toolchains/cgompi.py +++ b/easybuild/toolchains/cgompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgoolf.py b/easybuild/toolchains/cgoolf.py index 7b7b91990d..6e1fff1d8a 100644 --- a/easybuild/toolchains/cgoolf.py +++ b/easybuild/toolchains/cgoolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/clanggcc.py b/easybuild/toolchains/clanggcc.py index d7cbeb9a67..15a67af146 100644 --- a/easybuild/toolchains/clanggcc.py +++ b/easybuild/toolchains/clanggcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/__init__.py b/easybuild/toolchains/compiler/__init__.py index 96308aa9a1..cff9f74a89 100644 --- a/easybuild/toolchains/compiler/__init__.py +++ b/easybuild/toolchains/compiler/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 8402cae0e1..c13a0978ae 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py index 698a60e4e8..ad81ff0363 100644 --- a/easybuild/toolchains/compiler/craype.py +++ b/easybuild/toolchains/compiler/craype.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index 5a75bc302e..2c4de92ed5 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/dummycompiler.py b/easybuild/toolchains/compiler/dummycompiler.py index 4558e05b7e..5316104b7a 100644 --- a/easybuild/toolchains/compiler/dummycompiler.py +++ b/easybuild/toolchains/compiler/dummycompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py new file mode 100644 index 0000000000..92efd02577 --- /dev/null +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -0,0 +1,116 @@ +## +# Copyright 2014-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for the Fujitsu compiler drivers (aka fcc, frt). + +The basic concept is the same as for the Cray Programming Environment. + +:author: Miguel Dias Costa (National University of Singapore) +""" +import os +import re + +import easybuild.tools.environment as env +import easybuild.tools.systemtools as systemtools +from easybuild.tools.toolchain.compiler import Compiler, DEFAULT_OPT_LEVEL + +TC_CONSTANT_FUJITSU = 'Fujitsu' +TC_CONSTANT_MODULE_NAME = 'lang' +TC_CONSTANT_MODULE_VAR = 'FJSVXTCLANGA' + + +class FujitsuCompiler(Compiler): + """Generic support for using Fujitsu compiler drivers.""" + TOOLCHAIN_FAMILY = TC_CONSTANT_FUJITSU + + COMPILER_MODULE_NAME = [TC_CONSTANT_MODULE_NAME] + COMPILER_FAMILY = TC_CONSTANT_FUJITSU + + COMPILER_CC = 'fcc' + COMPILER_CXX = 'FCC' + + COMPILER_F77 = 'frt' + COMPILER_F90 = 'frt' + COMPILER_FC = 'frt' + + COMPILER_UNIQUE_OPTION_MAP = { + DEFAULT_OPT_LEVEL: 'O2', + 'lowopt': 'O1', + 'noopt': 'O0', + 'opt': 'Kfast', # -O3 -Keval,fast_matmul,fp_contract,fp_relaxed,fz,ilfunc,mfunc,omitfp,simd_packed_promotion + 'optarch': '', # Fujitsu compiler by default generates code for the arch it is running on + 'openmp': 'Kopenmp', + 'unroll': 'funroll-loops', + # apparently the -Kfp_precision flag doesn't work in clang mode, will need to look into these later + # also at strict vs precise and loose vs veryloose + 'strict': ['Knoeval,nofast_matmul,nofp_contract,nofp_relaxed,noilfunc'], # ['Kfp_precision'], + 'precise': ['Knoeval,nofast_matmul,nofp_contract,nofp_relaxed,noilfunc'], # ['Kfp_precision'], + 'defaultprec': [], + 'loose': ['Kfp_relaxed'], + 'veryloose': ['Kfp_relaxed'], + # apparently the -K[NO]SVE flags don't work in clang mode + # SVE is enabled by default, -Knosimd seems to disable it + 'vectorize': {False: 'Knosimd', True: ''}, + } + + # used when 'optarch' toolchain option is enabled (and --optarch is not specified) + COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { + # -march=archi[+features]. At least on Fugaku, these are set by default (-march=armv8.3-a+sve and -mcpu=a64fx) + (systemtools.AARCH64, systemtools.ARM): '', + } + + # used with --optarch=GENERIC + COMPILER_GENERIC_OPTION = { + (systemtools.AARCH64, systemtools.ARM): '-mcpu=generic -mtune=generic', + } + + def prepare(self, *args, **kwargs): + super(FujitsuCompiler, self).prepare(*args, **kwargs) + + # fcc doesn't accept e.g. -std=c++11 or -std=gnu++11, only -std=c11 or -std=gnu11 + pattern = r'-std=(gnu|c)\+\+(\d+)' + if re.search(pattern, self.vars['CFLAGS']): + self.log.debug("Found '-std=(gnu|c)++' in CFLAGS, fcc doesn't accept '++' here, removing it") + self.vars['CFLAGS'] = re.sub(pattern, r'-std=\1\2', self.vars['CFLAGS']) + self._setenv_variables() + + # make sure the fujitsu module libraries are found (and added to rpath by wrapper) + library_path = os.getenv('LIBRARY_PATH', '') + libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') + if libdir not in library_path: + self.log.debug("Adding %s to $LIBRARY_PATH" % libdir) + env.setvar('LIBRARY_PATH', os.pathsep.join([library_path, libdir])) + + def _set_compiler_vars(self): + super(FujitsuCompiler, self)._set_compiler_vars() + + # enable clang compatibility mode + self.variables.nappend('CFLAGS', ['Nclang']) + self.variables.nappend('CXXFLAGS', ['Nclang']) + + # also add fujitsu module library path to LDFLAGS + libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64') + self.log.debug("Adding %s to $LDFLAGS" % libdir) + self.variables.nappend('LDFLAGS', [libdir]) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index affcf588b8..c7966acb7e 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py new file mode 100644 index 0000000000..b2a571a17d --- /dev/null +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -0,0 +1,63 @@ +## +# Copyright 2021-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Intel compilers (icc, ifort) as toolchain compilers, version 2021.x and newer (oneAPI). + +:author: Kenneth Hoste (Ghent University) +""" +import os + +from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort +from easybuild.tools.toolchain.compiler import Compiler + + +class IntelCompilers(IntelIccIfort): + """ + Compiler class for Intel oneAPI compilers + """ + + COMPILER_MODULE_NAME = ['intel-compilers'] + + def _set_compiler_vars(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + + # skip IntelIccIfort._set_compiler_vars (no longer relevant for recent versions) + Compiler._set_compiler_vars(self) + + root = self.get_software_root(self.COMPILER_MODULE_NAME)[0] + version = self.get_software_version(self.COMPILER_MODULE_NAME)[0] + + libbase = os.path.join('compiler', version, 'linux') + libpaths = [ + os.path.join(libbase, 'compiler', 'lib', 'intel64'), + ] + + self.variables.append_subdirs("LDFLAGS", root, subdirs=libpaths) + + def set_variables(self): + """Set the variables.""" + + # skip IntelIccIfort.set_variables (no longer relevant for recent versions) + Compiler.set_variables(self) diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 8ac23e837b..2dc2faacbd 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/systemcompiler.py b/easybuild/toolchains/compiler/systemcompiler.py index f6b4ccd979..74edcb51e9 100644 --- a/easybuild/toolchains/compiler/systemcompiler.py +++ b/easybuild/toolchains/compiler/systemcompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craycce.py b/easybuild/toolchains/craycce.py index 799ed94853..93dd97c440 100644 --- a/easybuild/toolchains/craycce.py +++ b/easybuild/toolchains/craycce.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craygnu.py b/easybuild/toolchains/craygnu.py index 9e69e9a1f2..a53e90f931 100644 --- a/easybuild/toolchains/craygnu.py +++ b/easybuild/toolchains/craygnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/crayintel.py b/easybuild/toolchains/crayintel.py index c56bd4e0da..89e40cdc32 100644 --- a/easybuild/toolchains/crayintel.py +++ b/easybuild/toolchains/crayintel.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craypgi.py b/easybuild/toolchains/craypgi.py index ed5a8e66ba..9cd0735dc2 100644 --- a/easybuild/toolchains/craypgi.py +++ b/easybuild/toolchains/craypgi.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/dummy.py b/easybuild/toolchains/dummy.py index 44eddfa402..fd48d650a3 100644 --- a/easybuild/toolchains/dummy.py +++ b/easybuild/toolchains/dummy.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fcc.py b/easybuild/toolchains/fcc.py new file mode 100644 index 0000000000..eb20230f81 --- /dev/null +++ b/easybuild/toolchains/fcc.py @@ -0,0 +1,78 @@ +## +# Copyright 2012-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for Fujitsu Compiler toolchain. + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.compiler.fujitsu import FujitsuCompiler +from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME + + +class FCC(FujitsuCompiler): + """Compiler toolchain with Fujitsu Compiler.""" + NAME = 'FCC' + SUBTOOLCHAIN = SYSTEM_TOOLCHAIN_NAME + OPTIONAL = False + + # override in order to add an exception for the Fujitsu lang/tcsds module + def _add_dependency_variables(self, names=None, cpp=None, ld=None): + """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies + names should be a list of strings containing the name of the dependency + """ + cpp_paths = ['include'] + ld_paths = ['lib'] + if not self.options.get('32bit', None): + ld_paths.insert(0, 'lib64') + + if cpp is not None: + for p in cpp: + if p not in cpp_paths: + cpp_paths.append(p) + if ld is not None: + for p in ld: + if p not in ld_paths: + ld_paths.append(p) + + if not names: + deps = self.dependencies + else: + deps = [{'name': name} for name in names if name is not None] + + # collect software install prefixes for dependencies + roots = [] + for dep in deps: + if dep.get('external_module', False): + # for software names provided via external modules, install prefix may be unknown + names = dep['external_module_metadata'].get('name', []) + roots.extend([root for root in self.get_software_root(names) if root is not None]) + else: + roots.extend(self.get_software_root(dep['name'])) + + for root in roots: + # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode + if 'tcsds' not in root: + self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths) + self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths) diff --git a/easybuild/toolchains/ffmpi.py b/easybuild/toolchains/ffmpi.py new file mode 100644 index 0000000000..68e1799389 --- /dev/null +++ b/easybuild/toolchains/ffmpi.py @@ -0,0 +1,38 @@ +## +# Copyright 2012-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for ffmpi compiler toolchain (includes Fujitsu Compiler and MPI). + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.fcc import FCC +from easybuild.toolchains.mpi.fujitsumpi import FujitsuMPI + + +class Ffmpi(FCC, FujitsuMPI): + """Compiler toolchain with Fujitsu Compiler and MPI.""" + NAME = 'ffmpi' + SUBTOOLCHAIN = FCC.NAME + COMPILER_MODULE_NAME = [FCC.NAME] diff --git a/easybuild/toolchains/fft/__init__.py b/easybuild/toolchains/fft/__init__.py index b8d6cfce97..4353145f26 100644 --- a/easybuild/toolchains/fft/__init__.py +++ b/easybuild/toolchains/fft/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index 6597d1ead5..ee83197634 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,26 +33,35 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.fft import Fft +from easybuild.tools.modules import get_software_root class Fftw(Fft): """FFTW FFT library""" FFT_MODULE_NAME = ['FFTW'] + FFTW_API_VERSION = '' def _set_fftw_variables(self): - suffix = '' - version = self.get_software_version(self.FFT_MODULE_NAME)[0] - if LooseVersion(version) < LooseVersion('2') or LooseVersion(version) >= LooseVersion('4'): - raise EasyBuildError("_set_fft_variables: FFTW unsupported version %s (major should be 2 or 3)", version) - elif LooseVersion(version) > LooseVersion('2'): - suffix = '3' + suffix = self.FFTW_API_VERSION + if not suffix: + version = self.get_software_version(self.FFT_MODULE_NAME)[0] + if LooseVersion(version) < LooseVersion('2') or LooseVersion(version) >= LooseVersion('4'): + raise EasyBuildError("_set_fft_variables: FFTW unsupported version %s (major should be 2 or 3)", + version) + elif LooseVersion(version) > LooseVersion('2'): + suffix = '3' # order matters! fftw_libs = ["fftw%s" % suffix] if self.options.get('usempi', False): fftw_libs.insert(0, "fftw%s_mpi" % suffix) + fftwmpiroot = get_software_root('FFTW.MPI') + if fftwmpiroot: + # get libfft%_mpi via the FFTW.MPI module + self.FFT_MODULE_NAME = ['FFTW.MPI'] + fftw_libs_mt = ["fftw%s" % suffix] if self.options.get('openmp', False): fftw_libs_mt.insert(0, "fftw%s_omp" % suffix) @@ -68,7 +77,7 @@ def _set_fft_variables(self): # TODO can these be replaced with the FFT ones? self.variables.join('FFTW_INC_DIR', 'FFT_INC_DIR') self.variables.join('FFTW_LIB_DIR', 'FFT_LIB_DIR') - if 'FFT_STATIC_LIBS' in self.variables: - self.variables.join('FFTW_STATIC_LIBS', 'FFT_STATIC_LIBS') - if 'FFT_STATIC_LIBS_MT' in self.variables: - self.variables.join('FFTW_STATIC_LIBS_MT', 'FFT_STATIC_LIBS_MT') + + for key in ('SHARED_LIBS', 'SHARED_LIBS_MT', 'STATIC_LIBS', 'STATIC_LIBS_MT'): + if 'FFT_' + key in self.variables: + self.variables.join('FFTW_' + key, 'FFT_' + key) diff --git a/easybuild/toolchains/fft/fujitsufftw.py b/easybuild/toolchains/fft/fujitsufftw.py new file mode 100644 index 0000000000..33eb185708 --- /dev/null +++ b/easybuild/toolchains/fft/fujitsufftw.py @@ -0,0 +1,36 @@ +## +# Copyright 2012-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Fujitsu FFTW as toolchain FFT library. + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.fft.fftw import Fftw + + +class FujitsuFFTW(Fftw): + """Fujitsu FFTW FFT library""" + + FFTW_API_VERSION = '3' diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 037acf823e..2e969241b8 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -49,7 +49,10 @@ def _set_fftw_variables(self): if not hasattr(self, 'BLAS_LIB_DIR'): raise EasyBuildError("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)") + imklroot = get_software_root(self.FFT_MODULE_NAME[0]) imklver = get_software_version(self.FFT_MODULE_NAME[0]) + self.FFT_LIB_DIR = self.BLAS_LIB_DIR + self.FFT_INCLUDE_DIR = [os.path.join(d, 'fftw') for d in self.BLAS_INCLUDE_DIR] picsuff = '' if self.options.get('pic', None): @@ -57,22 +60,38 @@ def _set_fftw_variables(self): bitsuff = '_lp64' if self.options.get('i8', None): bitsuff = '_ilp64' - compsuff = '_intel' - if get_software_root('icc') is None: - if get_software_root('PGI'): - compsuff = '_pgi' - elif get_software_root('GCC'): - compsuff = '_gnu' - else: - error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries." - raise EasyBuildError(error_msg) + + if get_software_root('icc') or get_software_root('intel-compilers'): + compsuff = '_intel' + elif get_software_root('PGI'): + compsuff = '_pgi' + elif get_software_root('GCC') or get_software_root('GCCcore'): + compsuff = '_gnu' + else: + error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries." + raise EasyBuildError(error_msg) interface_lib = "fftw3xc%s%s" % (compsuff, picsuff) - fftw_libs = [interface_lib] - cluster_interface_lib = None + fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR] + + def fftw_lib_exists(libname): + """Helper function to check whether FFTW library with specified name exists.""" + return any(os.path.exists(os.path.join(d, "lib%s.a" % libname)) for d in fft_lib_dirs) + + # interface libs can be optional: + # MKL >= 10.2 include fftw3xc and fftw3xf interfaces in LIBBLAS=libmkl_gf/libmkl_intel + # See https://software.intel.com/en-us/articles/intel-mkl-main-libraries-contain-fftw3-interfaces + # The cluster interface libs (libfftw3x_cdft*) can be omitted if the toolchain does not provide MPI-FFTW + # interfaces. + fftw_libs = [] + if fftw_lib_exists(interface_lib) or LooseVersion(imklver) < LooseVersion("10.2"): + fftw_libs = [interface_lib] + if self.options.get('usempi', False): # add cluster interface for recent imkl versions - if LooseVersion(imklver) >= LooseVersion('10.3'): + # only get cluster_interface_lib from seperate module imkl-FFTW, rest via libmkl_gf/libmkl_intel + imklfftwroot = get_software_root('imkl-FFTW') + if LooseVersion(imklver) >= LooseVersion('10.3') and (fftw_libs or imklfftwroot): suff = picsuff if LooseVersion(imklver) >= LooseVersion('11.0.2'): suff = bitsuff + suff @@ -80,6 +99,9 @@ def _set_fftw_variables(self): fftw_libs.append(cluster_interface_lib) fftw_libs.append("mkl_cdft_core") # add cluster dft fftw_libs.extend(self.variables['LIBBLACS'].flatten()) # add BLACS; use flatten because ListOfList + if imklfftwroot: + fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')] + self.FFT_LIB_DIR = [os.path.join(imklfftwroot, 'lib')] fftw_mt_libs = fftw_libs + [x % self.BLAS_LIB_MAP for x in self.BLAS_LIB_MT] @@ -87,38 +109,19 @@ def _set_fftw_variables(self): fftw_libs.extend(self.variables['LIBBLAS'].flatten()) # add BLAS libs (contains dft) self.log.debug('fftw_libs %s' % fftw_libs.__repr__()) - self.FFT_LIB_DIR = self.BLAS_LIB_DIR - self.FFT_INCLUDE_DIR = [os.path.join(d, 'fftw') for d in self.BLAS_INCLUDE_DIR] - # building the FFTW interfaces is optional, # so make sure libraries are there before FFT_LIB is set - imklroot = get_software_root(self.FFT_MODULE_NAME[0]) - fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR] - - def fftw_lib_exists(libname): - """Helper function to check whether FFTW library with specified name exists.""" - return any([os.path.exists(os.path.join(d, "lib%s.a" % libname)) for d in fft_lib_dirs]) - - if not fftw_lib_exists(interface_lib) and LooseVersion(imklver) >= LooseVersion("10.2"): - # interface libs can be optional: - # MKL >= 10.2 include fftw3xc and fftw3xf interfaces in LIBBLAS=libmkl_gf/libmkl_intel - # See https://software.intel.com/en-us/articles/intel-mkl-main-libraries-contain-fftw3-interfaces - # The cluster interface libs (libfftw3x_cdft*) can be omitted if the toolchain does not provide MPI-FFTW - # interfaces. - fftw_libs = [lib for lib in fftw_libs if lib not in [interface_lib, cluster_interface_lib]] - fftw_mt_libs = [lib for lib in fftw_mt_libs if lib not in [interface_lib, cluster_interface_lib]] - # filter out libraries from list of FFTW libraries to check for if they are not provided by Intel MKL - check_fftw_libs = [lib for lib in fftw_libs if lib not in ['dl', 'gfortran']] + check_fftw_libs = [lib for lib in fftw_libs + fftw_mt_libs if lib not in ['dl', 'gfortran']] - if all([fftw_lib_exists(lib) for lib in check_fftw_libs]): - self.FFT_LIB = fftw_libs - else: + missing_fftw_libs = [lib for lib in check_fftw_libs if not fftw_lib_exists(lib)] + if missing_fftw_libs: msg = "Not all FFTW interface libraries %s are found in %s" % (check_fftw_libs, fft_lib_dirs) - msg += ", can't set $FFT_LIB." + msg += ", can't set $FFT_LIB. Missing: %s" % (missing_fftw_libs) if self.dry_run: dry_run_warning(msg, silent=build_option('silent')) else: raise EasyBuildError(msg) - - self.FFT_LIB_MT = fftw_mt_libs + else: + self.FFT_LIB = fftw_libs + self.FFT_LIB_MT = fftw_mt_libs diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index 794bb05485..970a2ea9b3 100644 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,27 +32,60 @@ from easybuild.toolchains.gompi import Gompi from easybuild.toolchains.golf import Golf from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK -class Foss(Gompi, OpenBLAS, ScaLAPACK, Fftw): +class Foss(Gompi, OpenBLAS, FlexiBLAS, ScaLAPACK, Fftw): """Compiler toolchain with GCC, OpenMPI, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'foss' SUBTOOLCHAIN = [Gompi.NAME, Golf.NAME] - def is_deprecated(self): - """Return whether or not this toolchain is deprecated.""" - # need to transform a version like '2016a' with something that is safe to compare with '2000' + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + super(Foss, self).__init__(*args, **kwargs) + + # need to transform a version like '2018b' with something that is safe to compare with '2019' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) version = self.version.replace('a', '.01').replace('b', '.07') - # foss toolchains older than foss/2016a are deprecated - # take into account that foss/2016.x is always < foss/2016a according to LooseVersion; - # foss/2016.01 & co are not deprecated yet... - foss_ver = LooseVersion(version) - if foss_ver < LooseVersion('2016.01'): + self.looseversion = LooseVersion(version) + + constants = ('BLAS_MODULE_NAME', 'BLAS_LIB', 'BLAS_LIB_MT', 'BLAS_FAMILY', + 'LAPACK_MODULE_NAME', 'LAPACK_IS_BLAS', 'LAPACK_FAMILY') + + if self.looseversion > LooseVersion('2021.0'): + for constant in constants: + setattr(self, constant, getattr(FlexiBLAS, constant)) + else: + for constant in constants: + setattr(self, constant, getattr(OpenBLAS, constant)) + + def banned_linked_shared_libs(self): + """ + List of shared libraries (names, file names, paths) which are + not allowed to be linked in any installed binary/library. + """ + res = [] + res.extend(Gompi.banned_linked_shared_libs(self)) + + if self.looseversion >= LooseVersion('2021.0'): + res.extend(FlexiBLAS.banned_linked_shared_libs(self)) + else: + res.extend(OpenBLAS.banned_linked_shared_libs(self)) + + res.extend(ScaLAPACK.banned_linked_shared_libs(self)) + res.extend(Fftw.banned_linked_shared_libs(self)) + + return res + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + + # foss toolchains older than foss/2019a are deprecated since EasyBuild v4.5.0; + if self.looseversion < LooseVersion('2019'): deprecated = True else: deprecated = False diff --git a/easybuild/toolchains/fosscuda.py b/easybuild/toolchains/fosscuda.py index e2e98d54f1..9edea341d2 100644 --- a/easybuild/toolchains/fosscuda.py +++ b/easybuild/toolchains/fosscuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py new file mode 100644 index 0000000000..ef0663e6af --- /dev/null +++ b/easybuild/toolchains/fujitsu.py @@ -0,0 +1,38 @@ +## +# Copyright 2014-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Fujitsu toolchain: Fujitsu compilers and MPI + Fujitsu SSL2 and Fujitsu FFTW + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.ffmpi import Ffmpi +from easybuild.toolchains.fft.fujitsufftw import FujitsuFFTW +from easybuild.toolchains.linalg.fujitsussl import FujitsuSSL + + +class Fujitsu(Ffmpi, FujitsuFFTW, FujitsuSSL): + """Compiler toolchain for Fujitsu.""" + NAME = 'Fujitsu' + SUBTOOLCHAIN = Ffmpi.NAME diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py index 30f3891f58..df610d745b 100644 --- a/easybuild/toolchains/gcc.py +++ b/easybuild/toolchains/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,6 +27,8 @@ :author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion +import re from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME @@ -38,3 +40,14 @@ class GccToolchain(GCCcore): COMPILER_MODULE_NAME = [NAME] SUBTOOLCHAIN = [GCCcore.NAME, SYSTEM_TOOLCHAIN_NAME] OPTIONAL = False + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # GCC toolchains older than GCC version 8.x are deprecated since EasyBuild v4.5.0 + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', self.version) and LooseVersion(self.version) < LooseVersion('8.0'): + deprecated = True + else: + deprecated = False + + return deprecated diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py index a95f0dcbdb..ace116975a 100644 --- a/easybuild/toolchains/gcccore.py +++ b/easybuild/toolchains/gcccore.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,6 +27,8 @@ :author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion +import re from easybuild.toolchains.compiler.gcc import Gcc from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME @@ -41,3 +43,14 @@ class GCCcore(Gcc): # GCCcore is only guaranteed to be present in recent toolchains # for old versions of some toolchains (GCC, intel) it is not part of the hierarchy and hence optional OPTIONAL = True + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # GCC toolchains older than GCC version 8.x are deprecated since EasyBuild v4.5.0 + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', self.version) and LooseVersion(self.version) < LooseVersion('8.0'): + deprecated = True + else: + deprecated = False + + return deprecated diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py index e7a1d5b8c8..babf11b776 100644 --- a/easybuild/toolchains/gcccuda.py +++ b/easybuild/toolchains/gcccuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,6 +27,8 @@ :author: Kenneth Hoste (Ghent University) """ +import re +from distutils.version import LooseVersion from easybuild.toolchains.compiler.cuda import Cuda from easybuild.toolchains.gcc import GccToolchain @@ -38,3 +40,20 @@ class GccCUDA(GccToolchain, Cuda): COMPILER_MODULE_NAME = ['GCC', 'CUDA'] SUBTOOLCHAIN = GccToolchain.NAME + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + deprecated = False + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version): + # gcccuda toolchains older than gcccuda/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(version) < LooseVersion('2019'): + deprecated = True + + return deprecated diff --git a/easybuild/toolchains/gfbf.py b/easybuild/toolchains/gfbf.py new file mode 100644 index 0000000000..cebe49ccf2 --- /dev/null +++ b/easybuild/toolchains/gfbf.py @@ -0,0 +1,41 @@ +## +# Copyright 2021-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gfbf compiler toolchain (includes GCC, FlexiBLAS and FFTW) + +:author: Kenneth Hoste (Ghent University) +:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" + +from easybuild.toolchains.gcc import GccToolchain +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS + + +class Gfbf(GccToolchain, FlexiBLAS, Fftw): + """Compiler toolchain with GCC, FlexiBLAS and FFTW.""" + NAME = 'gfbf' + SUBTOOLCHAIN = GccToolchain.NAME + OPTIONAL = True diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py index 986bdb978a..3afdb9659e 100644 --- a/easybuild/toolchains/gimkl.py +++ b/easybuild/toolchains/gimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py index ce93b4acd2..346e5d3055 100644 --- a/easybuild/toolchains/gimpi.py +++ b/easybuild/toolchains/gimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,6 +28,8 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +import re +from distutils.version import LooseVersion from easybuild.toolchains.gcc import GccToolchain from easybuild.toolchains.mpi.intelmpi import IntelMPI @@ -37,3 +39,20 @@ class Gimpi(GccToolchain, IntelMPI): """Compiler toolchain with GCC and Intel MPI.""" NAME = 'gimpi' SUBTOOLCHAIN = GccToolchain.NAME + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + deprecated = False + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version): + # gimpi toolchains older than gimpi/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(version) < LooseVersion('2019'): + deprecated = True + + return deprecated diff --git a/easybuild/toolchains/gimpic.py b/easybuild/toolchains/gimpic.py index a4fc060503..eddf0d6cb1 100644 --- a/easybuild/toolchains/gimpic.py +++ b/easybuild/toolchains/gimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolf.py b/easybuild/toolchains/giolf.py index 75b6bf01f0..f42b8e6a41 100644 --- a/easybuild/toolchains/giolf.py +++ b/easybuild/toolchains/giolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolfc.py b/easybuild/toolchains/giolfc.py index 6c6d08dc3b..af78d412c8 100644 --- a/easybuild/toolchains/giolfc.py +++ b/easybuild/toolchains/giolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmacml.py b/easybuild/toolchains/gmacml.py index 4e876b1b7a..6bcc9593ab 100644 --- a/easybuild/toolchains/gmacml.py +++ b/easybuild/toolchains/gmacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmkl.py b/easybuild/toolchains/gmkl.py index 3505cd1e66..f23a7befcb 100644 --- a/easybuild/toolchains/gmkl.py +++ b/easybuild/toolchains/gmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py index 8105e056cd..6481685708 100644 --- a/easybuild/toolchains/gmklc.py +++ b/easybuild/toolchains/gmklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py index 338c548154..2484163c5d 100644 --- a/easybuild/toolchains/gmpich.py +++ b/easybuild/toolchains/gmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich2.py b/easybuild/toolchains/gmpich2.py index 8a58f4fa03..562426b138 100644 --- a/easybuild/toolchains/gmpich2.py +++ b/easybuild/toolchains/gmpich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpit.py b/easybuild/toolchains/gmpit.py new file mode 100644 index 0000000000..5541f688fb --- /dev/null +++ b/easybuild/toolchains/gmpit.py @@ -0,0 +1,56 @@ +## +# Copyright 2022-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmpit compiler toolchain (includes GCC and MPItrampoline). + +:author: Alan O'Cais (CECAM) +""" +from distutils.version import LooseVersion +import re + +from easybuild.toolchains.gcc import GccToolchain +from easybuild.toolchains.mpi.mpitrampoline import MPItrampoline + + +class Gmpit(GccToolchain, MPItrampoline): + """Compiler toolchain with GCC and MPItrampoline.""" + NAME = 'gmpit' + SUBTOOLCHAIN = GccToolchain.NAME + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + deprecated = False + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version): + if LooseVersion(version) < LooseVersion('2019'): + deprecated = True + + return deprecated diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py index f36f46a9cb..4686a77ca8 100644 --- a/easybuild/toolchains/gmpolf.py +++ b/easybuild/toolchains/gmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gmvapich2.py b/easybuild/toolchains/gmvapich2.py index 7c2d615105..f2e36d1df8 100644 --- a/easybuild/toolchains/gmvapich2.py +++ b/easybuild/toolchains/gmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py index 7e555dddad..075b714c87 100644 --- a/easybuild/toolchains/gmvolf.py +++ b/easybuild/toolchains/gmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gnu.py b/easybuild/toolchains/gnu.py index e4b7db94fa..1366d333f3 100644 --- a/easybuild/toolchains/gnu.py +++ b/easybuild/toolchains/gnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goalf.py b/easybuild/toolchains/goalf.py index c7d1771d5b..c6003aa334 100644 --- a/easybuild/toolchains/goalf.py +++ b/easybuild/toolchains/goalf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gobff.py b/easybuild/toolchains/gobff.py index 26474f281d..4597d28e48 100644 --- a/easybuild/toolchains/gobff.py +++ b/easybuild/toolchains/gobff.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goblf.py b/easybuild/toolchains/goblf.py index 75c782b862..b4cb71e4d4 100644 --- a/easybuild/toolchains/goblf.py +++ b/easybuild/toolchains/goblf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -23,7 +23,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for foss compiler toolchain (includes GCC, OpenMPI, BLIS, LAPACK, ScaLAPACK and FFTW). +EasyBuild support for goblf compiler toolchain (includes GCC, OpenMPI, BLIS, LAPACK, ScaLAPACK and FFTW). :author: Kenneth Hoste (Ghent University) :author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) diff --git a/easybuild/toolchains/gofbf.py b/easybuild/toolchains/gofbf.py new file mode 100644 index 0000000000..8e448c89a1 --- /dev/null +++ b/easybuild/toolchains/gofbf.py @@ -0,0 +1,40 @@ +## +# Copyright 2021-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gofbf compiler toolchain (includes GCC, OpenMPI, FlexiBLAS, ScaLAPACK and FFTW) + +:author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Gofbf(Gompi, FlexiBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, OpenMPI, FlexiBLAS, ScaLAPACK and FFTW.""" + NAME = 'gofbf' + SUBTOOLCHAIN = Gompi.NAME diff --git a/easybuild/toolchains/golf.py b/easybuild/toolchains/golf.py index dbabc3c63a..f01d6ea7d7 100644 --- a/easybuild/toolchains/golf.py +++ b/easybuild/toolchains/golf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/golfc.py b/easybuild/toolchains/golfc.py index 9542026168..7d4073f1a8 100644 --- a/easybuild/toolchains/golfc.py +++ b/easybuild/toolchains/golfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gomkl.py b/easybuild/toolchains/gomkl.py index 48794536e2..c240628884 100644 --- a/easybuild/toolchains/gomkl.py +++ b/easybuild/toolchains/gomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py index 2ae5a40ee4..42e490e631 100644 --- a/easybuild/toolchains/gomklc.py +++ b/easybuild/toolchains/gomklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py index a5bbc4c7b9..7493539c5a 100644 --- a/easybuild/toolchains/gompi.py +++ b/easybuild/toolchains/gompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -41,7 +41,7 @@ class Gompi(GccToolchain, OpenMPI): def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" - # need to transform a version like '2016a' with something that is safe to compare with '2000' + # need to transform a version like '2018b' with something that is safe to compare with '2019' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) version = self.version.replace('a', '.01').replace('b', '.07') @@ -50,14 +50,8 @@ def is_deprecated(self): # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion if re.match('^[0-9]', version): - gompi_ver = LooseVersion(version) - # deprecate oldest gompi toolchains (versions 1.x) - if gompi_ver < LooseVersion('2000'): - deprecated = True - # gompi toolchains older than gompi/2016a are deprecated - # take into account that gompi/2016.x is always < gompi/2016a according to LooseVersion; - # gompi/2016.01 & co are not deprecated yet... - elif gompi_ver < LooseVersion('2016.01'): + # gompi toolchains older than gompi/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(version) < LooseVersion('2019'): deprecated = True return deprecated diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py index 65ca8e1f65..4f23b81b40 100644 --- a/easybuild/toolchains/gompic.py +++ b/easybuild/toolchains/gompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py index 0cf23145ce..fdf564e376 100644 --- a/easybuild/toolchains/goolf.py +++ b/easybuild/toolchains/goolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py index e32d342780..7eae7f4dbd 100644 --- a/easybuild/toolchains/goolfc.py +++ b/easybuild/toolchains/goolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py index cd9e019246..ecea4e4620 100644 --- a/easybuild/toolchains/gpsmpi.py +++ b/easybuild/toolchains/gpsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py index 24a313eb2d..282e7c8c00 100644 --- a/easybuild/toolchains/gpsolf.py +++ b/easybuild/toolchains/gpsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gqacml.py b/easybuild/toolchains/gqacml.py index 8a438f5462..ee03ad92bb 100644 --- a/easybuild/toolchains/gqacml.py +++ b/easybuild/toolchains/gqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsmpi.py b/easybuild/toolchains/gsmpi.py index 071b4ee0b9..7925f6fad0 100644 --- a/easybuild/toolchains/gsmpi.py +++ b/easybuild/toolchains/gsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py index 22e84fef4e..98c412beb6 100644 --- a/easybuild/toolchains/gsolf.py +++ b/easybuild/toolchains/gsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py index ee6f39846c..0b281c9c27 100644 --- a/easybuild/toolchains/iccifort.py +++ b/easybuild/toolchains/iccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -50,14 +50,15 @@ class IccIfort(IntelIccIfort): def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" - # need to transform a version like '2016a' with something that is safe to compare with '2016.01' + # need to transform a version like '2018b' with something that is safe to compare with '2019.0' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) version = self.version.replace('a', '.01').replace('b', '.07') - # iccifort toolchains older than iccifort/2016.1.150-* are deprecated + # iccifort toolchains older than iccifort/2019.0.117-* are deprecated; + # note: intel/2019a uses iccifort 2019.1.144; # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion - if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2016.1'): + if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019.0'): deprecated = True else: deprecated = False diff --git a/easybuild/toolchains/iccifortcuda.py b/easybuild/toolchains/iccifortcuda.py index 2fd32e4224..add77b3a0c 100644 --- a/easybuild/toolchains/iccifortcuda.py +++ b/easybuild/toolchains/iccifortcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py index 38fa8ffb76..4b26095d9b 100644 --- a/easybuild/toolchains/ictce.py +++ b/easybuild/toolchains/ictce.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iibff.py b/easybuild/toolchains/iibff.py new file mode 100644 index 0000000000..25a761ff2c --- /dev/null +++ b/easybuild/toolchains/iibff.py @@ -0,0 +1,41 @@ +## +# Copyright 2013-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iibff compiler toolchain (includes Intel compilers + MPI, BLIS, libFLAME, ScaLAPACK and FFTW). + +:author: Kenneth Hoste (HPC-UGent) +""" + +from easybuild.toolchains.iimpi import Iimpi +from easybuild.toolchains.linalg.blis import Blis +from easybuild.toolchains.linalg.flame import Flame +from easybuild.toolchains.linalg.scalapack import ScaLAPACK +from easybuild.toolchains.fft.fftw import Fftw + + +class Iibff(Iimpi, Blis, Flame, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, OpenMPI, BLIS, libFLAME, ScaLAPACK and FFTW.""" + NAME = 'iibff' + SUBTOOLCHAIN = Iimpi.NAME diff --git a/easybuild/toolchains/iimkl.py b/easybuild/toolchains/iimkl.py index 6a23f1912f..ba6292ffb2 100644 --- a/easybuild/toolchains/iimkl.py +++ b/easybuild/toolchains/iimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py index 9a1c0e5d36..08ebaf99cc 100644 --- a/easybuild/toolchains/iimklc.py +++ b/easybuild/toolchains/iimklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index d36e0ba843..ef19c0a5e0 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,32 +32,77 @@ import re from easybuild.toolchains.iccifort import IccIfort +from easybuild.toolchains.intel_compilers import IntelCompilersToolchain from easybuild.toolchains.mpi.intelmpi import IntelMPI -class Iimpi(IccIfort, IntelMPI): +class Iimpi(IccIfort, IntelCompilersToolchain, IntelMPI): """ Compiler toolchain with Intel compilers (icc/ifort), Intel MPI. """ NAME = 'iimpi' - SUBTOOLCHAIN = IccIfort.NAME + # compiler-only subtoolchain can't be determine statically + # since depends on toolchain version (see below), + # so register both here as possible alternatives (which is taken into account elsewhere) + SUBTOOLCHAIN = [(IntelCompilersToolchain.NAME, IccIfort.NAME)] + + def __init__(self, *args, **kwargs): + """Constructor for Iimpi toolchain class.""" + + super(Iimpi, self).__init__(*args, **kwargs) + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', self.version): + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) + # (good enough for this purpose) + self.iimpi_ver = self.version.replace('a', '.01').replace('b', '.07') + + if LooseVersion(self.iimpi_ver) >= LooseVersion('2020.12'): + self.oneapi_gen = True + self.SUBTOOLCHAIN = IntelCompilersToolchain.NAME + self.COMPILER_MODULE_NAME = IntelCompilersToolchain.COMPILER_MODULE_NAME + else: + self.oneapi_gen = False + self.SUBTOOLCHAIN = IccIfort.NAME + self.COMPILER_MODULE_NAME = IccIfort.COMPILER_MODULE_NAME + else: + self.iimpi_ver = self.version + self.oneapi_gen = False def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" - # need to transform a version like '2016a' with something that is safe to compare with '8.0', '2000', '2016.01' - # comparing subversions that include letters causes TypeErrors in Python 3 - # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) - version = self.version.replace('a', '.01').replace('b', '.07') deprecated = False + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion - if re.match('^[0-9]', version): - iimpi_ver = LooseVersion(version) - # iimpi toolchains older than iimpi/2016.01 are deprecated - # iimpi 8.1.5 is an exception, since it used in intel/2016a (which is not deprecated yet) - if iimpi_ver < LooseVersion('8.0'): - deprecated = True - elif iimpi_ver > LooseVersion('2000') and iimpi_ver < LooseVersion('2016.01'): + if re.match('^[0-9]', str(self.iimpi_ver)): + # iimpi toolchains older than iimpi/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(self.iimpi_ver) < LooseVersion('2019'): deprecated = True return deprecated + + def is_dep_in_toolchain_module(self, *args, **kwargs): + """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" + if self.oneapi_gen: + res = IntelCompilersToolchain.is_dep_in_toolchain_module(self, *args, **kwargs) + else: + res = IccIfort.is_dep_in_toolchain_module(self, *args, **kwargs) + + return res + + def _set_compiler_vars(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + if self.oneapi_gen: + IntelCompilersToolchain._set_compiler_vars(self) + else: + IccIfort._set_compiler_vars(self) + + def set_variables(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + if self.oneapi_gen: + IntelCompilersToolchain.set_variables(self) + else: + IccIfort.set_variables(self) diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py index 1940e20a30..a27c9eb97a 100644 --- a/easybuild/toolchains/iimpic.py +++ b/easybuild/toolchains/iimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,6 +27,8 @@ :author: Ake Sandgren (HPC2N) """ +import re +from distutils.version import LooseVersion from easybuild.toolchains.iccifortcuda import IccIfortCUDA from easybuild.toolchains.mpi.intelmpi import IntelMPI @@ -36,3 +38,20 @@ class Iimpic(IccIfortCUDA, IntelMPI): """Compiler toolchain with Intel compilers (icc/ifort), Intel MPI and CUDA.""" NAME = 'iimpic' SUBTOOLCHAIN = IccIfortCUDA.NAME + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + deprecated = False + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version): + # iimpic toolchains older than iimpic/2019a are deprecated since EasyBuild v4.5.0 + if LooseVersion(version) < LooseVersion('2019'): + deprecated = True + + return deprecated diff --git a/easybuild/toolchains/iiqmpi.py b/easybuild/toolchains/iiqmpi.py index b76fd58696..5de35f89bf 100644 --- a/easybuild/toolchains/iiqmpi.py +++ b/easybuild/toolchains/iiqmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py index 86d0f1ebb3..063a1f2c73 100644 --- a/easybuild/toolchains/impich.py +++ b/easybuild/toolchains/impich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py index 166de95d5b..bb550d1051 100644 --- a/easybuild/toolchains/impmkl.py +++ b/easybuild/toolchains/impmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py index 96b1f0372f..cbb02d98a4 100644 --- a/easybuild/toolchains/intel-para.py +++ b/easybuild/toolchains/intel-para.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py index bb0550baee..773cdf12a7 100644 --- a/easybuild/toolchains/intel.py +++ b/easybuild/toolchains/intel.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -48,16 +48,14 @@ class Intel(Iimpi, IntelMKL, IntelFFTW): def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" - # need to transform a version like '2016a' with something that is safe to compare with '2016.01' + # need to transform a version like '2018b' with something that is safe to compare with '2019' # comparing subversions that include letters causes TypeErrors in Python 3 # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) version = self.version.replace('a', '.01').replace('b', '.07') - # intel toolchains older than intel/2016a are deprecated - # take into account that intel/2016.x is always < intel/2016a according to LooseVersion; - # intel/2016.01 & co are not deprecated yet... + # intel toolchains older than intel/2019a are deprecated since EasyBuild v4.5.0 # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion - if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2016.01'): + if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019'): deprecated = True else: deprecated = False diff --git a/easybuild/toolchains/intel_compilers.py b/easybuild/toolchains/intel_compilers.py new file mode 100644 index 0000000000..00f3c874ac --- /dev/null +++ b/easybuild/toolchains/intel_compilers.py @@ -0,0 +1,41 @@ +## +# Copyright 2021-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for Intel compilers toolchain (icc, ifort), v2021.x or newer (oneAPI). + +:author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.intel_compilers import IntelCompilers +from easybuild.toolchains.gcccore import GCCcore +from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME + + +class IntelCompilersToolchain(IntelCompilers): + """Compiler toolchain with Intel compilers (icc/ifort).""" + NAME = 'intel-compilers' + # use GCCcore as subtoolchain rather than GCC, since two 'real' compiler-only toolchains don't mix well, + # in particular in a hierarchical module naming scheme + SUBTOOLCHAIN = [GCCcore.NAME, SYSTEM_TOOLCHAIN_NAME] + OPTIONAL = False diff --git a/easybuild/toolchains/intelcuda.py b/easybuild/toolchains/intelcuda.py index 3b5a1538aa..ffc9899b03 100644 --- a/easybuild/toolchains/intelcuda.py +++ b/easybuild/toolchains/intelcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py index bc68ae7b24..3f6c2910f5 100644 --- a/easybuild/toolchains/iomkl.py +++ b/easybuild/toolchains/iomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,8 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion +import re from easybuild.toolchains.iompi import Iompi from easybuild.toolchains.iimkl import Iimkl @@ -43,3 +45,19 @@ class Iomkl(Iompi, IntelMKL, IntelFFTW): """ NAME = 'iomkl' SUBTOOLCHAIN = [Iompi.NAME, Iimkl.NAME] + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + # iomkl toolchains older than iomkl/2019a are deprecated since EasyBuild v4.5.0 + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019'): + deprecated = True + else: + deprecated = False + + return deprecated diff --git a/easybuild/toolchains/iomklc.py b/easybuild/toolchains/iomklc.py index 785a884616..d1399c503f 100644 --- a/easybuild/toolchains/iomklc.py +++ b/easybuild/toolchains/iomklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py index 240d5d29d0..08a69f6164 100644 --- a/easybuild/toolchains/iompi.py +++ b/easybuild/toolchains/iompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,14 +28,83 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion +import re from easybuild.toolchains.iccifort import IccIfort +from easybuild.toolchains.intel_compilers import IntelCompilersToolchain from easybuild.toolchains.mpi.openmpi import OpenMPI -class Iompi(IccIfort, OpenMPI): +class Iompi(IccIfort, IntelCompilersToolchain, OpenMPI): """ Compiler toolchain with Intel compilers (icc/ifort) and OpenMPI. """ NAME = 'iompi' - SUBTOOLCHAIN = IccIfort.NAME + # compiler-only subtoolchain can't be determine statically + # since depends on toolchain version (see below), + # so register both here as possible alternatives (which is taken into account elsewhere) + SUBTOOLCHAIN = [(IntelCompilersToolchain.NAME, IccIfort.NAME)] + + def __init__(self, *args, **kwargs): + """Constructor for Iompi toolchain class.""" + + super(Iompi, self).__init__(*args, **kwargs) + + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', self.version): + # need to transform a version like '2016a' with something that is safe to compare with '8.0', '2016.01' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) + # (good enough for this purpose) + self.iompi_ver = self.version.replace('a', '.01').replace('b', '.07') + if LooseVersion(self.iompi_ver) >= LooseVersion('2020.12'): + self.oneapi_gen = True + self.SUBTOOLCHAIN = IntelCompilersToolchain.NAME + self.COMPILER_MODULE_NAME = IntelCompilersToolchain.COMPILER_MODULE_NAME + else: + self.oneapi_gen = False + self.SUBTOOLCHAIN = IccIfort.NAME + self.COMPILER_MODULE_NAME = IccIfort.COMPILER_MODULE_NAME + else: + self.iompi_ver = self.version + self.oneapi_gen = False + + def is_dep_in_toolchain_module(self, *args, **kwargs): + """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" + if self.oneapi_gen: + res = IntelCompilersToolchain.is_dep_in_toolchain_module(self, *args, **kwargs) + else: + res = IccIfort.is_dep_in_toolchain_module(self, *args, **kwargs) + + return res + + def _set_compiler_vars(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + if self.oneapi_gen: + IntelCompilersToolchain._set_compiler_vars(self) + else: + IccIfort._set_compiler_vars(self) + + def set_variables(self): + """Intel compilers-specific adjustments after setting compiler variables.""" + if self.oneapi_gen: + IntelCompilersToolchain.set_variables(self) + else: + IccIfort.set_variables(self) + + def is_deprecated(self): + """Return whether or not this toolchain is deprecated.""" + # need to transform a version like '2018b' with something that is safe to compare with '2019' + # comparing subversions that include letters causes TypeErrors in Python 3 + # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose) + version = self.version.replace('a', '.01').replace('b', '.07') + + # iompi toolchains older than iompi/2019a are deprecated since EasyBuild v4.5.0 + # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion + if re.match('^[0-9]', version) and LooseVersion(version) < LooseVersion('2019'): + deprecated = True + else: + deprecated = False + + return deprecated diff --git a/easybuild/toolchains/iompic.py b/easybuild/toolchains/iompic.py index b2b4cab6b4..f5a47930f5 100644 --- a/easybuild/toolchains/iompic.py +++ b/easybuild/toolchains/iompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py index 8798e746ff..ebbf638c35 100644 --- a/easybuild/toolchains/ipsmpi.py +++ b/easybuild/toolchains/ipsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iqacml.py b/easybuild/toolchains/iqacml.py index 2ff17dee96..a7905e132e 100644 --- a/easybuild/toolchains/iqacml.py +++ b/easybuild/toolchains/iqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py index 80e0951969..db7fc3f11e 100644 --- a/easybuild/toolchains/ismkl.py +++ b/easybuild/toolchains/ismkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/__init__.py b/easybuild/toolchains/linalg/__init__.py index d77ec3e1e3..ef9096a3fd 100644 --- a/easybuild/toolchains/linalg/__init__.py +++ b/easybuild/toolchains/linalg/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 32de9fd6eb..16eed0fe36 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/atlas.py b/easybuild/toolchains/linalg/atlas.py index b77e84a5d5..f15829bfff 100644 --- a/easybuild/toolchains/linalg/atlas.py +++ b/easybuild/toolchains/linalg/atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blacs.py b/easybuild/toolchains/linalg/blacs.py index 47949dd821..dcf1950954 100644 --- a/easybuild/toolchains/linalg/blacs.py +++ b/easybuild/toolchains/linalg/blacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py index 924e6b1f78..d8f1c9a327 100644 --- a/easybuild/toolchains/linalg/blis.py +++ b/easybuild/toolchains/linalg/blis.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index 998b5c39d4..8ae465b395 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py new file mode 100644 index 0000000000..835a4b3191 --- /dev/null +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -0,0 +1,90 @@ +## +# Copyright 2021-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for FlexiBLAS as toolchain linear algebra library. + +:author: Kenneth Hoste (Ghent University) +""" +import os +import re + +from easybuild.tools.toolchain.linalg import LinAlg + +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext + + +TC_CONSTANT_FLEXIBLAS = 'FlexiBLAS' + + +def det_flexiblas_backend_libs(): + """Determine list of paths to FlexiBLAS backend libraries.""" + + # example output for 'flexiblas list': + # System-wide (config directory): + # OPENBLAS + # library = libflexiblas_openblas.so + out, _ = run_cmd("flexiblas list", simple=False, trace=False) + + shlib_ext = get_shared_lib_ext() + flexiblas_lib_regex = re.compile(r'library = (?Plib.*\.%s)' % shlib_ext, re.M) + flexiblas_libs = flexiblas_lib_regex.findall(out) + + backend_libs = [] + for flexiblas_lib in flexiblas_libs: + # assumption here is that the name of FlexiBLAS library (like 'libflexiblas_openblas.so') + # maps directly to name of the backend library ('libopenblas.so') + backend_lib = 'lib' + flexiblas_lib.replace('libflexiblas_', '') + backend_libs.append(backend_lib) + + return backend_libs + + +class FlexiBLAS(LinAlg): + """ + Trivial class, provides FlexiBLAS support. + """ + BLAS_MODULE_NAME = ['FlexiBLAS'] + BLAS_LIB = ['flexiblas'] + BLAS_INCLUDE_DIR = [os.path.join('include', 'flexiblas')] + BLAS_FAMILY = TC_CONSTANT_FLEXIBLAS + + LAPACK_MODULE_NAME = ['FlexiBLAS'] + LAPACK_IS_BLAS = True + LAPACK_INCLUDE_DIR = [os.path.join('include', 'flexiblas')] + LAPACK_FAMILY = TC_CONSTANT_FLEXIBLAS + + def banned_linked_shared_libs(self): + """ + List of shared libraries (names, file names, paths) which are + not allowed to be linked in any installed binary/library. + """ + banned_libs = super(FlexiBLAS, self).banned_linked_shared_libs() + + # register backends are banned shared libraries, + # to avoid that anything links to them directly (rather than to libflexiblas.so) + flexiblas_banned_libs = det_flexiblas_backend_libs() + + return banned_libs + flexiblas_banned_libs diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py new file mode 100644 index 0000000000..99766e56d0 --- /dev/null +++ b/easybuild/toolchains/linalg/fujitsussl.py @@ -0,0 +1,121 @@ +## +# Copyright 2014-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Fujitsu's SSL library, which provides BLAS/LAPACK support. + +:author: Miguel Dias Costa (National University of Singapore) +""" +import os + +from easybuild.toolchains.compiler.fujitsu import TC_CONSTANT_MODULE_NAME, TC_CONSTANT_MODULE_VAR +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.toolchain.constants import COMPILER_FLAGS +from easybuild.tools.toolchain.linalg import LinAlg + +FUJITSU_SSL_MODULE_NAME = None +TC_CONSTANT_FUJITSU_SSL = 'FujitsuSSL' + + +class FujitsuSSL(LinAlg): + """Support for Fujitsu's SSL library, which provides BLAS/LAPACK support.""" + # BLAS/LAPACK support + # via lang/tcsds module + BLAS_MODULE_NAME = [TC_CONSTANT_MODULE_NAME] + + # no need to specify libraries nor includes, only the compiler flags below + BLAS_LIB = [''] + BLAS_LIB_MT = [''] + BLAS_INCLUDE_DIR = [''] + BLAS_FAMILY = TC_CONSTANT_FUJITSU_SSL + + LAPACK_MODULE_NAME = None + LAPACK_IS_BLAS = True + LAPACK_LIB = [''] + LAPACK_LIB_MT = [''] + LAPACK_INCLUDE_DIR = [''] + LAPACK_FAMILY = TC_CONSTANT_FUJITSU_SSL + + BLACS_MODULE_NAME = None + BLACS_LIB = [''] + BLACS_LIB_MT = [''] + BLACS_INCLUDE_DIR = [''] + + SCALAPACK_MODULE_NAME = BLAS_MODULE_NAME + SCALAPACK_LIB = [''] + SCALAPACK_LIB_MT = [''] + SCALAPACK_INCLUDE_DIR = [''] + SCALAPACK_FAMILY = TC_CONSTANT_FUJITSU_SSL + + def _get_software_root(self, name, required=True): + """Get install prefix for specified software name; special treatment for Fujitsu modules.""" + if name == TC_CONSTANT_MODULE_NAME: + env_var = TC_CONSTANT_MODULE_VAR + root = os.getenv(env_var) + if root is None: + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) + else: + self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + else: + root = super(FujitsuSSL, self)._get_software_root(name, required=required) + + return root + + def _set_blas_variables(self): + """Setting FujitsuSSL specific BLAS related variables""" + super(FujitsuSSL, self)._set_blas_variables() + if self.options.get('openmp', None): + for flags_var, _ in COMPILER_FLAGS: + self.variables.nappend(flags_var, ['SSL2BLAMP']) + else: + for flags_var, _ in COMPILER_FLAGS: + self.variables.nappend(flags_var, ['SSL2']) + + def _set_scalapack_variables(self): + """Setting FujitsuSSL specific SCALAPACK related variables""" + super(FujitsuSSL, self)._set_scalapack_variables() + for flags_var, _ in COMPILER_FLAGS: + self.variables.nappend(flags_var, ['SCALAPACK']) + + def definition(self): + """ + Filter BLAS module from toolchain definition. + The SSL2 module is loaded indirectly (and versionless) via the lang module, + and thus is not a direct toolchain component. + """ + tc_def = super(FujitsuSSL, self).definition() + tc_def['BLAS'] = [] + tc_def['LAPACK'] = [] + tc_def['SCALAPACK'] = [] + return tc_def + + def set_variables(self): + """Set the variables""" + self._set_blas_variables() + self._set_lapack_variables() + self._set_scalapack_variables() + + self.log.devel('set_variables: LinAlg variables %s', self.variables) + + super(FujitsuSSL, self).set_variables() diff --git a/easybuild/toolchains/linalg/gotoblas.py b/easybuild/toolchains/linalg/gotoblas.py index 3df653fee9..93faed6328 100644 --- a/easybuild/toolchains/linalg/gotoblas.py +++ b/easybuild/toolchains/linalg/gotoblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index d115ca2a09..469356bbd0 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,6 +28,7 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ +import os from distutils.version import LooseVersion from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC @@ -152,13 +153,18 @@ def _set_blas_variables(self): raise EasyBuildError("_set_blas_variables: 32-bit libraries not supported yet for IMKL v%s (> v10.3)", found_version) else: - self.BLAS_LIB_DIR = ['mkl/lib/intel64'] - if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'): - self.BLAS_LIB_DIR.append('compiler/lib/intel64') + if ver >= LooseVersion('2021'): + basedir = os.path.join('mkl', found_version) else: - self.BLAS_LIB_DIR.append('lib/intel64') + basedir = 'mkl' + + self.BLAS_LIB_DIR = [os.path.join(basedir, 'lib', 'intel64')] + if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'): + self.BLAS_LIB_DIR.append(os.path.join('compiler', 'lib', 'intel64')) + elif ver < LooseVersion('2021'): + self.BLAS_LIB_DIR.append(os.path.join('lib', 'intel64')) - self.BLAS_INCLUDE_DIR = ['mkl/include'] + self.BLAS_INCLUDE_DIR = [os.path.join(basedir, 'include')] super(IntelMKL, self)._set_blas_variables() diff --git a/easybuild/toolchains/linalg/lapack.py b/easybuild/toolchains/linalg/lapack.py index 19052f668e..338e6dd4b1 100644 --- a/easybuild/toolchains/linalg/lapack.py +++ b/easybuild/toolchains/linalg/lapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py index f31c3bf28c..d3bbb48b2d 100644 --- a/easybuild/toolchains/linalg/libsci.py +++ b/easybuild/toolchains/linalg/libsci.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -59,18 +59,19 @@ class LibSci(LinAlg): BLACS_MODULE_NAME = [] SCALAPACK_MODULE_NAME = [] - def _get_software_root(self, name): + def _get_software_root(self, name, required=True): """Get install prefix for specified software name; special treatment for Cray modules.""" if name == 'cray-libsci': # Cray-provided LibSci module env_var = 'CRAY_LIBSCI_PREFIX_DIR' root = os.getenv(env_var, None) if root is None: - raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) + if required: + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) else: self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) else: - root = super(LibSci, self)._get_software_root(name) + root = super(LibSci, self)._get_software_root(name, required=required) return root diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py index 7cfd042a94..49d49de85f 100644 --- a/easybuild/toolchains/linalg/openblas.py +++ b/easybuild/toolchains/linalg/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,6 +40,7 @@ class OpenBLAS(LinAlg): """ BLAS_MODULE_NAME = ['OpenBLAS'] BLAS_LIB = ['openblas'] + BLAS_LIB_MT = ['openblas'] BLAS_FAMILY = TC_CONSTANT_OPENBLAS LAPACK_MODULE_NAME = ['OpenBLAS'] diff --git a/easybuild/toolchains/linalg/scalapack.py b/easybuild/toolchains/linalg/scalapack.py index d538dbb1b1..0a3c501d3b 100644 --- a/easybuild/toolchains/linalg/scalapack.py +++ b/easybuild/toolchains/linalg/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/__init__.py b/easybuild/toolchains/mpi/__init__.py index c0fccd6239..5bb9f0bb9f 100644 --- a/easybuild/toolchains/mpi/__init__.py +++ b/easybuild/toolchains/mpi/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py index 43e6bb080a..ea0312d930 100644 --- a/easybuild/toolchains/mpi/craympich.py +++ b/easybuild/toolchains/mpi/craympich.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py new file mode 100644 index 0000000000..03ee6f3a41 --- /dev/null +++ b/easybuild/toolchains/mpi/fujitsumpi.py @@ -0,0 +1,62 @@ +## +# Copyright 2014-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +MPI support for Fujitsu MPI. + +:author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.toolchains.compiler.fujitsu import FujitsuCompiler +from easybuild.toolchains.mpi.openmpi import TC_CONSTANT_OPENMPI, TC_CONSTANT_MPI_TYPE_OPENMPI +from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_VARIABLES +from easybuild.tools.toolchain.mpi import Mpi +from easybuild.tools.toolchain.variables import CommandFlagList + + +class FujitsuMPI(Mpi): + """Generic support for using Fujitsu compiler wrappers""" + # MPI support + # no separate module, Fujitsu compiler drivers always provide MPI support + MPI_MODULE_NAME = None + MPI_FAMILY = TC_CONSTANT_OPENMPI + MPI_TYPE = TC_CONSTANT_MPI_TYPE_OPENMPI + + # OpenMPI reads from CC etc env variables + MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + + MPI_LINK_INFO_OPTION = '-showme:link' + + def _set_mpi_compiler_variables(self): + """Define MPI wrapper commands and add OMPI_* variables to set.""" + self.MPI_COMPILER_MPICC = 'mpi' + FujitsuCompiler.COMPILER_CC + self.MPI_COMPILER_MPICXX = 'mpi' + FujitsuCompiler.COMPILER_CXX + self.MPI_COMPILER_MPIF77 = 'mpi' + FujitsuCompiler.COMPILER_F77 + self.MPI_COMPILER_MPIF90 = 'mpi' + FujitsuCompiler.COMPILER_F90 + self.MPI_COMPILER_MPIFC = 'mpi' + FujitsuCompiler.COMPILER_FC + + # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled + for var, _ in COMPILER_VARIABLES: + self.variables.nappend('OMPI_%s' % var, str(self.variables[var].get_first()), var_class=CommandFlagList) + + super(FujitsuMPI, self)._set_mpi_compiler_variables() diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index 104320635d..7f9d7ecfe2 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py index d7ee92006d..dea8450d46 100644 --- a/easybuild/toolchains/mpi/mpich.py +++ b/easybuild/toolchains/mpi/mpich.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich2.py b/easybuild/toolchains/mpi/mpich2.py index be2ec1a143..2038b0a98a 100644 --- a/easybuild/toolchains/mpi/mpich2.py +++ b/easybuild/toolchains/mpi/mpich2.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py new file mode 100644 index 0000000000..7dcd0c44a1 --- /dev/null +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -0,0 +1,76 @@ +## +# Copyright 2022-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for MPItrampoline as toolchain MPI library. + +:author: Alan O'Cais (CECAM) +""" + +from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_VARIABLES +from easybuild.tools.toolchain.mpi import Mpi +from easybuild.tools.toolchain.variables import CommandFlagList + + +TC_CONSTANT_MPITRAMPOLINE = "MPItrampoline" +TC_CONSTANT_MPI_TYPE_MPITRAMPOLINE = "MPI_TYPE_MPITRAMPOLINE" + + +class MPItrampoline(Mpi): + """MPItrampoline MPI class""" + MPI_MODULE_NAME = ['MPItrampoline'] + MPI_FAMILY = TC_CONSTANT_MPITRAMPOLINE + MPI_TYPE = TC_CONSTANT_MPI_TYPE_MPITRAMPOLINE + + MPI_LIBRARY_NAME = 'mpi' + + # May be version-dependent, so defined at runtime + MPI_COMPILER_MPIF77 = None + MPI_COMPILER_MPIF90 = None + MPI_COMPILER_MPIFC = None + + # MPItrampoline reads from CC etc env variables + MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + + MPI_LINK_INFO_OPTION = '-showme:link' + + def __init__(self, *args, **kwargs): + """Toolchain constructor""" + super(MPItrampoline, self).__init__(*args, **kwargs) + + def _set_mpi_compiler_variables(self): + """Define MPI wrapper commands and add MPITRAMPOLINE_* variables to set.""" + + self.MPI_COMPILER_MPIF77 = 'mpifort' + self.MPI_COMPILER_MPIF90 = 'mpifort' + self.MPI_COMPILER_MPIFC = 'mpifort' + + # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled + for var, _ in COMPILER_VARIABLES: + self.variables.nappend( + 'MPITRAMPOLINE_%s' % var, str(self.variables[var].get_first()), + var_class=CommandFlagList + ) + + super(MPItrampoline, self)._set_mpi_compiler_variables() diff --git a/easybuild/toolchains/mpi/mvapich2.py b/easybuild/toolchains/mpi/mvapich2.py index 0a384019dc..1181b89da6 100644 --- a/easybuild/toolchains/mpi/mvapich2.py +++ b/easybuild/toolchains/mpi/mvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index a7d5c59155..856ee13e66 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/psmpi.py b/easybuild/toolchains/mpi/psmpi.py index 38b8d22ff9..1f8df716c2 100644 --- a/easybuild/toolchains/mpi/psmpi.py +++ b/easybuild/toolchains/mpi/psmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/qlogicmpi.py b/easybuild/toolchains/mpi/qlogicmpi.py index eda309df48..8a6d9757af 100644 --- a/easybuild/toolchains/mpi/qlogicmpi.py +++ b/easybuild/toolchains/mpi/qlogicmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/spectrummpi.py b/easybuild/toolchains/mpi/spectrummpi.py index 76e87cafb8..a4adf96e7c 100644 --- a/easybuild/toolchains/mpi/spectrummpi.py +++ b/easybuild/toolchains/mpi/spectrummpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvompi.py b/easybuild/toolchains/nvompi.py new file mode 100644 index 0000000000..f8bd107adb --- /dev/null +++ b/easybuild/toolchains/nvompi.py @@ -0,0 +1,38 @@ +## +# Copyright 2013-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for nvompi compiler toolchain (NVHPC + Open MPI). + +:author: Robert Mijakovic (LuxProvide) +""" + +from easybuild.toolchains.nvhpc import NVHPCToolchain +from easybuild.toolchains.mpi.openmpi import OpenMPI + + +class Nvompi(NVHPCToolchain, OpenMPI): + """Compiler toolchain with NVHPC and Open MPI.""" + NAME = 'nvompi' + SUBTOOLCHAIN = NVHPCToolchain.NAME diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py new file mode 100644 index 0000000000..0977bca226 --- /dev/null +++ b/easybuild/toolchains/nvompic.py @@ -0,0 +1,43 @@ +## +# Copyright 2016-2022 Ghent University +# Copyright 2016-2022 Forschungszentrum Juelich +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for nvompic compiler toolchain (includes NVHPC and OpenMPI, and CUDA as dependency). + +:author: Damian Alvarez (Forschungszentrum Juelich) +:author: Sebastian Achilles (Forschungszentrum Juelich) +""" + +from easybuild.toolchains.nvhpc import NVHPCToolchain +# We pull in MPI and CUDA at once so this maps nicely to HMNS +from easybuild.toolchains.mpi.openmpi import OpenMPI +from easybuild.toolchains.compiler.cuda import Cuda + + +# Order matters! +class NVompic(NVHPCToolchain, Cuda, OpenMPI): + """Compiler toolchain with NVHPC and OpenMPI, with CUDA as dependency.""" + NAME = 'nvompic' + SUBTOOLCHAIN = NVHPCToolchain.NAME diff --git a/easybuild/toolchains/nvpsmpi.py b/easybuild/toolchains/nvpsmpi.py new file mode 100644 index 0000000000..c68b83d7d3 --- /dev/null +++ b/easybuild/toolchains/nvpsmpi.py @@ -0,0 +1,40 @@ +## +# Copyright 2016-2021 Ghent University +# Copyright 2016-2021 Forschungszentrum Juelich +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for nvsmpi compiler toolchain (includes NVHPC and ParaStationMPI). + +:author: Robert Mijakovic (LuxProvide) +""" + +from easybuild.toolchains.nvhpc import NVHPCToolchain +from easybuild.toolchains.mpi.psmpi import Psmpi + + +# Order matters! +class NVpsmpi(NVHPCToolchain, Psmpi): + """Compiler toolchain with NVHPC and ParaStationMPI.""" + NAME = 'nvpsmpi' + SUBTOOLCHAIN = NVHPCToolchain.NAME diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py new file mode 100644 index 0000000000..085f7da0c4 --- /dev/null +++ b/easybuild/toolchains/nvpsmpic.py @@ -0,0 +1,43 @@ +## +# Copyright 2016-2022 Ghent University +# Copyright 2016-2022 Forschungszentrum Juelich +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for npsmpi compiler toolchain (includes NVHPC and ParaStationMPI, and CUDA as dependency). + +:author: Damian Alvarez (Forschungszentrum Juelich) +:author: Sebastian Achilles (Forschungszentrum Juelich) +""" + +from easybuild.toolchains.nvhpc import NVHPCToolchain +# We pull in MPI and CUDA at once so this maps nicely to HMNS +from easybuild.toolchains.mpi.psmpi import Psmpi +from easybuild.toolchains.compiler.cuda import Cuda + + +# Order matters! +class NVpsmpic(NVHPCToolchain, Cuda, Psmpi): + """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" + NAME = 'nvpsmpic' + SUBTOOLCHAIN = NVHPCToolchain.NAME diff --git a/easybuild/toolchains/pmkl.py b/easybuild/toolchains/pmkl.py index b0cd7e58c2..3d8a2ae1fe 100644 --- a/easybuild/toolchains/pmkl.py +++ b/easybuild/toolchains/pmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pomkl.py b/easybuild/toolchains/pomkl.py index a00a4ea51a..eeb46e2749 100644 --- a/easybuild/toolchains/pomkl.py +++ b/easybuild/toolchains/pomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pompi.py b/easybuild/toolchains/pompi.py index 4557ea5db2..fd5657e0e6 100644 --- a/easybuild/toolchains/pompi.py +++ b/easybuild/toolchains/pompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/system.py b/easybuild/toolchains/system.py index 61affaa28c..61e58b9c99 100644 --- a/easybuild/toolchains/system.py +++ b/easybuild/toolchains/system.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index cfd32b007d..6c0bf4f023 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py index fa1d8a1a6c..458836c1ba 100644 --- a/easybuild/tools/asyncprocess.py +++ b/easybuild/tools/asyncprocess.py @@ -1,6 +1,6 @@ ## # Copyright 2005 Josiah Carlson -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson. # and released under the GPL v2 on March 14, 2012 diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index 6d6da8f266..cc4ca7fb0a 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -1,4 +1,4 @@ -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index db6d6ddc67..48fd643bc0 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -215,7 +215,7 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger. os.close(fd) fancylogger.logToFile(logfile, max_bytes=0) - print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=silent) + print_msg('Temporary log file in case of crash %s' % (logfile), log=None, silent=silent) log = fancylogger.getLogger(fname=False) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b97186f3c5..1c764cd56a 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -48,6 +48,12 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type +try: + import rich # noqa + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + _log = fancylogger.getLogger('config', fname=False) @@ -79,8 +85,11 @@ DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY DEFAULT_BRANCH = 'develop' +DEFAULT_ENV_FOR_SHEBANG = '/usr/bin/env' +DEFAULT_ENVVAR_USERS_MODULES = 'HOME' DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds) DEFAULT_JOB_BACKEND = 'GC3Pie' +DEFAULT_JOB_EB_CMD = 'eb' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5 DEFAULT_MINIMAL_BUILD_ENV = 'CC:gcc,CXX:g++' @@ -126,7 +135,7 @@ JOB_DEPS_TYPE_ABORT_ON_ERROR = 'abort_on_error' JOB_DEPS_TYPE_ALWAYS_RUN = 'always_run' -DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:16.04' +DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:20.04' DOCKER_BASE_IMAGE_CENTOS = 'centos:7' LOCAL_VAR_NAMING_CHECK_ERROR = 'error' @@ -135,6 +144,13 @@ LOCAL_VAR_NAMING_CHECKS = [LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN] +OUTPUT_STYLE_AUTO = 'auto' +OUTPUT_STYLE_BASIC = 'basic' +OUTPUT_STYLE_NO_COLOR = 'no_color' +OUTPUT_STYLE_RICH = 'rich' +OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH) + + class Singleton(ABCMeta): """Serves as metaclass for classes that should implement the Singleton pattern. @@ -161,7 +177,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): # build options that have a perfectly matching command line option, listed by default value BUILD_OPTIONS_CMDLINE = { None: [ - 'accept_eula', + 'accept_eula_for', 'aggregate_regtest', 'backup_modules', 'container_config', @@ -169,16 +185,22 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'container_image_name', 'container_template_recipe', 'container_tmpdir', + 'cuda_cache_dir', + 'cuda_cache_maxsize', 'cuda_compute_capabilities', 'download_timeout', 'dump_test_report', 'easyblock', + 'envvars_user_modules', 'extra_modules', 'filter_deps', + 'filter_ecs', 'filter_env_vars', 'hide_deps', 'hide_toolchains', + 'http_header_fields_urlpat', 'force_download', + 'insecure_download', 'from_pr', 'git_working_dirs_path', 'github_user', @@ -208,8 +230,11 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'pr_descr', 'pr_target_repo', 'pr_title', - 'rpath_filter', 'regtest_output_dir', + 'rpath_filter', + 'rpath_override_dirs', + 'banned_linked_shared_libs', + 'required_linked_shared_libs', 'silence_deprecation_warnings', 'skip', 'stop', @@ -239,25 +264,30 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'ignore_checksums', 'ignore_index', 'ignore_locks', + 'ignore_test_failure', 'install_latest_eb_release', 'logtostdout', 'minimal_toolchains', 'module_extensions', 'module_only', 'package', + 'parallel_extensions_install', 'read_only_installdir', 'remove_ghost_install_dirs', 'rebuild', 'robot', 'rpath', + 'sanity_check_only', 'search_paths', 'sequential', 'set_gid_bit', + 'skip_extensions', 'skip_test_cases', 'skip_test_step', 'generate_devel_module', 'sticky_bit', 'trace', + 'unit_testing_mode', 'upload_test_report', 'update_modules_tool_cache', 'use_ccache', @@ -272,12 +302,14 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'cleanup_tmpdir', 'extended_dry_run_ignore_errors', 'fixed_installdir_naming_scheme', + 'lib_lib64_symlink', 'lib64_fallback_sanity_check', 'lib64_lib_symlink', 'mpi_tests', 'map_toolchains', 'modules_tool_version_check', 'pre_create_installdir', + 'show_progress_bar', ], WARN: [ 'check_ebroot_env_vars', @@ -291,9 +323,15 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_BRANCH: [ 'pr_target_branch', ], + DEFAULT_ENV_FOR_SHEBANG: [ + 'env_for_shebang', + ], DEFAULT_INDEX_MAX_AGE: [ 'index_max_age', ], + DEFAULT_JOB_EB_CMD: [ + 'job_eb_cmd', + ], DEFAULT_MAX_FAIL_RATIO_PERMS: [ 'max_fail_ratio_adjust_permissions', ], @@ -324,6 +362,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_WAIT_ON_LOCK_INTERVAL: [ 'wait_on_lock_interval', ], + OUTPUT_STYLE_AUTO: [ + 'output_style', + ], } # build option that do not have a perfectly matching command line option BUILD_OPTIONS_OTHER = { @@ -331,7 +372,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'build_specs', 'command_line', 'external_modules_metadata', - 'pr_path', + 'pr_paths', 'robot_path', 'valid_module_classes', 'valid_stops', @@ -496,6 +537,10 @@ def init_build_options(build_options=None, cmdline_options=None): _log.info("Auto-enabling ignoring of OS dependencies") cmdline_options.ignore_osdeps = True + if not cmdline_options.accept_eula_for and cmdline_options.accept_eula: + _log.deprecated("Use accept-eula-for configuration setting rather than accept-eula.", '5.0') + cmdline_options.accept_eula_for = cmdline_options.accept_eula + cmdline_build_option_names = [k for ks in BUILD_OPTIONS_CMDLINE.values() for k in ks] active_build_options.update(dict([(key, getattr(cmdline_options, key)) for key in cmdline_build_option_names])) # other options which can be derived but have no perfectly matching cmdline option @@ -529,6 +574,9 @@ def build_option(key, **kwargs): build_options = BuildOptions() if key in build_options: return build_options[key] + elif key == 'accept_eula': + _log.deprecated("Use accept_eula_for build option rather than accept_eula.", '5.0') + return build_options['accept_eula_for'] elif 'default' in kwargs: return kwargs['default'] else: @@ -538,6 +586,36 @@ def build_option(key, **kwargs): raise EasyBuildError(error_msg) +def update_build_option(key, value): + """ + Update build option with specified name to given value. + + WARNING: Use this with care, the build options are not expected to be changed during an EasyBuild session! + """ + # BuildOptions() is a (singleton) frozen dict, so this is less straightforward that it seems... + build_options = BuildOptions() + orig_value = build_options._FrozenDict__dict[key] + build_options._FrozenDict__dict[key] = value + _log.warning("Build option '%s' was updated to: %s", key, build_option(key)) + + # Return original value, so it can be restored later if needed + return orig_value + + +def update_build_options(key_value_dict): + """ + Update build options as specified by the given dictionary (where keys are assumed to be build option names). + Returns dictionary with original values for the updated build options. + """ + orig_key_value_dict = {} + for key, value in key_value_dict.items(): + orig_key_value_dict[key] = update_build_option(key, value) + + # Return original key-value pairs in a dictionary. + # This way, they can later be restored by a single call to update_build_options(orig_key_value_dict) + return orig_key_value_dict + + def build_path(): """ Return the build path @@ -651,6 +729,22 @@ def get_module_syntax(): return ConfigurationVariables()['module_syntax'] +def get_output_style(): + """Return output style to use.""" + output_style = build_option('output_style') + + if output_style == OUTPUT_STYLE_AUTO: + if HAVE_RICH: + output_style = OUTPUT_STYLE_RICH + else: + output_style = OUTPUT_STYLE_BASIC + + if output_style == OUTPUT_STYLE_RICH and not HAVE_RICH: + raise EasyBuildError("Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH) + + return output_style + + def log_file_format(return_directory=False, ec=None, date=None, timestamp=None): """ Return the format for the logfile or the directory diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index ea418ce750..433238b95f 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -1213,9 +1213,8 @@ def _load(self, infile, configspec): if isinstance(infile, string_type): self.filename = infile if os.path.isfile(infile): - h = open(infile, 'r') - infile = h.read() or [] - h.close() + with open(infile, 'r') as fh: + infile = fh.read() or [] elif self.file_error: # raise an error if the file doesn't exist raise IOError('Config file not found: "%s".' % self.filename) @@ -1224,9 +1223,8 @@ def _load(self, infile, configspec): if self.create_empty: # this is a good test that the filename specified # isn't impossible - like on a non-existent device - h = open(infile, 'w') - h.write('') - h.close() + with open(infile, 'w') as fh: + fh.write('') infile = [] elif isinstance(infile, (list, tuple)): @@ -2025,7 +2023,7 @@ def write(self, outfile=None, section=None): # might need to encode # NOTE: This will *screw* UTF16, each line will start with the BOM if self.encoding: - out = [line.encode(self.encoding) for line in out] + out = [lne.encode(self.encoding) for lne in out] if (self.BOM and ((self.encoding is None) or (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))): # Add the UTF8 BOM @@ -2052,9 +2050,8 @@ def write(self, outfile=None, section=None): if outfile is not None: outfile.write(output) else: - h = open(self.filename, 'wb') - h.write(output) - h.close() + with open(self.filename, 'wb') as fh: + fh.write(output) def validate(self, validator, preserve_errors=False, copy=False, section=None): diff --git a/easybuild/tools/containers/__init__.py b/easybuild/tools/containers/__init__.py index f5d8fca779..0ce2d21dd3 100644 --- a/easybuild/tools/containers/__init__.py +++ b/easybuild/tools/containers/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/base.py b/easybuild/tools/containers/base.py index 7fbb9edd1d..80d3641f58 100644 --- a/easybuild/tools/containers/base.py +++ b/easybuild/tools/containers/base.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/common.py b/easybuild/tools/containers/common.py index b473d7fd73..852d079d88 100644 --- a/easybuild/tools/containers/common.py +++ b/easybuild/tools/containers/common.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index 104a96ec86..df94375381 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,6 +35,7 @@ from easybuild.tools.containers.base import ContainerGenerator from easybuild.tools.containers.utils import det_os_deps from easybuild.tools.filetools import remove_dir +from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS from easybuild.tools.run import run_cmd @@ -44,9 +45,9 @@ """ DOCKER_INSTALL_EASYBUILD = """\ -RUN pip install -U pip setuptools && \\ - hash -r pip && \\ - pip install -U easybuild +RUN pip3 install -U pip setuptools && \\ + hash -r pip3&& \\ + pip3 install -U easybuild RUN mkdir /app && \\ mkdir /scratch && \\ @@ -61,29 +62,35 @@ RUN set -x && \\ . /usr/share/lmod/lmod/init/sh && \\ - eb %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp + eb --robot %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp -RUN touch ${HOME}/.profile && \\ - echo '\\n# Added by easybuild docker packaging' >> ${HOME}/.profile && \\ - echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.profile && \\ - echo 'module use %(init_modulepath)s' >> ${HOME}/.profile && \\ - echo 'module load %(mod_names)s' >> ${HOME}/.profile +RUN touch ${HOME}/.bashrc && \\ + echo '' >> ${HOME}/.bashrc && \\ + echo '# Added by easybuild docker packaging' >> ${HOME}/.bashrc && \\ + echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.bashrc && \\ + echo 'module use %(init_modulepath)s' >> ${HOME}/.bashrc && \\ + echo 'module load %(mod_names)s' >> ${HOME}/.bashrc CMD ["/bin/bash", "-l"] """ -DOCKER_UBUNTU1604_INSTALL_DEPS = """\ +DOCKER_UBUNTU2004_INSTALL_DEPS = """\ RUN apt-get update && \\ - apt-get install -y python python-pip lmod curl wget + DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-pip lmod \\ + curl wget git bzip2 gzip tar zip unzip xz-utils \\ + patch automake git debianutils \\ + g++ libdata-dump-perl libthread-queue-any-perl libssl-dev RUN OS_DEPS='%(os_deps)s' && \\ - test -n "${OS_DEPS}" && \\ for dep in ${OS_DEPS}; do apt-get -qq install ${dep} || true; done """ DOCKER_CENTOS7_INSTALL_DEPS = """\ RUN yum install -y epel-release && \\ - yum install -y python python-pip Lmod curl wget git + yum install -y python3 python3-pip Lmod curl wget git \\ + bzip2 gzip tar zip unzip xz \\ + patch make git which \\ + gcc-c++ perl-Data-Dumper perl-Thread-Queue openssl-dev RUN OS_DEPS='%(os_deps)s' && \\ test -n "${OS_DEPS}" && \\ @@ -91,7 +98,7 @@ """ DOCKER_OS_INSTALL_DEPS_TMPLS = { - DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU1604_INSTALL_DEPS, + DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU2004_INSTALL_DEPS, DOCKER_BASE_IMAGE_CENTOS: DOCKER_CENTOS7_INSTALL_DEPS, } @@ -125,9 +132,12 @@ def resolve_template_data(self): ec = self.easyconfigs[-1]['ec'] - init_modulepath = os.path.join("/app/modules/all", *self.mns.det_init_modulepaths(ec)) + # We are using the default MNS inside the container + docker_mns = EasyBuildMNS() + + init_modulepath = os.path.join("/app/modules/all", *docker_mns.det_init_modulepaths(ec)) - mod_names = [e['ec'].full_mod_name for e in self.easyconfigs] + mod_names = [docker_mns.det_full_module_name(e['ec']) for e in self.easyconfigs] eb_opts = [os.path.basename(e['spec']) for e in self.easyconfigs] diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index 98c416414c..82c74a5eeb 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -1,4 +1,4 @@ -# Copyright 2017-2021 Ghent University +# Copyright 2017-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -272,9 +272,9 @@ def resolve_template_data(self): # EPEL is required for installing Lmod & python-pip 'epel-release', # EasyBuild requirements - 'python setuptools Lmod', - # pip is used to install EasyBuild packages - 'python-pip', + 'python3 setuptools Lmod', + # pip3 is used to install EasyBuild packages + 'python3-pip', # useful utilities 'bzip2 gzip tar zip unzip xz', # extracting sources 'curl wget', # downloading @@ -308,13 +308,13 @@ def resolve_template_data(self): template_data['install_os_deps'] = '\n'.join(install_os_deps) # install (latest) EasyBuild in container image - # use 'pip install', unless custom commands are specified via 'install_eb' keyword + # use 'pip3 install', unless custom commands are specified via 'install_eb' keyword if 'install_eb' not in template_data: template_data['install_eb'] = '\n'.join([ - "# install EasyBuild using pip", + "# install EasyBuild using pip3", # upgrade pip - "pip install -U pip", - "pip install easybuild", + "pip3 install -U pip", + "pip3 install easybuild", ]) # if no custom value is specified for 'post_commands' keyword, diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py index 27b9ee1cdd..29f81fe349 100644 --- a/easybuild/tools/containers/utils.py +++ b/easybuild/tools/containers/utils.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py index de4d0984c1..74b3aca266 100644 --- a/easybuild/tools/convert.py +++ b/easybuild/tools/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index a052e176e2..2f6358c02f 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -46,9 +46,9 @@ from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, process_easyconfig from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import EasyConfigParser -from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_EASYCONFIG +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_DYNAMIC +from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_NAMES_EASYCONFIG from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_LOWER, TEMPLATE_NAMES_LOWER_TEMPLATE -from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_CONSTANTS from easybuild.framework.easyconfig.templates import TEMPLATE_SOFTWARE_VERSIONS, template_constant_dict from easybuild.framework.easyconfig.tools import avail_easyblocks from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs @@ -344,6 +344,12 @@ def avail_easyconfig_templates_txt(): for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + # some template values are only defined dynamically, + # see template_constant_dict function in easybuild.framework.easyconfigs.templates + doc.append('Template values which are defined dynamically') + for name in TEMPLATE_NAMES_DYNAMIC: + doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + doc.append('Template constants that can be used in easyconfigs') for cst in TEMPLATE_CONSTANTS: doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, cst[0], cst[2], cst[1])) @@ -395,6 +401,13 @@ def avail_easyconfig_templates_rst(): ] doc.extend(rst_title_and_table(title, table_titles, table_values)) + title = 'Template values which are defined dynamically' + table_values = [ + ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC], + [name[1] for name in TEMPLATE_NAMES_DYNAMIC], + ] + doc.extend(rst_title_and_table(title, table_titles, table_values)) + title = 'Template constants that can be used in easyconfigs' titles = ['Constant', 'Template value', 'Template name'] table_values = [ diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index 7e5eb636ef..e99d64da2f 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -50,17 +50,11 @@ def write_changes(filename): """ Write current changes to filename and reset environment afterwards """ - script = None try: - script = open(filename, 'w') - - for key in _changes: - script.write('export %s=%s\n' % (key, shell_quote(_changes[key]))) - - script.close() + with open(filename, 'w') as script: + for key in _changes: + script.write('export %s=%s\n' % (key, shell_quote(_changes[key]))) except IOError as err: - if script is not None: - script.close() raise EasyBuildError("Failed to write to %s: %s", filename, err) reset_changes() diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 51e5bc9d03..4a4c8b060a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,28 +40,32 @@ """ import datetime import difflib -import fileinput import glob import hashlib import imp import inspect +import itertools import os import re import shutil import signal import stat +import ssl import sys import tempfile import time import zlib +from functools import partial from easybuild.base import fancylogger from easybuild.tools import run # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning -from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path +from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN +from easybuild.tools.config import build_option, install_path +from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type -from easybuild.tools.utilities import nub, remove_unwanted_chars +from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars try: import requests @@ -141,9 +145,11 @@ '.tb2': "tar xjf %(filepath)s", '.tbz': "tar xjf %(filepath)s", '.tbz2': "tar xjf %(filepath)s", - # xzipped or xzipped tarball - '.tar.xz': "unxz %(filepath)s --stdout | tar x", - '.txz': "unxz %(filepath)s --stdout | tar x", + # xzipped or xzipped tarball; + # need to make sure that $TAPE is not set to avoid 'tar x' command failing, + # see https://github.com/easybuilders/easybuild-framework/issues/3652 + '.tar.xz': "unset TAPE; unxz %(filepath)s --stdout | tar x", + '.txz': "unset TAPE; unxz %(filepath)s --stdout | tar x", '.xz': "unxz %(filepath)s", # tarball '.tar': "tar xf %(filepath)s", @@ -157,6 +163,8 @@ '.sh': "cp -a %(filepath)s .", } +ZIPPED_PATCH_EXTS = ('.bz2', '.gz', '.xz') + # global set of names of locks that were created in this session global_lock_names = set() @@ -189,11 +197,21 @@ def is_readable(path): raise EasyBuildError("Failed to check whether %s is readable: %s", path, err) +def open_file(path, mode): + """Open a (usually) text file. If mode is not binary, then utf-8 encoding will be used for Python 3.x""" + # This is required for text files in Python 3, especially until Python 3.7 which implements PEP 540. + # This PEP opens files in UTF-8 mode if the C locale is used, see https://www.python.org/dev/peps/pep-0540 + if sys.version_info[0] >= 3 and 'b' not in mode: + return open(path, mode, encoding='utf-8') + else: + return open(path, mode) + + def read_file(path, log_error=True, mode='r'): """Read contents of file at given path, in a robust way.""" txt = None try: - with open(path, mode) as handle: + with open_file(path, mode) as handle: txt = handle.read() except IOError as err: if log_error: @@ -202,18 +220,21 @@ def read_file(path, log_error=True, mode='r'): return txt -def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False): +def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False, + show_progress=False, size=None): """ Write given contents to file at given path; overwrites current file contents without backup by default! :param path: location of file - :param data: contents to write to file + :param data: contents to write to file. Can be a file-like object of binary data :param append: append to existing file rather than overwrite :param forced: force actually writing file in (extended) dry run mode :param backup: back up existing file before overwriting or modifying it :param always_overwrite: don't require --force to overwrite an existing file :param verbose: be verbose, i.e. inform where backup file was created + :param show_progress: show progress bar while writing file + :param size: size (in bytes) of data to write (used for progress bar) """ # early exit in 'dry run' mode if not forced and build_option('extended_dry_run'): @@ -237,15 +258,36 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over # cfr. https://docs.python.org/3/library/functions.html#open mode = 'a' if append else 'w' + data_is_file_obj = hasattr(data, 'read') + # special care must be taken with binary data in Python 3 - if sys.version_info[0] >= 3 and isinstance(data, bytes): + if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj): mode += 'b' + # don't bother showing a progress bar for small files (< 10MB) + if size and size < 10 * (1024 ** 2): + _log.info("Not showing progress bar for downloading small file (size %s)", size) + show_progress = False + + if show_progress: + start_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, size, label=os.path.basename(path)) + # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: mkdir(os.path.dirname(path), parents=True) - with open(path, mode) as handle: - handle.write(data) + with open_file(path, mode) as fh: + if data_is_file_obj: + # if a file-like object was provided, read file in 1MB chunks + for chunk in iter(partial(data.read, 1024 ** 2), b''): + fh.write(chunk) + if show_progress: + update_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, progress_size=len(chunk)) + else: + fh.write(data) + + if show_progress: + stop_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE) + except IOError as err: raise EasyBuildError("Failed to write to %s: %s", path, err) @@ -282,11 +324,19 @@ def symlink(source_path, symlink_path, use_abspath_source=True): if use_abspath_source: source_path = os.path.abspath(source_path) - try: - os.symlink(source_path, symlink_path) - _log.info("Symlinked %s to %s", source_path, symlink_path) - except OSError as err: - raise EasyBuildError("Symlinking %s to %s failed: %s", source_path, symlink_path, err) + if os.path.exists(symlink_path): + abs_source_path = os.path.abspath(source_path) + symlink_target_path = os.path.abspath(os.readlink(symlink_path)) + if abs_source_path != symlink_target_path: + raise EasyBuildError("Trying to symlink %s to %s, but the symlink already exists and points to %s.", + source_path, symlink_path, symlink_target_path) + _log.info("Skipping symlinking %s to %s, link already exists", source_path, symlink_path) + else: + try: + os.symlink(source_path, symlink_path) + _log.info("Symlinked %s to %s", source_path, symlink_path) + except OSError as err: + raise EasyBuildError("Symlinking %s to %s failed: %s", source_path, symlink_path, err) def remove_file(path): @@ -408,11 +458,14 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced _log.debug("Unpacking %s in directory %s", fn, abs_dest) cwd = change_dir(abs_dest) - if not cmd: - cmd = extract_cmd(fn, overwrite=overwrite) - else: + if cmd: # complete command template with filename cmd = cmd % fn + _log.debug("Using specified command to unpack %s: %s", fn, cmd) + else: + cmd = extract_cmd(fn, overwrite=overwrite) + _log.debug("Using command derived from file extension to unpack %s: %s", fn, cmd) + if not cmd: raise EasyBuildError("Can't extract file %s with unknown filetype", fn) @@ -435,15 +488,29 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced return base_dir -def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=True): +def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, on_error=None): """ Return (first) path in $PATH for specified command, or None if command is not found :param retain_all: returns *all* locations to the specified command in $PATH, not just the first one :param check_perms: check whether candidate path has read/exec permissions before accepting it as a match :param log_ok: Log an info message where the command has been found (if any) - :param log_error: Log a warning message when command hasn't been found - """ + :param on_error: What to do if the command was not found, default: WARN. Possible values: IGNORE, WARN, ERROR + """ + if log_error is not None: + _log.deprecated("'log_error' named argument in which function has been replaced by 'on_error'", '5.0') + # If set, make sure on_error is at least WARN + if log_error and on_error == IGNORE: + on_error = WARN + elif not log_error and on_error is None: # If set to False, use IGNORE unless on_error is also set + on_error = IGNORE + # Set default + # TODO: After removal of log_error from the parameters, on_error=WARN can be used instead of this + if on_error is None: + on_error = WARN + if on_error not in (IGNORE, WARN, ERROR): + raise EasyBuildError("Invalid value for 'on_error': %s", on_error) + if retain_all: res = [] else: @@ -467,8 +534,12 @@ def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=True): res = cmd_path break - if not res and log_error: - _log.warning("Could not find command '%s' (with permissions to read/execute it) in $PATH (%s)" % (cmd, paths)) + if not res and on_error != IGNORE: + msg = "Could not find command '%s' (with permissions to read/execute it) in $PATH (%s)" % (cmd, paths) + if on_error == WARN: + _log.warning(msg) + else: + raise EasyBuildError(msg) return res @@ -484,7 +555,7 @@ def det_common_path_prefix(paths): found_common = False while not found_common and prefix != os.path.dirname(prefix): prefix = os.path.dirname(prefix) - found_common = all([p.startswith(prefix) for p in paths]) + found_common = all(p.startswith(prefix) for p in paths) if found_common: # prefix may be empty string for relative paths with a non-common prefix @@ -493,6 +564,24 @@ def det_common_path_prefix(paths): return None +def normalize_path(path): + """Normalize path removing empty and dot components. + + Similar to os.path.normpath but does not resolve '..' which may return a wrong path when symlinks are used + """ + # In POSIX 3 or more leading slashes are equivalent to 1 + if path.startswith(os.path.sep): + if path.startswith(os.path.sep * 2) and not path.startswith(os.path.sep * 3): + start_slashes = os.path.sep * 2 + else: + start_slashes = os.path.sep + else: + start_slashes = '' + + filtered_comps = (comp for comp in path.split(os.path.sep) if comp and comp != '.') + return start_slashes + os.path.sep.join(filtered_comps) + + def is_alt_pypi_url(url): """Determine whether specified URL is already an alternate PyPI URL, i.e. whether it contains a hash.""" # example: .../packages/5b/03/e135b19fadeb9b1ccb45eac9f60ca2dc3afe72d099f6bd84e03cb131f9bf/easybuild-2.7.0.tar.gz @@ -569,9 +658,96 @@ def derive_alt_pypi_url(url): return alt_pypi_url +def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_headers_collection=None, maxdepth=3): + """ + Recurse into multi-line string "[URLPAT::][HEADER:]FILE|FIELD" where FILE may be another such string or file + containing lines matching the same format, such as "^https://www.example.com::/path/to/headers.txt", and flatten + the result to dict e.g. {'^https://www.example.com': ['Authorization: Basic token', 'User-Agent: Special Agent']} + """ + if urlpat_headers_collection is None: + # this function call is not a recursive call + urlpat_headers = {} + else: + # copy existing header data to avoid modifying it + urlpat_headers = urlpat_headers_collection.copy() + + # stop infinite recursion that might happen if a file.txt refers to itself + if maxdepth < 0: + raise EasyBuildError("Failed to parse_http_header_fields_urlpat (recursion limit)") + + if not isinstance(arg, str): + raise EasyBuildError("Failed to parse_http_header_fields_urlpat (argument not a string)") + + # HTTP header fields are separated by CRLF but splitting on LF is more convenient + for argline in arg.split('\n'): + argline = argline.strip() # remove optional whitespace (e.g. remaining CR) + if argline == '' or '#' in argline[0]: + continue # permit comment lines: ignore them + + if os.path.isfile(os.path.join(os.getcwd(), argline)): + # expand existing relative path to absolute + argline = os.path.join(os.path.join(os.getcwd(), argline)) + if os.path.isfile(argline): + # argline is a file path, so read that instead + _log.debug('File included in parse_http_header_fields_urlpat: %s' % argline) + argline = read_file(argline) + urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, header, urlpat_headers, maxdepth - 1) + continue + + # URL pattern is separated by '::' from a HTTP header field + if '::' in argline: + [urlpat, argline] = argline.split('::', 1) # get the urlpat + # the remainder may be another parseable argument, recurse with same depth + urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, header, urlpat_headers, maxdepth) + continue + + # Header field has format HEADER: FIELD, and FIELD may be another parseable argument + # except if FIELD contains colons, then argline is the final HEADER: FIELD to be returned + if ':' in argline and argline.count(':') == 1: + [argheader, argline] = argline.split(':', 1) # get the header and the remainder + # the remainder may be another parseable argument, recurse with same depth + # note that argheader would be forgotten in favor of the urlpat_headers returned by recursion, + # so pass on the header for reconstruction just in case there was nothing to recurse in + urlpat_headers = parse_http_header_fields_urlpat(argline, urlpat, argheader, urlpat_headers, maxdepth) + continue + + if header is not None: + # parent caller didn't want to forget about the header, reconstruct as recursion stops here. + argline = header.strip() + ':' + argline + + if urlpat is not None: + if urlpat in urlpat_headers.keys(): + urlpat_headers[urlpat].append(argline) # add headers to the list + else: + urlpat_headers[urlpat] = list([argline]) # new list headers for this urlpat + else: + _log.warning("Non-empty argument to http-header-fields-urlpat ignored (missing URL pattern)") + + # return a dict full of {urlpat: [list, of, headers]} + return urlpat_headers + + +def det_file_size(http_header): + """ + Determine size of file from provided HTTP header info (without downloading it). + """ + res = None + len_key = 'Content-Length' + if len_key in http_header: + size = http_header[len_key] + try: + res = int(size) + except (ValueError, TypeError) as err: + _log.warning("Failed to interpret size '%s' as integer value: %s", size, err) + + return res + + def download_file(filename, url, path, forced=False): """Download a file from the given URL, to the specified path.""" + insecure = build_option('insecure_download') + _log.debug("Trying to download %s from %s to %s", filename, url, path) timeout = build_option('download_timeout') @@ -581,6 +757,15 @@ def download_file(filename, url, path, forced=False): timeout = 10 _log.debug("Using timeout of %s seconds for initiating download" % timeout) + # parse option HTTP header fields for URLs containing a pattern + http_header_fields_urlpat = build_option('http_header_fields_urlpat') + # compile a dict full of {urlpat: [header, list]} + urlpat_headers = dict() + if http_header_fields_urlpat is not None: + # there may be multiple options given, parse them all, while updating urlpat_headers + for arg in http_header_fields_urlpat: + urlpat_headers.update(parse_http_header_fields_urlpat(arg)) + # make sure directory exists basedir = os.path.dirname(path) mkdir(basedir, parents=True) @@ -592,6 +777,17 @@ def download_file(filename, url, path, forced=False): # use custom HTTP header headers = {'User-Agent': 'EasyBuild', 'Accept': '*/*'} + + # permit additional or override headers via http_headers_fields_urlpat option + # only append/override HTTP header fields that match current url + if urlpat_headers is not None: + for urlpatkey, http_header_fields in urlpat_headers.items(): + if re.search(urlpatkey, url): + extraheaders = dict(hf.split(':', 1) for hf in http_header_fields) + for key, val in extraheaders.items(): + headers[key] = val + _log.debug("Custom HTTP header field set: %s (value omitted from log)", key) + # for backward compatibility, and to avoid relying on 3rd party Python library 'requests' url_req = std_urllib.Request(url, headers=headers) used_urllib = std_urllib @@ -600,20 +796,34 @@ def download_file(filename, url, path, forced=False): while not downloaded and attempt_cnt < max_attempts: attempt_cnt += 1 try: + if insecure: + print_warning("Not checking server certificates while downloading %s from %s." % (filename, url)) if used_urllib is std_urllib: # urllib2 (Python 2) / urllib.request (Python 3) does the right thing for http proxy setups, # urllib does not! - url_fd = std_urllib.urlopen(url_req, timeout=timeout) + if insecure: + url_fd = std_urllib.urlopen(url_req, timeout=timeout, context=ssl._create_unverified_context()) + else: + url_fd = std_urllib.urlopen(url_req, timeout=timeout) status_code = url_fd.getcode() + size = det_file_size(url_fd.info()) else: - response = requests.get(url, headers=headers, stream=True, timeout=timeout) + response = requests.get(url, headers=headers, stream=True, timeout=timeout, verify=(not insecure)) status_code = response.status_code response.raise_for_status() + size = det_file_size(response.headers) url_fd = response.raw url_fd.decode_content = True - _log.debug('response code for given url %s: %s' % (url, status_code)) - write_file(path, url_fd.read(), forced=forced, backup=True) - _log.info("Downloaded file %s from url %s to %s" % (filename, url, path)) + + _log.debug("HTTP response code for given url %s: %s", url, status_code) + _log.info("File size for %s: %s", url, size) + + # note: we pass the file object to write_file rather than reading the file first, + # to ensure the data is read in chunks (which prevents problems in Python 3.9+); + # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455 + # and https://bugs.python.org/issue42853 + write_file(path, url_fd, forced=forced, backup=True, show_progress=True, size=size) + _log.info("Downloaded file %s from url %s to %s", filename, url, path) downloaded = True url_fd.close() except used_urllib.HTTPError as err: @@ -902,8 +1112,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen if not terse: print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) - path_index = load_index(path, ignore_dirs=ignore_dirs) - if path_index is None or build_option('ignore_index'): + if build_option('ignore_index'): + path_index = None + else: + path_index = load_index(path, ignore_dirs=ignore_dirs) + if path_index is None: if os.path.exists(path): _log.info("No index found for %s, creating one...", path) path_index = create_index(path, ignore_dirs=ignore_dirs) @@ -923,22 +1136,31 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen else: path_hits.append(os.path.join(path, filepath)) - path_hits = sorted(path_hits) + path_hits = sorted(path_hits, key=natural_keys) if path_hits: - common_prefix = det_common_path_prefix(path_hits) - if not terse and short and common_prefix is not None and len(common_prefix) > len(var) * 2: - var_defs.append((var, common_prefix)) - hits.extend([os.path.join('$%s' % var, fn[len(common_prefix) + 1:]) for fn in path_hits]) - else: - hits.extend(path_hits) + if not terse and short: + common_prefix = det_common_path_prefix(path_hits) + if common_prefix is not None and len(common_prefix) > len(var) * 2: + var_defs.append((var, common_prefix)) + var_spec = '$' + var + # Replace the common prefix by var_spec + path_hits = (var_spec + fn[len(common_prefix):] for fn in path_hits) + hits.extend(path_hits) return var_defs, hits -def dir_contains_files(path): - """Return True if the given directory does contain any file in itself or any subdirectory""" - return any(files for _root, _dirs, files in os.walk(path)) +def dir_contains_files(path, recursive=True): + """ + Return True if the given directory does contain any file + + :recursive If False only the path itself is considered, else all subdirectories are also searched + """ + if recursive: + return any(files for _root, _dirs, files in os.walk(path)) + else: + return any(os.path.isfile(os.path.join(path, x)) for x in os.listdir(path)) def find_eb_script(script_name): @@ -1011,10 +1233,9 @@ def calc_block_checksum(path, algorithm): _log.debug("Using blocksize %s for calculating the checksum" % blocksize) try: - f = open(path, 'rb') - for block in iter(lambda: f.read(blocksize), b''): - algorithm.update(block) - f.close() + with open(path, 'rb') as fh: + for block in iter(lambda: fh.read(blocksize), b''): + algorithm.update(block) except IOError as err: raise EasyBuildError("Failed to read %s: %s", path, err) @@ -1150,7 +1371,7 @@ def find_extension(filename): if res: ext = res.group('ext') else: - raise EasyBuildError('Unknown file type for file %s', filename) + raise EasyBuildError("%s has unknown file extension", filename) return ext @@ -1161,9 +1382,11 @@ def extract_cmd(filepath, overwrite=False): """ filename = os.path.basename(filepath) ext = find_extension(filename) - target = filename.rstrip(ext) + target = filename[:-len(ext)] + # find_extension will either return an extension listed in EXTRACT_CMDS, or raise an error cmd_tmpl = EXTRACT_CMDS[ext.lower()] + if overwrite: if 'unzip -qq' in cmd_tmpl: cmd_tmpl = cmd_tmpl.replace('unzip -qq', 'unzip -qq -o') @@ -1242,6 +1465,49 @@ def guess_patch_level(patched_files, parent_dir): return patch_level +def create_patch_info(patch_spec): + """ + Create info dictionary from specified patch spec. + """ + if isinstance(patch_spec, (list, tuple)): + if not len(patch_spec) == 2: + error_msg = "Unknown patch specification '%s', only 2-element lists/tuples are supported!" + raise EasyBuildError(error_msg, str(patch_spec)) + + patch_info = {'name': patch_spec[0]} + + patch_arg = patch_spec[1] + # patch level *must* be of type int, nothing else (not True/False!) + # note that 'isinstance(..., int)' returns True for True/False values... + if isinstance(patch_arg, int) and not isinstance(patch_arg, bool): + patch_info['level'] = patch_arg + + # string value as patch argument can be either path where patch should be applied, + # or path to where a non-patch file should be copied + elif isinstance(patch_arg, string_type): + if patch_spec[0].endswith('.patch'): + patch_info['sourcepath'] = patch_arg + # non-patch files are assumed to be files to copy + else: + patch_info['copy'] = patch_arg + else: + raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element", + str(patch_spec)) + + elif isinstance(patch_spec, string_type): + allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS] + if not any(patch_spec.endswith(x) for x in allowed_patch_exts): + msg = "Use of patch file with filename that doesn't end with correct extension: %s " % patch_spec + msg += "(should be any of: %s)" % (', '.join(allowed_patch_exts)) + _log.deprecated(msg, '5.0') + patch_info = {'name': patch_spec} + else: + error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s" + raise EasyBuildError(error_msg, patch_spec) + + return patch_info + + def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False, use_git=False): """ Apply a patch to source code in directory dest @@ -1287,7 +1553,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa # split in stem (filename w/o extension) + extension patch_stem, patch_extension = os.path.splitext(os.path.split(abs_patch_file)[1]) # Supports only bz2, gz and xz. zip can be archives which are not supported. - if patch_extension in ['.gz', '.bz2', '.xz']: + if patch_extension in ZIPPED_PATCH_EXTS: # split again to get the second extension patch_subextension = os.path.splitext(patch_stem)[1] if patch_subextension == ".patch": @@ -1335,14 +1601,24 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa return True -def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): +def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_match=None): """ Apply specified list of regex substitutions. :param paths: list of paths to files to patch (or just a single filepath) :param regex_subs: list of substitutions to apply, specified as (, ) :param backup: create backup of original file with specified suffix (no backup if value evaluates to False) + :param on_missing_match: Define what to do when no match was found in the file. + Can be 'error' to raise an error, 'warn' to print a warning or 'ignore' to do nothing + Defaults to the value of --strict """ + if on_missing_match is None: + on_missing_match = build_option('strict') + allowed_values = (ERROR, IGNORE, WARN) + if on_missing_match not in allowed_values: + raise EasyBuildError('Invalid value passed to on_missing_match: %s (allowed: %s)', + on_missing_match, ', '.join(allowed_values)) + if isinstance(paths, string_type): paths = [paths] @@ -1356,39 +1632,50 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): else: _log.info("Applying following regex substitutions to %s: %s", paths, regex_subs) - compiled_regex_subs = [] - for regex, subtxt in regex_subs: - compiled_regex_subs.append((re.compile(regex), subtxt)) - - if backup: - backup_ext = backup - else: - # no (persistent) backup file is created if empty string value is passed to 'backup' in fileinput.input - backup_ext = '' + compiled_regex_subs = [(re.compile(regex), subtxt) for (regex, subtxt) in regex_subs] for path in paths: try: # make sure that file can be opened in text mode; # it's possible this fails with UnicodeDecodeError when running EasyBuild with Python 3 try: - with open(path, 'r') as fp: - _ = fp.read() + with open_file(path, 'r') as fp: + txt_utf8 = fp.read() except UnicodeDecodeError as err: _log.info("Encountered UnicodeDecodeError when opening %s in text mode: %s", path, err) path_backup = back_up_file(path) _log.info("Editing %s to strip out non-UTF-8 characters (backup at %s)", path, path_backup) txt = read_file(path, mode='rb') txt_utf8 = txt.decode(encoding='utf-8', errors='replace') + del txt write_file(path, txt_utf8) - for line_id, line in enumerate(fileinput.input(path, inplace=1, backup=backup_ext)): - for regex, subtxt in compiled_regex_subs: - match = regex.search(line) - if match: - origtxt = match.group(0) - _log.info("Replacing line %d in %s: '%s' -> '%s'", (line_id + 1), path, origtxt, subtxt) - line = regex.sub(subtxt, line) - sys.stdout.write(line) + if backup: + copy_file(path, path + backup) + replacement_msgs = [] + with open_file(path, 'w') as out_file: + lines = txt_utf8.split('\n') + del txt_utf8 + for line_id, line in enumerate(lines): + for regex, subtxt in compiled_regex_subs: + match = regex.search(line) + if match: + origtxt = match.group(0) + replacement_msgs.append("Replaced in line %d: '%s' -> '%s'" % + (line_id + 1, origtxt, subtxt)) + line = regex.sub(subtxt, line) + lines[line_id] = line + out_file.write('\n'.join(lines)) + if replacement_msgs: + _log.info('Applied the following substitutions to %s:\n%s', path, '\n'.join(replacement_msgs)) + else: + msg = 'Nothing found to replace in %s' % path + if on_missing_match == ERROR: + raise EasyBuildError(msg) + elif on_missing_match == WARN: + _log.warning(msg) + else: + _log.info(msg) except (IOError, OSError) as err: raise EasyBuildError("Failed to patch %s: %s", path, err) @@ -1553,6 +1840,25 @@ def patch_perl_script_autoflush(path): write_file(path, newtxt) +def set_gid_sticky_bits(path, set_gid=None, sticky=None, recursive=False): + """Set GID/sticky bits on specified path.""" + if set_gid is None: + set_gid = build_option('set_gid_bit') + if sticky is None: + sticky = build_option('sticky_bit') + + bits = 0 + if set_gid: + bits |= stat.S_ISGID + if sticky: + bits |= stat.S_ISVTX + if bits: + try: + adjust_permissions(path, bits, add=True, relative=True, recursive=recursive, onlydirs=True) + except OSError as err: + raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err) + + def mkdir(path, parents=False, set_gid=None, sticky=None): """ Create a directory @@ -1563,16 +1869,16 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): :param sticky: set the sticky bit on this directory (a.k.a. the restricted deletion flag), to avoid users can removing/renaming files in this directory """ - if set_gid is None: - set_gid = build_option('set_gid_bit') - if sticky is None: - sticky = build_option('sticky_bit') - if not os.path.isabs(path): path = os.path.abspath(path) # exit early if path already exists if not os.path.exists(path): + if set_gid is None: + set_gid = build_option('set_gid_bit') + if sticky is None: + sticky = build_option('sticky_bit') + _log.info("Creating directory %s (parents: %s, set_gid: %s, sticky: %s)", path, parents, set_gid, sticky) # set_gid and sticky bits are only set on new directories, so we need to determine the existing parent path existing_parent_path = os.path.dirname(path) @@ -1588,18 +1894,9 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): raise EasyBuildError("Failed to create directory %s: %s", path, err) # set group ID and sticky bits, if desired - bits = 0 - if set_gid: - bits |= stat.S_ISGID - if sticky: - bits |= stat.S_ISVTX - if bits: - try: - new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep) - new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0]) - adjust_permissions(new_path, bits, add=True, relative=True, recursive=True, onlydirs=True) - except OSError as err: - raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err) + new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep) + new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0]) + set_gid_sticky_bits(new_path, set_gid, sticky, recursive=True) else: _log.debug("Not creating existing path %s" % path) @@ -1815,8 +2112,8 @@ def back_up_file(src_file, backup_extension='bak', hidden=False, strip_fn=None): fn_suffix = '.%s' % backup_extension src_dir, src_fn = os.path.split(src_file) - if strip_fn: - src_fn = src_fn.rstrip(strip_fn) + if strip_fn and src_fn.endswith(strip_fn): + src_fn = src_fn[:-len(strip_fn)] backup_fp = find_backup_name_candidate(os.path.join(src_dir, fn_prefix + src_fn + fn_suffix)) @@ -2042,7 +2339,8 @@ def find_flexlm_license(custom_env_vars=None, lic_specs=None): if lic_files: for lic_file in lic_files: try: - open(lic_file, 'r') + # just try to open file for reading, no need to actually read it + open(lic_file, 'rb').close() valid_lic_specs.append(lic_file) except IOError as err: _log.warning("License file %s found, but failed to open it for reading: %s", lic_file, err) @@ -2072,11 +2370,17 @@ def copy_file(path, target_path, force_in_dry_run=False): :param force_in_dry_run: force copying of file during dry run """ if not force_in_dry_run and build_option('extended_dry_run'): + # If in dry run mode, do not copy any files, just lie about it dry_run_msg("copied file %s to %s" % (path, target_path)) + elif not os.path.exists(path) and not os.path.islink(path): + # NOTE: 'exists' will return False if 'path' is a broken symlink + raise EasyBuildError("Could not copy '%s' it does not exist!", path) else: try: + # check whether path to copy exists (we could be copying a broken symlink, which is supported) + path_exists = os.path.exists(path) target_exists = os.path.exists(target_path) - if target_exists and os.path.samefile(path, target_path): + if target_exists and path_exists and os.path.samefile(path, target_path): _log.debug("Not copying %s to %s since files are identical", path, target_path) # if target file exists and is owned by someone else than the current user, # try using shutil.copyfile to just copy the file contents @@ -2086,13 +2390,19 @@ def copy_file(path, target_path, force_in_dry_run=False): _log.info("Copied contents of file %s to %s", path, target_path) else: mkdir(os.path.dirname(target_path), parents=True) - if os.path.exists(path): + if path_exists: shutil.copy2(path, target_path) + _log.info("%s copied to %s", path, target_path) elif os.path.islink(path): + if os.path.isdir(target_path): + target_path = os.path.join(target_path, os.path.basename(path)) + _log.info("target_path changed to %s", target_path) # special care for copying broken symlinks link_target = os.readlink(path) - symlink(link_target, target_path) - _log.info("%s copied to %s", path, target_path) + symlink(link_target, target_path, use_abspath_source=False) + _log.info("created symlink %s to %s", link_target, target_path) + else: + raise EasyBuildError("Specified path %s is not an existing file or a symbolic link!", path) except (IOError, OSError, shutil.Error) as err: raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) @@ -2142,7 +2452,28 @@ def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=Fa raise EasyBuildError("One or more files to copy should be specified!") -def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): +def has_recursive_symlinks(path): + """ + Check the given directory for recursive symlinks. + + That means symlinks to folders inside the path which would cause infinite loops when traversed regularily. + + :param path: Path to directory to check + """ + for dirpath, dirnames, filenames in os.walk(path, followlinks=True): + for name in itertools.chain(dirnames, filenames): + fullpath = os.path.join(dirpath, name) + if os.path.islink(fullpath): + linkpath = os.path.realpath(fullpath) + fullpath += os.sep # To catch the case where both are equal + if fullpath.startswith(linkpath + os.sep): + _log.info("Recursive symlink detected at %s", fullpath) + return True + return False + + +def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, check_for_recursive_symlinks=True, + **kwargs): """ Copy a directory from specified location to specified location @@ -2150,6 +2481,7 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k :param target_path: path to copy the directory to :param force_in_dry_run: force running the command during dry run :param dirs_exist_ok: boolean indicating whether it's OK if the target directory already exists + :param check_for_recursive_symlinks: If symlink arg is not given or False check for recursive symlinks first shutil.copytree is used if the target path does not exist yet; if the target path already exists, the 'copy' function will be used to copy the contents of @@ -2161,6 +2493,13 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k dry_run_msg("copied directory %s to %s" % (path, target_path)) else: try: + if check_for_recursive_symlinks and not kwargs.get('symlinks'): + if has_recursive_symlinks(path): + raise EasyBuildError("Recursive symlinks detected in %s. " + "Will not try copying this unless `symlinks=True` is passed", + path) + else: + _log.debug("No recursive symlinks in %s", path) if not dirs_exist_ok and os.path.exists(target_path): raise EasyBuildError("Target location %s to copy %s to already exists", target_path, path) @@ -2188,7 +2527,9 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k paths_to_copy = [os.path.join(path, x) for x in entries] copy(paths_to_copy, target_path, - force_in_dry_run=force_in_dry_run, dirs_exist_ok=dirs_exist_ok, **kwargs) + force_in_dry_run=force_in_dry_run, dirs_exist_ok=dirs_exist_ok, + check_for_recursive_symlinks=False, # Don't check again + **kwargs) else: # if dirs_exist_ok is not enabled or target directory doesn't exist, just use shutil.copytree @@ -2245,6 +2586,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): repo_name = git_config.pop('repo_name', None) commit = git_config.pop('commit', None) recursive = git_config.pop('recursive', False) + clone_into = git_config.pop('clone_into', False) keep_git_dir = git_config.pop('keep_git_dir', False) # input validation of git_config dict @@ -2273,17 +2615,31 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # compose 'git clone' command, and run it clone_cmd = ['git', 'clone'] + if not keep_git_dir and not commit: + # Speed up cloning by only fetching the most recent commit, not the whole history + # When we don't want to keep the .git folder there won't be a difference in the result + clone_cmd.extend(['--depth', '1']) + if tag: clone_cmd.extend(['--branch', tag]) - - if recursive: - clone_cmd.append('--recursive') + if recursive: + clone_cmd.append('--recursive') + else: + # checkout is done separately below for specific commits + clone_cmd.append('--no-checkout') clone_cmd.append('%s/%s.git' % (url, repo_name)) + if clone_into: + clone_cmd.append('%s' % clone_into) + tmpdir = tempfile.mkdtemp() cwd = change_dir(tmpdir) - run.run_cmd(' '.join(clone_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run.run_cmd(' '.join(clone_cmd), log_all=True, simple=True, regexp=False) + + # If the clone is done into a specified name, change repo_name + if clone_into: + repo_name = clone_into # if a specific commit is asked for, check it out if commit: @@ -2291,14 +2647,40 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) - run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) + run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) + + elif not build_option('extended_dry_run'): + # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name + # This doesn't make sense in dry-run mode as we don't have anything to check + cmd = 'git describe --exact-match --tags HEAD' + # Note: Disable logging to also disable the error handling in run_cmd + (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name) + if ec != 0 or tag not in out.splitlines(): + print_warning('Tag %s was not downloaded in the first try due to %s/%s containing a branch' + ' with the same name. You might want to alert the maintainers of %s about that issue.', + tag, url, repo_name, repo_name) + cmds = [] + + if not keep_git_dir: + # make the repo unshallow first; + # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+ + # (first fetch seems to do nothing, unclear why) + cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647') + + cmds.append('git checkout refs/tags/' + tag) + # Clean all untracked files, e.g. from left-over submodules + cmds.append('git clean --force -d -x') + if recursive: + cmds.append('git submodule update --init --recursive') + for cmd in cmds: + run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name) # create an archive and delete the git repo directory if keep_git_dir: tar_cmd = ['tar', 'cfvz', targetpath, repo_name] else: tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] - run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False) # cleanup (repo_name dir does not exist in dry run mode) change_dir(cwd) @@ -2376,7 +2758,7 @@ def install_fake_vsc(): fake_vsc_init_path = os.path.join(fake_vsc_path, 'vsc', '__init__.py') if not os.path.exists(os.path.dirname(fake_vsc_init_path)): os.makedirs(os.path.dirname(fake_vsc_init_path)) - with open(fake_vsc_init_path, 'w') as fp: + with open_file(fake_vsc_init_path, 'w') as fp: fp.write(fake_vsc_init) sys.path.insert(0, fake_vsc_path) @@ -2473,3 +2855,32 @@ def copy_framework_files(paths, target_dir): raise EasyBuildError("Couldn't find parent folder of updated file: %s", path) return file_info + + +def create_unused_dir(parent_folder, name): + """ + Create a new folder in parent_folder using name as the name. + When a folder of that name already exists, '_0' is appended which is retried for increasing numbers until + an unused name was found + """ + if not os.path.isabs(parent_folder): + parent_folder = os.path.abspath(parent_folder) + + start_path = os.path.join(parent_folder, name) + for number in range(-1, 10000): # Start with no suffix and limit the number of attempts + if number < 0: + path = start_path + else: + path = start_path + '_' + str(number) + try: + os.mkdir(path) + break + except OSError as err: + # Distinguish between error due to existing folder and anything else + if not os.path.exists(path): + raise EasyBuildError("Failed to create directory %s: %s", path, err) + + # set group ID and sticky bits, if desired + set_gid_sticky_bits(path, recursive=True) + + return path diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 6be474bcd0..46fe56c703 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ import getpass import glob import functools +import itertools import os import random import re @@ -85,6 +86,8 @@ GITHUB_URL = 'https://github.com' GITHUB_API_URL = 'https://api.github.com' +GITHUB_BRANCH_MAIN = 'main' +GITHUB_BRANCH_MASTER = 'master' GITHUB_DIR_TYPE = u'dir' GITHUB_EB_MAIN = 'easybuilders' GITHUB_EASYBLOCKS_REPO = 'easybuild-easyblocks' @@ -120,18 +123,33 @@ } +def pick_default_branch(github_owner): + """Determine default name to use.""" + # use 'main' as default branch for 'easybuilders' organisation, + # otherwise use 'master' + if github_owner == GITHUB_EB_MAIN: + branch = GITHUB_BRANCH_MAIN + else: + branch = GITHUB_BRANCH_MASTER + + return branch + + class Githubfs(object): """This class implements some higher level functionality on top of the Github api""" - def __init__(self, githubuser, reponame, branchname="master", username=None, password=None, token=None): + def __init__(self, githubuser, reponame, branchname=None, username=None, password=None, token=None): """Construct a new githubfs object :param githubuser: the github user's repo we want to use. :param reponame: The name of the repository we want to use. - :param branchname: Then name of the branch to use (defaults to master) + :param branchname: Then name of the branch to use (defaults to 'main' for easybuilders org, 'master' otherwise) :param username: (optional) your github username. :param password: (optional) your github password. :param token: (optional) a github api token. """ + if branchname is None: + branchname = pick_default_branch(githubuser) + if token is None: token = fetch_github_token(username) self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) @@ -218,7 +236,7 @@ def read(self, path, api=True): """Read the contents of a file and return it Or, if api=False it will download the file and return the location of the downloaded file""" # we don't need use the api for this, but can also use raw.github.com - # https://raw.github.com/easybuilders/easybuild/master/README.rst + # https://raw.github.com/easybuilders/easybuild/main/README.rst if not api: outfile = tempfile.mkstemp()[1] url = '/'.join([GITHUB_RAW, self.githubuser, self.reponame, self.branchname, path]) @@ -301,7 +319,7 @@ def github_api_put_request(request_f, github_user=None, token=None, **kwargs): return (status, data) -def fetch_latest_commit_sha(repo, account, branch='master', github_user=None, token=None): +def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token=None): """ Fetch latest SHA1 for a specified repository and branch. :param repo: GitHub repository @@ -311,6 +329,9 @@ def fetch_latest_commit_sha(repo, account, branch='master', github_user=None, to :param token: GitHub token to use :return: latest SHA1 """ + if branch is None: + branch = pick_default_branch(account) + status, data = github_api_get_request(lambda x: x.repos[account][repo].branches, github_user=github_user, token=token, per_page=GITHUB_MAX_PER_PAGE) if status != HTTP_STATUS_OK: @@ -332,7 +353,7 @@ def fetch_latest_commit_sha(repo, account, branch='master', github_user=None, to return res -def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_EB_MAIN, path=None, github_user=None): +def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_MAIN, path=None, github_user=None): """ Download entire GitHub repo as a tar.gz archive, and extract it into specified path. :param repo: repo to download @@ -341,6 +362,9 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ :param path: path to extract to :param github_user: name of GitHub user to use """ + if branch is None: + branch = pick_default_branch(account) + # make sure path exists, create it if necessary if path is None: path = tempfile.mkdtemp() @@ -424,7 +448,15 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi if path is None: if github_repo == GITHUB_EASYCONFIGS_REPO: - path = build_option('pr_path') + pr_paths = build_option('pr_paths') + if pr_paths: + # figure out directory for this specific PR (see also alt_easyconfig_paths) + cands = [p for p in pr_paths if p.endswith('files_pr%s' % pr)] + if len(cands) == 1: + path = cands[0] + else: + raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, pr_paths) + elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr) else: @@ -653,6 +685,9 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa """ _log.debug("Cloning from %s", github_url) + if target_account is None: + raise EasyBuildError("target_account not specified in setup_repo_from!") + # salt to use for names of remotes/branches that are created salt = ''.join(random.choice(ascii_letters) for _ in range(5)) @@ -665,10 +700,12 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa # git fetch # can't use --depth to only fetch a shallow copy, since pushing to another repo from a shallow copy doesn't work print_msg("fetching branch '%s' from %s..." % (branch_name, github_url), silent=silent) + res = None try: res = origin.fetch() except GitCommandError as err: raise EasyBuildError("Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err) + if res: if res[0].flags & res[0].ERROR: raise EasyBuildError("Fetching branch '%s' from remote %s failed: %s", branch_name, origin, res[0].note) @@ -784,6 +821,10 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # if start branch is not specified, we're opening a new PR # account to use is determined by active EasyBuild configuration (--github-org or --github-user) target_account = build_option('github_org') or build_option('github_user') + + if target_account is None: + raise EasyBuildError("--github-org or --github-user must be specified!") + # if branch to start from is specified, we're updating an existing PR start_branch = build_option('pr_target_branch') else: @@ -799,7 +840,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # copy easyconfig files to right place target_dir = os.path.join(git_working_dir, pr_target_repo) print_msg("copying files to %s..." % target_dir) - file_info = COPY_FUNCTIONS[pr_target_repo](ec_paths, os.path.join(git_working_dir, pr_target_repo)) + file_info = COPY_FUNCTIONS[pr_target_repo](ec_paths, target_dir) # figure out commit message to use if commit_msg: @@ -861,6 +902,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if pr_branch is None: if ec_paths and pr_target_repo == GITHUB_EASYCONFIGS_REPO: label = file_info['ecs'][0].name + re.sub('[.-]', '', file_info['ecs'][0].version) + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and paths.get('py_files'): + label = os.path.splitext(os.path.basename(paths['py_files'][0]))[0] else: label = ''.join(random.choice(ascii_letters) for _ in range(10)) pr_branch = '%s_new_pr_%s' % (time.strftime("%Y%m%d%H%M%S"), label) @@ -936,6 +979,8 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch): :param target_repo: repository name :param branch: name of branch to push """ + if target_account is None: + raise EasyBuildError("target_account not specified in push_branch_to_github!") # push to GitHub remote = create_remote(git_repo, target_account, target_repo) @@ -971,10 +1016,14 @@ def is_patch_for(patch_name, ec): patches = copy.copy(ec['patches']) - for ext in ec['exts_list']: - if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict): - ext_options = ext[2] - patches.extend(ext_options.get('patches', [])) + with ec.disable_templating(): + # take into account both list of extensions (via exts_list) and components (cfr. Bundle easyblock) + for entry in itertools.chain(ec['exts_list'], ec.get('components', [])): + if isinstance(entry, (list, tuple)) and len(entry) == 3 and isinstance(entry[2], dict): + templates = {'name': entry[0], 'version': entry[1]} + options = entry[2] + patches.extend(p[0] % templates if isinstance(p, (tuple, list)) else p % templates + for p in options.get('patches', [])) for patch in patches: if isinstance(patch, (tuple, list)): @@ -1027,21 +1076,43 @@ def find_software_name_for_patch(patch_name, ec_dirs): soft_name = None + ignore_dirs = build_option('ignore_dirs') all_ecs = [] for ec_dir in ec_dirs: - for (dirpath, _, filenames) in os.walk(ec_dir): + for (dirpath, dirnames, filenames) in os.walk(ec_dir): + # Exclude ignored dirs + if ignore_dirs: + dirnames[:] = [i for i in dirnames if i not in ignore_dirs] for fn in filenames: - if fn != 'TEMPLATE.eb' and not fn.endswith('.py'): + # TODO: In EasyBuild 5.x only check for '*.eb' files + if fn != 'TEMPLATE.eb' and os.path.splitext(fn)[1] not in ('.py', '.patch'): path = os.path.join(dirpath, fn) rawtxt = read_file(path) if 'patches' in rawtxt: all_ecs.append(path) + # Usual patch names are -_fix_foo.patch + # So search those ECs first + patch_stem = os.path.splitext(patch_name)[0] + # Extract possible sw name and version according to above scheme + # Those might be the same as the whole patch stem, which is OK + possible_sw_name = patch_stem.split('-')[0].lower() + possible_sw_name_version = patch_stem.split('_')[0].lower() + + def ec_key(path): + filename = os.path.basename(path).lower() + # Put files with one of those as the prefix first, then sort by name + return ( + not filename.startswith(possible_sw_name_version), + not filename.startswith(possible_sw_name), + filename + ) + all_ecs.sort(key=ec_key) + nr_of_ecs = len(all_ecs) for idx, path in enumerate(all_ecs): if soft_name: break - rawtxt = read_file(path) try: ecs = process_easyconfig(path, validate=False) for ec in ecs: @@ -1084,12 +1155,14 @@ def not_eligible(msg): # check test suite result, Travis must give green light msg_tmpl = "* test suite passes: %s" + failed_status_last_commit = False if pr_data['status_last_commit'] == STATUS_SUCCESS: print_msg(msg_tmpl % 'OK', prefix=False) elif pr_data['status_last_commit'] == STATUS_PENDING: res = not_eligible(msg_tmpl % "pending...") else: res = not_eligible(msg_tmpl % "(status: %s)" % pr_data['status_last_commit']) + failed_status_last_commit = True if pr_data['base']['repo']['name'] == GITHUB_EASYCONFIGS_REPO: # check for successful test report (checked in reverse order) @@ -1119,6 +1192,19 @@ def not_eligible(msg): if review['state'] == 'APPROVED': approved_review_by.append(review['user']['login']) + # check for requested changes + changes_requested_by = [] + for review in pr_data['reviews']: + if review['state'] == 'CHANGES_REQUESTED': + if review['user']['login'] not in approved_review_by + changes_requested_by: + changes_requested_by.append(review['user']['login']) + + msg_tmpl = "* no pending change requests: %s" + if changes_requested_by: + res = not_eligible(msg_tmpl % 'FAILED (changes requested by %s)' % ', '.join(changes_requested_by)) + else: + print_msg(msg_tmpl % 'OK', prefix=False) + msg_tmpl = "* approved review: %s" if approved_review_by: print_msg(msg_tmpl % 'OK (by %s)' % ', '.join(approved_review_by), prefix=False) @@ -1128,10 +1214,29 @@ def not_eligible(msg): # check whether a milestone is set msg_tmpl = "* milestone is set: %s" if pr_data['milestone']: - print_msg(msg_tmpl % "OK (%s)" % pr_data['milestone']['title'], prefix=False) + milestone = pr_data['milestone']['title'] + if '.x' in milestone: + milestone += ", please change to the next release milestone once the PR is merged" + print_msg(msg_tmpl % "OK (%s)" % milestone, prefix=False) else: res = not_eligible(msg_tmpl % 'no milestone found') + # check github mergeable state + msg_tmpl = "* mergeable state is clean: %s" + mergeable = False + if pr_data['merged']: + print_msg(msg_tmpl % "PR is already merged", prefix=False) + elif pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN: + print_msg(msg_tmpl % "OK", prefix=False) + mergeable = True + else: + reason = "FAILED (mergeable state is '%s')" % pr_data['mergeable_state'] + res = not_eligible(msg_tmpl % reason) + + if failed_status_last_commit and mergeable: + print_msg("\nThis PR is mergeable but the test suite has a failed status. Try syncing the PR with the " + "develop branch using 'eb --sync-pr-with-develop %s'" % pr_data['number'], prefix=False) + return res @@ -1167,7 +1272,7 @@ def reasons_for_closing(pr_data): robot_paths = build_option('robot_path') - pr_files = [path for path in fetch_easyconfigs_from_pr(pr_data['number']) if path.endswith('.eb')] + pr_files = [p for p in fetch_easyconfigs_from_pr(pr_data['number']) if p.endswith('.eb')] obsoleted = [] uses_archived_tc = [] @@ -1207,7 +1312,7 @@ def reasons_for_closing(pr_data): if uses_archived_tc: possible_reasons.append('archived') - if any([e['name'] in pr_data['title'] for e in obsoleted]): + if any(e['name'] in pr_data['title'] for e in obsoleted): possible_reasons.append('obsolete') return possible_reasons @@ -1324,6 +1429,7 @@ def merge_pr(pr): msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_data['user']['login']) msg += "you are using GitHub account '%s'\n" % github_user + msg += "\nPR title: %s\n\n" % pr_data['title'] print_msg(msg, prefix=False) if pr_data['user']['login'] == github_user: raise EasyBuildError("Please do not merge your own PRs!") @@ -1359,6 +1465,95 @@ def merge_url(gh): print_warning("Review indicates this PR should not be merged (use -f/--force to do so anyway)") +def det_pr_labels(file_info, pr_target_repo): + """ + Determine labels for a pull request based on provided information on files changed by that pull request. + """ + labels = [] + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + if any(file_info['new_folder']): + labels.append('new') + if any(file_info['new_file_in_existing_folder']): + labels.append('update') + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: + if any(file_info['new']): + labels.append('new') + return labels + + +def post_pr_labels(pr, labels): + """ + Update PR labels + """ + pr_target_account = build_option('pr_target_account') + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO + + # fetch GitHub token if available + github_user = build_option('github_user') + if github_user is None: + _log.info("GitHub user not specified, not adding labels to PR# %s" % pr) + return False + + github_token = fetch_github_token(github_user) + if github_token is None: + _log.info("GitHub token for user '%s' not found, not adding labels to PR# %s" % (github_user, pr)) + return False + + dry_run = build_option('dry_run') or build_option('extended_dry_run') + + if not dry_run: + g = RestClient(GITHUB_API_URL, username=github_user, token=github_token) + + pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr] + try: + status, data = pr_url.labels.post(body=labels) + if status == HTTP_STATUS_OK: + print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False) + return True + except HTTPError as err: + _log.info("Failed to add labels to PR# %s: %s." % (pr, err)) + return False + else: + return True + + +def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH): + """ + Try to determine and add labels to PR. + :param pr: pull request number in easybuild-easyconfigs repo + :param branch: easybuild-easyconfigs branch to compare with + """ + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO + if pr_target_repo != GITHUB_EASYCONFIGS_REPO: + raise EasyBuildError("Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet") + + tmpdir = tempfile.mkdtemp() + + download_repo_path = download_repo(branch=branch, path=tmpdir) + + pr_files = [p for p in fetch_easyconfigs_from_pr(pr) if p.endswith('.eb')] + + file_info = det_file_info(pr_files, download_repo_path) + + pr_target_account = build_option('pr_target_account') + github_user = build_option('github_user') + pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) + pr_labels = [x['name'] for x in pr_data['labels']] + + expected_labels = det_pr_labels(file_info, pr_target_repo) + missing_labels = [x for x in expected_labels if x not in pr_labels] + + dry_run = build_option('dry_run') or build_option('extended_dry_run') + + if missing_labels: + missing_labels_txt = ', '.join(["'%s'" % ml for ml in missing_labels]) + print_msg("PR #%s should be labelled %s" % (pr, missing_labels_txt), log=_log, prefix=False) + if not dry_run and not post_pr_labels(pr, missing_labels): + print_msg("Could not add labels %s to PR #%s" % (missing_labels_txt, pr), log=_log, prefix=False) + else: + print_msg("Could not determine any missing labels for PR #%s" % pr, log=_log, prefix=False) + + @only_if_module_is_available('git', pkgname='GitPython') def new_branch_github(paths, ecs, commit_msg=None): """ @@ -1461,7 +1656,7 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, msg.extend([" " + x for x in patch_paths]) if deleted_paths: msg.append("* %d deleted file(s)" % len(deleted_paths)) - msg.append([" " + x for x in deleted_paths]) + msg.extend([" " + x for x in deleted_paths]) print_msg('\n'.join(msg), log=_log) else: @@ -1486,14 +1681,9 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, file_info = det_file_info(ec_paths, target_dir) - labels = [] - if pr_target_repo == GITHUB_EASYCONFIGS_REPO: - # label easyconfigs for new software and/or new easyconfigs for existing software - if any(file_info['new_folder']): - labels.append('new') - if any(file_info['new_file_in_existing_folder']): - labels.append('update') + labels = det_pr_labels(file_info, pr_target_repo) + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: # only use most common toolchain(s) in toolchain label of PR title toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)]) @@ -1503,9 +1693,6 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, classes = [ec['moduleclass'] for ec in file_info['ecs']] classes_counted = sorted([(classes.count(c), c) for c in nub(classes)]) class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]]) - elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: - if any(file_info['new']): - labels.append('new') if title is None: if pr_target_repo == GITHUB_EASYCONFIGS_REPO: @@ -1581,15 +1768,9 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, print_msg("Opened pull request: %s" % data['html_url'], log=_log, prefix=False) if labels: - # post labels pr = data['html_url'].split('/')[-1] - pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr] - try: - status, data = pr_url.labels.post(body=labels) - if status == HTTP_STATUS_OK: - print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False) - except HTTPError as err: - _log.info("Failed to add labels to PR# %s: %s." % (pr, err)) + if not post_pr_labels(pr, labels): + print_msg("This PR should be labelled %s" % ', '.join(labels), log=_log, prefix=False) def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): @@ -1610,6 +1791,16 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): res = new_branch_github(paths, ecs, commit_msg=commit_msg) file_info, deleted_paths, _, branch_name, diff_stat, pr_target_repo = res + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + for ec, ec_path in zip(file_info['ecs'], file_info['paths_in_repo']): + for patch in ec.asdict()['patches']: + if isinstance(patch, tuple): + patch = patch[0] + if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path), + patch)): + print_warning("new patch file %s, referenced by %s, is not included in this PR" % + (patch, ec.filename())) + new_pr_from_branch(branch_name, title=title, descr=descr, pr_target_repo=pr_target_repo, pr_metadata=(file_info, deleted_paths, diff_stat), commit_msg=commit_msg) @@ -1660,7 +1851,7 @@ def det_pr_target_repo(paths): # if all Python files are easyblocks, target repo should be easyblocks; # otherwise, target repo is assumed to be framework - if all([get_easyblock_class_name(path) for path in py_files]): + if all(get_easyblock_class_name(path) for path in py_files): pr_target_repo = GITHUB_EASYBLOCKS_REPO _log.info("All Python files are easyblocks, target repository is assumed to be %s", pr_target_repo) else: @@ -1865,7 +2056,7 @@ def check_github(): branch_name = 'test_branch_%s' % ''.join(random.choice(ascii_letters) for _ in range(5)) try: git_repo = init_repo(git_working_dir, GITHUB_EASYCONFIGS_REPO, silent=not debug) - remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, 'master', + remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, GITHUB_DEVELOP_BRANCH, silent=not debug, git_only=True) git_repo.create_head(branch_name) res = getattr(git_repo.remotes, remote_name).push(branch_name) @@ -1884,6 +2075,8 @@ def check_github(): ver, req_ver = git.__version__, '1.0' if LooseVersion(ver) < LooseVersion(req_ver): check_res = "FAIL (GitPython version %s is too old, should be version %s or newer)" % (ver, req_ver) + elif "Could not read from remote repository" in str(push_err): + check_res = "FAIL (GitHub SSH key missing? %s)" % push_err else: check_res = "FAIL (unexpected exception: %s)" % push_err else: @@ -2035,22 +2228,32 @@ def validate_github_token(token, github_user): * see if it conforms expectations (only [a-f]+[0-9] characters, length of 40) * see if it can be used for authenticated access """ - sha_regex = re.compile('^[0-9a-f]{40}') + # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ + token_regex = re.compile('^ghp_[a-zA-Z0-9]{36}$') + token_regex_old_format = re.compile('^[0-9a-f]{40}$') # token should be 40 characters long, and only contain characters in [0-9a-f] - sanity_check = bool(sha_regex.match(token)) + sanity_check = bool(token_regex.match(token)) if sanity_check: _log.info("Sanity check on token passed") else: - _log.warning("Sanity check on token failed; token doesn't match pattern '%s'", sha_regex.pattern) + _log.warning("Sanity check on token failed; token doesn't match pattern '%s'", token_regex.pattern) + sanity_check = bool(token_regex_old_format.match(token)) + if sanity_check: + _log.info("Sanity check on token (old format) passed") + else: + _log.warning("Sanity check on token failed; token doesn't match pattern '%s'", + token_regex_old_format.pattern) # try and determine sha of latest commit in easybuilders/easybuild-easyconfigs repo through authenticated access sha = None try: - sha = fetch_latest_commit_sha(GITHUB_EASYCONFIGS_REPO, GITHUB_EB_MAIN, github_user=github_user, token=token) + sha = fetch_latest_commit_sha(GITHUB_EASYCONFIGS_REPO, GITHUB_EB_MAIN, + branch=GITHUB_DEVELOP_BRANCH, github_user=github_user, token=token) except Exception as err: _log.warning("An exception occurred when trying to use token for authenticated GitHub access: %s", err) + sha_regex = re.compile('^[0-9a-f]{40}$') token_test = bool(sha_regex.match(sha or '')) if token_test: _log.info("GitHub token can be used for authenticated GitHub access, validation passed") @@ -2064,7 +2267,8 @@ def find_easybuild_easyconfig(github_user=None): :param github_user: name of GitHub user to use when querying GitHub """ - dev_repo = download_repo(GITHUB_EASYCONFIGS_REPO, branch='develop', account=GITHUB_EB_MAIN, github_user=github_user) + dev_repo = download_repo(GITHUB_EASYCONFIGS_REPO, branch=GITHUB_DEVELOP_BRANCH, + account=GITHUB_EB_MAIN, github_user=github_user) eb_parent_path = os.path.join(dev_repo, 'easybuild', 'easyconfigs', 'e', 'EasyBuild') files = os.listdir(eb_parent_path) @@ -2223,7 +2427,7 @@ def sync_with_develop(git_repo, branch_name, github_account, github_repo): remote = create_remote(git_repo, github_account, github_repo, https=True) # fetch latest version of develop branch - pull_out = git_repo.git.pull(remote.name, GITHUB_DEVELOP_BRANCH) + pull_out = git_repo.git.pull(remote.name, GITHUB_DEVELOP_BRANCH, no_rebase=True) _log.debug("Output of 'git pull %s %s': %s", remote.name, GITHUB_DEVELOP_BRANCH, pull_out) # fetch to make sure we can check out the 'develop' branch diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 85407fb993..e12882374a 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2021 Ghent University +# Copyright 2017-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,6 +33,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.config import build_option _log = fancylogger.getLogger('hooks', fname=False) @@ -58,6 +59,7 @@ START = 'start' PARSE = 'parse' +MODULE_WRITE = 'module_write' END = 'end' PRE_PREF = 'pre_' @@ -69,7 +71,7 @@ INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP] -HOOK_NAMES = [START, PARSE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END] +HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] @@ -99,7 +101,7 @@ def load_hooks(hooks_path): if attr.endswith(HOOK_SUFF): hook = getattr(imported_hooks, attr) if callable(hook): - hooks.update({attr: hook}) + hooks[attr] = hook else: _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr) _log.info("Found hooks: %s", sorted(hooks.keys())) @@ -119,11 +121,8 @@ def load_hooks(hooks_path): def verify_hooks(hooks): - """Check whether list of obtained hooks only includes known hooks.""" - unknown_hooks = [] - for key in sorted(hooks): - if key not in KNOWN_HOOKS: - unknown_hooks.append(key) + """Check whether obtained hooks only includes known hooks.""" + unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS] if unknown_hooks: error_lines = ["Found one or more unknown hooks:"] @@ -147,7 +146,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): Find hook with specified label. :param label: name of hook - :param hooks: list of defined hooks + :param hooks: dict of defined hooks :param pre_step_hook: indicates whether hook to run is a pre-step hook :param post_step_hook: indicates whether hook to run is a post-step hook """ @@ -162,27 +161,26 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): hook_name = hook_prefix + label + HOOK_SUFF - for key in hooks: - if key == hook_name: - _log.info("Found %s hook", hook_name) - res = hooks[key] - break + res = hooks.get(hook_name) + if res: + _log.info("Found %s hook", hook_name) return res def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None): """ - Run hook with specified label. + Run hook with specified label and return result of calling the hook or None. :param label: name of hook - :param hooks: list of defined hooks + :param hooks: dict of defined hooks :param pre_step_hook: indicates whether hook to run is a pre-step hook :param post_step_hook: indicates whether hook to run is a post-step hook :param args: arguments to pass to hook function :param msg: custom message that is printed when hook is called """ hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) + res = None if hook: if args is None: args = [] @@ -194,7 +192,9 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if msg is None: msg = "Running %s hook..." % label - print_msg(msg) + if build_option('debug'): + print_msg(msg) _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args) - hook(*args) + res = hook(*args) + return res diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 58464a4d25..5aeb74d51c 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -72,9 +72,11 @@ for subdir in subdirs: __path__ = pkgutil.extend_path(__path__, '%s.%s' % (__name__, subdir)) -del l, subdir, subdirs +del subdir, subdirs +if 'char' in dir(): + del char -__path__ = __import__('pkgutil').extend_path(__path__, __name__) +__path__ = pkgutil.extend_path(__path__, __name__) """ @@ -126,15 +128,17 @@ def set_up_eb_package(parent_path, eb_pkg_name, subpkgs=None, pkg_init_body=None def verify_imports(pymods, pypkg, from_path): """Verify that import of specified modules from specified package and expected location works.""" - for pymod in pymods: - pymod_spec = '%s.%s' % (pypkg, pymod) - + pymod_specs = ['%s.%s' % (pypkg, pymod) for pymod in pymods] + for pymod_spec in pymod_specs: # force re-import if the specified modules was already imported; # this is required to ensure that an easyblock that is included via --include-easyblocks-from-pr # gets preference over one that is included via --include-easyblocks if pymod_spec in sys.modules: del sys.modules[pymod_spec] + # After all modules to be reloaded have been removed, import them again + # Note that removing them here may delete transitively loaded modules and not import them again + for pymod_spec in pymod_specs: try: pymod = __import__(pymod_spec, fromlist=[pypkg]) # different types of exceptions may be thrown, not only ImportErrors @@ -178,8 +182,8 @@ def include_easyblocks(tmpdir, paths): if not os.path.exists(target_path): symlink(easyblock_module, target_path) - included_ebs = [x for x in os.listdir(easyblocks_dir) if x not in ['__init__.py', 'generic']] - included_generic_ebs = [x for x in os.listdir(os.path.join(easyblocks_dir, 'generic')) if x != '__init__.py'] + included_ebs = sorted(x for x in os.listdir(easyblocks_dir) if x not in ['__init__.py', 'generic']) + included_generic_ebs = sorted(x for x in os.listdir(os.path.join(easyblocks_dir, 'generic')) if x != '__init__.py') _log.debug("Included generic easyblocks: %s", included_generic_ebs) _log.debug("Included software-specific easyblocks: %s", included_ebs) diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py index 859e0030a5..b299667e01 100644 --- a/easybuild/tools/jenkins.py +++ b/easybuild/tools/jenkins.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -105,9 +105,8 @@ def create_success(name, stats): root.firstChild.appendChild(el) try: - output_file = open(filename, "w") - root.writexml(output_file) - output_file.close() + with open(filename, "w") as output_file: + root.writexml(output_file) except IOError as err: raise EasyBuildError("Failed to write out XML file %s: %s", filename, err) @@ -162,9 +161,8 @@ def aggregate_xml_in_dirs(base_dir, output_filename): comment = root.createComment("%s out of %s builds succeeded" % (succes, total)) root.firstChild.insertBefore(comment, properties) try: - output_file = open(output_filename, "w") - root.writexml(output_file, addindent="\t", newl="\n") - output_file.close() + with open(output_filename, "w") as output_file: + root.writexml(output_file, addindent="\t", newl="\n") except IOError as err: raise EasyBuildError("Failed to write out XML file %s: %s", output_filename, err) diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py index 5e5e5ceacb..868edb7fd4 100644 --- a/easybuild/tools/job/backend.py +++ b/easybuild/tools/job/backend.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index 806d21544c..a8c921e723 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # Copyright 2015 S3IT, University of Zurich # # This file is part of EasyBuild, diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index 28659ab91c..77fabb9ffe 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py index 43531980d3..924c46aaf1 100644 --- a/easybuild/tools/job/slurm.py +++ b/easybuild/tools/job/slurm.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2021 Ghent University +# Copyright 2018-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index c031f14d83..9991f9ba01 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,11 +37,12 @@ import os import re import tempfile +from contextlib import contextmanager from distutils.version import LooseVersion from textwrap import wrap from easybuild.base import fancylogger -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option, get_module_syntax, install_path from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool @@ -136,17 +137,29 @@ def __init__(self, application, fake=False): self.fake_mod_path = tempfile.mkdtemp() self.modules_tool = modules_tool() + self.added_paths_per_key = None - def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): + @contextmanager + def start_module_creation(self): """ - Generate append-path statements for the given list of paths. + Prepares creating a module and returns the file header (shebang) if any including the newline - :param key: environment variable to append paths to - :param paths: list of paths to append - :param allow_abs: allow providing of absolute paths - :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) + Meant to be used in a with statement: + with generator.start_module_creation() as txt: + # Write txt """ - return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths) + if self.added_paths_per_key is not None: + raise EasyBuildError('Module creation already in process. ' + 'You cannot create multiple modules at the same time!') + # Mapping of keys/env vars to paths already added + self.added_paths_per_key = dict() + txt = self.MODULE_SHEBANG + if txt: + txt += '\n' + try: + yield txt + finally: + self.added_paths_per_key = None def create_symlinks(self, mod_symlink_paths, fake=False): """Create moduleclass symlink(s) to actual module file.""" @@ -191,6 +204,49 @@ def get_modules_path(self, fake=False, mod_path_suffix=None): return os.path.join(mod_path, mod_path_suffix) + def _filter_paths(self, key, paths): + """Filter out paths already added to key and return the remaining ones""" + if self.added_paths_per_key is None: + # For compatibility this is only a warning for now and we don't filter any paths + print_warning('Module creation has not been started. Call start_module_creation first!') + return paths + + added_paths = self.added_paths_per_key.setdefault(key, set()) + # paths can be a string + if isinstance(paths, string_type): + if paths in added_paths: + filtered_paths = None + else: + added_paths.add(paths) + filtered_paths = paths + else: + # Coerce any iterable/generator into a list + if not isinstance(paths, list): + paths = list(paths) + filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)] + if filtered_paths != paths: + removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths] + print_warning("Suppressed adding the following path(s) to $%s of the module as they were already added: %s", + key, removed_paths, + log=self.log) + if not filtered_paths: + filtered_paths = None + return filtered_paths + + def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): + """ + Generate append-path statements for the given list of paths. + + :param key: environment variable to append paths to + :param paths: list of paths to append + :param allow_abs: allow providing of absolute paths + :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) + """ + paths = self._filter_paths(key, paths) + if paths is None: + return '' + return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths) + def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): """ Generate prepend-path statements for the given list of paths. @@ -200,6 +256,9 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): :param allow_abs: allow providing of absolute paths :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ + paths = self._filter_paths(key, paths) + if paths is None: + return '' return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths) def _modulerc_check_module_version(self, module_version): @@ -394,7 +453,7 @@ def get_description(self, conflict=True): """ raise NotImplementedError - def getenv_cmd(self, envvar): + def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ @@ -559,6 +618,9 @@ def _generate_help_text(self): # Examples (optional) lines.extend(self._generate_section('Examples', self.app.cfg['examples'], strip=True)) + # Citing (optional) + lines.extend(self._generate_section('Citing', self.app.cfg['citing'], strip=True)) + # Additional information: homepage + (if available) doc paths/urls, upstream/site contact lines.extend(self._generate_section("More information", " - Homepage: %s" % self.app.cfg['homepage'])) @@ -776,18 +838,27 @@ def get_description(self, conflict=True): return txt - def getenv_cmd(self, envvar): + def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ - return '$::env(%s)' % envvar + if default is None: + cmd = '$::env(%s)' % envvar + else: + values = { + 'default': default, + 'envvar': '::env(%s)' % envvar, + } + cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values + return cmd - def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): + def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_modules=None, multi_dep_mods=None): """ Generate load statement for specified module. :param mod_name: name of module to generate load statement for :param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload + (if None: enable if recursive_mod_unload build option or depends_on is True) :param depends_on: use depends_on statements rather than (guarded) load statements :param unload_modules: name(s) of module to unload first :param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement @@ -807,7 +878,11 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload depends_on = load_template == self.LOAD_TEMPLATE_DEPENDS_ON cond_tmpl = None - if build_option('recursive_mod_unload') or recursive_unload or depends_on: + + if recursive_unload is None: + recursive_unload = build_option('recursive_mod_unload') or depends_on + + if recursive_unload: # wrapping the 'module load' statement with an 'is-loaded or mode == unload' # guard ensures recursive unloading while avoiding load storms; # when "module unload" is called on the module in which the @@ -1196,18 +1271,23 @@ def get_description(self, conflict=True): return txt - def getenv_cmd(self, envvar): + def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ - return 'os.getenv("%s")' % envvar + if default is None: + cmd = 'os.getenv("%s")' % envvar + else: + cmd = 'os.getenv("%s") or "%s"' % (envvar, default) + return cmd - def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): + def load_module(self, mod_name, recursive_unload=None, depends_on=False, unload_modules=None, multi_dep_mods=None): """ Generate load statement for specified module. :param mod_name: name of module to generate load statement for :param recursive_unload: boolean indicating whether the 'load' statement should be reverted on unload + (if None: enable if recursive_mod_unload build option or depends_on is True) :param depends_on: use depends_on statements rather than (guarded) load statements :param unload_modules: name(s) of module to unload first :param multi_dep_mods: list of module names in multi_deps context, to use for guarding load statement @@ -1228,7 +1308,11 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload depends_on = load_template == self.LOAD_TEMPLATE_DEPENDS_ON cond_tmpl = None - if build_option('recursive_mod_unload') or recursive_unload or depends_on: + + if recursive_unload is None: + recursive_unload = build_option('recursive_mod_unload') or depends_on + + if recursive_unload: # wrapping the 'module load' statement with an 'is-loaded or mode == unload' # guard ensures recursive unloading while avoiding load storms; # when "module unload" is called on the module in which the @@ -1415,7 +1499,10 @@ def swap_module(self, mod_name_out, mod_name_in, guarded=True): :param mod_name_in: name of module to load (swap in) :param guarded: guard 'swap' statement, fall back to 'load' if module being swapped out is not loaded """ - body = 'swap("%s", "%s")' % (mod_name_out, mod_name_in) + body = '\n'.join([ + 'unload("%s")' % mod_name_out, + 'load("%s")' % mod_name_in, + ]) if guarded: alt_body = self.LOAD_TEMPLATE % {'mod_name': mod_name_in} swap_statement = [self.conditional_statement(self.is_loaded(mod_name_out), body, else_body=alt_body)] diff --git a/easybuild/tools/module_naming_scheme/__init__.py b/easybuild/tools/module_naming_scheme/__init__.py index 8a4b4975ab..c20c98fc8a 100644 --- a/easybuild/tools/module_naming_scheme/__init__.py +++ b/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/categorized_mns.py b/easybuild/tools/module_naming_scheme/categorized_mns.py index f81f9436a9..d11a7cd2bd 100644 --- a/easybuild/tools/module_naming_scheme/categorized_mns.py +++ b/easybuild/tools/module_naming_scheme/categorized_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/easybuild_mns.py b/easybuild/tools/module_naming_scheme/easybuild_mns.py index b2c6c577eb..7f6c243632 100644 --- a/easybuild/tools/module_naming_scheme/easybuild_mns.py +++ b/easybuild/tools/module_naming_scheme/easybuild_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 33a2e32eb7..5b5228c839 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -54,6 +54,8 @@ # required for use of iccifort toolchain 'icc,ifort': ('intel', '%(icc)s'), 'iccifort': ('intel', '%(iccifort)s'), + # required for use of intel-compilers toolchain (OneAPI compilers) + 'intel-compilers': ('intel', '%(intel-compilers)s'), # required for use of ClangGCC toolchain 'Clang,GCC': ('Clang-GCC', '%(Clang)s-%(GCC)s'), # required for use of gcccuda toolchain, and for CUDA installed with GCC toolchain @@ -68,6 +70,12 @@ 'xlc,xlf': ('xlcxlf', '%(xlc)s'), } +# possible prefixes for Cray toolchain names +# example: CrayGNU, CrayCCE, cpeGNU, cpeCCE, ...; +# important for determining $MODULEPATH extensions in det_modpath_extensions, +# cfr. https://github.com/easybuilders/easybuild-framework/issues/3575 +CRAY_TOOLCHAIN_NAME_PREFIXES = ('Cray', 'cpe') + class HierarchicalMNS(ModuleNamingScheme): """Class implementing an example hierarchical module naming scheme.""" @@ -108,26 +116,28 @@ def det_toolchain_compilers_name_version(self, tc_comps): if tc_comps is None: # no compiler in toolchain, system toolchain res = None - elif len(tc_comps) == 1: - tc_comp = tc_comps[0] - if tc_comp is None: - res = None - else: - res = (tc_comp['name'], self.det_full_version(tc_comp)) else: - comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps]) - comp_names = comp_versions.keys() - key = ','.join(sorted(comp_names)) - if key in COMP_NAME_VERSION_TEMPLATES: - tc_comp_name, tc_comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key] - tc_comp_ver = tc_comp_ver_tmpl % comp_versions - # make sure that icc/ifort versions match (unless not existing as separate modules) - if tc_comp_name == 'intel' and comp_versions.get('icc') != comp_versions.get('ifort'): - raise EasyBuildError("Bumped into different versions for Intel compilers: %s", comp_versions) + if len(tc_comps) > 0 and tc_comps[0]: + comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps]) + comp_names = comp_versions.keys() + key = ','.join(sorted(comp_names)) + if key in COMP_NAME_VERSION_TEMPLATES: + tc_comp_name, tc_comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key] + tc_comp_ver = tc_comp_ver_tmpl % comp_versions + # make sure that icc/ifort versions match (unless not existing as separate modules) + if tc_comp_name == 'intel' and comp_versions.get('icc') != comp_versions.get('ifort'): + raise EasyBuildError("Bumped into different versions for Intel compilers: %s", comp_versions) + res = (tc_comp_name, tc_comp_ver) + else: + if len(tc_comps) == 1: + tc_comp = tc_comps[0] + res = (tc_comp['name'], self.det_full_version(tc_comp)) + else: + raise EasyBuildError("Unknown set of toolchain compilers, module naming scheme needs work: %s", + comp_names) else: - raise EasyBuildError("Unknown set of toolchain compilers, module naming scheme needs work: %s", - comp_names) - res = (tc_comp_name, tc_comp_ver) + res = None + return res def det_module_subdir(self, ec): @@ -234,8 +244,9 @@ def det_modpath_extensions(self, ec): paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver)) # special case for Cray toolchains - elif modclass == MODULECLASS_TOOLCHAIN and tc_comp_info is None and ec.name.startswith('Cray'): - paths.append(os.path.join(TOOLCHAIN, ec.name, ec.version)) + elif modclass == MODULECLASS_TOOLCHAIN and tc_comp_info is None: + if any(ec.name.startswith(x) for x in CRAY_TOOLCHAIN_NAME_PREFIXES): + paths.append(os.path.join(TOOLCHAIN, ec.name, ec.version)) return paths diff --git a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py index e45c257537..99bb4c6d0a 100644 --- a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py +++ b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index 7465691e7f..e2d95677b3 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index 3948b41603..4f270f1252 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index 0fc1e16b32..0b01037f34 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index a2a7739222..1478f9299f 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -46,7 +46,7 @@ from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars -from easybuild.tools.filetools import convert_name, mkdir, path_matches, read_file, which +from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.py2vs3 import subprocess_popen_text from easybuild.tools.run import run_cmd @@ -181,13 +181,12 @@ def __init__(self, mod_paths=None, testing=False): self.set_mod_paths(mod_paths) if env_cmd_path: - cmd_path = which(self.cmd, log_ok=False, log_error=False) + cmd_path = which(self.cmd, log_ok=False, on_error=IGNORE) # only use command path in environment variable if command in not available in $PATH if cmd_path is None: self.cmd = env_cmd_path self.log.debug("Set %s command via environment variable %s: %s", self.NAME, self.COMMAND_ENVIRONMENT, self.cmd) - # check whether paths obtained via $PATH and $LMOD_CMD are different elif cmd_path != env_cmd_path: self.log.debug("Different paths found for %s command '%s' via which/$PATH and $%s: %s vs %s", self.NAME, self.COMMAND, self.COMMAND_ENVIRONMENT, cmd_path, env_cmd_path) @@ -208,6 +207,15 @@ def __init__(self, mod_paths=None, testing=False): self.set_and_check_version() self.supports_depends_on = False + def __str__(self): + """String representation of this ModulesTool instance.""" + res = self.NAME + if self.version: + res += ' ' + self.version + else: + res += ' (unknown version)' + return res + def buildstats(self): """Return tuple with data to be included in buildstats""" return (self.NAME, self.cmd, self.version) @@ -362,8 +370,12 @@ def use(self, path, priority=None): self.log.info("Ignoring specified priority '%s' when running 'module use %s' (Lmod-specific)", priority, path) - # make sure path exists before we add it - mkdir(path, parents=True) + if not path: + raise EasyBuildError("Cannot add empty path to $MODULEPATH") + if not os.path.exists(path): + self.log.deprecated("Path '%s' for module.use should exist" % path, '5.0') + # make sure path exists before we add it + mkdir(path, parents=True) self.run_module(['use', path]) def unuse(self, path): @@ -377,7 +389,8 @@ def add_module_path(self, path, set_mod_paths=True): :param path: path to add to $MODULEPATH via 'use' :param set_mod_paths: (re)set self.mod_paths """ - if path not in curr_module_paths(): + path = normalize_path(path) + if path not in curr_module_paths(normalize=True): # add module path via 'module use' and make sure self.mod_paths is synced self.use(path) if set_mod_paths: @@ -391,8 +404,14 @@ def remove_module_path(self, path, set_mod_paths=True): :param set_mod_paths: (re)set self.mod_paths """ # remove module path via 'module unuse' and make sure self.mod_paths is synced - if path in curr_module_paths(): - self.unuse(path) + path = normalize_path(path) + try: + # Unuse the path that is actually present in the environment + module_path = next(p for p in curr_module_paths() if normalize_path(p) == path) + except StopIteration: + pass + else: + self.unuse(module_path) if set_mod_paths: self.set_mod_paths() @@ -431,6 +450,7 @@ def check_module_path(self): eb_modpath = os.path.join(install_path(typ='modules'), build_option('suffix_modules_path')) # make sure EasyBuild module path is in 1st place + mkdir(eb_modpath, parents=True) self.prepend_module_path(eb_modpath) self.log.info("Prepended list of module paths with path used by EasyBuild: %s" % eb_modpath) @@ -665,7 +685,8 @@ def load(self, modules, mod_paths=None, purge=False, init_env=None, allow_reload # extend $MODULEPATH if needed for mod_path in mod_paths: full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path) - self.prepend_module_path(full_mod_path) + if os.path.exists(full_mod_path): + self.prepend_module_path(full_mod_path) loaded_modules = self.loaded_modules() for mod in modules: @@ -788,14 +809,17 @@ def run_module(self, *args, **kwargs): else: args = list(args) - self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) + self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) # restore selected original environment variables before running module command environ = os.environ.copy() for key in LD_ENV_VAR_KEYS: - environ[key] = ORIG_OS_ENVIRON.get(key, '') - self.log.debug("Changing %s from '%s' to '%s' in environment for module command", - key, os.environ.get(key, ''), environ[key]) + old_value = environ.get(key, '') + new_value = ORIG_OS_ENVIRON.get(key, '') + if old_value != new_value: + environ[key] = new_value + self.log.debug("Changing %s from '%s' to '%s' in environment for module command", + key, old_value, new_value) cmd_list = self.compose_cmd_list(args) full_cmd = ' '.join(cmd_list) @@ -842,11 +866,13 @@ def run_module(self, *args, **kwargs): # correct values of selected environment variables as yielded by the adjustments made # make sure we get the order right (reverse lists with [::-1]) for key in LD_ENV_VAR_KEYS: - curr_ld_val = os.environ.get(key, '').split(os.pathsep) + curr_ld_val = os.environ.get(key, '') + curr_ld_val = curr_ld_val.split(os.pathsep) if curr_ld_val else [] # Take care of empty/unset values new_ld_val = [x for x in nub(prev_ld_values[key] + curr_ld_val[::-1]) if x][::-1] - self.log.debug("Correcting paths in $%s from %s to %s" % (key, curr_ld_val, new_ld_val)) - self.set_path_env_var(key, new_ld_val) + if new_ld_val != curr_ld_val: + self.log.debug("Correcting paths in $%s from %s to %s" % (key, curr_ld_val, new_ld_val)) + self.set_path_env_var(key, new_ld_val) # Process stderr result = [] @@ -1160,7 +1186,7 @@ def update(self): class EnvironmentModulesC(ModulesTool): """Interface to (C) environment modules (modulecmd).""" - NAME = "Environment Modules v3" + NAME = "Environment Modules" COMMAND = "modulecmd" REQ_VERSION = '3.2.10' MAX_VERSION = '3.99' @@ -1281,16 +1307,23 @@ def remove_module_path(self, path, set_mod_paths=True): # remove module path via 'module use' and make sure self.mod_paths is synced # modulecmd.tcl keeps track of how often a path was added via 'module use', # so we need to check to make sure it's really removed - while path in curr_module_paths(): - self.unuse(path) + path = normalize_path(path) + while True: + try: + # Unuse the path that is actually present in the environment + module_path = next(p for p in curr_module_paths() if normalize_path(p) == path) + except StopIteration: + break + self.unuse(module_path) if set_mod_paths: self.set_mod_paths() class EnvironmentModules(EnvironmentModulesTcl): """Interface to environment modules 4.0+""" - NAME = "Environment Modules v4" + NAME = "Environment Modules" COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') + COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' @@ -1403,17 +1436,12 @@ def update(self): # don't actually update local cache when testing, just return the cache contents return stdout else: - try: - cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua') - self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd))) - cache_dir = os.path.dirname(cache_fp) - if not os.path.exists(cache_dir): - mkdir(cache_dir, parents=True) - cache_file = open(cache_fp, 'w') - cache_file.write(stdout) - cache_file.close() - except (IOError, OSError) as err: - raise EasyBuildError("Failed to update Lmod spider cache %s: %s", cache_fp, err) + cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua') + self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd))) + cache_dir = os.path.dirname(cache_fp) + if not os.path.exists(cache_dir): + mkdir(cache_dir, parents=True) + write_file(cache_fp, stdout) def use(self, path, priority=None): """ @@ -1422,13 +1450,47 @@ def use(self, path, priority=None): :param path: path to add to $MODULEPATH :param priority: priority for this path in $MODULEPATH (Lmod-specific) """ - # make sure path exists before we add it - mkdir(path, parents=True) + if not path: + raise EasyBuildError("Cannot add empty path to $MODULEPATH") + if not os.path.exists(path): + self.log.deprecated("Path '%s' for module.use should exist" % path, '5.0') + # make sure path exists before we add it + mkdir(path, parents=True) if priority: self.run_module(['use', '--priority', str(priority), path]) else: - self.run_module(['use', path]) + # LMod allows modifying MODULEPATH directly. So do that to avoid the costly module use + # unless priorities are in use already + if os.environ.get('__LMOD_Priority_MODULEPATH'): + self.run_module(['use', path]) + else: + path = normalize_path(path) + cur_mod_path = os.environ.get('MODULEPATH') + if cur_mod_path is None: + new_mod_path = path + else: + new_mod_path = [path] + [p for p in cur_mod_path.split(':') if normalize_path(p) != path] + new_mod_path = ':'.join(new_mod_path) + self.log.debug('Changing MODULEPATH from %s to %s' % + ('' if cur_mod_path is None else cur_mod_path, new_mod_path)) + os.environ['MODULEPATH'] = new_mod_path + + def unuse(self, path): + """Remove a module path""" + # We can simply remove the path from MODULEPATH to avoid the costly module call + cur_mod_path = os.environ.get('MODULEPATH') + if cur_mod_path is not None: + # Removing the last entry unsets the variable + if cur_mod_path == path: + self.log.debug('Changing MODULEPATH from %s to ' % cur_mod_path) + del os.environ['MODULEPATH'] + else: + path = normalize_path(path) + new_mod_path = ':'.join(p for p in cur_mod_path.split(':') if normalize_path(p) != path) + if new_mod_path != cur_mod_path: + self.log.debug('Changing MODULEPATH from %s to %s' % (cur_mod_path, new_mod_path)) + os.environ['MODULEPATH'] = new_mod_path def prepend_module_path(self, path, set_mod_paths=True, priority=None): """ @@ -1575,12 +1637,17 @@ def get_software_version(name): return version -def curr_module_paths(): +def curr_module_paths(normalize=False): """ Return a list of current module paths. + + :param normalize: Normalize the paths """ # avoid empty or nonexistent paths, which don't make any sense - return [p for p in os.environ.get('MODULEPATH', '').split(':') if p and os.path.exists(p)] + module_paths = (p for p in os.environ.get('MODULEPATH', '').split(':') if p and os.path.exists(p)) + if normalize: + module_paths = (normalize_path(p) for p in module_paths) + return list(module_paths) def mk_module_path(paths): diff --git a/easybuild/tools/multidiff.py b/easybuild/tools/multidiff.py index a8a9d30b6c..b1f7f526b9 100644 --- a/easybuild/tools/multidiff.py +++ b/easybuild/tools/multidiff.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a363cde325..dae1d35974 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -60,15 +60,17 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES -from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE -from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS +from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES +from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE, DEFAULT_JOB_BACKEND +from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, WARN +from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS +from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST @@ -97,7 +99,7 @@ from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family -from easybuild.tools.systemtools import get_cpu_features, get_system_info +from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_system_info from easybuild.tools.version import this_is_easybuild @@ -333,7 +335,9 @@ def override_options(self): descr = ("Override options", "Override default EasyBuild behavior.") opts = OrderedDict({ - 'accept-eula': ("Accept EULA for specified software", 'strlist', 'store', []), + 'accept-eula': ("Accept EULA for specified software [DEPRECATED, use --accept-eula-for instead!]", + 'strlist', 'store', []), + 'accept-eula-for': ("Accept EULA for specified software", 'strlist', 'store', []), 'add-dummy-to-minimal-toolchains': ("Include dummy toolchain in minimal toolchain searches " "[DEPRECATED, use --add-system-to-minimal-toolchains instead!)", None, 'store_true', False), @@ -347,6 +351,9 @@ def override_options(self): None, 'store_true', False), 'backup-modules': ("Back up an existing module file, if any. Only works when using --module-only", None, 'store_true', None), # default None to allow auto-enabling if not disabled + 'banned-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " + "which are not allowed to be linked in any installed binary/library", + 'strlist', 'extend', None), 'check-ebroot-env-vars': ("Action to take when defined $EBROOT* environment variables are found " "for which there is no matching loaded module; " "supported values: %s" % ', '.join(EBROOT_ENV_VAR_ACTIONS), None, 'store', WARN), @@ -357,6 +364,11 @@ def override_options(self): 'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False), 'containerize': ("Generate container recipe/image", None, 'store_true', False, 'C'), 'copy-ec': ("Copy specified easyconfig(s) to specified location", None, 'store_true', False), + 'cuda-cache-dir': ("Path to CUDA cache dir to use if enabled. Defaults to a path inside the build dir.", + str, 'store', None, {'metavar': "PATH"}), + 'cuda-cache-maxsize': ("Maximum size of the CUDA cache (in MiB) used for JIT compilation of PTX code. " + "Leave value empty to let EasyBuild choose a value or '0' to disable the cache", + int, 'store_or_None', None), 'cuda-compute-capabilities': ("List of CUDA compute capabilities to use when building GPU software; " "values should be specified as digits separated by a dot, " "for example: 3.5,5.0,7.2", 'strlist', 'extend', None), @@ -374,6 +386,8 @@ def override_options(self): None, 'store', None, 'e', {'metavar': 'CLASS'}), 'enforce-checksums': ("Enforce availability of checksums for all sources/patches, so they can be verified", None, 'store_true', False), + 'env-for-shebang': ("Define the env command to use when fixing shebangs", None, 'store', + DEFAULT_ENV_FOR_SHEBANG), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", @@ -383,6 +397,9 @@ def override_options(self): 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", 'strlist', 'extend', None), + 'filter-ecs': ("List of easyconfigs (given as glob patterns) to *ignore* when given on command line " + "or auto-selected when building with --from-pr. (e.g. --filter-ecs=*intel*)", + 'strlist', 'extend', None), 'filter-env-vars': ("List of names of environment variables that should *not* be defined/updated by " "module files generated by EasyBuild", 'strlist', 'extend', None), 'fixed-installdir-naming-scheme': ("Use fixed naming scheme for installation directories", None, @@ -390,6 +407,8 @@ def override_options(self): 'force-download': ("Force re-downloading of sources and/or patches, " "even if they are available already in source path", 'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES), + 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", + None, 'store_true', True), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), 'group-writable-installdir': ("Enable group write permissions on installation directory after installation", None, 'store_true', False), @@ -399,9 +418,19 @@ def override_options(self): "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), 'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, " "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None), + 'http-header-fields-urlpat': ("Set extra HTTP header FIELDs when downloading files from URL PATterns. " + "To not log sensitive values, specify a file containing newline separated " + "FIELDs. e.g. \"^https://www.example.com::/path/to/headers.txt\" or " + "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\".", + None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}), 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), + 'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), + 'insecure-download': ("Don't check the server certificate against the available certificate authorities.", + None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), + 'lib-lib64-symlink': ("Automatically create symlinks for lib/ pointing to lib64/ if the former is missing", + None, 'store_true', True), 'lib64-fallback-sanity-check': ("Fallback in sanity check to lib64/ equivalent for missing libraries", None, 'store_true', True), 'lib64-lib-symlink': ("Automatically create symlinks for lib64/ pointing to lib/ if the former is missing", @@ -422,8 +451,13 @@ def override_options(self): 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_TXT, FORMAT_RST]), + 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " + "with fallback to basic colored output", + 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), 'parallel': ("Specify (maximum) level of parallellism used during build procedure", 'int', 'store', None), + 'parallel-extensions-install': ("Install list of extensions in parallel (if supported)", + None, 'store_true', False), 'pre-create-installdir': ("Create installation directory before submitting build jobs", None, 'store_true', True), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), @@ -433,16 +467,23 @@ def override_options(self): 'remove-ghost-install-dirs': ("Remove ghost installation directories when --force or --rebuild is used, " "rather than just warning about them", None, 'store_true', False), + 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " + "which must be linked in all installed binaries/libraries", + 'strlist', 'extend', None), 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', False), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), + 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", + None, 'store', None), + 'sanity-check-only': ("Only run sanity check (module is expected to be installed already", + None, 'store_true', False), 'set-default-module': ("Set the generated module as default", None, 'store_true', False), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), + 'show-progress-bar': ("Show progress bar in terminal output", None, 'store_true', True), 'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None), - 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), + 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), - 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", - None, 'store_true', True), + 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include", None, 'store', None), 'trace': ("Provide more information in output to stdout on progress", None, 'store_true', False, 'T'), @@ -450,6 +491,7 @@ def override_options(self): None, 'store', None), 'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file", None, 'store_true', False), + 'unit-testing-mode': ("Run in unit test mode", None, 'store_true', False), 'use-ccache': ("Enable use of ccache to speed up compilation, with specified cache dir", str, 'store', False, {'metavar': "PATH"}), 'use-f90cache': ("Enable use of f90cache to speed up compilation, with specified cache dir", @@ -489,6 +531,9 @@ def config_options(self): 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), 'containerpath': ("Location where container recipe & image will be stored", None, 'store', mk_full_default_path('containerpath')), + 'envvars-user-modules': ("List of environment variables that hold the base paths for which user-specific " + "modules will be installed relative to", 'strlist', 'store', + [DEFAULT_ENVVAR_USERS_MODULES]), 'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata " "for external modules (INI format)", 'strlist', 'store', None), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), @@ -547,7 +592,8 @@ def config_options(self): 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), - 'subdir-user-modules': ("Base path of user-specific modules relative to their $HOME", None, 'store', None), + 'subdir-user-modules': ("Base path of user-specific modules relative to --envvars-user-modules", + None, 'store', None), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage @@ -580,6 +626,8 @@ def informative_options(self): 'avail-hooks': ("Show list of known hooks", None, 'store_true', False), 'avail-toolchain-opts': ("Show options for toolchain", 'str', 'store', None), 'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False), + 'check-eb-deps': ("Check presence and version of (required and optional) EasyBuild dependencies", + None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.'}), 'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies", None, 'store_true', False), @@ -619,13 +667,14 @@ def github_options(self): descr = ("GitHub integration options", "Integration with GitHub") opts = OrderedDict({ + 'add-pr-labels': ("Try to add labels to PR based on files changed", int, 'store', None, {'metavar': 'PR#'}), 'check-github': ("Check status of GitHub integration, and report back", None, 'store_true', False), 'check-contrib': ("Runs checks to see whether the given easyconfigs are ready to be contributed back", None, 'store_true', False), 'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False), 'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), - 'from-pr': ("Obtain easyconfigs from specified PR", int, 'store', None, {'metavar': 'PR#'}), + 'from-pr': ("Obtain easyconfigs from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None), 'github-user': ("GitHub username", str, 'store', None), 'github-org': ("GitHub organization", str, 'store', None), @@ -657,6 +706,9 @@ def github_options(self): 'sync-pr-with-develop': ("Sync pull request with current 'develop' branch", int, 'store', None, {'metavar': 'PR#'}), 'review-pr': ("Review specified pull request", int, 'store', None, {'metavar': 'PR#'}), + 'review-pr-filter': ("Regex used to filter out easyconfigs to diff against in --review-pr", + None, 'regex', None), + 'review-pr-max': ("Maximum number of easyconfigs to diff against in --review-pr", int, 'store', None), 'test-report-env-filter': ("Regex used to filter out variables in environment dump of test report", None, 'regex', None), 'update-branch-github': ("Update specified branch in GitHub", str, 'store', None), @@ -746,6 +798,7 @@ def job_options(self): 'cores': ("Number of cores to request per job", 'int', 'store', None), 'deps-type': ("Type of dependency to set between jobs (default depends on job backend)", 'choice', 'store', None, [JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN]), + 'eb-cmd': ("EasyBuild command to use in jobs", 'str', 'store', DEFAULT_JOB_EB_CMD), 'max-jobs': ("Maximum number of concurrent jobs (queued and running, 0 = unlimited)", 'int', 'store', 0), 'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24), 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()), @@ -844,19 +897,23 @@ def postprocess(self): # set tmpdir self.tmpdir = set_tmpdir(self.options.tmpdir) + # early check for opt-in to installing extensions in parallel (experimental feature) + if self.options.parallel_extensions_install: + self.log.experimental("installing extensions in parallel") + # take --include options into account (unless instructed otherwise) if self.with_include: self._postprocess_include() # prepare for --list/--avail - if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, + if any((self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, self.options.show_default_configfiles, self.options.avail_toolchain_opts, self.options.avail_hooks, self.options.show_system_info, - ]): + )): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() @@ -1009,8 +1066,50 @@ def _postprocess_checks(self): self.log.info("Checks on configuration options passed") + def get_cfg_opt_abs_path(self, opt_name, path): + """Get path value of configuration option as absolute path.""" + if os.path.isabs(path) or path.startswith('git@'): + abs_path = path + else: + abs_path = os.path.abspath(path) + self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s", + opt_name, abs_path) + return abs_path + + def _ensure_abs_path(self, opt_name): + """Ensure that path value for specified configuration option is an absolute path.""" + + opt_val = getattr(self.options, opt_name) + if opt_val: + if isinstance(opt_val, string_type): + setattr(self.options, opt_name, self.get_cfg_opt_abs_path(opt_name, opt_val)) + elif isinstance(opt_val, list): + abs_paths = [self.get_cfg_opt_abs_path(opt_name, p) for p in opt_val] + setattr(self.options, opt_name, abs_paths) + else: + error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)" + raise EasyBuildError(error_msg, opt_name, type(opt_val)) + def _postprocess_config(self): """Postprocessing of configuration options""" + + # resolve relative paths for configuration options that specify a location, + # to avoid incorrect paths being used when EasyBuild changes the current working directory + # (see https://github.com/easybuilders/easybuild-framework/issues/3619); + # ensuring absolute paths for 'robot' is handled separately below, + # because we need to be careful with the argument pass to --robot; + # note: repositorypath is purposely not listed here, because it's a special case: + # - the value could consist of a 2-tuple (, ); + # - the could also specify the location of a *remote* (Git( repository, + # which can be done in variety of formats (git@:/), https://, etc.) + # (see also https://github.com/easybuilders/easybuild-framework/issues/3892); + path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', + 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', + 'robot_paths', 'sourcepath'] + + for opt_name in path_opt_names: + self._ensure_abs_path(opt_name) + if self.options.prefix is not None: # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account # in the legacy-style configuration, repository is initialised in configuration file itself @@ -1053,7 +1152,7 @@ def _postprocess_config(self): # paths specified to --robot have preference over --robot-paths # keep both values in sync if robot is enabled, which implies enabling dependency resolver - self.options.robot_paths = [os.path.abspath(path) for path in self.options.robot + self.options.robot_paths] + self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot] + self.options.robot_paths self.options.robot = self.options.robot_paths # Update the search_paths (if any) to absolute paths @@ -1204,6 +1303,7 @@ def show_system_info(self): """Show system information.""" system_info = get_system_info() cpu_features = get_cpu_features() + gpu_info = get_gpu_info() cpu_arch_name = system_info['cpu_arch_name'] lines = [ "System information (%s):" % system_info['hostname'], @@ -1229,6 +1329,19 @@ def show_system_info(self): " -> speed: %s" % system_info['cpu_speed'], " -> cores: %s" % system_info['core_count'], " -> features: %s" % ','.join(cpu_features), + ]) + + if gpu_info: + lines.extend([ + '', + "* GPU:", + ]) + for vendor in gpu_info: + lines.append(" -> %s" % vendor) + for gpu, num in gpu_info[vendor].items(): + lines.append(" -> %sx %s" % (num, gpu)) + + lines.extend([ '', "* software:", " -> glibc version: %s" % system_info['glibc_version'], @@ -1358,7 +1471,7 @@ def parse_options(args=None, with_include=True): eb_go = EasyBuildOptions(usage=usage, description=description, prog='eb', envvar_prefix=CONFIG_ENV_VAR_PREFIX, go_args=eb_args, error_env_options=True, error_env_option_method=raise_easybuilderror, with_include=with_include) - except Exception as err: + except EasyBuildError as err: raise EasyBuildError("Failed to parse configuration options: %s" % err) return eb_go @@ -1429,13 +1542,25 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) + # map list of strings --from-pr value to list of integers + try: + from_prs = [int(x) for x in eb_go.options.from_pr] + except ValueError: + raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + + try: + review_pr = (lambda x: int(x) if x else None)(eb_go.options.review_pr) + except ValueError: + raise EasyBuildError("Argument to --review-pr must be an integer PR #.") + # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs - tweaked_ecs_paths, pr_path = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) + tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs, + review_pr=review_pr) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query - robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot) - log.debug("Full robot path: %s" % robot_path) + robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_paths, auto_robot=auto_robot) + log.debug("Full robot path: %s", robot_path) if not robot_path: print_warning("Robot search path is empty!") @@ -1448,7 +1573,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): 'build_specs': build_specs, 'command_line': eb_cmd_line, 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata), - 'pr_path': pr_path, + 'pr_paths': pr_paths, 'robot_path': robot_path, 'silent': testing or new_update_opt, 'try_to_generate': try_to_generate, @@ -1461,7 +1586,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): # done here instead of in _postprocess_include because github integration requires build_options to be initialized if eb_go.options.include_easyblocks_from_pr: try: - easyblock_prs = map(int, eb_go.options.include_easyblocks_from_pr) + easyblock_prs = [int(x) for x in eb_go.options.include_easyblocks_from_pr] except ValueError: raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") @@ -1504,7 +1629,8 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): sys.path.remove(fake_vsc_path) sys.path.insert(0, new_fake_vsc_path) - return eb_go, (build_specs, log, logfile, robot_path, search_query, tmpdir, try_to_generate, tweaked_ecs_paths) + return eb_go, (build_specs, log, logfile, robot_path, search_query, tmpdir, try_to_generate, + from_prs, tweaked_ecs_paths) def process_software_build_specs(options): @@ -1715,24 +1841,38 @@ def set_tmpdir(tmpdir=None, raise_error=False): # reset to make sure tempfile picks up new temporary directory to use tempfile.tempdir = None - # test if temporary directory allows to execute files, warn if it doesn't - try: - fd, tmptest_file = tempfile.mkstemp() - os.close(fd) - os.chmod(tmptest_file, 0o700) - if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, - stream_output=False): - msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() - msg += "This can cause problems in the build process, consider using --tmpdir." - if raise_error: - raise EasyBuildError(msg) + # cache for checked paths, via function attribute + executable_tmp_paths = getattr(set_tmpdir, 'executable_tmp_paths', []) + + # Skip the executable check if it already succeeded for any parent folder + # Especially important for the unit test suite, less so for actual execution + if not any(current_tmpdir.startswith(path) for path in executable_tmp_paths): + + # test if temporary directory allows to execute files, warn if it doesn't + try: + fd, tmptest_file = tempfile.mkstemp() + os.close(fd) + os.chmod(tmptest_file, 0o700) + if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, + stream_output=False): + msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() + msg += "This can cause problems in the build process, consider using --tmpdir." + if raise_error: + raise EasyBuildError(msg) + else: + _log.warning(msg) else: - _log.warning(msg) - else: - _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) - os.remove(tmptest_file) + _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) - except OSError as err: - raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) + # Put this folder into the cache + executable_tmp_paths.append(current_tmpdir) + + # set function attribute so we can retrieve cache later + set_tmpdir.executable_tmp_paths = executable_tmp_paths + + os.remove(tmptest_file) + + except OSError as err: + raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) return current_tmpdir diff --git a/easybuild/tools/ordereddict.py b/easybuild/tools/ordereddict.py deleted file mode 100644 index 34b2a771fa..0000000000 --- a/easybuild/tools/ordereddict.py +++ /dev/null @@ -1,276 +0,0 @@ -# http://code.activestate.com/recipes/576693/ (r9) -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. - -# Copyright (C) 2009 Raymond Hettinger - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -""" -Module provided an ordered dictionary class. - -:author: Raymond Hettinger -""" -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(self, *args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 1: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (1 + len(args),)) - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 1: - other = args[0] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self) == len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py new file mode 100644 index 0000000000..9968edf19f --- /dev/null +++ b/easybuild/tools/output.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +# # +# Copyright 2021-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Tools for controlling output to terminal produced by EasyBuild. + +:author: Kenneth Hoste (Ghent University) +:author: Jørgen Nordmoen (University of Oslo) +""" +import functools + +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style +from easybuild.tools.py2vs3 import OrderedDict + +try: + from rich.console import Console, Group + from rich.live import Live + from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn + from rich.progress import DownloadColumn, FileSizeColumn, TransferSpeedColumn, TimeRemainingColumn + from rich.table import Column, Table +except ImportError: + pass + + +COLOR_GREEN = 'green' +COLOR_RED = 'red' +COLOR_YELLOW = 'yellow' + +# map known colors to ANSII color codes +KNOWN_COLORS = { + COLOR_GREEN: '\033[0;32m', + COLOR_RED: '\033[0;31m', + COLOR_YELLOW: '\033[1;33m', +} +COLOR_END = '\033[0m' + +PROGRESS_BAR_DOWNLOAD_ALL = 'download_all' +PROGRESS_BAR_DOWNLOAD_ONE = 'download_one' +PROGRESS_BAR_EXTENSIONS = 'extensions' +PROGRESS_BAR_EASYCONFIG = 'easyconfig' +STATUS_BAR = 'status' + +_progress_bar_cache = {} + + +def colorize(txt, color): + """ + Colorize given text, with specified color. + """ + if color in KNOWN_COLORS: + if use_rich(): + coltxt = '[bold %s]%s[/bold %s]' % (color, txt, color) + else: + coltxt = KNOWN_COLORS[color] + txt + COLOR_END + else: + raise EasyBuildError("Unknown color: %s", color) + + return coltxt + + +class DummyRich(object): + """ + Dummy shim for Rich classes. + Used in case Rich is not available, or when EasyBuild is not configured to use rich output style. + """ + + # __enter__ and __exit__ must be implemented to allow use as context manager + def __enter__(self, *args, **kwargs): + pass + + def __exit__(self, *args, **kwargs): + pass + + # dummy implementations for methods supported by rich.progress.Progress class + def add_task(self, *args, **kwargs): + pass + + def stop_task(self, *args, **kwargs): + pass + + def update(self, *args, **kwargs): + pass + + # internal Rich methods + def __rich_console__(self, *args, **kwargs): + pass + + +def use_rich(): + """ + Return whether or not to use Rich to produce rich output. + """ + return get_output_style() == OUTPUT_STYLE_RICH + + +def show_progress_bars(): + """ + Return whether or not to show progress bars. + """ + return use_rich() and build_option('show_progress_bar') and not build_option('extended_dry_run') + + +def rich_live_cm(): + """ + Return Live instance to use as context manager. + """ + if show_progress_bars(): + pbar_group = Group( + download_one_progress_bar(), + download_one_progress_bar_unknown_size(), + download_all_progress_bar(), + extensions_progress_bar(), + easyconfig_progress_bar(), + status_bar(), + ) + live = Live(pbar_group) + else: + live = DummyRich() + + return live + + +def progress_bar_cache(func): + """ + Function decorator to cache created progress bars for easy retrieval. + """ + @functools.wraps(func) + def new_func(ignore_cache=False): + if hasattr(func, 'cached') and not ignore_cache: + progress_bar = func.cached + elif use_rich() and build_option('show_progress_bar'): + progress_bar = func() + else: + progress_bar = DummyRich() + + func.cached = progress_bar + return func.cached + + return new_func + + +@progress_bar_cache +def status_bar(): + """ + Get progress bar to display overall progress. + """ + progress_bar = Progress( + TimeElapsedColumn(Column(min_width=7, no_wrap=True)), + TextColumn("{task.completed} out of {task.total} easyconfigs done{task.description}"), + ) + + return progress_bar + + +@progress_bar_cache +def easyconfig_progress_bar(): + """ + Get progress bar to display progress for installing a single easyconfig file. + """ + progress_bar = Progress( + SpinnerColumn('point', speed=0.2), + TextColumn("[bold green]{task.description} ({task.completed} out of {task.total} steps done)"), + BarColumn(), + TimeElapsedColumn(), + refresh_per_second=1, + ) + + return progress_bar + + +@progress_bar_cache +def download_all_progress_bar(): + """ + Get progress bar to show progress on downloading of all source files. + """ + progress_bar = Progress( + TextColumn("[bold blue]Fetching files: {task.percentage:>3.0f}% ({task.completed}/{task.total})"), + BarColumn(), + TimeElapsedColumn(), + TextColumn("({task.description})"), + ) + + return progress_bar + + +@progress_bar_cache +def download_one_progress_bar(): + """ + Get progress bar to show progress for downloading a file of known size. + """ + progress_bar = Progress( + TextColumn('[bold yellow]Downloading {task.description}'), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TimeRemainingColumn(), + ) + + return progress_bar + + +@progress_bar_cache +def download_one_progress_bar_unknown_size(): + """ + Get progress bar to show progress for downloading a file of unknown size. + """ + progress_bar = Progress( + TextColumn('[bold yellow]Downloading {task.description}'), + FileSizeColumn(), + TransferSpeedColumn(), + ) + + return progress_bar + + +@progress_bar_cache +def extensions_progress_bar(): + """ + Get progress bar to show progress for installing extensions. + """ + progress_bar = Progress( + TextColumn("[bold blue]{task.description}"), + BarColumn(), + TimeElapsedColumn(), + ) + + return progress_bar + + +def get_progress_bar(bar_type, ignore_cache=False, size=None): + """ + Get progress bar of given type. + """ + + if bar_type == PROGRESS_BAR_DOWNLOAD_ONE and not size: + pbar = download_one_progress_bar_unknown_size(ignore_cache=ignore_cache) + elif bar_type in PROGRESS_BAR_TYPES: + pbar = PROGRESS_BAR_TYPES[bar_type](ignore_cache=ignore_cache) + else: + raise EasyBuildError("Unknown progress bar type: %s", bar_type) + + return pbar + + +def start_progress_bar(bar_type, size, label=None): + """ + Start progress bar of given type. + + :param label: label for progress bar + :param size: total target size of progress bar + """ + pbar = get_progress_bar(bar_type, size=size) + task_id = pbar.add_task('') + _progress_bar_cache[bar_type] = (pbar, task_id) + + # don't bother showing progress bar if there's only 1 item to make progress on + if size == 1: + pbar.update(task_id, visible=False) + elif size: + pbar.update(task_id, total=size) + + if label: + pbar.update(task_id, description=label) + + +def update_progress_bar(bar_type, label=None, progress_size=1, total=None): + """ + Update progress bar of given type (if it was started), add progress of given size. + + :param bar_type: type of progress bar + :param label: label for progress bar + :param progress_size: amount of progress made + """ + if bar_type in _progress_bar_cache: + (pbar, task_id) = _progress_bar_cache[bar_type] + if label: + pbar.update(task_id, description=label) + if progress_size: + pbar.update(task_id, advance=progress_size) + if total: + pbar.update(task_id, total=total) + + +def stop_progress_bar(bar_type, visible=False): + """ + Stop progress bar of given type. + """ + if bar_type in _progress_bar_cache: + (pbar, task_id) = _progress_bar_cache[bar_type] + pbar.stop_task(task_id) + if not visible: + pbar.update(task_id, visible=False) + else: + raise EasyBuildError("Failed to stop %s progress bar, since it was never started?!", bar_type) + + +def print_checks(checks_data): + """Print overview of checks that were made.""" + + col_titles = checks_data.pop('col_titles', ('name', 'info', 'description')) + + col2_label = col_titles[1] + + if use_rich(): + console = Console() + # don't use console.print, which causes SyntaxError in Python 2 + console_print = getattr(console, 'print') # noqa: B009 + console_print('') + + for section in checks_data: + section_checks = checks_data[section] + + if use_rich(): + table = Table(title=section) + table.add_column(col_titles[0]) + table.add_column(col_titles[1]) + # only add 3rd column if there's any information to include in it + if any(x[1] for x in section_checks.values()): + table.add_column(col_titles[2]) + else: + lines = [ + '', + section + ':', + '-' * (len(section) + 1), + '', + ] + + if isinstance(section_checks, OrderedDict): + check_names = section_checks.keys() + else: + check_names = sorted(section_checks, key=lambda x: x.lower()) + + if use_rich(): + for check_name in check_names: + (info, descr) = checks_data[section][check_name] + if info is None: + info = ':yellow_circle: [yellow]%s?!' % col2_label + elif info is False: + info = ':cross_mark: [red]not found' + else: + info = ':white_heavy_check_mark: [green]%s' % info + if descr: + table.add_row(check_name.rstrip(':'), info, descr) + else: + table.add_row(check_name.rstrip(':'), info) + else: + for check_name in check_names: + (info, descr) = checks_data[section][check_name] + if info is None: + info = '(found, UNKNOWN %s)' % col2_label + elif info is False: + info = '(NOT FOUND)' + line = "* %s %s" % (check_name, info) + if descr: + line = line.ljust(40) + '[%s]' % descr + lines.append(line) + lines.append('') + + if use_rich(): + console_print(table) + else: + print('\n'.join(lines)) + + +# this constant must be defined at the end, since functions used as values need to be defined +PROGRESS_BAR_TYPES = { + PROGRESS_BAR_DOWNLOAD_ALL: download_all_progress_bar, + PROGRESS_BAR_DOWNLOAD_ONE: download_one_progress_bar, + PROGRESS_BAR_EXTENSIONS: extensions_progress_bar, + PROGRESS_BAR_EASYCONFIG: easyconfig_progress_bar, + STATUS_BAR: status_bar, +} diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py index 7116c002a4..670941b804 100644 --- a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py +++ b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/pns.py b/easybuild/tools/package/package_naming_scheme/pns.py index 8556abb019..bb00f550fc 100644 --- a/easybuild/tools/package/package_naming_scheme/pns.py +++ b/easybuild/tools/package/package_naming_scheme/pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 3fcd5d0ed1..97e83592b4 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index df151aa3a1..6f5b5ee0cb 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -127,7 +127,7 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True): curdir = os.getcwd() # regex pattern for options to ignore (help options can't reach here) - ignore_opts = re.compile('^--robot$|^--job|^--try-.*$') + ignore_opts = re.compile('^--robot$|^--job|^--try-.*$|^--easystack$') # generate_cmd_line returns the options in form --longopt=value opts = [o for o in cmd_line_opts if not ignore_opts.match(o.split('=')[0])] @@ -141,8 +141,14 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True): # compose string with command line options, properly quoted and with '%' characters escaped opts_str = ' '.join(opts).replace('%', '%%') - command = "unset TMPDIR && cd %s && eb %%(spec)s %s %%(add_opts)s --testoutput=%%(output_dir)s" % (curdir, opts_str) - _log.info("Command template for jobs: %s" % command) + eb_cmd = build_option('job_eb_cmd') + + command = ' && '.join([ + "unset TMPDIR", + "cd %s" % curdir, + "%s %%(spec)s %s %%(add_opts)s --testoutput=%%(output_dir)s" % (eb_cmd, opts_str), + ]) + _log.info("Command template for jobs: %s", command) if testing: _log.debug("Skipping actual submission of jobs since testing mode is enabled") return command diff --git a/easybuild/tools/py2vs3/__init__.py b/easybuild/tools/py2vs3/__init__.py index 2c6e58e8bf..dd3db6dfff 100644 --- a/easybuild/tools/py2vs3/__init__.py +++ b/easybuild/tools/py2vs3/__init__.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index 00c7382316..4177c636b4 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ import json import subprocess import urllib2 as std_urllib # noqa +from collections import Mapping, OrderedDict # noqa from HTMLParser import HTMLParser # noqa from string import letters as ascii_letters # noqa from string import lowercase as ascii_lowercase # noqa @@ -41,12 +42,6 @@ from urllib import urlencode # noqa from urllib2 import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa -try: - # Python 2.7 - from collections import OrderedDict # noqa -except ImportError: - # only needed to keep supporting Python 2.6 - from easybuild.tools.ordereddict import OrderedDict # noqa # reload function (built-in in Python 2) reload = reload diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index ba5f46043f..307642fdec 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2021 Ghent University +# Copyright 2019-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,6 +36,7 @@ import sys import urllib.request as std_urllib # noqa from collections import OrderedDict # noqa +from collections.abc import Mapping # noqa from distutils.version import LooseVersion from functools import cmp_to_key from html.parser import HTMLParser # noqa diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py index 2870802d84..27d0ae9ebe 100644 --- a/easybuild/tools/repository/filerepo.py +++ b/easybuild/tools/repository/filerepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py index 576afebd48..863ebf4534 100644 --- a/easybuild/tools/repository/gitrepo.py +++ b/easybuild/tools/repository/gitrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/hgrepo.py b/easybuild/tools/repository/hgrepo.py index 61c09f163a..30bebd04c1 100644 --- a/easybuild/tools/repository/hgrepo.py +++ b/easybuild/tools/repository/hgrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index 84d8c89abc..b67b4b2cc2 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index 8902843184..ca2b9a26d8 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 896a213b22..510cf8c23f 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -52,10 +52,10 @@ _log = fancylogger.getLogger('tools.robot', fname=False) -def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_path, auto_robot=False): +def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_paths, auto_robot=False): """Determine robot path.""" robot_path = robot_paths_option[:] - _log.info("Using robot path(s): %s" % robot_path) + _log.info("Using robot path(s): %s", robot_path) tweaked_ecs_path, tweaked_ecs_deps_path = None, None # paths to tweaked easyconfigs or easyconfigs downloaded from a PR have priority @@ -67,9 +67,10 @@ def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_path, auto_robot=Fa robot_path.append(tweaked_ecs_deps_path) _log.info("Prepended list of robot search paths with %s and appended with %s: %s", tweaked_ecs_path, tweaked_ecs_deps_path, robot_path) - if pr_path is not None: - robot_path.append(pr_path) - _log.info("Appended list of robot search paths with %s: %s" % (pr_path, robot_path)) + + if pr_paths is not None: + robot_path.extend(pr_paths) + _log.info("Extended list of robot search paths with %s: %s", pr_paths, robot_path) return robot_path diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 67977156b0..519d161747 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -107,7 +107,10 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): """ if asynchronous: - output = asyncprocess.recv_some(proc) + # e=False is set to avoid raising an exception when command has completed; + # that's needed to ensure we get all output, + # see https://github.com/easybuilders/easybuild-framework/issues/3593 + output = asyncprocess.recv_some(proc, e=False) elif read_size: output = proc.stdout.read(read_size) else: @@ -125,7 +128,7 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): @run_cmd_cache def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, - force_in_dry_run=False, verbose=True, shell=True, trace=True, stream_output=None): + force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False): """ Run specified command (in a subshell) :param cmd: command to run @@ -138,9 +141,10 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True :param path: path to execute the command in; current working directory is used if unspecified :param force_in_dry_run: force running the command during dry run :param verbose: include message on running the command in dry run output - :param shell: allow commands to not run in a shell (especially useful for cmd lists) + :param shell: allow commands to not run in a shell (especially useful for cmd lists), defaults to True :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout + :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) """ cwd = os.getcwd() @@ -151,6 +155,13 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True else: raise EasyBuildError("Unknown command type ('%s'): %s", type(cmd), cmd) + if shell is None: + shell = True + if isinstance(cmd, list): + raise EasyBuildError("When passing cmd as a list then `shell` must be set explictely! " + "Note that all elements of the list but the first are treated as arguments " + "to the shell and NOT to the command to be executed!") + if log_output or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd-') @@ -225,10 +236,76 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True stdin=subprocess.PIPE, close_fds=True, executable=exec_cmd) except OSError as err: raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) + if inp: proc.stdin.write(inp.encode()) proc.stdin.close() + if asynchronous: + return (proc, cmd, cwd, start_time, cmd_log) + else: + return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple, + regexp=regexp, stream_output=stream_output, trace=trace) + + +def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''): + """ + Check status of command that was started asynchronously. + + :param proc: subprocess.Popen instance representing asynchronous command + :param cmd: command being run + :param owd: original working directory + :param start_time: start time of command (datetime instance) + :param cmd_log: log file to print command output to + :param fail_on_error: raise EasyBuildError when command exited with an error + :param output_read_size: number of bytes to read from output + :param output: already collected output for this command + + :result: dict value with result of the check (boolean 'done', 'exit_code', 'output') + """ + # use small read size, to avoid waiting for a long time until sufficient output is produced + if output_read_size: + if not isinstance(output_read_size, int) or output_read_size < 0: + raise EasyBuildError("Number of output bytes to read should be a positive integer value (or zero)") + add_out = get_output_from_process(proc, read_size=output_read_size) + _log.debug("Additional output from asynchronous command '%s': %s" % (cmd, add_out)) + output += add_out + + exit_code = proc.poll() + if exit_code is None: + _log.debug("Asynchronous command '%s' still running..." % cmd) + done = False + else: + _log.debug("Asynchronous command '%s' completed!", cmd) + output, _ = complete_cmd(proc, cmd, owd, start_time, cmd_log, output=output, + simple=False, trace=False, log_ok=fail_on_error) + done = True + + res = { + 'done': done, + 'exit_code': exit_code, + 'output': output, + } + return res + + +def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, + regexp=True, stream_output=None, trace=True, output=''): + """ + Complete running of command represented by passed subprocess.Popen instance. + + :param proc: subprocess.Popen instance representing running command + :param cmd: command being run + :param owd: original working directory + :param start_time: start time of command (datetime instance) + :param cmd_log: log file to print command output to + :param log_ok: only run output/exit code for failing commands (exit code non-zero) + :param log_all: always log command output and exit code + :param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) + :param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) + :param stream_output: enable streaming command output to stdout + :param trace: print command being executed as part of trace output + """ # use small read size when streaming output, to make it stream more fluently # read size should not be too small though, to avoid too much overhead if stream_output: @@ -236,8 +313,9 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True else: read_size = 1024 * 8 + stdouterr = output + ec = proc.poll() - stdouterr = '' while ec is None: # need to read from time to time. # - otherwise the stdout/stderr buffer gets filled and it all stops working @@ -251,6 +329,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True # read remaining data (all of it) output = get_output_from_process(proc) + proc.stdout.close() if cmd_log: cmd_log.write(output) cmd_log.close() @@ -262,9 +341,9 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) try: - os.chdir(cwd) + os.chdir(owd) except OSError as err: - raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) + raise EasyBuildError("Failed to return to %s after executing command: %s", owd, err) return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp) @@ -287,6 +366,11 @@ def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, re """ cwd = os.getcwd() + if not isinstance(cmd, string_type) and len(cmd) > 1: + # We use shell=True and hence we should really pass the command as a string + # When using a list then every element past the first is passed to the shell itself, not the command! + raise EasyBuildError("The command passed must be a string!") + if log_all or (trace and build_option('trace')): # collect output of running command in temporary log file, if desired fd, cmd_log_fn = tempfile.mkstemp(suffix='.log', prefix='easybuild-run_cmd_qa-') @@ -411,6 +495,7 @@ def check_answers_list(answers): # - otherwise the stdout/stderr buffer gets filled and it all stops working try: out = get_output_from_process(proc, asynchronous=True) + if cmd_log: cmd_log.write(out) stdout_err += out diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index d58cfc0abf..f9e319779f 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,7 @@ @auther: Ward Poelmans (Ghent University) """ import ctypes +import errno import fcntl import grp # @UnresolvedImport import os @@ -41,11 +42,24 @@ from ctypes.util import find_library from socket import gethostname +# pkg_resources is provided by the setuptools Python package, +# which we really want to keep as an *optional* dependency +try: + import pkg_resources + HAVE_PKG_RESOURCES = True +except ImportError: + HAVE_PKG_RESOURCES = False + +try: + # only needed on macOS, may not be available on Linux + import ctypes.macholib.dyld +except ImportError: + pass + from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import build_option from easybuild.tools.filetools import is_readable, read_file, which -from easybuild.tools.py2vs3 import string_type +from easybuild.tools.py2vs3 import OrderedDict, string_type from easybuild.tools.run import run_cmd @@ -72,10 +86,18 @@ AARCH64 = 'AArch64' POWER = 'POWER' X86_64 = 'x86_64' +RISCV32 = 'RISC-V-32' +RISCV64 = 'RISC-V-64' + +# known values for ARCH constant (determined by _get_arch_constant in easybuild.framework.easyconfig.constants) +KNOWN_ARCH_CONSTANTS = ('aarch64', 'ppc64le', 'riscv64', 'x86_64') + +ARCH_KEY_PREFIX = 'arch=' # Vendor constants AMD = 'AMD' APM = 'Applied Micro' +APPLE = 'Apple' ARM = 'ARM' BROADCOM = 'Broadcom' CAVIUM = 'Cavium' @@ -90,6 +112,7 @@ # Family constants POWER_LE = 'POWER little-endian' +RISCV = 'RISC-V' # OS constants LINUX = 'Linux' @@ -97,13 +120,14 @@ UNKNOWN = 'UNKNOWN' +ETC_OS_RELEASE = '/etc/os-release' MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' PROC_CPUINFO_FP = '/proc/cpuinfo' PROC_MEMINFO_FP = '/proc/meminfo' -CPU_ARCHITECTURES = [AARCH32, AARCH64, POWER, X86_64] -CPU_FAMILIES = [AMD, ARM, INTEL, POWER, POWER_LE] -CPU_VENDORS = [AMD, APM, ARM, BROADCOM, CAVIUM, DEC, IBM, INTEL, MARVELL, MOTOROLA, NVIDIA, QUALCOMM] +CPU_ARCHITECTURES = [AARCH32, AARCH64, POWER, RISCV32, RISCV64, X86_64] +CPU_FAMILIES = [AMD, ARM, INTEL, POWER, POWER_LE, RISCV] +CPU_VENDORS = [AMD, APM, APPLE, ARM, BROADCOM, CAVIUM, DEC, IBM, INTEL, MARVELL, MOTOROLA, NVIDIA, QUALCOMM] # ARM implementer IDs (i.e., the hexadeximal keys) taken from ARMv8-A Architecture Reference Manual # (ARM DDI 0487A.j, Section G6.2.102, Page G6-4493) VENDOR_IDS = { @@ -143,6 +167,48 @@ # OS package handler name constants RPM = 'rpm' DPKG = 'dpkg' +ZYPPER = 'zypper' + +SYSTEM_TOOLS = { + '7z': "extracting sources (.iso)", + 'bunzip2': "decompressing sources (.bz2, .tbz, .tbz2, ...)", + DPKG: "checking OS dependencies (Debian, Ubuntu, ...)", + 'git': "downloading sources using 'git clone'", + 'gunzip': "decompressing source files (.gz, .tgz, ...)", + 'make': "build tool", + 'patch': "applying patch files", + RPM: "checking OS dependencies (CentOS, RHEL, OpenSuSE, SLES, ...)", + 'sed': "runtime patching", + 'Slurm': "backend for --job (sbatch command)", + 'tar': "unpacking source files (.tar)", + 'unxz': "decompressing source files (.xz, .txz)", + 'unzip': "decompressing files (.zip)", + ZYPPER: "checking OS dependencies (openSUSE)", +} + +SYSTEM_TOOL_CMDS = { + 'Slurm': 'sbatch', +} + +EASYBUILD_OPTIONAL_DEPENDENCIES = { + 'archspec': (None, "determining name of CPU microarchitecture"), + 'autopep8': (None, "auto-formatting for dumped easyconfigs"), + 'GC3Pie': ('gc3libs', "backend for --job"), + 'GitPython': ('git', "GitHub integration + using Git repository as easyconfigs archive"), + 'graphviz-python': ('gv', "rendering dependency graph with Graphviz: --dep-graph"), + 'keyring': (None, "storing GitHub token"), + 'pbs-python': ('pbs', "using Torque as --job backend"), + 'pep8': (None, "fallback for code style checking: --check-style, --check-contrib"), + 'pycodestyle': (None, "code style checking: --check-style, --check-contrib"), + 'pysvn': (None, "using SVN repository as easyconfigs archive"), + 'python-graph-core': ('pygraph.classes.digraph', "creating dependency graph: --dep-graph"), + 'python-graph-dot': ('pygraph.readwrite.dot', "saving dependency graph as dot file: --dep-graph"), + 'python-hglib': ('hglib', "using Mercurial repository as easyconfigs archive"), + 'requests': (None, "fallback library for downloading files"), + 'Rich': (None, "eb command rich terminal output"), + 'PyYAML': ('yaml', "easystack files and .yeb easyconfig format"), + 'setuptools': ('pkg_resources', "obtaining information on Python packages via pkg_resources module"), +} class SystemToolsException(Exception): @@ -152,24 +218,35 @@ class SystemToolsException(Exception): def sched_getaffinity(): """Determine list of available cores for current process.""" cpu_mask_t = ctypes.c_ulong - cpu_setsize = 1024 n_cpu_bits = 8 * ctypes.sizeof(cpu_mask_t) - n_mask_bits = cpu_setsize // n_cpu_bits - - class cpu_set_t(ctypes.Structure): - """Class that implements the cpu_set_t struct.""" - _fields_ = [('bits', cpu_mask_t * n_mask_bits)] _libc_lib = find_library('c') - _libc = ctypes.cdll.LoadLibrary(_libc_lib) + _libc = ctypes.CDLL(_libc_lib, use_errno=True) pid = os.getpid() - cs = cpu_set_t() - ec = _libc.sched_getaffinity(os.getpid(), ctypes.sizeof(cpu_set_t), ctypes.pointer(cs)) - if ec == 0: - _log.debug("sched_getaffinity for pid %s successful", pid) - else: - raise EasyBuildError("sched_getaffinity failed for pid %s ec %s", pid, ec) + + cpu_setsize = 1024 # Max number of CPUs currently detectable + max_cpu_setsize = cpu_mask_t(-1).value // 4 # (INT_MAX / 2) + # Limit it to something reasonable but still big enough + max_cpu_setsize = min(max_cpu_setsize, 1e9) + while cpu_setsize < max_cpu_setsize: + n_mask_bits = cpu_setsize // n_cpu_bits + + class cpu_set_t(ctypes.Structure): + """Class that implements the cpu_set_t struct.""" + _fields_ = [('bits', cpu_mask_t * n_mask_bits)] + + cs = cpu_set_t() + ec = _libc.sched_getaffinity(pid, ctypes.sizeof(cpu_set_t), ctypes.pointer(cs)) + if ec == 0: + _log.debug("sched_getaffinity for pid %s successful", pid) + break + elif ctypes.get_errno() != errno.EINVAL: + raise EasyBuildError("sched_getaffinity failed for pid %s errno %s", pid, ctypes.get_errno()) + cpu_setsize *= 2 + + if ec != 0: + raise EasyBuildError("sched_getaffinity failed finding a large enough cpuset for pid %s", pid) cpus = [] for bitmask in cs.bits: @@ -246,9 +323,11 @@ def get_cpu_architecture(): :return: a value from the CPU_ARCHITECTURES list """ - power_regex = re.compile("ppc64.*") - aarch64_regex = re.compile("aarch64.*") aarch32_regex = re.compile("arm.*") + aarch64_regex = re.compile("(aarch64|arm64).*") + power_regex = re.compile("ppc64.*") + riscv32_regex = re.compile("riscv32.*") + riscv64_regex = re.compile("riscv64.*") system, node, release, version, machine, processor = platform.uname() @@ -261,6 +340,10 @@ def get_cpu_architecture(): arch = AARCH64 elif aarch32_regex.match(machine): arch = AARCH32 + elif riscv64_regex.match(machine): + arch = RISCV64 + elif riscv32_regex.match(machine): + arch = RISCV32 if arch == UNKNOWN: _log.warning("Failed to determine CPU architecture, returning %s", arch) @@ -305,11 +388,18 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) out = out.strip() if ec == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) + else: + cmd = "sysctl -n machdep.cpu.brand_string" + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) + out = out.strip().split(' ')[0] + if ec == 0 and out in CPU_VENDORS: + vendor = out + _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) if vendor is None: vendor = UNKNOWN @@ -344,6 +434,9 @@ def get_cpu_family(): if powerle_regex.search(machine): family = POWER_LE + elif arch in [RISCV32, RISCV64]: + family = RISCV + if family is None: family = UNKNOWN _log.warning("Failed to determine CPU family, returning %s" % family) @@ -451,9 +544,11 @@ def get_cpu_speed(): cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) - if ec == 0: + out = out.strip() + cpu_freq = None + if ec == 0 and out: # returns clock frequency in cycles/sec, but we want MHz - cpu_freq = float(out.strip()) // (1000 ** 2) + cpu_freq = float(out) // (1000 ** 2) else: raise SystemToolsException("Could not determine CPU clock frequency (OS: %s)." % os_type) @@ -496,7 +591,7 @@ def get_cpu_features(): for feature_set in ['extfeatures', 'features', 'leaf7_features']: cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) if ec == 0: cpu_feat.extend(out.strip().lower().split()) @@ -508,6 +603,58 @@ def get_cpu_features(): return cpu_feat +def get_gpu_info(): + """ + Get the GPU info + """ + gpu_info = {} + os_type = get_os_type() + + if os_type == LINUX: + try: + cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" + _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + if ec == 0: + for line in out.strip().split('\n'): + nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) + nvidia_gpu_info.setdefault(line, 0) + nvidia_gpu_info[line] += 1 + else: + _log.debug("None zero exit (%s) from nvidia-smi: %s", ec, out) + except Exception as err: + _log.debug("Exception was raised when running nvidia-smi: %s", err) + _log.info("No NVIDIA GPUs detected") + + try: + cmd = "rocm-smi --showdriverversion --csv" + _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + if ec == 0: + amd_driver = out.strip().split('\n')[1].split(',')[1] + + cmd = "rocm-smi --showproductname --csv" + _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + if ec == 0: + for line in out.strip().split('\n')[1:]: + amd_card_series = line.split(',')[1] + amd_card_model = line.split(',')[2] + amd_gpu = "%s (model: %s, driver: %s)" % (amd_card_series, amd_card_model, amd_driver) + amd_gpu_info = gpu_info.setdefault('AMD', {}) + amd_gpu_info.setdefault(amd_gpu, 0) + amd_gpu_info[amd_gpu] += 1 + else: + _log.debug("None zero exit (%s) from rocm-smi: %s", ec, out) + except Exception as err: + _log.debug("Exception was raised when running rocm-smi: %s", err) + _log.info("No AMD GPUs detected") + else: + _log.info("Only know how to get GPU info on Linux, assuming no GPUs are present") + + return gpu_info + + def get_kernel_name(): """NO LONGER SUPPORTED: use get_os_type() instead""" _log.nosupport("get_kernel_name() is replaced by get_os_type()", '2.0') @@ -575,14 +722,21 @@ def get_os_name(): if hasattr(platform, 'linux_distribution'): # platform.linux_distribution is more useful, but only available since Python 2.6 # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) - os_name = platform.linux_distribution()[0].strip().lower() - elif HAVE_DISTRO: + os_name = platform.linux_distribution()[0].strip() + + # take into account that on some OSs, platform.distribution returns an empty string as OS name, + # for example on OpenSUSE Leap 15.2 + if not os_name and HAVE_DISTRO: # distro package is the recommended alternative to platform.linux_distribution, # see https://pypi.org/project/distro os_name = distro.name() - else: - # no easy way to determine name of Linux distribution - os_name = None + + if not os_name and os.path.exists(ETC_OS_RELEASE): + os_release_txt = read_file(ETC_OS_RELEASE) + name_regex = re.compile('^NAME="?(?P[^"\n]+)"?$', re.M) + res = name_regex.search(os_release_txt) + if res: + os_name = res.group('name') os_name_map = { 'red hat enterprise linux server': 'RHEL', @@ -593,7 +747,7 @@ def get_os_name(): } if os_name: - return os_name_map.get(os_name, os_name) + return os_name_map.get(os_name.lower(), os_name) else: return UNKNOWN @@ -601,49 +755,57 @@ def get_os_name(): def get_os_version(): """Determine system version.""" + os_version = None + # platform.dist was removed in Python 3.8 if hasattr(platform, 'dist'): os_version = platform.dist()[1] - elif HAVE_DISTRO: + + # take into account that on some OSs, platform.dist returns an empty string as OS version, + # for example on OpenSUSE Leap 15.2 + if not os_version and HAVE_DISTRO: os_version = distro.version() - else: - os_version = None - if os_version: - if get_os_name() in ["suse", "SLES"]: - - # SLES subversions can only be told apart based on kernel version, - # see http://wiki.novell.com/index.php/Kernel_versions - version_suffixes = { - '11': [ - ('2.6.27', ''), - ('2.6.32', '_SP1'), - ('3.0.101-63', '_SP4'), - # not 100% correct, since early SP3 had 3.0.76 - 3.0.93, but close enough? - ('3.0.101', '_SP3'), - # SP2 kernel versions range from 3.0.13 - 3.0.101 - ('3.0', '_SP2'), - ], - - '12': [ - ('3.12.28', ''), - ('3.12.49', '_SP1'), - ], - } + if not os_version and os.path.exists(ETC_OS_RELEASE): + os_release_txt = read_file(ETC_OS_RELEASE) + version_regex = re.compile('^VERSION="?(?P[^"\n]+)"?$', re.M) + res = version_regex.search(os_release_txt) + if res: + os_version = res.group('version') + else: + # VERSION may not always be defined (for example on Gentoo), + # fall back to VERSION_ID in that case + version_regex = re.compile('^VERSION_ID="?(?P[^"\n]+)"?$', re.M) + res = version_regex.search(os_release_txt) + if res: + os_version = res.group('version') + if os_version: + # older SLES subversions can only be told apart based on kernel version, + # see http://wiki.novell.com/index.php/Kernel_versions + sles_version_suffixes = { + '11': [ + ('2.6.27', ''), + ('2.6.32', '_SP1'), + ('3.0.101-63', '_SP4'), + # not 100% correct, since early SP3 had 3.0.76 - 3.0.93, but close enough? + ('3.0.101', '_SP3'), + # SP2 kernel versions range from 3.0.13 - 3.0.101 + ('3.0', '_SP2'), + ], + + '12': [ + ('3.12.28', ''), + ('3.12.49', '_SP1'), + ], + } + if get_os_name() in ['suse', 'SLES'] and os_version in sles_version_suffixes: # append suitable suffix to system version - if os_version in version_suffixes.keys(): - kernel_version = platform.uname()[2] - known_sp = False - for (kver, suff) in version_suffixes[os_version]: - if kernel_version.startswith(kver): - os_version += suff - known_sp = True - break - if not known_sp: - suff = '_UNKNOWN_SP' - else: - raise EasyBuildError("Don't know how to determine subversions for SLES %s", os_version) + kernel_version = platform.uname()[2] + for (kver, suff) in sles_version_suffixes[os_version]: + if kernel_version.startswith(kver): + os_version += suff + break return os_version else: @@ -662,14 +824,17 @@ def check_os_dependency(dep): os_to_pkg_cmd_map = { 'centos': RPM, 'debian': DPKG, + 'opensuse': ZYPPER, 'redhat': RPM, + 'rhel': RPM, 'ubuntu': DPKG, } pkg_cmd_flag = { DPKG: '-s', RPM: '-q', + ZYPPER: 'search -i', } - os_name = get_os_name() + os_name = get_os_name().lower().split(' ')[0] if os_name in os_to_pkg_cmd_map: pkg_cmds = [os_to_pkg_cmd_map[os_name]] else: @@ -703,14 +868,14 @@ def check_os_dependency(dep): return found -def get_tool_version(tool, version_option='--version'): +def get_tool_version(tool, version_option='--version', ignore_ec=False): """ Get output of running version option for specific command line tool. Output is returned as a single-line string (newlines are replaced by '; '). """ out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True, trace=False, stream_output=False) - if ec: + if not ignore_ec and ec: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out)) return UNKNOWN else: @@ -770,6 +935,140 @@ def get_glibc_version(): return glibc_ver +def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None): + """ + Check for (lack of) patterns in linked shared libraries for binary/library at specified path. + Uses 'ldd' on Linux and 'otool -L' on macOS to determine linked shared libraries. + + Returns True or False for dynamically linked binaries and shared libraries to indicate + whether all patterns match and antipatterns don't match. + + Returns None if given path is not a dynamically linked binary or library. + """ + if required_patterns is None: + required_regexs = [] + else: + required_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in required_patterns] + + if banned_patterns is None: + banned_regexs = [] + else: + banned_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in banned_patterns] + + # resolve symbolic links (unless they're broken) + if os.path.islink(path) and os.path.exists(path): + path = os.path.realpath(path) + + file_cmd_out, _ = run_cmd("file %s" % path, simple=False, trace=False) + + os_type = get_os_type() + + # check whether specified path is a dynamically linked binary or a shared library + if os_type == LINUX: + # example output for dynamically linked binaries: + # /usr/bin/ls: ELF 64-bit LSB executable, x86-64, ..., dynamically linked (uses shared libs), ... + # example output for shared libraries: + # /lib64/libc-2.17.so: ELF 64-bit LSB shared object, x86-64, ..., dynamically linked (uses shared libs), ... + if "dynamically linked" in file_cmd_out: + linked_libs_out, _ = run_cmd("ldd %s" % path, simple=False, trace=False) + else: + return None + + elif os_type == DARWIN: + # example output for dynamically linked binaries: + # /bin/ls: Mach-O 64-bit executable x86_64 + # example output for shared libraries: + # /usr/lib/libz.dylib: Mach-O 64-bit dynamically linked shared library x86_64 + bin_lib_regex = re.compile('(Mach-O .* executable)|(dynamically linked)', re.M) + if bin_lib_regex.search(file_cmd_out): + linked_libs_out, _ = run_cmd("otool -L %s" % path, simple=False, trace=False) + else: + return None + else: + raise EasyBuildError("Unknown OS type: %s", os_type) + + found_banned_patterns = [] + missing_required_patterns = [] + for regex in required_regexs: + if not regex.search(linked_libs_out): + missing_required_patterns.append(regex.pattern) + + for regex in banned_regexs: + if regex.search(linked_libs_out): + found_banned_patterns.append(regex.pattern) + + if missing_required_patterns: + patterns = ', '.join("'%s'" % p for p in missing_required_patterns) + _log.warning("Required patterns not found in linked libraries output for %s: %s", path, patterns) + + if found_banned_patterns: + patterns = ', '.join("'%s'" % p for p in found_banned_patterns) + _log.warning("Banned patterns found in linked libraries output for %s: %s", path, patterns) + + return not (found_banned_patterns or missing_required_patterns) + + +def locate_solib(libobj): + """ + Return absolute path to loaded library using dlinfo + Based on https://stackoverflow.com/a/35683698 + + :params libobj: ctypes CDLL object + """ + # early return if we're not on a Linux system + if get_os_type() != LINUX: + return None + + class LINKMAP(ctypes.Structure): + _fields_ = [ + ("l_addr", ctypes.c_void_p), + ("l_name", ctypes.c_char_p) + ] + + libdl = ctypes.cdll.LoadLibrary(ctypes.util.find_library('dl')) + + dlinfo = libdl.dlinfo + dlinfo.argtypes = ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p + dlinfo.restype = ctypes.c_int + + libpointer = ctypes.c_void_p() + dlinfo(libobj._handle, 2, ctypes.byref(libpointer)) + libpath = ctypes.cast(libpointer, ctypes.POINTER(LINKMAP)).contents.l_name + + return libpath.decode('utf-8') + + +def find_library_path(lib_filename): + """ + Search library by file name in the system + Return absolute path to existing libraries + + :params lib_filename: name of library file + """ + + lib_abspath = None + os_type = get_os_type() + + try: + lib_obj = ctypes.cdll.LoadLibrary(lib_filename) + except OSError: + _log.info("Library '%s' not found in host system", lib_filename) + else: + # ctypes.util.find_library only accepts unversioned library names + if os_type == LINUX: + # find path to library with dlinfo + lib_abspath = locate_solib(lib_obj) + elif os_type == DARWIN: + # ctypes.macholib.dyld.dyld_find accepts file names and returns full path + lib_abspath = ctypes.macholib.dyld.dyld_find(lib_filename) + else: + raise EasyBuildError("Unknown host OS type: %s", os_type) + + _log.info("Found absolute path to %s: %s", lib_filename, lib_abspath) + + return lib_abspath + + def get_system_info(): """Return a dictionary with system information.""" python_version = '; '.join(sys.version.split('\n')) @@ -823,31 +1122,42 @@ def det_parallelism(par=None, maxpar=None): Determine level of parallelism that should be used. Default: educated guess based on # cores and 'ulimit -u' setting: min(# cores, ((ulimit -u) - 15) // 6) """ - if par is not None: - if not isinstance(par, int): + def get_default_parallelism(): + try: + # Get cache value if any + par = det_parallelism._default_parallelism + except AttributeError: + # No cache -> Calculate value from current system values + par = get_avail_core_count() + # check ulimit -u + out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) try: - par = int(par) + if out.startswith("unlimited"): + maxuserproc = 2 ** 32 - 1 + else: + maxuserproc = int(out) except ValueError as err: - raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) - else: - par = get_avail_core_count() - # check ulimit -u - out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) - try: - if out.startswith("unlimited"): - out = 2 ** 32 - 1 - maxuserproc = int(out) + raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) # assume 6 processes per build thread + 15 overhead - par_guess = int((maxuserproc - 15) // 6) + par_guess = (maxuserproc - 15) // 6 if par_guess < par: par = par_guess - _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) + _log.info("Limit parallel builds to %s because max user processes is %s", par, out) + # Cache value + det_parallelism._default_parallelism = par + return par + + if par is None: + par = get_default_parallelism() + else: + try: + par = int(par) except ValueError as err: - raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) + raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: - _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) - par = min(par, maxpar) + _log.info("Limiting parallellism from %s to %s", par, maxpar) + par = maxpar return par @@ -878,17 +1188,9 @@ def check_python_version(): python_ver = '%d.%d' % (python_maj_ver, python_min_ver) _log.info("Found Python version %s", python_ver) - silence_deprecation_warnings = build_option('silence_deprecation_warnings') or [] - if python_maj_ver == 2: - if python_min_ver < 6: - raise EasyBuildError("Python 2.6 or higher is required when using Python 2, found Python %s", python_ver) - elif python_min_ver == 6: - depr_msg = "Running EasyBuild with Python 2.6 is deprecated" - if 'Python26' in silence_deprecation_warnings: - _log.warning(depr_msg) - else: - _log.deprecated(depr_msg, '5.0') + if python_min_ver < 7: + raise EasyBuildError("Python 2.7 is required when using Python 2, found Python %s", python_ver) else: _log.info("Running EasyBuild with Python 2 (version %s)", python_ver) @@ -921,20 +1223,126 @@ def pick_dep_version(dep_version): result = None elif isinstance(dep_version, dict): - # figure out matches based on dict keys (after splitting on '=') - my_arch_key = 'arch=%s' % get_cpu_architecture() - arch_keys = [x for x in dep_version.keys() if x.startswith('arch=')] + arch_keys = [x for x in dep_version.keys() if x.startswith(ARCH_KEY_PREFIX)] other_keys = [x for x in dep_version.keys() if x not in arch_keys] if other_keys: - raise EasyBuildError("Unexpected keys in version: %s. Only 'arch=' keys are supported", other_keys) + other_keys = ','.join(sorted(other_keys)) + raise EasyBuildError("Unexpected keys in version: %s (only 'arch=' keys are supported)", other_keys) if arch_keys: - if my_arch_key in dep_version: - result = dep_version[my_arch_key] - _log.info("Version selected from %s using key %s: %s", dep_version, my_arch_key, result) + host_arch_key = ARCH_KEY_PREFIX + get_cpu_architecture() + star_arch_key = ARCH_KEY_PREFIX + '*' + # check for specific 'arch=' key first + if host_arch_key in dep_version: + result = dep_version[host_arch_key] + _log.info("Version selected from %s using key %s: %s", dep_version, host_arch_key, result) + # fall back to 'arch=*' + elif star_arch_key in dep_version: + result = dep_version[star_arch_key] + _log.info("Version selected for %s using fallback key %s: %s", dep_version, star_arch_key, result) else: - raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, my_arch_key) + raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, host_arch_key) + else: + raise EasyBuildError("Found empty dict as version!") else: - raise EasyBuildError("Unknown value type for version: %s", dep_version) + typ = type(dep_version) + raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version) return result + + +def det_pypkg_version(pkg_name, imported_pkg, import_name=None): + """Determine version of a Python package.""" + + version = None + + if HAVE_PKG_RESOURCES: + if import_name: + try: + version = pkg_resources.get_distribution(import_name).version + except pkg_resources.DistributionNotFound as err: + _log.debug("%s Python package not found: %s", import_name, err) + + if version is None: + try: + version = pkg_resources.get_distribution(pkg_name).version + except pkg_resources.DistributionNotFound as err: + _log.debug("%s Python package not found: %s", pkg_name, err) + + if version is None and hasattr(imported_pkg, '__version__'): + version = imported_pkg.__version__ + + return version + + +def check_easybuild_deps(modtool): + """ + Check presence and version of required and optional EasyBuild dependencies, and report back to terminal. + """ + version_regex = re.compile(r'\s(?P[0-9][0-9.]+[a-z]*)') + + checks_data = OrderedDict() + + def extract_version(tool): + """Helper function to extract (only) version for specific command line tool.""" + out = get_tool_version(tool, ignore_ec=True) + res = version_regex.search(out) + if res: + version = res.group('version') + else: + version = "UNKNOWN version" + + return version + + python_version = extract_version(sys.executable) + + opt_dep_versions = {} + for key in EASYBUILD_OPTIONAL_DEPENDENCIES: + + pkg = EASYBUILD_OPTIONAL_DEPENDENCIES[key][0] + if pkg is None: + pkg = key.lower() + + try: + mod = __import__(pkg) + except ImportError: + mod = None + + if mod: + dep_version = det_pypkg_version(key, mod, import_name=pkg) + else: + dep_version = False + + opt_dep_versions[key] = dep_version + + checks_data['col_titles'] = ('name', 'version', 'used for') + + req_deps_key = "Required dependencies" + checks_data[req_deps_key] = OrderedDict() + checks_data[req_deps_key]['Python'] = (python_version, None) + checks_data[req_deps_key]['modules tool:'] = (str(modtool), None) + + opt_deps_key = "Optional dependencies" + checks_data[opt_deps_key] = {} + + for key in opt_dep_versions: + checks_data[opt_deps_key][key] = (opt_dep_versions[key], EASYBUILD_OPTIONAL_DEPENDENCIES[key][1]) + + sys_tools_key = "System tools" + checks_data[sys_tools_key] = {} + + for tool in SYSTEM_TOOLS: + tool_info = None + cmd = SYSTEM_TOOL_CMDS.get(tool, tool) + if which(cmd): + version = extract_version(cmd) + if version.startswith('UNKNOWN'): + tool_info = None + else: + tool_info = version + else: + tool_info = False + + checks_data[sys_tools_key][tool] = (tool_info, None) + + return checks_data diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index cf93034570..8765e3a0c0 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -50,7 +50,7 @@ from easybuild.tools.jenkins import aggregate_xml_in_dirs from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel from easybuild.tools.robot import resolve_dependencies -from easybuild.tools.systemtools import UNKNOWN, get_system_info +from easybuild.tools.systemtools import UNKNOWN, get_gpu_info, get_system_info from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION @@ -138,20 +138,30 @@ def session_state(): } -def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False): +def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_log=False, easyblock_pr_nrs=None): """Create test report for easyconfigs PR, in Markdown format.""" github_user = build_option('github_user') pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO + pr_target_repo = build_option('pr_target_repo') end_time = gmtime() # create a gist with a full test report test_report = [] - if pr_nr is not None: + pr_list = [] + if pr_nrs: + repo = pr_target_repo or GITHUB_EASYCONFIGS_REPO + pr_urls = ["https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, x) for x in pr_nrs] + pr_list.append("PR(s) %s" % ', '.join(pr_urls)) + if easyblock_pr_nrs: + repo = pr_target_repo or GITHUB_EASYBLOCKS_REPO + easyblock_pr_urls = ["https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, x) + for x in easyblock_pr_nrs] + pr_list.append("easyblock PR(s) %s" % ', '.join(easyblock_pr_urls)) + if pr_list: test_report.extend([ - "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, pr_target_repo, pr_nr), + "Test report for %s" % ', '.join(pr_list), "", ]) test_report.extend([ @@ -182,8 +192,13 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l logtxt = read_file(ec_res['log_file']) partial_log_txt = '\n'.join(logtxt.split('\n')[-500:]) descr = "(partial) EasyBuild log for failed build of %s" % ec['spec'] - if pr_nr is not None: - descr += " (PR #%s)" % pr_nr + + if pr_nrs: + descr += " (PR(s) #%s)" % ', #'.join(str(x) for x in pr_nrs) + + if easyblock_pr_nrs: + descr += " (easyblock PR(s) #%s)" % ', #'.join(str(x) for x in easyblock_pr_nrs) + fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3] gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user) test_log = "(partial log available at %s)" % gist_url @@ -249,31 +264,51 @@ def upload_test_report_as_gist(test_report, descr=None, fn=None): return gist_url -def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, success): +def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state, success): """Post test report in a gist, and submit comment in easyconfigs or easyblocks PR.""" + # make sure pr_nrs is a list of strings + if isinstance(pr_nrs, str): + pr_nrs = [pr_nrs] + elif isinstance(pr_nrs, int): + pr_nrs = [str(pr_nrs)] + else: + try: + pr_nrs = [str(x) for x in pr_nrs] + except ValueError: + raise EasyBuildError("Can't convert %s to a list of PR #s." % pr_nrs) + github_user = build_option('github_user') pr_target_account = build_option('pr_target_account') pr_target_repo = build_option('pr_target_repo') or repo_type # create gist with test report - descr = "EasyBuild test report for %s/%s PR #%s" % (pr_target_account, pr_target_repo, pr_nr) + descr = "EasyBuild test report for %s/%s PR(s) #%s" % (pr_target_account, pr_target_repo, ', #'.join(pr_nrs)) timestamp = strftime("%Y%M%d-UTC-%H-%M-%S", gmtime()) - fn = 'easybuild_test_report_%s_%s_pr%s_%s.md' % (pr_nr, pr_target_account, pr_target_repo, timestamp) + fn = 'easybuild_test_report_%s_%s_pr%s_%s.md' % ('_'.join(pr_nrs), pr_target_account, pr_target_repo, timestamp) gist_url = upload_test_report_as_gist(test_report['full'], descr=descr, fn=fn) # post comment to report test result - system_info = init_session_state['system_info'] + system_info = init_session_state['system_info'].copy() # also mention CPU architecture name, but only if it's known if system_info['cpu_arch_name'] != UNKNOWN: system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name'] + # add GPU info, if known + gpu_info = get_gpu_info() + gpu_str = "" + if gpu_info: + for vendor in gpu_info: + for gpu, num in gpu_info[vendor].items(): + gpu_str += ", %s x %s %s" % (num, vendor, gpu) + os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info - short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s, Python %(pyver)s" % { + short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s%(gpu)s, Python %(pyver)s" % { 'os_info': os_info, 'cpu_arch': system_info['cpu_arch'], 'cpu_model': system_info['cpu_model'], + 'gpu': gpu_str, 'pyver': system_info['python_version'].split(' ')[0], } @@ -281,7 +316,7 @@ def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, if build_option('include_easyblocks_from_pr'): if repo_type == GITHUB_EASYCONFIGS_REPO: - easyblocks_pr_nrs = map(int, build_option('include_easyblocks_from_pr')) + easyblocks_pr_nrs = [int(x) for x in build_option('include_easyblocks_from_pr')] comment_lines.append("Using easyblocks from PR(s) %s" % ", ".join(["https://github.com/%s/%s/pull/%s" % (pr_target_account, GITHUB_EASYBLOCKS_REPO, easyblocks_pr_nr) @@ -301,9 +336,11 @@ def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, ]) comment = '\n'.join(comment_lines) - post_comment_in_issue(pr_nr, comment, account=pr_target_account, repo=pr_target_repo, github_user=github_user) + for pr_nr in pr_nrs: + post_comment_in_issue(pr_nr, comment, account=pr_target_account, repo=pr_target_repo, github_user=github_user) - msg = "Test report uploaded to %s and mentioned in a comment in %s PR#%s" % (gist_url, pr_target_repo, pr_nr) + msg = "Test report uploaded to %s and mentioned in a comment in %s PR(s) #%s" % (gist_url, pr_target_repo, + ', #'.join(pr_nrs)) return msg @@ -317,21 +354,33 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state :param init_session_state: initial session state info to include in test report """ dump_path = build_option('dump_test_report') - pr_nr = build_option('from_pr') - eb_pr_nrs = build_option('include_easyblocks_from_pr') + + try: + pr_nrs = [int(x) for x in build_option('from_pr')] + except ValueError: + raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + + try: + easyblock_pr_nrs = [int(x) for x in build_option('include_easyblocks_from_pr')] + except ValueError: + raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") + upload = build_option('upload_test_report') if upload: msg = msg + " (%d easyconfigs in total)" % orig_cnt - test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True) - if pr_nr: - # upload test report to gist and issue a comment in the PR to notify - txt = post_pr_test_report(pr_nr, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, success) - elif eb_pr_nrs: - # upload test report to gist and issue a comment in the easyblocks PR to notify - for eb_pr_nr in map(int, eb_pr_nrs): - txt = post_pr_test_report(eb_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, init_session_state, - success) + + test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=pr_nrs, gist_log=True, + easyblock_pr_nrs=easyblock_pr_nrs) + if pr_nrs: + # upload test report to gist and issue a comment in the PR(s) to notify + txt = post_pr_test_report(pr_nrs, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, + success) + elif easyblock_pr_nrs: + # upload test report to gist and issue a comment in the easyblocks PR(s) to notify + txt = post_pr_test_report(easyblock_pr_nrs, GITHUB_EASYBLOCKS_REPO, test_report, msg, + init_session_state, success) + else: # only upload test report as a gist gist_url = upload_test_report_as_gist(test_report['full']) diff --git a/easybuild/tools/toolchain/__init__.py b/easybuild/tools/toolchain/__init__.py index e51e6d33e0..56bb55edf0 100644 --- a/easybuild/tools/toolchain/__init__.py +++ b/easybuild/tools/toolchain/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index b8337b9b57..745e32423b 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -296,10 +296,10 @@ def _set_compiler_flags(self): self.variables.nextend(var, fflags) extra = 'extra_' + var.lower() if self.options.get(extra): - flags = self.options.option(extra) - if not flags or flags[0] != '-': - raise EasyBuildError("toolchainopts %s: '%s' must start with a '-'." % (extra, flags)) - self.variables.nappend_el(var, flags[1:]) + extraflags = self.options.option(extra) + if not extraflags or extraflags[0] != '-': + raise EasyBuildError("toolchainopts %s: '%s' must start with a '-'." % (extra, extraflags)) + self.variables.nappend_el(var, extraflags[1:]) def _set_optimal_architecture(self, default_optarch=None): """ diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index 307827dd1e..f6f3beb2d6 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,8 +30,8 @@ """ from easybuild.tools.variables import AbsPathList -from easybuild.tools.toolchain.variables import LinkLibraryPaths, IncludePaths, CommandFlagList, CommaStaticLibs -from easybuild.tools.toolchain.variables import FlagList, LibraryList +from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs +from easybuild.tools.toolchain.variables import FlagList, IncludePaths, LibraryList, LinkLibraryPaths COMPILER_VARIABLES = [ @@ -114,6 +114,10 @@ ('LIBBLAS', 'BLAS libraries'), ('LIBBLAS_MT', 'multithreaded BLAS libraries'), ], + CommaSharedLibs: [ + ('BLAS_SHARED_LIBS', 'Comma-separated list of shared BLAS libraries'), + ('BLAS_MT_SHARED_LIBS', 'Comma-separated list of shared multithreaded BLAS libraries'), + ], CommaStaticLibs: [ ('BLAS_STATIC_LIBS', 'Comma-separated list of static BLAS libraries'), ('BLAS_MT_STATIC_LIBS', 'Comma-separated list of static multithreaded BLAS libraries'), @@ -132,6 +136,12 @@ ('LIBLAPACK', 'LAPACK libraries'), ('LIBLAPACK_MT', 'multithreaded LAPACK libraries'), ], + CommaSharedLibs: [ + ('LAPACK_SHARED_LIBS', 'Comma-separated list of shared LAPACK libraries'), + ('LAPACK_MT_SHARED_LIBS', 'Comma-separated list of shared LAPACK libraries'), + ('BLAS_LAPACK_SHARED_LIBS', 'Comma-separated list of shared BLAS and LAPACK libraries'), + ('BLAS_LAPACK_MT_SHARED_LIBS', 'Comma-separated list of shared BLAS and LAPACK libraries'), + ], CommaStaticLibs: [ ('LAPACK_STATIC_LIBS', 'Comma-separated list of static LAPACK libraries'), ('LAPACK_MT_STATIC_LIBS', 'Comma-separated list of static LAPACK libraries'), @@ -166,6 +176,10 @@ ('LIBSCALAPACK', 'SCALAPACK libraries'), ('LIBSCALAPACK_MT', 'multithreaded SCALAPACK libraries'), ], + CommaSharedLibs: [ + ('SCALAPACK_SHARED_LIBS', 'Comma-separated list of shared SCALAPACK libraries'), + ('SCALAPACK_MT_SHARED_LIBS', 'Comma-separated list of shared SCALAPACK libraries'), + ], CommaStaticLibs: [ ('SCALAPACK_STATIC_LIBS', 'Comma-separated list of static SCALAPACK libraries'), ('SCALAPACK_MT_STATIC_LIBS', 'Comma-separated list of static SCALAPACK libraries'), @@ -181,6 +195,10 @@ ('LIBFFT', 'FFT libraries'), ('LIBFFT_MT', 'Multithreaded FFT libraries'), ], + CommaSharedLibs: [ + ('FFT_SHARED_LIBS', 'Comma-separated list of shared FFT libraries'), + ('FFT_SHARED_LIBS_MT', 'Comma-separated list of shared multithreaded FFT libraries'), + ], CommaStaticLibs: [ ('FFT_STATIC_LIBS', 'Comma-separated list of static FFT libraries'), ('FFT_STATIC_LIBS_MT', 'Comma-separated list of static multithreaded FFT libraries'), @@ -192,6 +210,10 @@ ('FFTW_LIB_DIR', 'FFTW library directory'), ('FFTW_INC_DIR', 'FFTW include directory'), ], + CommaSharedLibs: [ + ('FFTW_SHARED_LIBS', 'Comma-separated list of shared FFTW libraries'), + ('FFTW_SHARED_LIBS_MT', 'Comma-separated list of shared multithreaded FFTW libraries'), + ], CommaStaticLibs: [ ('FFTW_STATIC_LIBS', 'Comma-separated list of static FFTW libraries'), ('FFTW_STATIC_LIBS_MT', 'Comma-separated list of static multithreaded FFTW libraries'), diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py index 4facafc7c2..e3fcda405e 100644 --- a/easybuild/tools/toolchain/fft.py +++ b/easybuild/tools/toolchain/fft.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -68,7 +68,9 @@ def _set_fft_variables(self): if getattr(self, 'LIB_MULTITHREAD', None) is not None: self.variables.nappend('LIBFFT_MT', self.LIB_MULTITHREAD) + self.variables.join('FFT_SHARED_LIBS', 'LIBFFT') self.variables.join('FFT_STATIC_LIBS', 'LIBFFT') + self.variables.join('FFT_SHARED_LIBS_MT', 'LIBFFT_MT') self.variables.join('FFT_STATIC_LIBS_MT', 'LIBFFT_MT') for root in self.get_software_root(self.FFT_MODULE_NAME): diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 7d27350996..573d7ae750 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -127,7 +127,9 @@ def _set_blas_variables(self): self.variables.nappend('LIBBLAS', self.LIB_EXTRA, position=20) self.variables.nappend('LIBBLAS_MT', self.LIB_EXTRA, position=20) + self.variables.join('BLAS_SHARED_LIBS', 'LIBBLAS') self.variables.join('BLAS_STATIC_LIBS', 'LIBBLAS') + self.variables.join('BLAS_MT_SHARED_LIBS', 'LIBBLAS_MT') self.variables.join('BLAS_MT_STATIC_LIBS', 'LIBBLAS_MT') for root in self.get_software_root(self.BLAS_MODULE_NAME): self.variables.append_exists('BLAS_LIB_DIR', root, self.BLAS_LIB_DIR) @@ -147,7 +149,9 @@ def _set_lapack_variables(self): self.variables.join('LIBLAPACK_MT_ONLY', 'LIBBLAS_MT') self.variables.join('LIBLAPACK', 'LIBBLAS') self.variables.join('LIBLAPACK_MT', 'LIBBLAS_MT') + self.variables.join('LAPACK_SHARED_LIBS', 'BLAS_SHARED_LIBS') self.variables.join('LAPACK_STATIC_LIBS', 'BLAS_STATIC_LIBS') + self.variables.join('LAPACK_MT_SHARED_LIBS', 'BLAS_MT_SHARED_LIBS') self.variables.join('LAPACK_MT_STATIC_LIBS', 'BLAS_MT_STATIC_LIBS') self.variables.join('LAPACK_LIB_DIR', 'BLAS_LIB_DIR') self.variables.join('LAPACK_INC_DIR', 'BLAS_INC_DIR') @@ -183,7 +187,9 @@ def _set_lapack_variables(self): self.variables.nappend('LIBLAPACK', self.LIB_EXTRA, position=20) self.variables.nappend('LIBLAPACK_MT', self.LIB_EXTRA, position=20) + self.variables.join('LAPACK_SHARED_LIBS', 'LIBLAPACK') self.variables.join('LAPACK_STATIC_LIBS', 'LIBLAPACK') + self.variables.join('LAPACK_MT_SHARED_LIBS', 'LIBLAPACK_MT') self.variables.join('LAPACK_MT_STATIC_LIBS', 'LIBLAPACK_MT') for root in self.get_software_root(self.LAPACK_MODULE_NAME): @@ -192,7 +198,9 @@ def _set_lapack_variables(self): self.variables.join('BLAS_LAPACK_LIB_DIR', 'LAPACK_LIB_DIR', 'BLAS_LIB_DIR') self.variables.join('BLAS_LAPACK_INC_DIR', 'LAPACK_INC_DIR', 'BLAS_INC_DIR') + self.variables.join('BLAS_LAPACK_SHARED_LIBS', 'LAPACK_SHARED_LIBS', 'BLAS_SHARED_LIBS') self.variables.join('BLAS_LAPACK_STATIC_LIBS', 'LAPACK_STATIC_LIBS', 'BLAS_STATIC_LIBS') + self.variables.join('BLAS_LAPACK_MT_SHARED_LIBS', 'LAPACK_MT_SHARED_LIBS', 'BLAS_MT_SHARED_LIBS') self.variables.join('BLAS_LAPACK_MT_STATIC_LIBS', 'LAPACK_MT_STATIC_LIBS', 'BLAS_MT_STATIC_LIBS') # add general dependency variables @@ -293,7 +301,9 @@ def _set_scalapack_variables(self): self.variables.nappend('LIBSCALAPACK', self.LIB_EXTRA, position=20) self.variables.nappend('LIBSCALAPACK_MT', self.LIB_EXTRA, position=20) + self.variables.join('SCALAPACK_SHARED_LIBS', 'LIBSCALAPACK') self.variables.join('SCALAPACK_STATIC_LIBS', 'LIBSCALAPACK') + self.variables.join('SCALAPACK_MT_SHARED_LIBS', 'LIBSCALAPACK_MT') self.variables.join('SCALAPACK_MT_STATIC_LIBS', 'LIBSCALAPACK_MT') for root in self.get_software_root(self.SCALAPACK_MODULE_NAME): self.variables.append_exists('SCALAPACK_LIB_DIR', root, self.SCALAPACK_LIB_DIR) diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 8121dbd5c8..56c76ebead 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -68,6 +68,7 @@ def get_mpi_cmd_template(mpi_family, params, mpi_version=None): toolchain.MVAPICH2: mpirun_n_cmd, toolchain.MPICH: mpirun_n_cmd, toolchain.MPICH2: mpirun_n_cmd, + toolchain.MPITRAMPOLINE: "mpiexec -n %(nr_ranks)s %(cmd)s", } # Intel MPI mpirun needs more work @@ -293,10 +294,9 @@ def mpi_cmd_for(self, cmd, nr_ranks): # for Intel MPI, try to determine impi version # this fails when it's done too early (before modules for toolchain/dependencies are loaded), # but it's safe to ignore this - try: - mpi_version = self.get_software_version(self.MPI_MODULE_NAME)[0] - except EasyBuildError as err: - self.log.debug("Ignoring error when trying to determine %s version: %s", self.MPI_MODULE_NAME, err) + mpi_version = self.get_software_version(self.MPI_MODULE_NAME, required=False)[0] + if not mpi_version: + self.log.debug("Ignoring error when trying to determine %s version", self.MPI_MODULE_NAME) # impi version is required to determine correct MPI command template, # so we have to return early if we couldn't determine the impi version... return None diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py index 423d7aa339..ae6eb436f1 100644 --- a/easybuild/tools/toolchain/options.py +++ b/easybuild/tools/toolchain/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index ca2f732824..e21bd66869 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -403,33 +403,35 @@ def get_software_root(self, names): """Try to get the software root for all names""" return self._get_software_multiple(names, self._get_software_root) - def get_software_version(self, names): + def get_software_version(self, names, required=True): """Try to get the software version for all names""" - return self._get_software_multiple(names, self._get_software_version) + return self._get_software_multiple(names, self._get_software_version, required=required) - def _get_software_multiple(self, names, function): + def _get_software_multiple(self, names, function, required=True): """Execute function of each of names""" if isinstance(names, (str,)): names = [names] res = [] for name in names: - res.append(function(name)) + res.append(function(name, required=required)) return res - def _get_software_root(self, name): + def _get_software_root(self, name, required=True): """Try to get the software root for name""" root = get_software_root(name) if root is None: - raise EasyBuildError("get_software_root software root for %s was not found in environment", name) + if required: + raise EasyBuildError("get_software_root software root for %s was not found in environment", name) else: self.log.debug("get_software_root software root %s for %s was found in environment", root, name) return root - def _get_software_version(self, name): + def _get_software_version(self, name, required=True): """Try to get the software version for name""" version = get_software_version(name) if version is None: - raise EasyBuildError("get_software_version software version for %s was not found in environment", name) + if required: + raise EasyBuildError("get_software_version software version for %s was not found in environment", name) else: self.log.debug("get_software_version software version %s for %s was found in environment", version, name) @@ -634,7 +636,9 @@ def _load_toolchain_module(self, silent=False): if self.init_modpaths: mod_path_suffix = build_option('suffix_modules_path') for modpath in self.init_modpaths: - self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath)) + modpath = os.path.join(install_path('mod'), mod_path_suffix, modpath) + if os.path.exists(modpath): + self.modules_tool.prepend_module_path(modpath) # load modules for all dependencies self.log.debug("Loading module for toolchain: %s", tc_mod) @@ -923,7 +927,7 @@ def prepare_compiler_cache(self, cache_tool): cache_path = which(cache_tool) if cache_path is None: - raise EasyBuildError("%s binary not found in $PATH, required by --use-compiler-cache", cache_tool) + raise EasyBuildError("%s binary not found in $PATH, required by --use-ccache", cache_tool) else: self.symlink_commands({cache_tool: (cache_path, compilers)}) @@ -956,9 +960,11 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None # always include filter for 'stubs' library directory, # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683 - lib_stubs_pattern = '.*/lib(64)?/stubs/?' - if lib_stubs_pattern not in rpath_filter_dirs: - rpath_filter_dirs.append(lib_stubs_pattern) + # (since CUDA 11.something the stubs are in $EBROOTCUDA/stubs/lib64) + lib_stubs_patterns = ['.*/lib(64)?/stubs/?', '.*/stubs/lib(64)?/?'] + for lib_stubs_pattern in lib_stubs_patterns: + if lib_stubs_pattern not in rpath_filter_dirs: + rpath_filter_dirs.append(lib_stubs_pattern) # directory where all wrappers will be placed wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) @@ -1130,17 +1136,31 @@ def comp_family(self): raise NotImplementedError def blas_family(self): - "Return type of BLAS library used in this toolchain, or 'None' if BLAS is not supported." + """Return type of BLAS library used in this toolchain, or 'None' if BLAS is not supported.""" return None def lapack_family(self): - "Return type of LAPACK library used in this toolchain, or 'None' if LAPACK is not supported." + """Return type of LAPACK library used in this toolchain, or 'None' if LAPACK is not supported.""" return None def mpi_family(self): - "Return type of MPI library used in this toolchain, or 'None' if MPI is not supported." + """Return type of MPI library used in this toolchain, or 'None' if MPI is not supported.""" return None + def banned_linked_shared_libs(self): + """ + List of shared libraries (names, file names, paths) which are + not allowed to be linked in any installed binary/library. + """ + return [] + + def required_linked_shared_libs(self): + """ + List of shared libraries (names, file names, paths) which + must be linked in all installed binaries/libraries. + """ + return [] + def cleanup(self): """Clean up after using this toolchain""" pass diff --git a/easybuild/tools/toolchain/toolchainvariables.py b/easybuild/tools/toolchain/toolchainvariables.py index d1a25f65df..82e5550658 100644 --- a/easybuild/tools/toolchain/toolchainvariables.py +++ b/easybuild/tools/toolchain/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index ff6f01cbff..d9a6c02412 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index 30a365ad5b..a0987694e9 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,6 +30,7 @@ """ from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.variables import StrList, AbsPathList @@ -111,8 +112,16 @@ def change(self, separator=None, separator_begin_end=None, prefix=None, prefix_b self.END.PREFIX = prefix_begin_end +class CommaSharedLibs(LibraryList): + """Comma-separated list of shared libraries""" + SEPARATOR = ',' + + PREFIX = 'lib' + SUFFIX = '.' + get_shared_lib_ext() + + class CommaStaticLibs(LibraryList): - """Comma-separated list""" + """Comma-separated list of static libraries""" SEPARATOR = ',' PREFIX = 'lib' diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index e1b19b2985..b7fc9fd657 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -300,19 +300,23 @@ def time2str(delta): if not isinstance(delta, datetime.timedelta): raise EasyBuildError("Incorrect value type provided to time2str, should be datetime.timedelta: %s", type(delta)) - delta_secs = delta.days * 3600 * 24 + delta.seconds + delta.microseconds / 10**6 + delta_secs = delta.total_seconds() - if delta_secs < 60: - res = '%d sec' % int(delta_secs) - elif delta_secs < 3600: - mins = int(delta_secs / 60) - secs = int(delta_secs - (mins * 60)) - res = '%d min %d sec' % (mins, secs) - else: - hours = int(delta_secs / 3600) - mins = int((delta_secs - hours * 3600) / 60) - secs = int(delta_secs - (hours * 3600) - (mins * 60)) - hours_str = 'hours' if hours > 1 else 'hour' - res = '%d %s %d min %d sec' % (hours, hours_str, mins, secs) + hours, remainder = divmod(delta_secs, 3600) + mins, secs = divmod(remainder, 60) - return res + res = [] + if hours: + res.append('%d %s' % (hours, 'hour' if hours == 1 else 'hours')) + if mins or hours: + res.append('%d %s' % (mins, 'min' if mins == 1 else 'mins')) + res.append('%d %s' % (secs, 'sec' if secs == 1 else 'secs')) + + return ' '.join(res) + + +def natural_keys(key): + """Can be used as the sort key in list.sort(key=natural_keys) to sort in natural order (i.e. respecting numbers)""" + def try_to_int(key_part): + return int(key_part) if key_part.isdigit() else key_part + return [try_to_int(key_part) for key_part in re.split(r'(\d+)', key)] diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py index 8cd8dcd3cb..9db0aeb6fa 100644 --- a/easybuild/tools/variables.py +++ b/easybuild/tools/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 125344e4d3..0559fab2b3 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.3.dev0') +VERSION = LooseVersion('4.5.6.dev0') UNKNOWN = 'UNKNOWN' diff --git a/eb b/eb index 8968892dbe..b0b625f19d 100755 --- a/eb +++ b/eb @@ -1,6 +1,6 @@ #!/bin/bash ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,10 +33,22 @@ # @author: Pieter De Baets (Ghent University) # @author: Jens Timmerman (Ghent University) +keyboard_interrupt() { + echo "Keyboard interrupt!" + exit 1 +} + +trap keyboard_interrupt SIGINT + # Python 2.6+ or 3.5+ required REQ_MIN_PY2VER=6 REQ_MIN_PY3VER=5 +EASYBUILD_MAIN='easybuild.main' + +# easybuild module to import to check whether EasyBuild framework is available; +# don't use easybuild.main here, since that's a very expensive module to import (it makes the 'eb' command slow) +EASYBUILD_IMPORT_TEST='easybuild.framework' function verbose() { if [ ! -z ${EB_VERBOSE} ]; then echo ">> $1"; fi @@ -52,6 +64,8 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; verbose "Considering '$python_cmd'..." + # check whether python* command being considered is available + # (using 'command -v', since 'which' implies an extra dependency) command -v $python_cmd &> /dev/null if [ $? -eq 0 ]; then @@ -63,10 +77,25 @@ for python_cmd in ${EB_PYTHON} ${EB_INSTALLPYTHON} 'python' 'python3' 'python2'; if [ $pyver_maj -eq 2 ] && [ $pyver_min -ge $REQ_MIN_PY2VER ]; then verbose "'$python_cmd' version: $pyver, which matches Python 2 version requirement (>= 2.$REQ_MIN_PY2VER)" PYTHON=$python_cmd - break elif [ $pyver_maj -eq 3 ] && [ $pyver_min -ge $REQ_MIN_PY3VER ]; then verbose "'$python_cmd' version: $pyver, which matches Python 3 version requirement (>= 3.$REQ_MIN_PY3VER)" PYTHON=$python_cmd + fi + + if [ ! -z $PYTHON ]; then + # check whether EasyBuild framework is available for selected python command + $PYTHON -c "import $EASYBUILD_IMPORT_TEST" 2> /dev/null + if [ $? -eq 0 ]; then + verbose "'$python_cmd' is able to import '$EASYBUILD_IMPORT_TEST', so retaining it" + else + # if EasyBuild framework is not available, don't use this python command, keep searching... + verbose "'$python_cmd' is NOT able to import '$EASYBUILD_IMPORT_TEST' so NOT retaining it" + unset PYTHON + fi + fi + + # break out of for loop if we've found a valid python command + if [ ! -z $PYTHON ]; then break fi else @@ -79,7 +108,7 @@ if [ -z $PYTHON ]; then echo "(EasyBuild requires Python 2.${REQ_MIN_PY2VER}+ or 3.${REQ_MIN_PY3VER}+)" >&2 exit 1 else - verbose "Selected Python command: $python_cmd (`which $python_cmd`)" + verbose "Selected Python command: $python_cmd (`command -v $python_cmd`)" fi # enable optimization, unless $PYTHONOPTIMIZE is defined (use "export PYTHONOPTIMIZE=0" to disable optimization) @@ -97,5 +126,5 @@ fi export EB_SCRIPT_PATH=$0 -verbose "$PYTHON -m easybuild.main `echo \"$@\"`" -$PYTHON -m easybuild.main "$@" +verbose "$PYTHON -m $EASYBUILD_MAIN `echo \"$@\"`" +$PYTHON -m $EASYBUILD_MAIN "$@" diff --git a/eb_bash_completion_local.bash b/eb_bash_completion_local.bash new file mode 100644 index 0000000000..c85ca740b8 --- /dev/null +++ b/eb_bash_completion_local.bash @@ -0,0 +1,12 @@ +_eb() +{ + local cur prev quoted + _get_comp_words_by_ref cur prev + _quote_readline_by_ref "$cur" quoted + + case $cur in + --*) _optcomplete "$@"; return 0 ;; + *) COMPREPLY=( $(compgen -f -X '!*.eb' -- $cur ) ) ;; + esac +} +complete -F _eb eb diff --git a/requirements.txt b/requirements.txt index 0defb13c7c..cc7c9b4fb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,59 +1,35 @@ # keyring is required to provide GitHub token to EasyBuild; -# keyring v5.7.1 is last version to be compatible with py2.6; # for recent versions of keyring, keyrings.alt must be installed too -keyring==5.7.1; python_version < '2.7' -keyring<=9.1; python_version >= '2.7' -keyrings.alt; python_version >= '2.7' +keyring +keyrings.alt -# GitDB 4.0.1 no longer supports Python 2.6 -gitdb==0.6.4; python_version < '2.7' -gitdb; python_version >= '2.7' +# GitPython 3.1.15 deprecates Python 3.5 +GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' +GitPython; python_version >= '3.6' or python_version <= '3.0' -# GitPython 2.1.9 no longer supports Python 2.6 -GitPython==2.1.8; python_version < '2.7' -GitPython; python_version >= '2.7' +# autopep8 +autopep8 -# pydot (dep for python-graph-dot) 1.2.0 and more recent doesn't work with Python 2.6 -pydot==1.1.0; python_version < '2.7' -pydot; python_version >= '2.7' - -# pycparser 2.19 (dep for paramiko) doesn't work with Python 2.6 -pycparser<2.19; python_version < '2.7' - -# idna 2.8 (dep for paramiko) & more recent doesn't work with Python 2.6 -idna<2.8; python_version < '2.7' - -# paramiko 2.4.0 (dep for GC3Pie) & more recent doesn't work with Python 2.6 -paramiko<2.4.0; python_version < '2.7' - -# SQLAlchemy 1.2.0 (dep for GC3Pie) & more recent doesn't work with Python 2.6 -SQLAlchemy<1.2.0; python_version < '2.7' - -# python 2.0 (dep for GC3Pie) & more recent doesn't work with Python 2.6 -python-daemon<2.0; python_version < '2.7' - -# autopep8 1.3.4 is last one to support Python 2.6 -autopep8<1.3.5; python_version < '2.7' -autopep8; python_version >= '2.7' - -# PyYAML 5.x no longer supports Python 2.6 -PyYAML<5.0; python_version < '2.7' -PyYAML; python_version >= '2.7' +# PyYAML +PyYAML # optional Python packages for EasyBuild # flake8 is a superset of pycodestyle -pycodestyle; python_version < '2.7' -flake8; python_version >= '2.7' +flake8 -GC3Pie +# 2.6.7 uses invalid Python 2 syntax +GC3Pie!=2.6.7; python_version < '3.0' +GC3Pie; python_version >= '3.0' python-graph-dot python-hglib requests -archspec; python_version >= '2.7' +archspec -# cryptography 3.0 deprecates Python 2.7 (but v3.2.1 still works with Python 2.7); -# cryptography is not needed at all for Python 2.6 -cryptography==3.2.1; python_version == '2.7' +# cryptography 3.4.0 no longer supports Python 2.7 +cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' + +# rich is only supported for Python 3.6+ +rich; python_version >= '3.6' diff --git a/setup.py b/setup.py index 1fec1be8c9..f3ae4e58cc 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -84,7 +84,7 @@ def find_rel_test(): implement support for installing particular (groups of) software packages.""", license="GPLv2", keywords="software build building installation installing compilation HPC scientific", - url="https://easybuilders.github.io/easybuild", + url="https://easybuild.io", packages=easybuild_packages, package_dir={'test.framework': 'test/framework'}, package_data={'test.framework': find_rel_test()}, @@ -101,6 +101,7 @@ def find_rel_test(): data_files=[ ('easybuild/scripts', glob.glob('easybuild/scripts/*')), ('etc', glob.glob('etc/*')), + ('contrib/hooks', glob.glob('contrib/hooks/*')), ], long_description=read('README.rst'), classifiers=[ @@ -109,11 +110,13 @@ def find_rel_test(): "Intended Audience :: System Administrators", "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Build Tools", ], platforms="Linux", diff --git a/test/__init__.py b/test/__init__.py index 5a26aaf317..5770afd893 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/__init__.py b/test/framework/__init__.py index ff095ace59..77c470bf68 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index 1d4adb9fe3..c3cf41316c 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 32aefc7b74..357773e4b0 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -70,7 +70,7 @@ def test_easybuilderror(self): logToFile(tmplog, enable=False) log_re = re.compile(r"^fancyroot ::.* BOOM \(at .*:[0-9]+ in [a-z_]+\)$", re.M) - logtxt = open(tmplog, 'r').read() + logtxt = read_file(tmplog, 'r') self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt)) # test formatting of message @@ -419,7 +419,7 @@ def test_init_logging(self): self.assertTrue(os.path.exists(logfile)) self.assertEqual(os.path.dirname(logfile), tmpdir) self.assertTrue(isinstance(log, EasyBuildLog)) - self.assertTrue(stdout.startswith("== temporary log file in case of crash")) + self.assertTrue(stdout.startswith("== Temporary log file in case of crash")) stop_logging(logfile) diff --git a/test/framework/config.py b/test/framework/config.py index 0b17a4e876..e0cc07991e 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -41,6 +41,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, build_path, get_build_log_path, get_log_filename, get_repositorypath from easybuild.tools.config import install_path, log_file_format, log_path, source_paths +from easybuild.tools.config import update_build_option, update_build_options from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, init_build_options from easybuild.tools.filetools import copy_dir, mkdir, write_file @@ -244,6 +245,9 @@ def test_generaloption_config_file(self): cfgtxt = '\n'.join([ '[config]', 'installpath = %s' % testpath2, + # special case: configuration option to a value starting with '--' + '[override]', + 'optarch = --test', ]) write_file(config_file, cfgtxt) @@ -261,6 +265,8 @@ def test_generaloption_config_file(self): self.assertEqual(install_path(), installpath_software) # via cmdline arg self.assertEqual(install_path('mod'), os.path.join(testpath2, 'modules')) # via config file + self.assertEqual(options.optarch, '--test') # via config file + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory # to check whether easyconfigs install path is auto-included in robot path tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') @@ -672,6 +678,52 @@ def test_get_build_log_path(self): init_config(args=['--tmp-logdir=%s' % build_log_path]) self.assertEqual(get_build_log_path(), build_log_path) + def test_update_build_option(self): + """Test updating of a build option.""" + self.assertEqual(build_option('banned_linked_shared_libs'), None) + orig_banned_linked_shared_libs = update_build_option('banned_linked_shared_libs', '/usr/lib64/libssl.so.1.1') + self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') + self.assertEqual(orig_banned_linked_shared_libs, None) + + self.assertTrue(build_option('cleanup_builddir')) + orig_cleanup_builddir = update_build_option('cleanup_builddir', False) + self.assertFalse(build_option('cleanup_builddir')) + self.assertTrue(orig_cleanup_builddir) + + self.assertEqual(build_option('pr_target_account'), 'easybuilders') + orig_pr_target_account = update_build_option('pr_target_account', 'test_pr_target_account') + self.assertEqual(build_option('pr_target_account'), 'test_pr_target_account') + self.assertEqual(orig_pr_target_account, 'easybuilders') + + def test_update_build_options(self): + """Test updating of a dictionary of build options.""" + # Check if original defaults are as expected: + self.assertEqual(build_option('banned_linked_shared_libs'), None) + self.assertEqual(build_option('filter_env_vars'), None) + self.assertTrue(build_option('cleanup_builddir')) + self.assertEqual(build_option('pr_target_account'), 'easybuilders') + + # Update build options based on dictionary + new_opt_dict = { + 'banned_linked_shared_libs': '/usr/lib64/libssl.so.1.1', + 'filter_env_vars': 'LD_LIBRARY_PATH', + 'cleanup_builddir': False, + 'pr_target_account': 'test_pr_target_account', + } + original_opt_dict = update_build_options(new_opt_dict) + self.assertEqual(build_option('banned_linked_shared_libs'), '/usr/lib64/libssl.so.1.1') + self.assertEqual(build_option('filter_env_vars'), 'LD_LIBRARY_PATH') + self.assertFalse(build_option('cleanup_builddir')) + self.assertEqual(build_option('pr_target_account'), 'test_pr_target_account') + + # Check the returned dictionary by simply restoring the variables and checking if the build + # options have their original values again + update_build_options(original_opt_dict) + self.assertEqual(build_option('banned_linked_shared_libs'), None) + self.assertEqual(build_option('filter_env_vars'), None) + self.assertTrue(build_option('cleanup_builddir')) + self.assertEqual(build_option('pr_target_account'), 'easybuilders') + def suite(): return TestLoaderFiltered().loadTestsFromTestCase(EasyBuildConfigTest, sys.argv[1:]) diff --git a/test/framework/containers.py b/test/framework/containers.py index da7b5c32d3..16cb5cfd34 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2021 Ghent University +# Copyright 2018-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -141,8 +141,8 @@ def test_end2end_singularity_recipe_config(self): self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) pip_patterns = [ - # EasyBuild is installed with pip by default - "pip install easybuild", + # EasyBuild is installed with pip3 by default + "pip3 install easybuild", ] post_commands_patterns = [ # easybuild user is added if it doesn't exist yet @@ -386,7 +386,7 @@ def test_end2end_dockerfile(self): base_args + ['--container-config=not-supported'], raise_error=True) - for cont_base in ['ubuntu:16.04', 'centos:7']: + for cont_base in ['ubuntu:20.04', 'centos:7']: stdout, stderr = self.run_main(base_args + ['--container-config=%s' % cont_base]) self.assertFalse(stderr) regexs = ["^== Dockerfile definition file created at %s/containers/Dockerfile.toy-0.0" % self.test_prefix] @@ -406,11 +406,11 @@ def test_end2end_dockerfile(self): remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb')) - self.run_main(base_args + ['--container-config=ubuntu:16.04']) + self.run_main(base_args + ['--container-config=ubuntu:20.04']) def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0')) regexs = [ - "FROM ubuntu:16.04", - "eb toy-0.0.eb GCC-4.9.2.eb", + "FROM ubuntu:20.04", + "eb --robot toy-0.0.eb GCC-4.9.2.eb", "module load toy/0.0 GCC/4.9.2", ] self.check_regexs(regexs, def_file) @@ -435,7 +435,7 @@ def test_end2end_docker_image(self): '-C', # equivalent with --containerize '--experimental', '--container-type=docker', - '--container-config=ubuntu:16.04', + '--container-config=ubuntu:20.04', '--container-build-image', ] diff --git a/test/framework/docs.py b/test/framework/docs.py index 075ec58908..fb3e2a2b44 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -260,7 +260,7 @@ def test_list_software(self): ] txt = list_software(output_format='txt', detailed=True) lines = txt.split('\n') - expected_found = any([lines[i:i + len(expected)] == expected for i in range(len(lines))]) + expected_found = any(lines[i:i + len(expected)] == expected for i in range(len(lines))) self.assertTrue(expected_found, "%s found in: %s" % (expected, lines)) expected = [ @@ -283,7 +283,7 @@ def test_list_software(self): ] txt = list_software(output_format='rst', detailed=True) lines = txt.split('\n') - expected_found = any([lines[i:i + len(expected)] == expected for i in range(len(lines))]) + expected_found = any(lines[i:i + len(expected)] == expected for i in range(len(lines))) self.assertTrue(expected_found, "%s found in: %s" % (expected, lines)) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 399a269597..f77823fcec 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,6 @@ import sys import tempfile from inspect import cleandoc -from datetime import datetime from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -46,10 +45,10 @@ from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax -from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_file, write_file +from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_file +from easybuild.tools.filetools import verify_checksum, write_file from easybuild.tools.module_generator import module_generator from easybuild.tools.modules import reset_module_caches -from easybuild.tools.utilities import time2str from easybuild.tools.version import get_git_revision, this_is_easybuild from easybuild.tools.py2vs3 import string_type @@ -128,6 +127,8 @@ def check_extra_options_format(extra_options): extra_options = exeb1.extra_options() check_extra_options_format(extra_options) self.assertTrue('options' in extra_options) + # Reporting test failure should work also for the extension EB + self.assertRaises(EasyBuildError, exeb1.report_test_failure, "Fails") # test extensioneasyblock, as easyblock exeb2 = ExtensionEasyBlock(ec) @@ -136,6 +137,8 @@ def check_extra_options_format(extra_options): extra_options = exeb2.extra_options() check_extra_options_format(extra_options) self.assertTrue('options' in extra_options) + # Reporting test failure should work also for the extension EB + self.assertRaises(EasyBuildError, exeb2.report_test_failure, "Fails") class TestExtension(ExtensionEasyBlock): @staticmethod @@ -247,6 +250,9 @@ def test_fake_module_load(self): def test_make_module_extend_modpath(self): """Test for make_module_extend_modpath""" + + module_syntax = get_module_syntax() + self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', @@ -262,7 +268,7 @@ def test_make_module_extend_modpath(self): # no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS) self.assertEqual(eb.make_module_extend_modpath(), '') - usermodsdir = 'my/own/modules' + usermodsdir = 'my_own_modules' modclasses = ['compiler', 'tools'] os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS' build_options = { @@ -275,9 +281,10 @@ def test_make_module_extend_modpath(self): eb.installdir = config.install_path() txt = eb.make_module_extend_modpath() - if get_module_syntax() == 'Tcl': + if module_syntax == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - home = r'\$::env\(HOME\)' + home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } ' + home += r'else { concat "HOME_NOT_DEFINED" } \]' fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -285,9 +292,9 @@ def test_make_module_extend_modpath(self): # no per-moduleclass extension for user modules r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir), ]) - elif get_module_syntax() == 'Lua': + elif module_syntax == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] - home = r'os.getenv\("HOME"\)' + home = r'os.getenv\("HOME"\) or "HOME_NOT_DEFINED"' pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -296,11 +303,113 @@ def test_make_module_extend_modpath(self): r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir), ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(False, "Unknown module syntax: %s" % module_syntax) + for regex in regexs: regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + # Repeat this but using an alternate envvars (instead of $HOME) + list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS'] + + build_options = { + 'envvars_user_modules': list_of_envvars, + 'subdir_user_modules': usermodsdir, + 'valid_module_classes': modclasses, + 'suffix_modules_path': 'funky', + } + init_config(build_options=build_options) + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.installdir = config.install_path() + + txt = eb.make_module_extend_modpath() + for envvar in list_of_envvars: + if module_syntax == 'Tcl': + regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] + module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar + module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar + module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED') + fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir + regexs.extend([ + # extension for user modules is guarded + r'if { \[ file isdirectory \[ file join %s \[ %s \] \] \] } {$' % (module_envvar, fj_usermodsdir), + # no per-moduleclass extension for user modules + r'^\s+module use \[ file join %s \[ %s \] \]$' % (module_envvar, fj_usermodsdir), + ]) + elif module_syntax == 'Lua': + regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c + for c in modclasses] + module_envvar = r'os.getenv\("%s"\) or "%s"' % (envvar, envvar + "_NOT_DEFINED") + pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir + regexs.extend([ + # extension for user modules is guarded + r'if isDir\(pathJoin\(%s, %s\)\) then' % (module_envvar, pj_usermodsdir), + # no per-moduleclass extension for user modules + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (module_envvar, pj_usermodsdir), + ]) + else: + self.assertTrue(False, "Unknown module syntax: %s" % module_syntax) + + for regex in regexs: + regex = re.compile(regex, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + os.unsetenv(envvar) + + # Check behaviour when directories do and do not exist + usermodsdir_extension = os.path.join(usermodsdir, "funky", "Compiler/pi/3.14") + site_install_path = os.path.join(config.install_path(), 'site') + site_modules = os.path.join(site_install_path, usermodsdir_extension) + user_install_path = os.path.join(config.install_path(), 'user') + user_modules = os.path.join(user_install_path, usermodsdir_extension) + + # make a modules directory so that we can create our module files + temp_module_file_dir = os.path.join(site_install_path, usermodsdir, "temp_module_files") + mkdir(temp_module_file_dir, parents=True) + + # write out a module file + if module_syntax == 'Tcl': + module_file = os.path.join(temp_module_file_dir, "mytest") + module_txt = "#%Module\n" + txt + elif module_syntax == 'Lua': + module_file = os.path.join(temp_module_file_dir, "mytest.lua") + module_txt = txt + write_file(module_file, module_txt) + + # Set MODULEPATH and check the effect of `module load` + os.environ['MODULEPATH'] = temp_module_file_dir + + # Let's switch to a dir where the paths we will use exist to make sure they can + # not be accidentally picked up if the variable is not defined but the paths exist + # relative to the current directory + cwd = os.getcwd() + mkdir(os.path.join(config.install_path(), "existing_dir", usermodsdir_extension), parents=True) + change_dir(os.path.join(config.install_path(), "existing_dir")) + self.modtool.run_module('load', 'mytest') + self.assertFalse(usermodsdir_extension in os.environ['MODULEPATH']) + self.modtool.run_module('unload', 'mytest') + change_dir(cwd) + + # Now define our environment variables + os.environ['SITE_INSTALLS'] = site_install_path + os.environ['USER_INSTALLS'] = user_install_path + + # Check MODULEPATH when neither directories exist + self.modtool.run_module('load', 'mytest') + self.assertFalse(site_modules in os.environ['MODULEPATH']) + self.assertFalse(user_modules in os.environ['MODULEPATH']) + self.modtool.run_module('unload', 'mytest') + # Now create the directory for site modules + mkdir(site_modules, parents=True) + self.modtool.run_module('load', 'mytest') + self.assertTrue(os.environ['MODULEPATH'].startswith(site_modules)) + self.assertFalse(user_modules in os.environ['MODULEPATH']) + self.modtool.run_module('unload', 'mytest') + # Now create the directory for user modules + mkdir(user_modules, parents=True) + self.modtool.run_module('load', 'mytest') + self.assertTrue(os.environ['MODULEPATH'].startswith(user_modules + ":" + site_modules)) + self.modtool.run_module('unload', 'mytest') + def test_make_module_req(self): """Testcase for make_module_req""" self.contents = '\n'.join([ @@ -317,8 +426,8 @@ def test_make_module_req(self): # create fake directories and files that should be guessed os.makedirs(eb.installdir) - open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar') - open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar') + write_file(os.path.join(eb.installdir, 'foo.jar'), 'foo.jar') + write_file(os.path.join(eb.installdir, 'bla.jar'), 'bla.jar') for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'): if isinstance(path, string_type): path = (path, ) @@ -326,7 +435,8 @@ def test_make_module_req(self): # this is not a path that should be picked up os.mkdir(os.path.join(eb.installdir, 'CPATH')) - guess = eb.make_module_req() + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M)) @@ -352,8 +462,9 @@ def test_make_module_req(self): self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) # check that bin is only added to PATH if there are files in there - open(os.path.join(eb.installdir, 'bin', 'test'), 'w').write('test') - guess = eb.make_module_req() + write_file(os.path.join(eb.installdir, 'bin', 'test'), 'test') + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M)) self.assertFalse(re.search(r"^prepend-path\s+PATH\s+\$root/sbin$", guess, re.M)) @@ -371,17 +482,19 @@ def test_make_module_req(self): elif get_module_syntax() == 'Lua': self.assertFalse('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) # -- With files - open(os.path.join(eb.installdir, 'lib64', 'libfoo.so'), 'w').write('test') - guess = eb.make_module_req() + write_file(os.path.join(eb.installdir, 'lib64', 'libfoo.so'), 'test') + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) elif get_module_syntax() == 'Lua': self.assertTrue('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) # -- With files in lib and lib64 symlinks to lib - open(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'w').write('test') + write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'test') shutil.rmtree(os.path.join(eb.installdir, 'lib64')) os.symlink('lib', os.path.join(eb.installdir, 'lib64')) - guess = eb.make_module_req() + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) elif get_module_syntax() == 'Lua': @@ -400,7 +513,8 @@ def test_make_module_req(self): # check for behavior when a string value is used as dict value by make_module_req_guesses eb.make_module_req_guess = lambda: {'PATH': 'bin'} - txt = eb.make_module_req() + with eb.module_generator.start_module_creation(): + txt = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M)) elif get_module_syntax() == 'Lua': @@ -411,7 +525,8 @@ def test_make_module_req(self): # check for correct behaviour if empty string is specified as one of the values # prepend-path statements should be included for both the 'bin' subdir and the install root eb.make_module_req_guess = lambda: {'PATH': ['bin', '']} - txt = eb.make_module_req() + with eb.module_generator.start_module_creation(): + txt = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root/bin\n", txt, re.M)) self.assertTrue(re.search(r"\nprepend-path\s+PATH\s+\$root\n", txt, re.M)) @@ -425,8 +540,9 @@ def test_make_module_req(self): eb.make_module_req_guess = lambda: {'LD_LIBRARY_PATH': ['lib/pathC', 'lib/pathA', 'lib/pathB', 'lib/pathA']} for path in ['pathA', 'pathB', 'pathC']: os.mkdir(os.path.join(eb.installdir, 'lib', path)) - open(os.path.join(eb.installdir, 'lib', path, 'libfoo.so'), 'w').write('test') - txt = eb.make_module_req() + write_file(os.path.join(eb.installdir, 'lib', path, 'libfoo.so'), 'test') + with eb.module_generator.start_module_creation(): + txt = eb.make_module_req() if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r"\nprepend-path\s+LD_LIBRARY_PATH\s+\$root/lib/pathC\n" + r"prepend-path\s+LD_LIBRARY_PATH\s+\$root/lib/pathA\n" + @@ -446,6 +562,25 @@ def test_make_module_req(self): else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + # If PATH or LD_LIBRARY_PATH contain only folders, do not add an entry + sub_lib_path = os.path.join('lib', 'path_folders') + sub_path_path = os.path.join('bin', 'path_folders') + eb.make_module_req_guess = lambda: {'LD_LIBRARY_PATH': sub_lib_path, 'PATH': sub_path_path} + for path in (sub_lib_path, sub_path_path): + full_path = os.path.join(eb.installdir, path, 'subpath') + os.makedirs(full_path) + write_file(os.path.join(full_path, 'any.file'), 'test') + txt = eb.make_module_req() + if get_module_syntax() == 'Tcl': + self.assertFalse(re.search(r"prepend-path\s+LD_LIBRARY_PATH\s+\$%s\n" % sub_lib_path, + txt, re.M)) + self.assertFalse(re.search(r"prepend-path\s+PATH\s+\$%s\n" % sub_path_path, txt, re.M)) + else: + assert get_module_syntax() == 'Lua' + self.assertFalse(re.search(r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "%s"\)\)\n' % sub_lib_path, + txt, re.M)) + self.assertFalse(re.search(r'prepend_path\("PATH", pathJoin\(root, "%s"\)\)\n' % sub_path_path, txt, re.M)) + # cleanup eb.close_log() os.remove(eb.logfile) @@ -888,6 +1023,69 @@ def test_extensions_step(self): eb.close_log() os.remove(eb.logfile) + def test_init_extensions(self): + """Test creating extension instances.""" + + testdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec_txt = read_file(toy_ec_file) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = toy_ec_txt.replace("('barbar', '0.0', {", "('barbar', '0.0', {'easyblock': 'DummyExtension',") + write_file(test_ec, test_ec_txt) + ec = process_easyconfig(test_ec)[0] + eb = get_easyblock_instance(ec) + + eb.prepare_for_extensions() + eb.init_ext_instances() + ext_inst_class_names = [x.__class__.__name__ for x in eb.ext_instances] + expected = [ + 'Toy_Extension', # 'ls' extension + 'Toy_Extension', # 'bar' extension + 'DummyExtension', # 'barbar' extension + 'EB_toy', # 'toy' extension + ] + self.assertEqual(ext_inst_class_names, expected) + + # check what happen if we specify an easyblock that doesn't derive from Extension, + # and hence can't be used to install extensions... + test_ec = os.path.join(self.test_prefix, 'test_broken.eb') + test_ec_txt = test_ec_txt.replace('DummyExtension', 'ConfigureMake') + write_file(test_ec, test_ec_txt) + ec = process_easyconfig(test_ec)[0] + eb = get_easyblock_instance(ec) + + eb.prepare_for_extensions() + error_pattern = "ConfigureMake easyblock can not be used to install extensions" + self.assertErrorRegex(EasyBuildError, error_pattern, eb.init_ext_instances) + + def test_extension_source_tmpl(self): + """Test type checking for 'source_tmpl' value of an extension.""" + self.contents = '\n'.join([ + "easyblock = 'ConfigureMake'", + "name = 'toy'", + "version = '0.0'", + "homepage = 'https://example.com'", + "description = 'test'", + "toolchain = SYSTEM", + "exts_list = [", + " ('bar', '0.0', {", + " 'source_tmpl': [SOURCE_TAR_GZ],", + " }),", + "]", + ]) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + error_pattern = r"source_tmpl value must be a string! " + error_pattern += r"\(found value of type 'list'\): \['bar-0\.0\.tar\.gz'\]" + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + + self.contents = self.contents.replace("'source_tmpl': [SOURCE_TAR_GZ]", "'source_tmpl': SOURCE_TAR_GZ") + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.fetch_step() + def test_skip_extensions_step(self): """Test the skip_extensions_step""" @@ -961,7 +1159,7 @@ def test_make_module_step(self): # purposely use a 'nasty' description, that includes (unbalanced) special chars: [, ], {, } descr = "This {is a}} [fancy]] [[description]]. {{[[TEST}]" modextravars = {'PI': '3.1415', 'FOO': 'bar'} - modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'} + modextrapaths = {'PATH': ('bin', 'pibin'), 'CPATH': 'pi/include'} self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, @@ -986,6 +1184,10 @@ def test_make_module_step(self): eb.make_builddir() eb.prepare_step() + # Create a dummy file in bin to test if the duplicate entry of modextrapaths is ignored + os.makedirs(os.path.join(eb.installdir, 'bin')) + write_file(os.path.join(eb.installdir, 'bin', 'dummy_exe'), 'hello') + modpath = os.path.join(eb.make_module_step(), name, version) if get_module_syntax() == 'Lua': modpath += '.lua' @@ -1019,14 +1221,20 @@ def test_make_module_step(self): self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) - for (key, val) in modextrapaths.items(): - if get_module_syntax() == 'Tcl': - regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M) - elif get_module_syntax() == 'Lua': - regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) - else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) - self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + for (key, vals) in modextrapaths.items(): + if isinstance(vals, string_type): + vals = [vals] + for val in vals: + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + # Check for duplicates + num_prepends = len(regex.findall(txt)) + self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) for (name, ver) in [('GCC', '6.4.0-2.28')]: if get_module_syntax() == 'Tcl': @@ -1272,6 +1480,125 @@ def test_fetch_sources(self): error_pattern = "Found one or more unexpected keys in 'sources' specification: {'nosuchkey': 'foobar'}" self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_sources, sources, checksums=[]) + def test_download_instructions(self): + """Test use of download_instructions easyconfig parameter.""" + orig_test_ec = '\n'.join([ + "easyblock = 'ConfigureMake'", + "name = 'software_with_missing_sources'", + "version = '0.0'", + "homepage = 'https://example.com'", + "description = 'test'", + "toolchain = SYSTEM", + "sources = [SOURCE_TAR_GZ]", + "exts_list = [", + " ('ext_with_missing_sources', '0.0', {", + " 'sources': [SOURCE_TAR_GZ],", + " }),", + "]", + ]) + self.contents = orig_test_ec + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + common_error_pattern = "^Couldn't find file software_with_missing_sources-0.0.tar.gz anywhere" + error_pattern = common_error_pattern + ", and downloading it didn't work either" + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + + download_instructions = "download_instructions = 'Manual download from example.com required'" + sources = "sources = [SOURCE_TAR_GZ]" + self.contents = self.contents.replace(sources, download_instructions + '\n' + sources) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + error_pattern = common_error_pattern + ", please follow the download instructions above" + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required") + self.assertEqual(stdout, '') + + # create dummy source file + write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '') + + # now downloading of sources for extension should fail + # top-level download instructions are printed (because there's nothing else) + error_pattern = "^Couldn't find file ext_with_missing_sources-0.0.tar.gz anywhere" + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required") + self.assertEqual(stdout, '') + + # wipe top-level download instructions, try again + self.contents = self.contents.replace(download_instructions, '') + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + # no download instructions printed anymore now + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stdout, '') + + # inject download instructions for extension + download_instructions = ' ' * 8 + "'download_instructions': " + download_instructions += "'Extension sources must be downloaded via example.com'," + sources = "'sources': [SOURCE_TAR_GZ]," + self.contents = self.contents.replace(sources, sources + '\n' + download_instructions) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nExtension sources must be downloaded via example.com") + self.assertEqual(stdout, '') + + # download instructions should also be printed if 'source_tmpl' is used to specify extension sources + self.contents = self.contents.replace(sources, "'source_tmpl': SOURCE_TAR_GZ,") + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, "Download instructions:\n\nExtension sources must be downloaded via example.com") + self.assertEqual(stdout, '') + + # create dummy source file for extension + write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '') + + # no more errors, all source files found (so no download instructions printed either) + self.mock_stderr(True) + self.mock_stdout(True) + eb.fetch_step() + stderr = self.get_stderr().strip() + stdout = self.get_stdout().strip() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + self.assertEqual(stdout, '') + def test_fetch_patches(self): """Test fetch_patches method.""" testdir = os.path.abspath(os.path.dirname(__file__)) @@ -1367,6 +1694,12 @@ def test_obtain_file(self): error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir]) + # also test triggering error when downloading from a URL that includes URL-encoded characters + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4005 + url = 'file://%s' % os.path.dirname(tmpdir_subdir) + url += '%2F' + os.path.basename(tmpdir_subdir) + self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=[url]) + # file specifications via URL also work, are downloaded to (first) sourcepath init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)]) urls = [ @@ -1388,7 +1721,7 @@ def test_obtain_file(self): loc = os.path.join(tmpdir, 't', 'toy', fn) self.assertEqual(res, loc) self.assertTrue(os.path.exists(loc), "%s file is found at %s" % (fn, loc)) - txt = open(loc, 'r').read() + txt = read_file(loc) eb_regex = re.compile("EasyBuild: building software with ease") self.assertTrue(eb_regex.search(txt), "Pattern '%s' found in: %s" % (eb_regex.pattern, txt)) else: @@ -1396,6 +1729,95 @@ def test_obtain_file(self): shutil.rmtree(tmpdir) + def test_fallback_source_url(self): + """Check whether downloading from fallback source URL https://sources.easybuild.io works.""" + # cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11951 + + init_config(args=["--sourcepath=%s" % self.test_prefix]) + + udunits_ec = os.path.join(self.test_prefix, 'UDUNITS.eb') + udunits_ec_txt = '\n'.join([ + "easyblock = 'ConfigureMake'", + "name = 'UDUNITS'", + "version = '2.2.26'", + "homepage = 'https://www.unidata.ucar.edu/software/udunits'", + "description = 'UDUNITS'", + "toolchain = {'name': 'GCC', 'version': '4.8.2'}", + "source_urls = ['https://broken.source.urls/nosuchdirectory']", + "sources = [SOURCELOWER_TAR_GZ]", + "checksums = ['368f4869c9c7d50d2920fa8c58654124e9ed0d8d2a8c714a9d7fdadc08c7356d']", + ]) + write_file(udunits_ec, udunits_ec_txt) + + ec = process_easyconfig(udunits_ec)[0] + eb = EasyBlock(ec['ec']) + + eb.fetch_step() + + expected_path = os.path.join(self.test_prefix, 'u', 'UDUNITS', 'udunits-2.2.26.tar.gz') + self.assertTrue(os.path.samefile(eb.src[0]['path'], expected_path)) + + self.assertTrue(verify_checksum(expected_path, eb.cfg['checksums'][0])) + + def test_collect_exts_file_info(self): + """Test collect_exts_file_info method.""" + testdir = os.path.abspath(os.path.dirname(__file__)) + toy_sources = os.path.join(testdir, 'sandbox', 'sources', 'toy') + toy_ext_sources = os.path.join(toy_sources, 'extensions') + toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + toy_ec = process_easyconfig(toy_ec_file)[0] + toy_eb = EasyBlock(toy_ec['ec']) + + exts_file_info = toy_eb.collect_exts_file_info() + + self.assertTrue(isinstance(exts_file_info, list)) + self.assertEqual(len(exts_file_info), 4) + + self.assertEqual(exts_file_info[0], {'name': 'ls'}) + + self.assertEqual(exts_file_info[1]['name'], 'bar') + self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz')) + bar_patch1 = 'bar-0.0_fix-silly-typo-in-printf-statement.patch' + self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1) + self.assertEqual(exts_file_info[1]['patches'][0]['path'], os.path.join(toy_ext_sources, bar_patch1)) + bar_patch2 = 'bar-0.0_fix-very-silly-typo-in-printf-statement.patch' + self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2) + self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2)) + + self.assertEqual(exts_file_info[2]['name'], 'barbar') + self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz')) + self.assertFalse('patches' in exts_file_info[2]) + + self.assertEqual(exts_file_info[3]['name'], 'toy') + self.assertEqual(exts_file_info[3]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz')) + self.assertFalse('patches' in exts_file_info[3]) + + # location of files is missing when fetch_files is set to False + exts_file_info = toy_eb.collect_exts_file_info(fetch_files=False, verify_checksums=False) + + self.assertTrue(isinstance(exts_file_info, list)) + self.assertEqual(len(exts_file_info), 4) + + self.assertEqual(exts_file_info[0], {'name': 'ls'}) + + self.assertEqual(exts_file_info[1]['name'], 'bar') + self.assertFalse('src' in exts_file_info[1]) + self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1) + self.assertFalse('path' in exts_file_info[1]['patches'][0]) + self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2) + self.assertFalse('path' in exts_file_info[1]['patches'][1]) + + self.assertEqual(exts_file_info[2]['name'], 'barbar') + self.assertFalse('src' in exts_file_info[2]) + self.assertFalse('patches' in exts_file_info[2]) + + self.assertEqual(exts_file_info[3]['name'], 'toy') + self.assertFalse('src' in exts_file_info[3]) + self.assertFalse('patches' in exts_file_info[3]) + + error_msg = "Can't verify checksums for extension files if they are not being fetched" + self.assertErrorRegex(EasyBuildError, error_msg, toy_eb.collect_exts_file_info, fetch_files=False) + def test_obtain_file_extension(self): """Test use of obtain_file method on an extension.""" @@ -1430,9 +1852,7 @@ def test_check_readiness(self): tmpdir = tempfile.mkdtemp() shutil.copy2(ec_path, tmpdir) ec_path = os.path.join(tmpdir, ec_file) - f = open(ec_path, 'a') - f.write("\ndependencies += [('nosuchsoftware', '1.2.3')]\n") - f.close() + write_file(ec_path, "\ndependencies += [('nosuchsoftware', '1.2.3')]\n", append=True) ec = EasyConfig(ec_path) eb = EasyBlock(ec) try: @@ -1544,16 +1964,18 @@ def test_extensions_sanity_check(self): test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + # Do this before loading the easyblock to check the non-translated output below + os.environ['LC_ALL'] = 'C' + # this import only works here, since EB_toy is a test easyblock from easybuild.easyblocks.toy import EB_toy # purposely inject failing custom extension filter for last extension toy_ec = EasyConfig(toy_ec_fn) - toy_ec.enable_templating = False - exts_list = toy_ec['exts_list'] - exts_list[-1][2]['exts_filter'] = ("thisshouldfail", '') - toy_ec['exts_list'] = exts_list - toy_ec.enable_templating = True + with toy_ec.disable_templating(): + exts_list = toy_ec['exts_list'] + exts_list[-1][2]['exts_filter'] = ("thisshouldfail", '') + toy_ec['exts_list'] = exts_list eb = EB_toy(toy_ec) eb.silent = True @@ -1761,6 +2183,60 @@ def test_prepare_step_hmns(self): self.assertEqual(len(loaded_modules), 1) self.assertEqual(loaded_modules[0]['mod_name'], 'GCC/6.4.0-2.28') + def test_prepare_step_cuda_cache(self): + """Test handling cuda-cache-* options.""" + + init_config(build_options={'cuda_cache_maxsize': None}) # Automatic mode + + test_ecs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + ec = process_easyconfig(toy_ec)[0] + eb = EasyBlock(ec['ec']) + eb.silent = True + eb.make_builddir() + + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertNotIn('Disabling CUDA PTX cache', logtxt) + self.assertNotIn('Enabling CUDA PTX cache', logtxt) + + # Now with CUDA + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ectxt = re.sub('^toolchain = .*', "toolchain = {'name': 'gcccuda', 'version': '2018a'}", + read_file(toy_ec), flags=re.M) + write_file(test_ec, test_ectxt) + ec = process_easyconfig(test_ec)[0] + eb = EasyBlock(ec['ec']) + eb.silent = True + eb.make_builddir() + + write_file(eb.logfile, '') + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertNotIn('Disabling CUDA PTX cache', logtxt) + self.assertIn('Enabling CUDA PTX cache', logtxt) + self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '0') + + init_config(build_options={'cuda_cache_maxsize': 0}) # Disable + write_file(eb.logfile, '') + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertIn('Disabling CUDA PTX cache', logtxt) + self.assertNotIn('Enabling CUDA PTX cache', logtxt) + self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '1') + + # Specified size and location + cuda_cache_dir = os.path.join(self.test_prefix, 'custom-cuda-cache') + init_config(build_options={'cuda_cache_maxsize': 1234, 'cuda_cache_dir': cuda_cache_dir}) + write_file(eb.logfile, '') + eb.prepare_step(start_dir=False) + logtxt = read_file(eb.logfile) + self.assertNotIn('Disabling CUDA PTX cache', logtxt) + self.assertIn('Enabling CUDA PTX cache', logtxt) + self.assertEqual(os.environ['CUDA_CACHE_DISABLE'], '0') + self.assertEqual(os.environ['CUDA_CACHE_MAXSIZE'], str(1234 * 1024 * 1024)) + self.assertEqual(os.environ['CUDA_CACHE_PATH'], cuda_cache_dir) + def test_checksum_step(self): """Test checksum step""" testdir = os.path.abspath(os.path.dirname(__file__)) @@ -1787,9 +2263,16 @@ def test_checksum_step(self): error_msg = "Checksum verification for .*/toy-0.0.tar.gz using .* failed" self.assertErrorRegex(EasyBuildError, error_msg, eb.checksum_step) - # also check verification of checksums for extensions, which is part of fetch_extension_sources + # also check verification of checksums for extensions, which is part of collect_exts_file_info error_msg = "Checksum verification for extension source bar-0.0.tar.gz failed" + self.assertErrorRegex(EasyBuildError, error_msg, eb.collect_exts_file_info) + + # also check with deprecated fetch_extension_sources method + self.allow_deprecated_behaviour() + self.mock_stderr(True) self.assertErrorRegex(EasyBuildError, error_msg, eb.fetch_extension_sources) + self.mock_stderr(False) + self.disallow_deprecated_behaviour() # if --ignore-checksums is enabled, faulty checksums are reported but otherwise ignored (no error) build_options = { @@ -1809,7 +2292,7 @@ def test_checksum_step(self): self.mock_stderr(True) self.mock_stdout(True) - eb.fetch_extension_sources() + eb.collect_exts_file_info() stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) @@ -1817,6 +2300,19 @@ def test_checksum_step(self): self.assertEqual(stdout, '') self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz") + # also check with deprecated fetch_extension_sources method + self.allow_deprecated_behaviour() + self.mock_stderr(True) + self.mock_stdout(True) + eb.fetch_extension_sources() + stderr = self.get_stderr() + stdout = self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stdout, '') + self.assertTrue(stderr.strip().endswith("WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz")) + self.disallow_deprecated_behaviour() + def test_check_checksums(self): """Test for check_checksums_for and check_checksums methods.""" testdir = os.path.abspath(os.path.dirname(__file__)) @@ -1884,6 +2380,21 @@ def run_checks(): # no checksum issues self.assertEqual(eb.check_checksums(), []) + # checksums as dict for some files + eb.cfg['checksums'] = [ + { + 'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', + 'toy-0.1.tar.gz': '123456789abcdef123456789abcdef123456789abcdef123456789abcdef1234', + }, + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487', # toy-*.patch + '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458', # toy-extra.txt + ] + self.assertEqual(eb.check_checksums(), []) + + # sources can also have dict entries + eb.cfg['sources'] = [{'filename': 'toy-0.0.tar.gz', 'download_fileame': 'toy.tar.gz'}] + self.assertEqual(eb.check_checksums(), []) + def test_this_is_easybuild(self): """Test 'this_is_easybuild' function (and get_git_revision function used by it).""" # make sure both return a non-Unicode string @@ -1973,34 +2484,6 @@ def test_avail_easyblocks(self): self.assertEqual(hpl['class'], 'EB_HPL') self.assertTrue(hpl['loc'].endswith('sandbox/easybuild/easyblocks/h/hpl.py')) - def test_time2str(self): - """Test time2str function.""" - - start = datetime(2019, 7, 30, 5, 14, 23) - - test_cases = [ - (start, "0 sec"), - (datetime(2019, 7, 30, 5, 14, 37), "14 sec"), - (datetime(2019, 7, 30, 5, 15, 22), "59 sec"), - (datetime(2019, 7, 30, 5, 15, 23), "1 min 0 sec"), - (datetime(2019, 7, 30, 5, 16, 22), "1 min 59 sec"), - (datetime(2019, 7, 30, 5, 37, 26), "23 min 3 sec"), - (datetime(2019, 7, 30, 6, 14, 22), "59 min 59 sec"), - (datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 min 0 sec"), - (datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 min 51 sec"), - (datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 min 0 sec"), - (datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 min 36 sec"), - (datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 min 1 sec"), - (datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 min 59 sec"), - (datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 min 0 sec"), - (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 min 21 sec"), - ] - for end, expected in test_cases: - self.assertEqual(time2str(end - start), expected) - - error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>" - self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123) - def test_sanity_check_paths_verification(self): """Test verification of sanity_check_paths w.r.t. keys & values.""" diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 4e47085936..a40c8c1df5 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,6 +37,7 @@ import stat import sys import tempfile +import textwrap from distutils.version import LooseVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -62,7 +63,7 @@ from easybuild.framework.extension import resolve_exts_filter_template from easybuild.toolchains.system import SystemToolchain from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import module_classes +from easybuild.tools.config import build_option, get_module_syntax, module_classes, update_build_option from easybuild.tools.configobj import ConfigObj from easybuild.tools.docs import avail_easyconfig_constants, avail_easyconfig_templates from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, read_file @@ -72,7 +73,8 @@ from easybuild.tools.options import parse_external_modules_metadata from easybuild.tools.py2vs3 import OrderedDict, reload from easybuild.tools.robot import resolve_dependencies -from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64 +from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import quote_str, quote_py_str from test.framework.utilities import find_full_path @@ -104,6 +106,7 @@ class EasyConfigTest(EnhancedTestCase): def setUp(self): """Set up everything for running a unit test.""" super(EasyConfigTest, self).setUp() + self.orig_get_cpu_architecture = st.get_cpu_architecture self.cwd = os.getcwd() self.all_stops = [x[0] for x in EasyBlock.get_steps()] @@ -122,6 +125,7 @@ def prep(self): def tearDown(self): """ make sure to remove the temporary file """ + st.get_cpu_architecture = self.orig_get_cpu_architecture super(EasyConfigTest, self).tearDown() if os.path.exists(self.eb_file): os.remove(self.eb_file) @@ -194,7 +198,13 @@ def test_validation(self): self.contents += "\nsyntax_error'" self.prep() - error_pattern = "Parsing easyconfig file failed: EOL while scanning string literal" + + # exact error message depends on Python version (different starting with Python 3.10) + if sys.version_info >= (3, 10): + error_pattern = "Parsing easyconfig file failed: unterminated string literal" + else: + error_pattern = "Parsing easyconfig file failed: EOL while scanning string literal" + self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, self.eb_file) # introduce "TypeError: format requires mapping" issue" @@ -305,6 +315,69 @@ def test_dependency(self): self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, (EXTERNAL_MODULE_MARKER,)) self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, ('foo', '1.2.3', EXTERNAL_MODULE_MARKER)) + def test_false_dep_version(self): + """ + Test use False as dependency version via dict using 'arch=' keys, + which should result in filtering the dependency. + """ + # silence warnings about missing easyconfigs for dependencies, we don't care + init_config(build_options={'silent': True}) + + arch = get_cpu_architecture() + + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'versionsuffix = "-test"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name":"GCC", "version": "4.6.3"}', + 'builddependencies = [', + ' ("first_build", {"arch=%s": False}),' % arch, + ' ("second_build", "2.0"),', + ']', + 'dependencies = [' + ' ("first", "1.0"),', + ' ("second", {"arch=%s": False}),' % arch, + ']', + ]) + self.prep() + eb = EasyConfig(self.eb_file) + deps = eb.dependencies() + self.assertEqual(len(deps), 2) + self.assertEqual(deps[0]['name'], 'second_build') + self.assertEqual(deps[1]['name'], 'first') + + # more realistic example: only filter dep for POWER + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'versionsuffix = "-test"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name":"GCC", "version": "4.6.3"}', + 'dependencies = [' + ' ("not_on_power", {"arch=*": "1.2.3", "arch=POWER": False}),', + ']', + ]) + self.prep() + + # only non-POWER arch, dependency is retained + for arch in (AARCH64, X86_64): + st.get_cpu_architecture = lambda: arch + eb = EasyConfig(self.eb_file) + deps = eb.dependencies() + self.assertEqual(len(deps), 1) + self.assertEqual(deps[0]['name'], 'not_on_power') + + # only power, dependency gets filtered + st.get_cpu_architecture = lambda: POWER + eb = EasyConfig(self.eb_file) + deps = eb.dependencies() + self.assertEqual(deps, []) + def test_extra_options(self): """ extra_options should allow other variables to be stored """ init_config(build_options={'silent': True}) @@ -395,7 +468,7 @@ def test_exts_list(self): ' ("ext1", "1.0"),', ' ("ext2", "2.0", {', ' "source_urls": [("http://example.com", "suffix")],' - ' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail + ' "patches": [("toy-0.0.eb", ".")],', # dummy patch to avoid downloading fail ' "checksums": [', # SHA256 checksum for source (gzip-1.4.eb) ' "6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45",', @@ -408,7 +481,7 @@ def test_exts_list(self): self.prep() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) - exts_sources = eb.fetch_extension_sources() + exts_sources = eb.collect_exts_file_info() self.assertEqual(len(exts_sources), 2) self.assertEqual(exts_sources[0]['name'], 'ext1') @@ -422,7 +495,7 @@ def test_exts_list(self): self.assertEqual(exts_sources[1]['options'], { 'checksums': ['6a5abcab719cefa95dca4af0db0d2a9d205d68f775a33b452ec0f2b75b6a3a45', '2d964e0e8f05a7cce0dd83a3e68c9737da14b87b61b8b8b0291d58d4c8d1031c'], - 'patches': ['toy-0.0.eb'], + 'patches': [('toy-0.0.eb', '.')], 'source_tmpl': 'gzip-1.4.eb', 'source_urls': [('http://example.com', 'suffix')], }) @@ -565,6 +638,8 @@ def test_tweaking(self): 'version = "3.14"', 'toolchain = {"name": "GCC", "version": "4.6.3"}', 'patches = %s', + 'parallel = 1', + 'keepsymlinks = True', ]) % str(patches) self.prep() @@ -581,7 +656,17 @@ def test_tweaking(self): 'versionprefix': verpref, 'versionsuffix': versuff, 'toolchain_version': tcver, - 'patches': new_patches + 'patches': new_patches, + 'keepsymlinks': 'True', # Don't change this + # It should be possible to overwrite values with True/False/None as they often have special meaning + 'runtest': 'False', + 'hidden': 'True', + 'parallel': 'None', # Good example: parallel=None means "Auto detect" + # Adding new options (added only by easyblock) should also be possible + # and in case the string "True/False/None" is really wanted it is possible to quote it first + 'test_none': '"False"', + 'test_bool': '"True"', + 'test_123': '"None"', } tweak_one(self.eb_file, tweaked_fn, tweaks) @@ -591,15 +676,20 @@ def test_tweaking(self): self.assertEqual(eb['versionsuffix'], versuff) self.assertEqual(eb['toolchain']['version'], tcver) self.assertEqual(eb['patches'], new_patches) + self.assertTrue(eb['runtest'] is False) + self.assertTrue(eb['hidden'] is True) + self.assertTrue(eb['parallel'] is None) + self.assertEqual(eb['test_none'], 'False') + self.assertEqual(eb['test_bool'], 'True') + self.assertEqual(eb['test_123'], 'None') remove_file(tweaked_fn) eb = EasyConfig(self.eb_file) # eb['toolchain']['version'] = tcver does not work as expected with templating enabled - eb.enable_templating = False - eb['version'] = ver - eb['toolchain']['version'] = tcver - eb.enable_templating = True + with eb.disable_templating(): + eb['version'] = ver + eb['toolchain']['version'] = tcver eb.dump(self.eb_file) tweaks = { @@ -773,13 +863,6 @@ def test_obtain_easyconfig(self): self.assertEqual(ec['start_dir'], specs['start_dir']) remove_file(res[1]) - specs.update({ - 'foo': 'bar123' - }) - self.assertErrorRegex(EasyBuildError, "Unknown easyconfig parameter: foo", - obtain_ec_for, specs, [self.test_prefix], None) - del specs['foo'] - # should pick correct version, i.e. not newer than what's specified, if a choice needs to be made ver = '3.14' specs.update({'version': ver}) @@ -955,8 +1038,8 @@ def trim_path(path): self.assertEqual(ec['name'], specs['name']) os.remove(res[1]) - def test_templating(self): - """ test easyconfig templating """ + def test_templating_constants(self): + """Test use of template values and constants in an easyconfig file.""" inp = { 'name': 'PI', # purposely using minor version that starts with a 0, to check for correct version_minor value @@ -977,7 +1060,7 @@ def test_templating(self): 'sources = [SOURCE_TAR_GZ, (SOURCELOWER_TAR_BZ2, "%(cmd)s")]', 'sanity_check_paths = {', ' "files": ["bin/pi_%%(version_major)s_%%(version_minor)s", "lib/python%%(pyshortver)s/site-packages"],', - ' "dirs": ["libfoo.%%s" %% SHLIB_EXT, "lib/%%(arch)s"],', + ' "dirs": ["libfoo.%%s" %% SHLIB_EXT, "lib/%%(arch)s/" + SYS_PYTHON_VERSION, "include/" + ARCH],', '}', 'dependencies = [', ' ("CUDA", "10.1.105"),' @@ -1004,12 +1087,9 @@ def test_templating(self): eb.validate() # temporarily disable templating, just so we can check later whether it's *still* disabled - eb.enable_templating = False - - eb.generate_template_values() - - self.assertFalse(eb.enable_templating) - eb.enable_templating = True + with eb.disable_templating(): + eb.generate_template_values() + self.assertFalse(eb.enable_templating) self.assertEqual(eb['description'], "test easyconfig PI") self.assertEqual(eb['sources'][0], 'PI-3.04.tar.gz') @@ -1020,9 +1100,13 @@ def test_templating(self): self.assertEqual(eb['sanity_check_paths']['files'][0], 'bin/pi_3_04') self.assertEqual(eb['sanity_check_paths']['files'][1], 'lib/python2.7/site-packages') self.assertEqual(eb['sanity_check_paths']['dirs'][0], 'libfoo.%s' % get_shared_lib_ext()) - lib_arch_regex = re.compile('^lib/[a-z0-9_]+$') # should match lib/x86_64, lib/aarch64, lib/ppc64le, etc. + # should match lib/x86_64/2.7.18, lib/aarch64/3.8.6, lib/ppc64le/3.9.2, etc. + lib_arch_regex = re.compile(r'^lib/[a-z0-9_]+/[23]\.[0-9]+\.[0-9]+$') dirs1 = eb['sanity_check_paths']['dirs'][1] - self.assertTrue(lib_arch_regex.match(dirs1), "Pattern '%s' matches '%s'" % (lib_arch_regex.pattern, dirs1)) + self.assertTrue(lib_arch_regex.match(dirs1), "Pattern '%s' should match '%s'" % (lib_arch_regex.pattern, dirs1)) + inc_regex = re.compile('^include/(aarch64|ppc64le|x86_64)$') + dirs2 = eb['sanity_check_paths']['dirs'][2] + self.assertTrue(inc_regex.match(dirs2), "Pattern '%s' should match '%s'" % (inc_regex, dirs2)) self.assertEqual(eb['homepage'], "http://example.com/P/p/v3/") expected = ("CUDA: 10.1.105, 10, 1, 10.1; " "Java: 1.7.80, 1, 7, 1.7; " @@ -1050,6 +1134,51 @@ def test_templating(self): ec = EasyConfig(test_ec) self.assertEqual(ec['sanity_check_commands'], ['mpiexec -np 1 -- toy']) + def test_templating_cuda_toolchain(self): + """Test templates via toolchain component, like setting %(cudaver)s with fosscuda toolchain.""" + + build_options = {'robot_path': [self.test_prefix]} + init_config(build_options=build_options) + + # create fake easyconfig files, good enough to test with + cuda_ec = os.path.join(self.test_prefix, 'CUDA-10.1.243') + cuda_ec_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'CUDA'", + "version = '10.1.243'", + "homepage = 'https://example.com'", + "description = 'CUDA'", + "toolchain = SYSTEM", + ]) + write_file(cuda_ec, cuda_ec_txt) + + fosscuda_ec = os.path.join(self.test_prefix, 'fosscuda-2021.02.eb') + fosscuda_ec_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'fosscuda'", + "version = '2021.02'", + "homepage = 'https://example.com'", + "description = 'fosscuda toolchain'", + "toolchain = SYSTEM", + "dependencies = [('CUDA', '10.1.243')]", + ]) + write_file(fosscuda_ec, fosscuda_ec_txt) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'test'", + "version = '1.0'", + "homepage = 'https://example.com'", + "description = 'just a test'", + "toolchain = {'name': 'fosscuda', 'version': '2021.02'}", + ]) + write_file(test_ec, test_ec_txt) + ec = EasyConfig(test_ec) + self.assertEqual(ec.template_values['cudaver'], '10.1.243') + self.assertEqual(ec.template_values['cudamajver'], '10') + self.assertEqual(ec.template_values['cudashortver'], '10.1') + def test_java_wrapper_templating(self): """test templating when the Java wrapper is a dep""" self.contents = '\n'.join([ @@ -1073,6 +1202,37 @@ def test_java_wrapper_templating(self): self.assertEqual(eb['modloadmsg'], "Java: 11, 11, 11") + def test_python_whl_templating(self): + """test templating for Python wheels""" + + self.contents = textwrap.dedent(""" + easyblock = "ConfigureMake" + name = "Pi" + version = "3.14" + homepage = "https://example.com" + description = "test easyconfig" + toolchain = {"name":"GCC", "version": "4.6.3"} + sources = [ + SOURCE_WHL, + SOURCELOWER_WHL, + SOURCE_PY2_WHL, + SOURCELOWER_PY2_WHL, + SOURCE_PY3_WHL, + SOURCELOWER_PY3_WHL, + ] + """) + self.prep() + ec = EasyConfig(self.eb_file) + + sources = ec['sources'] + + self.assertEqual(sources[0], 'Pi-3.14-py2.py3-none-any.whl') + self.assertEqual(sources[1], 'pi-3.14-py2.py3-none-any.whl') + self.assertEqual(sources[2], 'Pi-3.14-py2-none-any.whl') + self.assertEqual(sources[3], 'pi-3.14-py2-none-any.whl') + self.assertEqual(sources[4], 'Pi-3.14-py3-none-any.whl') + self.assertEqual(sources[5], 'pi-3.14-py3-none-any.whl') + def test_templating_doc(self): """test templating documentation""" doc = avail_easyconfig_templates() @@ -1083,6 +1243,7 @@ def test_templating_doc(self): easyconfig.templates.TEMPLATE_NAMES_CONFIG, easyconfig.templates.TEMPLATE_NAMES_LOWER, easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, + easyconfig.templates.TEMPLATE_NAMES_DYNAMIC, easyconfig.templates.TEMPLATE_CONSTANTS, ] @@ -1181,8 +1342,7 @@ def test_buildininstalldir(self): self.prep() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) - eb.gen_builddir() - eb.gen_installdir() + eb.post_init() eb.make_builddir() eb.make_installdir() self.assertEqual(eb.builddir, eb.installdir) @@ -1512,7 +1672,6 @@ def test_external_dependencies(self): # by adding a couple of matching module files with some useful data in them # (use Tcl syntax, so it works with all varieties of module tools) mod_dir = os.path.join(self.test_prefix, 'modules') - self.modtool.use(mod_dir) pi_mod_txt = '\n'.join([ "#%Module", @@ -1537,6 +1696,8 @@ def test_external_dependencies(self): ]) write_file(os.path.join(mod_dir, 'foobar/2.3.4'), foobar_mod_txt) + self.modtool.use(mod_dir) + ec = EasyConfig(toy_ec) deps = ec.dependencies() @@ -1548,24 +1709,24 @@ def test_external_dependencies(self): self.assertEqual(deps[3]['full_mod_name'], 'foobar/1.2.3') foobar_metadata = { 'name': ['foobar'], - 'prefix': '/software/foobar/2.3.4', - 'version': ['2.3.4'], + 'prefix': 'CRAY_FOOBAR_DIR', + 'version': ['CRAY_FOOBAR_VERSION'], } self.assertEqual(deps[3]['external_module_metadata'], foobar_metadata) self.assertEqual(deps[5]['full_mod_name'], 'pi/3.14') pi_metadata = { 'name': ['pi'], - 'prefix': '/software/pi/3.14', - 'version': ['3.14'], + 'prefix': 'PI_ROOT', + 'version': ['PI_VERSION'], } self.assertEqual(deps[5]['external_module_metadata'], pi_metadata) self.assertEqual(deps[7]['full_mod_name'], 'cray-netcdf-hdf5parallel/1.10.6') cray_netcdf_metadata = { 'name': ['netcdf-hdf5parallel'], - 'prefix': '/software/cray-netcdf-hdf5parallel/1.10.6', - 'version': ['1.10.6'], + 'prefix': 'CRAY_NETCDF_HDF5PARALLEL_PREFIX', + 'version': ['CRAY_NETCDF_HDF5PARALLEL_VERSION'], } self.assertEqual(deps[7]['external_module_metadata'], cray_netcdf_metadata) @@ -1584,7 +1745,8 @@ def test_external_dependencies(self): 'name = TEST', '[cray-netcdf-hdf5parallel/1.10.6]', 'name = HDF5', - 'version = 1.10.6', + # purpose omit version, to see whether fallback of + # resolving $CRAY_NETCDF_HDF5PARALLEL_VERSION at runtime is used ]) write_file(metadata, metadatatxt) build_options = { @@ -1603,7 +1765,7 @@ def test_external_dependencies(self): self.assertEqual(deps[3]['full_mod_name'], 'foobar/1.2.3') foobar_metadata = { 'name': ['foobar'], # probed from 'foobar' module - 'prefix': '/software/foobar/2.3.4', # probed from 'foobar' module + 'prefix': 'CRAY_FOOBAR_DIR', # probed from 'foobar' module 'version': ['1.2.3'], # from [foobar/1.2.3] entry in metadata file } self.assertEqual(deps[3]['external_module_metadata'], foobar_metadata) @@ -1617,7 +1779,7 @@ def test_external_dependencies(self): self.assertEqual(deps[5]['full_mod_name'], 'pi/3.14') pi_metadata = { 'name': ['PI'], # from [pi/3.14] entry in metadata file - 'prefix': '/software/pi/3.14', # probed from 'pi/3.14' module + 'prefix': 'PI_ROOT', # probed from 'pi/3.14' module 'version': ['3.14.0'], # from [pi/3.14] entry in metadata file } self.assertEqual(deps[5]['external_module_metadata'], pi_metadata) @@ -1625,8 +1787,8 @@ def test_external_dependencies(self): self.assertEqual(deps[7]['full_mod_name'], 'cray-netcdf-hdf5parallel/1.10.6') cray_netcdf_metadata = { 'name': ['HDF5'], - 'prefix': '/software/cray-netcdf-hdf5parallel/1.10.6', - 'version': ['1.10.6'], + 'prefix': 'CRAY_NETCDF_HDF5PARALLEL_PREFIX', + 'version': ['CRAY_NETCDF_HDF5PARALLEL_VERSION'], } self.assertEqual(deps[7]['external_module_metadata'], cray_netcdf_metadata) @@ -2651,12 +2813,12 @@ def test_find_related_easyconfigs(self): # tweak version to 4.6.1, GCC/4.6.x easyconfigs are found as closest match ec['version'] = '4.6.1' res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] - self.assertEqual(res, ['GCC-4.6.3.eb', 'GCC-4.6.4.eb']) + self.assertEqual(res, ['GCC-4.6.4.eb', 'GCC-4.6.3.eb']) # tweak version to 4.5.0, GCC/4.x easyconfigs are found as closest match ec['version'] = '4.5.0' res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] - expected = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb'] + expected = ['GCC-4.9.2.eb', 'GCC-4.8.3.eb', 'GCC-4.8.2.eb', 'GCC-4.6.4.eb', 'GCC-4.6.3.eb'] self.assertEqual(res, expected) ec_file = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-deps.eb') @@ -2670,7 +2832,7 @@ def test_find_related_easyconfigs(self): ec['toolchain'] = {'name': 'gompi', 'version': '1.5.16'} ec['versionsuffix'] = '-foobar' res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] - self.assertEqual(res, ['toy-0.0-gompi-2018a-test.eb', 'toy-0.0-gompi-2018a.eb']) + self.assertEqual(res, ['toy-0.0-gompi-2018a.eb', 'toy-0.0-gompi-2018a-test.eb']) # restore original versionsuffix => matching versionsuffix wins over matching toolchain (name) ec['versionsuffix'] = '-deps' @@ -2896,6 +3058,10 @@ def test_template_constant_dict(self): self.assertEqual(res, expected) # mock get_avail_core_count which is used by set_parallel -> det_parallelism + try: + del st.det_parallelism._default_parallelism # Remove cache value + except AttributeError: + pass # Ignore if not present orig_get_avail_core_count = st.get_avail_core_count st.get_avail_core_count = lambda: 42 @@ -2927,26 +3093,29 @@ def test_template_constant_dict(self): " 'arch=fooarch': '1.8.0-foo',", " })", "]", + "builddependencies = [", + " ('CMake', '3.18.4'),", + "]", ]) test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, toy_ec_txt) - # only perform shallow/quick parse (as is done in list_software function) - ec = EasyConfigParser(filename=test_ec).get_config_dict() - expected = { + 'bitbucket_account': 'toy', + 'github_account': 'toy', 'javamajver': '1', 'javaminver': '8', 'javashortver': '1.8', 'javaver': '1.8.0_221', - 'module_name': None, + 'module_name': 'toy/0.01-deps', 'name': 'toy', 'namelower': 'toy', 'nameletter': 't', 'toolchain_name': 'system', 'toolchain_version': 'system', 'nameletterlower': 't', + 'parallel': None, 'pymajver': '3', 'pyminver': '7', 'pyshortver': '3.7', @@ -2955,9 +3124,38 @@ def test_template_constant_dict(self): 'version_major': '0', 'version_major_minor': '0.01', 'version_minor': '01', + 'versionprefix': '', 'versionsuffix': '-deps', } + + # proper EasyConfig instance + ec = EasyConfig(test_ec) + + # CMake should *not* be included, since it's a build-only dependency + dep_names = [x['name'] for x in ec['dependencies']] + self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) res = template_constant_dict(ec) + dep_names = [x['name'] for x in ec['dependencies']] + self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) + + self.assertTrue('arch' in res) + arch = res.pop('arch') + self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern)) + + self.assertEqual(res, expected) + + # only perform shallow/quick parse (as is done in list_software function) + ec = EasyConfigParser(filename=test_ec).get_config_dict() + + expected['module_name'] = None + for key in ('bitbucket_account', 'github_account', 'parallel', 'versionprefix'): + del expected[key] + + dep_names = [x[0] for x in ec['dependencies']] + self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) + res = template_constant_dict(ec) + dep_names = [x[0] for x in ec['dependencies']] + self.assertFalse('CMake' in dep_names, "CMake should not be included in list of dependencies: %s" % dep_names) self.assertTrue('arch' in res) arch = res.pop('arch') @@ -3079,10 +3277,11 @@ def test_categorize_files_by_type(self): configuremake = os.path.join(easyblocks_dir, 'generic', 'configuremake.py') toy_easyblock = os.path.join(easyblocks_dir, 't', 'toy.py') + gzip_ec = os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb') paths = [ 'bzip2-1.0.6.eb', toy_easyblock, - os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb'), + gzip_ec, toy_patch, 'foo', ':toy-0.0-deps.eb', @@ -3091,7 +3290,7 @@ def test_categorize_files_by_type(self): res = categorize_files_by_type(paths) expected = [ 'bzip2-1.0.6.eb', - os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb'), + gzip_ec, 'foo', ] self.assertEqual(res['easyconfigs'], expected) @@ -3099,6 +3298,23 @@ def test_categorize_files_by_type(self): self.assertEqual(res['patch_files'], [toy_patch]) self.assertEqual(res['py_files'], [toy_easyblock, configuremake]) + # Error cases + tmpdir = tempfile.mkdtemp() + non_existing = os.path.join(tmpdir, 'does_not_exist.patch') + self.assertErrorRegex(EasyBuildError, + "File %s does not exist" % non_existing, + categorize_files_by_type, [non_existing]) + patch_dir = os.path.join(tmpdir, 'folder.patch') + os.mkdir(patch_dir) + self.assertErrorRegex(EasyBuildError, + "File %s is expected to be a regular file" % patch_dir, + categorize_files_by_type, [patch_dir]) + invalid_patch = os.path.join(tmpdir, 'invalid.patch') + copy_file(gzip_ec, invalid_patch) + self.assertErrorRegex(EasyBuildError, + "%s is not detected as a valid patch file" % invalid_patch, + categorize_files_by_type, [invalid_patch]) + def test_resolve_template(self): """Test resolve_template function.""" self.assertEqual(resolve_template('', {}), '') @@ -3194,6 +3410,12 @@ def test_det_subtoolchain_version(self): for subtoolchain_name in subtoolchains[current_tc['name']]] self.assertEqual(versions, ['4.9.3', '']) + # test det_subtoolchain_version when two alternatives for subtoolchain are specified + current_tc = {'name': 'gompi', 'version': '2018b'} + cands = [{'name': 'GCC', 'version': '7.3.0-2.30'}] + subtc_version = det_subtoolchain_version(current_tc, ('GCCcore', 'GCC'), optional_toolchains, cands) + self.assertEqual(subtc_version, '7.3.0-2.30') + def test_verify_easyconfig_filename(self): """Test verify_easyconfig_filename function""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -3263,6 +3485,11 @@ def test_get_paths_for(self): self.mock_stderr(False) self.assertTrue(os.path.samefile(test_ecs, res[0])) + # Can't have EB_SCRIPT_PATH set (for some of) these tests + env_eb_script_path = os.getenv('EB_SCRIPT_PATH') + if env_eb_script_path: + del os.environ['EB_SCRIPT_PATH'] + # easyconfigs location can also be derived from location of 'eb' write_file(os.path.join(self.test_prefix, 'bin', 'eb'), "#!/bin/bash; echo 'This is a fake eb'") adjust_permissions(os.path.join(self.test_prefix, 'bin', 'eb'), stat.S_IXUSR) @@ -3279,6 +3506,10 @@ def test_get_paths_for(self): res = get_paths_for(subdir='easyconfigs', robot_path=None) self.assertTrue(os.path.samefile(test_ecs, res[-1])) + # Restore (temporarily) EB_SCRIPT_PATH value if set originally + if env_eb_script_path: + os.environ['EB_SCRIPT_PATH'] = env_eb_script_path + # also locations in sys.path are considered os.environ['PATH'] = orig_path sys.path.insert(0, self.test_prefix) @@ -3329,6 +3560,10 @@ def test_get_paths_for(self): self.assertTrue(os.path.exists(res[0])) self.assertTrue(os.path.samefile(res[0], os.path.join(someprefix, 'easybuild', 'easyconfigs'))) + # Finally restore EB_SCRIPT_PATH value if set + if env_eb_script_path: + os.environ['EB_SCRIPT_PATH'] = env_eb_script_path + def test_is_generic_easyblock(self): """Test for is_generic_easyblock function.""" @@ -3360,7 +3595,9 @@ def test_not_an_easyconfig(self): # cfr. https://github.com/easybuilders/easybuild-framework/issues/2383 not_an_ec = os.path.join(os.path.dirname(test_ecs_dir), 'sandbox', 'not_an_easyconfig.eb') - error_pattern = "Parsing easyconfig file failed: invalid syntax" + # from Python 3.10 onwards: invalid decimal literal + # older Python versions: invalid syntax + error_pattern = "Parsing easyconfig file failed: invalid" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, not_an_ec) def test_check_sha256_checksums(self): @@ -4007,6 +4244,8 @@ def test_cuda_compute_capabilities(self): "toolchain = SYSTEM", "cuda_compute_capabilities = ['5.1', '7.0', '7.1']", "installopts = '%(cuda_compute_capabilities)s'", + "preinstallopts = '%(cuda_cc_space_sep)s'", + "prebuildopts = '%(cuda_cc_semicolon_sep)s'", "configopts = '%(cuda_sm_comma_sep)s'", "preconfigopts = '%(cuda_sm_space_sep)s'", ]) @@ -4014,6 +4253,8 @@ def test_cuda_compute_capabilities(self): ec = EasyConfig(test_ec) self.assertEqual(ec['installopts'], '5.1,7.0,7.1') + self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') + self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') self.assertEqual(ec['configopts'], 'sm_51,sm_70,sm_71') self.assertEqual(ec['preconfigopts'], 'sm_51 sm_70 sm_71') @@ -4021,6 +4262,8 @@ def test_cuda_compute_capabilities(self): init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']}) ec = EasyConfig(test_ec) self.assertEqual(ec['installopts'], '4.2,6.3') + self.assertEqual(ec['preinstallopts'], '4.2 6.3') + self.assertEqual(ec['prebuildopts'], '4.2;6.3') self.assertEqual(ec['configopts'], 'sm_42,sm_63') self.assertEqual(ec['preconfigopts'], 'sm_42 sm_63') @@ -4102,6 +4345,264 @@ def test_det_copy_ec_specs(self): self.assertEqual(os.path.basename(paths[3]), bat_patch_fn) self.assertTrue(os.path.samefile(target_path, cwd)) + def test_recursive_module_unload(self): + """Test use of recursive_module_unload easyconfig parameter.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + write_file(test_ec, test_ec_txt) + + test_module = os.path.join(self.test_installpath, 'modules', 'all', 'foss', '2018a') + gcc_modname = 'GCC/6.4.0-2.28' + if get_module_syntax() == 'Lua': + test_module += '.lua' + guarded_load_pat = r'if not \( isloaded\("%(mod)s"\) \) then\n\s*load\("%(mod)s"\)' + recursive_unload_pat = r'if mode\(\) == "unload" or not \( isloaded\("%(mod)s"\) \) then\n' + recursive_unload_pat += r'\s*load\("%(mod)s"\)' + else: + guarded_load_pat = r'if { \!\[ is-loaded %(mod)s \] } {\n\s*module load %(mod)s' + recursive_unload_pat = r'if { \[ module-info mode remove \] \|\| \!\[ is-loaded %(mod)s \] } {\n' + recursive_unload_pat += r'\s*module load %(mod)s' + + guarded_load_regex = re.compile(guarded_load_pat % {'mod': gcc_modname}, re.M) + recursive_unload_regex = re.compile(recursive_unload_pat % {'mod': gcc_modname}, re.M) + + # by default, recursive module unloading is disabled everywhere + # (--recursive-module-unload configuration option is disabled, + # recursive_module_unload easyconfig parameter is None) + self.assertFalse(build_option('recursive_mod_unload')) + ec = EasyConfig(test_ec) + self.assertFalse(ec['recursive_module_unload']) + eb = EasyBlock(ec) + eb.builddir = self.test_prefix + eb.prepare_step() + eb.make_module_step() + modtxt = read_file(test_module) + fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertTrue(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg) + + remove_file(test_module) + + # recursive_module_unload easyconfig parameter is honored + test_ec_bis = os.path.join(self.test_prefix, 'test_bis.eb') + test_ec_bis_txt = read_file(toy_ec) + '\nrecursive_module_unload = True' + write_file(test_ec_bis, test_ec_bis_txt) + + ec_bis = EasyConfig(test_ec_bis) + self.assertTrue(ec_bis['recursive_module_unload']) + eb_bis = EasyBlock(ec_bis) + eb_bis.builddir = self.test_prefix + eb_bis.prepare_step() + eb_bis.make_module_step() + modtxt = read_file(test_module) + fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertFalse(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg) + + # recursive_mod_unload build option is honored + update_build_option('recursive_mod_unload', True) + eb = EasyBlock(ec) + eb.builddir = self.test_prefix + eb.prepare_step() + eb.make_module_step() + modtxt = read_file(test_module) + fail_msg = "Pattern '%s' should not be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertFalse(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertTrue(recursive_unload_regex.search(modtxt), fail_msg) + + # disabling via easyconfig parameter works even when recursive_mod_unload build option is enabled + self.assertTrue(build_option('recursive_mod_unload')) + test_ec_bis = os.path.join(self.test_prefix, 'test_bis.eb') + test_ec_bis_txt = read_file(toy_ec) + '\nrecursive_module_unload = False' + write_file(test_ec_bis, test_ec_bis_txt) + ec_bis = EasyConfig(test_ec_bis) + self.assertEqual(ec_bis['recursive_module_unload'], False) + eb_bis = EasyBlock(ec_bis) + eb_bis.builddir = self.test_prefix + eb_bis.prepare_step() + eb_bis.make_module_step() + modtxt = read_file(test_module) + fail_msg = "Pattern '%s' should be found in: %s" % (guarded_load_regex.pattern, modtxt) + self.assertTrue(guarded_load_regex.search(modtxt), fail_msg) + fail_msg = "Pattern '%s' should not be found in: %s" % (recursive_unload_regex.pattern, modtxt) + self.assertFalse(recursive_unload_regex.search(modtxt), fail_msg) + + def test_pure_ec(self): + """ + Test whether we can get a 'pure' view on the easyconfig file, + which correctly reflects what's defined in the easyconfig file. + """ + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) + + ec_dict = toy_ec.parser.get_config_dict() + self.assertEqual(ec_dict.get('version'), '0.0') + self.assertEqual(ec_dict.get('sources'), ['%(name)s-%(version)s.tar.gz']) + self.assertEqual(ec_dict.get('exts_default_options'), None) + self.assertEqual(ec_dict.get('sanity_check_paths'), {'dirs': ['bin'], 'files': [('bin/yot', 'bin/toy')]}) + + # manipulating easyconfig parameter values should not affect the result of parser.get_config_dict() + with toy_ec.disable_templating(): + toy_ec['version'] = '1.2.3' + toy_ec['sources'].append('test.tar.gz') + toy_ec['sanity_check_paths']['files'].append('bin/foobar.exe') + + ec_dict_bis = toy_ec.parser.get_config_dict() + self.assertEqual(ec_dict_bis.get('version'), '0.0') + self.assertEqual(ec_dict_bis.get('sources'), ['%(name)s-%(version)s.tar.gz']) + self.assertEqual(ec_dict_bis.get('exts_default_options'), None) + self.assertEqual(ec_dict.get('sanity_check_paths'), {'dirs': ['bin'], 'files': [('bin/yot', 'bin/toy')]}) + + def test_easyconfig_import(self): + """ + Test parsing of an easyconfig file that includes import statements. + """ + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "import os", + "local_test = os.getenv('TEST_TOY')", + "sanity_check_commands = ['toy | grep %s' % local_test]", + ]) + write_file(test_ec, test_ec_txt) + + os.environ['TEST_TOY'] = '123' + + ec = EasyConfig(test_ec) + + self.assertEqual(ec['sanity_check_commands'], ['toy | grep 123']) + + # inject weird stuff, like a class definition that creates a logger instance + # and a local variable with a list of imported modules, to check clean error handling + test_ec_txt += '\n' + '\n'.join([ + "import logging", + "class _TestClass(object):", + " def __init__(self):", + " self.log = logging.Logger('alogger')", + "local_test = _TestClass()", + "local_modules = [logging, os]", + ]) + write_file(test_ec, test_ec_txt) + + error_pattern = r"Failed to copy '.*' easyconfig parameter" + self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) + + def test_get_cuda_cc_template_value(self): + """ + Test getting template value based on --cuda-compute-capabilities / cuda_compute_capabilities. + """ + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + self.prep() + ec = EasyConfig(self.eb_file) + + error_pattern = "foobar is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities" + self.assertErrorRegex(EasyBuildError, error_pattern, ec.get_cuda_cc_template_value, 'foobar') + + error_pattern = r"Template value '%s' is not defined!\n" + error_pattern += r"Make sure that either the --cuda-compute-capabilities EasyBuild configuration " + error_pattern += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." + cuda_template_values = { + 'cuda_compute_capabilities': '6.5,7.0', + 'cuda_cc_space_sep': '6.5 7.0', + 'cuda_cc_semicolon_sep': '6.5;7.0', + 'cuda_sm_comma_sep': 'sm_65,sm_70', + 'cuda_sm_space_sep': 'sm_65 sm_70', + } + for key in cuda_template_values: + self.assertErrorRegex(EasyBuildError, error_pattern % key, ec.get_cuda_cc_template_value, key) + + update_build_option('cuda_compute_capabilities', ['6.5', '7.0']) + ec = EasyConfig(self.eb_file) + + for key in cuda_template_values: + self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key]) + + update_build_option('cuda_compute_capabilities', None) + ec = EasyConfig(self.eb_file) + + for key in cuda_template_values: + self.assertErrorRegex(EasyBuildError, error_pattern % key, ec.get_cuda_cc_template_value, key) + + self.contents += "\ncuda_compute_capabilities = ['6.5', '7.0']" + self.prep() + ec = EasyConfig(self.eb_file) + + for key in cuda_template_values: + self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key]) + + def test_count_files(self): + """Tests for EasyConfig.count_files method.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + + foss = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') + toy = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + toy_exts = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + + # no sources or patches for toolchain => 0 + foss_ec = EasyConfig(foss) + self.assertEqual(foss_ec['sources'], []) + self.assertEqual(foss_ec['patches'], []) + self.assertEqual(foss_ec.count_files(), 0) + # 1 source + 2 patches => 3 + toy_ec = EasyConfig(toy) + self.assertEqual(len(toy_ec['sources']), 1) + self.assertEqual(len(toy_ec['patches']), 2) + self.assertEqual(toy_ec['exts_list'], []) + self.assertEqual(toy_ec.count_files(), 3) + # 1 source + 1 patch + # 4 extensions + # * ls: no sources/patches (only name is specified) + # * bar: 1 source (implied, using default source_tmpl) + 2 patches + # * barbar: 1 source (implied, using default source_tmpl) + # * toy: 1 source (implied, using default source_tmpl) + # => 7 files in total + toy_exts_ec = EasyConfig(toy_exts) + self.assertEqual(len(toy_exts_ec['sources']), 1) + self.assertEqual(len(toy_exts_ec['patches']), 1) + self.assertEqual(len(toy_exts_ec['exts_list']), 4) + self.assertEqual(toy_exts_ec.count_files(), 7) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + copy_file(toy_exts, test_ec) + # add a couple of additional extensions to verify correct file count + test_ec_extra = '\n'.join([ + 'exts_list += [', + ' ("test-ext-one", "0.0", {', + ' "sources": ["test-ext-one-0.0-part1.tgz", "test-ext-one-0.0-part2.zip"],', + # if both 'sources' and 'source_tmpl' are specified, 'source_tmpl' is ignored, + # see EasyBlock.fetch_extension_sources, so it should be too when counting files + ' "source_tmpl": "test-ext-one-%(version)s.tar.gz",', + ' }),', + ' ("test-ext-two", "0.0", {', + ' "source_tmpl": "test-ext-two-0.0-part1.tgz",', + ' "patches": ["test-ext-two.patch"],', + ' }),', + ']', + ]) + write_file(test_ec, test_ec_extra, append=True) + test_ec = EasyConfig(test_ec) + self.assertEqual(test_ec.count_files(), 11) + + def test_ARCH(self): + """Test ARCH easyconfig constant.""" + arch = easyconfig.constants.EASYCONFIG_CONSTANTS['ARCH'][0] + self.assertTrue(arch in KNOWN_ARCH_CONSTANTS, "Unexpected value for ARCH constant: %s" % arch) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py index 777693a6ea..8c2ab8aa16 100644 --- a/test/framework/easyconfigformat.py +++ b/test/framework/easyconfigformat.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index 436858ffb2..1a6134c0d3 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -59,6 +59,14 @@ def test_v10(self): self.assertEqual(ec['name'], 'GCC') self.assertEqual(ec['version'], '4.6.3') + # changes to this dict should not affect the return value of the next call to get_config_dict + fn = 'test.tar.gz' + ec['sources'].append(fn) + + ec_bis = ecp.get_config_dict() + self.assertTrue(fn in ec['sources']) + self.assertFalse(fn in ec_bis['sources']) + def test_v20(self): """Test parsing of easyconfig in format v2.""" # hard enable experimental @@ -81,6 +89,14 @@ def test_v20(self): self.assertEqual(ec['name'], 'GCC') self.assertEqual(ec['version'], '4.6.2') + # changes to this dict should not affect the return value of the next call to get_config_dict + fn = 'test.tar.gz' + ec['sources'].append(fn) + + ec_bis = ecp.get_config_dict() + self.assertTrue(fn in ec['sources']) + self.assertFalse(fn in ec_bis['sources']) + # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental diff --git a/test/framework/easyconfigs/test_ecs/c/cpeGNU/cpeGNU-21.04.eb b/test/framework/easyconfigs/test_ecs/c/cpeGNU/cpeGNU-21.04.eb new file mode 100644 index 0000000000..cf9af655db --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/c/cpeGNU/cpeGNU-21.04.eb @@ -0,0 +1,16 @@ +easyblock = 'Toolchain' + +name = 'cpeGNU' +version = '21.04' + +homepage = 'https://pubs.cray.com' +description = """Toolchain using Cray compiler wrapper with gcc module (CPE release: %(version)s).""" + +toolchain = SYSTEM + +dependencies = [ + ('PrgEnv-gnu', EXTERNAL_MODULE), + ('cpe/%(version)s', EXTERNAL_MODULE), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2021a.eb b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2021a.eb new file mode 100644 index 0000000000..b5ebf60d37 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2021a.eb @@ -0,0 +1,18 @@ +# This is an easyconfig file for EasyBuild, see http://easybuilders.github.io/easybuild +easyblock = 'Toolchain' + +name = 'iimpi' +version = '2021a' + +homepage = 'https://software.intel.com/parallel-studio-xe' +description = """Intel C/C++ and Fortran compilers, alongside Intel MPI.""" + +toolchain = SYSTEM + +local_comp_ver = '2021.2.0' +dependencies = [ + ('intel-compilers', local_comp_ver), + ('impi', local_comp_ver, '', ('intel-compilers', local_comp_ver)), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/i/imkl/imkl-2021.2.0-iimpi-2021a.eb b/test/framework/easyconfigs/test_ecs/i/imkl/imkl-2021.2.0-iimpi-2021a.eb new file mode 100644 index 0000000000..4d047da06e --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/imkl/imkl-2021.2.0-iimpi-2021a.eb @@ -0,0 +1,17 @@ +# dummy easyconfig, only for use with easybuild-framwork test suite! +easyblock = 'Toolchain' + +name = 'imkl' +version = '2021.2.0' + +homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html' +description = "Intel oneAPI Math Kernel Library" + +toolchain = {'name': 'iimpi', 'version': '2021a'} + +# see https://software.intel.com/content/www/us/en/develop/articles/oneapi-standalone-components.html +source_urls = ['https://registrationcenter-download.intel.com/akdlm/irc_nas/tec/17757/'] +sources = ['l_onemkl_p_%(version)s.296_offline.sh'] +checksums = ['816e9df26ff331d6c0751b86ed5f7d243f9f172e76f14e83b32bf4d1d619dbae'] + +moduleclass = 'numlib' diff --git a/test/framework/easyconfigs/test_ecs/i/impi/impi-2021.2.0-intel-compilers-2021.2.0.eb b/test/framework/easyconfigs/test_ecs/i/impi/impi-2021.2.0-intel-compilers-2021.2.0.eb new file mode 100644 index 0000000000..acbee9cd10 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/impi/impi-2021.2.0-intel-compilers-2021.2.0.eb @@ -0,0 +1,22 @@ +# dummy easyconfig, only for use with easybuild-framwork test suite! +easyblock = 'Toolchain' + +name = 'impi' +version = '2021.2.0' + +homepage = 'https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html' +description = "Intel MPI Library, compatible with MPICH ABI" + +toolchain = {'name': 'intel-compilers', 'version': '2021.2.0'} + +# see https://software.intel.com/content/www/us/en/develop/articles/oneapi-standalone-components.html +source_urls = ['https://registrationcenter-download.intel.com/akdlm/irc_nas/17729/'] +sources = ['l_mpi_oneapi_p_%(version)s.215_offline.sh'] +checksums = ['d0d4cdd11edaff2e7285e38f537defccff38e37a3067c02f4af43a3629ad4aa3'] + +# dummy easyconfig, only for use with easybuild-framwork test suite! +dependencies = [ + # ('UCX', '1.10.0'), +] + +moduleclass = 'mpi' diff --git a/test/framework/easyconfigs/test_ecs/i/intel-compilers/intel-compilers-2021.2.0.eb b/test/framework/easyconfigs/test_ecs/i/intel-compilers/intel-compilers-2021.2.0.eb new file mode 100644 index 0000000000..fe9e39c7f2 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/intel-compilers/intel-compilers-2021.2.0.eb @@ -0,0 +1,36 @@ +# dummy easyconfig, only for use with easybuild-framwork test suite! +easyblock = 'Toolchain' + +name = 'intel-compilers' +version = '2021.2.0' + +homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html' +description = "Intel C, C++ & Fortran compilers (classic and oneAPI)" + +toolchain = SYSTEM + +# see https://software.intel.com/content/www/us/en/develop/articles/oneapi-standalone-components.html +sources = [ + { + 'source_urls': ['https://registrationcenter-download.intel.com/akdlm/irc_nas/17749/'], + 'filename': 'l_dpcpp-cpp-compiler_p_%(version)s.118_offline.sh', + }, + { + 'source_urls': ['https://registrationcenter-download.intel.com/akdlm/irc_nas/17756/'], + 'filename': 'l_fortran-compiler_p_%(version)s.136_offline.sh', + }, +] +checksums = [ + # l_dpcpp-cpp-compiler_p_2021.2.0.118_offline.sh + '5d01cbff1a574c3775510cd97ffddd27fdf56d06a6b0c89a826fb23da4336d59', + 'a62e04a80f6d2f05e67cd5acb03fa58857ee22c6bd581ec0651c0ccd5bdec5a1', # l_fortran-compiler_p_2021.2.0.136_offline.sh +] + +# dummy easyconfig, only for use with easybuild-framwork test suite! +local_gccver = '10.3.0' +dependencies = [ + # ('GCCcore', local_gccver), + # ('binutils', '2.36.1', '', ('GCCcore', local_gccver)), +] + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb b/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb new file mode 100644 index 0000000000..3d89f4bf2e --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/l/libtoy/libtoy-0.0.eb @@ -0,0 +1,17 @@ +name = 'libtoy' +version = '0.0' + +homepage = 'https://easybuild.io' +description = "Toy C library." + +toolchain = SYSTEM + +sources = [SOURCE_TAR_GZ] +checksums = ['1523195c806fb511c3f0b60de1e53890c39cb783c7061c2736e40a31a38fd017'] + +sanity_check_paths = { + 'files': ['bin/toy', 'lib/libtoy.%s' % SHLIB_EXT], + 'dirs': [], +} + +moduleclass = 'lib' diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py index c3037b8826..f119c054fa 100644 --- a/test/framework/easyconfigversion.py +++ b/test/framework/easyconfigversion.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easystack.py b/test/framework/easystack.py new file mode 100644 index 0000000000..78be785fb5 --- /dev/null +++ b/test/framework/easystack.py @@ -0,0 +1,207 @@ +# # +# Copyright 2013-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for easystack files + +@author: Denis Kristak (Inuits) +@author: Kenneth Hoste (Ghent University) +""" +import os +import sys +from unittest import TextTestRunner + +import easybuild.tools.build_log +from easybuild.framework.easystack import check_value, parse_easystack +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import write_file +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered + + +class EasyStackTest(EnhancedTestCase): + """Testcases for easystack files.""" + + logfile = None + + def setUp(self): + """Set up test.""" + super(EasyStackTest, self).setUp() + self.orig_experimental = easybuild.tools.build_log.EXPERIMENTAL + # easystack files are an experimental feature + easybuild.tools.build_log.EXPERIMENTAL = True + + def tearDown(self): + """Clean up after test.""" + easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental + super(EasyStackTest, self).tearDown() + + def test_parse_fail(self): + """Test for clean error when easystack file fails to parse.""" + test_yml = os.path.join(self.test_prefix, 'test.yml') + write_file(test_yml, 'software: %s') + error_pattern = "Failed to parse .*/test.yml: while scanning for the next token" + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_yml) + + def test_easystack_wrong_structure(self): + """Test for --easystack when yaml easystack has wrong structure""" + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') + + expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:" + expected_err += r"( 'float' object is not subscriptable[\S\s]*" + expected_err += r"| 'float' object is unsubscriptable" + expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*)" + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, test_easystack) + + def test_easystack_asterisk(self): + """Test for --easystack when yaml easystack contains asterisk (wildcard)""" + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml') + + expected_err = "EasyStack specifications of 'binutils' in .*/test_easystack_asterisk.yaml contain asterisk. " + expected_err += "Wildcard feature is not supported yet." + + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, test_easystack) + + def test_easystack_labels(self): + topdir = os.path.dirname(os.path.abspath(__file__)) + test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') + + error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. " + error_msg += "Labels aren't supported yet." + self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, test_easystack) + + def test_check_value(self): + """Test check_value function.""" + check_value('1.2.3', None) + check_value('1.2', None) + check_value('3.50', None) + check_value('100', None) + + context = "" + for version in (1.2, 100, None): + error_pattern = r"Value .* \(of type .*\) obtained for is not valid!" + self.assertErrorRegex(EasyBuildError, error_pattern, check_value, version, context) + + def test_easystack_versions(self): + """Test handling of versions in easystack files.""" + + test_easystack = os.path.join(self.test_prefix, 'test.yml') + tmpl_easystack_txt = '\n'.join([ + "software:", + " foo:", + " toolchains:", + " SYSTEM:", + " versions:", + ]) + + # normal versions, which are not treated special by YAML: no single quotes needed + versions = ('1.2.3', '1.2.30', '2021a', '1.2.3') + for version in versions: + write_file(test_easystack, tmpl_easystack_txt + ' ' + version) + ec_fns, _ = parse_easystack(test_easystack) + self.assertEqual(ec_fns, ['foo-%s.eb' % version]) + + # multiple versions as a list + test_easystack_txt = tmpl_easystack_txt + " [1.2.3, 3.2.1]" + write_file(test_easystack, test_easystack_txt) + ec_fns, _ = parse_easystack(test_easystack) + expected = ['foo-1.2.3.eb', 'foo-3.2.1.eb'] + self.assertEqual(sorted(ec_fns), sorted(expected)) + + # multiple versions listed with more info + test_easystack_txt = '\n'.join([ + tmpl_easystack_txt, + " 1.2.3:", + " 2021a:", + " 3.2.1:", + " versionsuffix: -foo", + ]) + write_file(test_easystack, test_easystack_txt) + ec_fns, _ = parse_easystack(test_easystack) + expected = ['foo-1.2.3.eb', 'foo-2021a.eb', 'foo-3.2.1-foo.eb'] + self.assertEqual(sorted(ec_fns), sorted(expected)) + + # versions that get interpreted by YAML as float or int, single quotes required + for version in ('1.2', '123', '3.50', '100', '2.44_01'): + error_pattern = r"Value .* \(of type .*\) obtained for foo \(with system toolchain\) is not valid\!" + + write_file(test_easystack, tmpl_easystack_txt + ' ' + version) + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) + + # all is fine when wrapping the value in single quotes + write_file(test_easystack, tmpl_easystack_txt + " '" + version + "'") + ec_fns, _ = parse_easystack(test_easystack) + self.assertEqual(ec_fns, ['foo-%s.eb' % version]) + + # one rotten apple in the basket is enough + test_easystack_txt = tmpl_easystack_txt + " [1.2.3, %s, 3.2.1]" % version + write_file(test_easystack, test_easystack_txt) + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) + + test_easystack_txt = '\n'.join([ + tmpl_easystack_txt, + " 1.2.3:", + " %s:" % version, + " 3.2.1:", + " versionsuffix: -foo", + ]) + write_file(test_easystack, test_easystack_txt) + self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) + + # single quotes to the rescue! + test_easystack_txt = '\n'.join([ + tmpl_easystack_txt, + " 1.2.3:", + " '%s':" % version, + " 3.2.1:", + " versionsuffix: -foo", + ]) + write_file(test_easystack, test_easystack_txt) + ec_fns, _ = parse_easystack(test_easystack) + expected = ['foo-1.2.3.eb', 'foo-%s.eb' % version, 'foo-3.2.1-foo.eb'] + self.assertEqual(sorted(ec_fns), sorted(expected)) + + # also check toolchain version that could be interpreted as a non-string value... + test_easystack_txt = '\n'.join([ + 'software:', + ' test:', + ' toolchains:', + ' intel-2021.03:', + " versions: [1.2.3, '2.3']", + ]) + write_file(test_easystack, test_easystack_txt) + ec_fns, _ = parse_easystack(test_easystack) + expected = ['test-1.2.3-intel-2021.03.eb', 'test-2.3-intel-2021.03.eb'] + self.assertEqual(sorted(ec_fns), sorted(expected)) + + +def suite(): + """ returns all the testcases in this module """ + return TestLoaderFiltered().loadTestsFromTestCase(EasyStackTest, sys.argv[1:]) + + +if __name__ == '__main__': + res = TextTestRunner(verbosity=1).run(suite()) + sys.exit(len(res.failures)) diff --git a/test/framework/easystacks/test_easystack_basic.yaml b/test/framework/easystacks/test_easystack_basic.yaml index 2de5dfd129..491f113f4a 100644 --- a/test/framework/easystacks/test_easystack_basic.yaml +++ b/test/framework/easystacks/test_easystack_basic.yaml @@ -3,8 +3,8 @@ software: toolchains: GCCcore-4.9.3: versions: - 2.25: - 2.26: + '2.25': + '2.26': foss: toolchains: SYSTEM: @@ -13,5 +13,5 @@ software: toolchains: gompi-2018a: versions: - 0.0: + '0.0': versionsuffix: '-test' diff --git a/test/framework/easystacks/test_easystack_labels.yaml b/test/framework/easystacks/test_easystack_labels.yaml index 51a113523f..f00db0e249 100644 --- a/test/framework/easystacks/test_easystack_labels.yaml +++ b/test/framework/easystacks/test_easystack_labels.yaml @@ -3,5 +3,5 @@ software: toolchains: GCCcore-4.9.3: versions: - 3.11: + '3.11': exclude-labels: arch:aarch64 diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py index e7f05ab83a..3f3e0074cc 100644 --- a/test/framework/ebconfigobj.py +++ b/test/framework/ebconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/environment.py b/test/framework/environment.py index 10f875e733..64ab3c8bad 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index def99f7550..7d890ed7e8 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -42,11 +42,12 @@ import time from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner - +from easybuild.tools import run import easybuild.tools.filetools as ft from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff -from easybuild.tools.py2vs3 import std_urllib +from easybuild.tools.py2vs3 import StringIO, std_urllib class FileToolsTest(EnhancedTestCase): @@ -65,12 +66,18 @@ def setUp(self): super(FileToolsTest, self).setUp() self.orig_filetools_std_urllib_urlopen = ft.std_urllib.urlopen + if ft.HAVE_REQUESTS: + self.orig_filetools_requests_get = ft.requests.get + self.orig_filetools_HAVE_REQUESTS = ft.HAVE_REQUESTS def tearDown(self): """Cleanup.""" super(FileToolsTest, self).tearDown() ft.std_urllib.urlopen = self.orig_filetools_std_urllib_urlopen + ft.HAVE_REQUESTS = self.orig_filetools_HAVE_REQUESTS + if ft.HAVE_REQUESTS: + ft.requests.get = self.orig_filetools_requests_get def test_extract_cmd(self): """Test various extract commands.""" @@ -91,11 +98,15 @@ def test_extract_cmd(self): ('untar.gz', "gunzip -c untar.gz > untar"), ("/some/path/test.gz", "gunzip -c /some/path/test.gz > test"), ('test.xz', "unxz test.xz"), - ('test.tar.xz', "unxz test.tar.xz --stdout | tar x"), - ('test.txz', "unxz test.txz --stdout | tar x"), + ('test.tar.xz', "unset TAPE; unxz test.tar.xz --stdout | tar x"), + ('test.txz', "unset TAPE; unxz test.txz --stdout | tar x"), ('test.iso', "7z x test.iso"), ('test.tar.Z', "tar xzf test.tar.Z"), ('test.foo.bar.sh', "cp -a test.foo.bar.sh ."), + # check whether extension is stripped correct to determine name of target file + # cfr. https://github.com/easybuilders/easybuild-framework/pull/3705 + ('testbz2.bz2', "bunzip2 -c testbz2.bz2 > testbz2"), + ('testgz.gz', "gunzip -c testgz.gz > testgz"), ] for (fn, expected_cmd) in tests: cmd = ft.extract_cmd(fn) @@ -103,6 +114,9 @@ def test_extract_cmd(self): self.assertEqual("unzip -qq -o test.zip", ft.extract_cmd('test.zip', True)) + error_pattern = "test.foo has unknown file extension" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.extract_cmd, 'test.foo') + def test_find_extension(self): """Test find_extension function.""" tests = [ @@ -213,8 +227,13 @@ def test_which(self): python = ft.which('python') self.assertTrue(python and os.path.exists(python) and os.path.isabs(python)) - path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available') + invalid_cmd = 'i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available' + path = ft.which(invalid_cmd) + self.assertTrue(path is None) + path = ft.which(invalid_cmd, on_error=IGNORE) self.assertTrue(path is None) + error_msg = "Could not find command '%s'" % invalid_cmd + self.assertErrorRegex(EasyBuildError, error_msg, ft.which, invalid_cmd, on_error=ERROR) os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.environ['PATH']) # put a directory 'foo' in place (should be ignored by 'which') @@ -351,6 +370,51 @@ def test_common_path_prefix(self): self.assertEqual(ft.det_common_path_prefix(['foo']), None) self.assertEqual(ft.det_common_path_prefix([]), None) + def test_normalize_path(self): + """Test normalize_path""" + self.assertEqual(ft.normalize_path(''), '') + self.assertEqual(ft.normalize_path('/'), '/') + self.assertEqual(ft.normalize_path('//'), '//') + self.assertEqual(ft.normalize_path('///'), '/') + self.assertEqual(ft.normalize_path('/foo/bar/baz'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('/foo//bar/././baz/'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('foo//bar/././baz/'), 'foo/bar/baz') + self.assertEqual(ft.normalize_path('//foo//bar/././baz/'), '//foo/bar/baz') + self.assertEqual(ft.normalize_path('///foo//bar/././baz/'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('////foo//bar/././baz/'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('/././foo//bar/././baz/'), '/foo/bar/baz') + self.assertEqual(ft.normalize_path('//././foo//bar/././baz/'), '//foo/bar/baz') + + def test_det_file_size(self): + """Test det_file_size function.""" + + self.assertEqual(ft.det_file_size({'Content-Length': '12345'}), 12345) + + # missing content length, or invalid value + self.assertEqual(ft.det_file_size({}), None) + self.assertEqual(ft.det_file_size({'Content-Length': 'foo'}), None) + + test_url = 'https://github.com/easybuilders/easybuild-framework/raw/develop/' + test_url += 'test/framework/sandbox/sources/toy/toy-0.0.tar.gz' + expected_size = 273 + + # also try with actual HTTP header + try: + fh = std_urllib.urlopen(test_url) + self.assertEqual(ft.det_file_size(fh.info()), expected_size) + fh.close() + + # also try using requests, which is used as a fallback in download_file + try: + import requests + res = requests.get(test_url) + self.assertEqual(ft.det_file_size(res.headers), expected_size) + res.close() + except ImportError: + pass + except std_urllib.URLError: + print("Skipping online test for det_file_size (working offline)") + def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' @@ -453,7 +517,6 @@ def fake_urllib_open(*args, **kwargs): # replaceurlopen with function that raises HTTP error 403 def fake_urllib_open(*args, **kwargs): - from easybuild.tools.py2vs3 import StringIO raise ft.std_urllib.HTTPError(url, 403, "Forbidden", "", StringIO()) ft.std_urllib.urlopen = fake_urllib_open @@ -468,6 +531,82 @@ def fake_urllib_open(*args, **kwargs): ft.HAVE_REQUESTS = False self.assertErrorRegex(EasyBuildError, "SSL issues with urllib2", ft.download_file, fn, url, target) + def test_download_file_insecure(self): + """ + Test downloading of file via insecure URL + """ + + self.assertFalse(build_option('insecure_download')) + + # replace urlopen with function that raises IOError + def fake_urllib_open(url, *args, **kwargs): + if kwargs.get('context') is None: + error_msg = " sub2, sub2/cycle_1 -> sub1, ... + test_folder = tempfile.mkdtemp() + sub_folder1 = os.path.join(test_folder, 'sub1') + sub_folder2 = sub_folder = os.path.join(test_folder, 'sub2') + os.mkdir(sub_folder1) + os.mkdir(sub_folder2) + os.symlink(os.path.join('..', 'sub2'), os.path.join(sub_folder1, 'cycle_1')) + os.symlink(os.path.join('..', 'sub1'), os.path.join(sub_folder2, 'cycle_2')) + self.assertTrue(ft.has_recursive_symlinks(test_folder)) + def test_copy_dir(self): """Test copy_dir function.""" testdir = os.path.dirname(os.path.abspath(__file__)) @@ -1748,6 +2083,15 @@ def ignore_func(_, names): ft.mkdir(subdir) ft.copy_dir(srcdir, target_dir, symlinks=True, dirs_exist_ok=True) + # Detect recursive symlinks by default instead of infinite loop during copy + ft.remove_dir(target_dir) + os.symlink('.', os.path.join(subdir, 'recursive_link')) + self.assertErrorRegex(EasyBuildError, 'Recursive symlinks detected', ft.copy_dir, srcdir, target_dir) + self.assertFalse(os.path.exists(target_dir)) + # Ok for symlinks=True + ft.copy_dir(srcdir, target_dir, symlinks=True) + self.assertTrue(os.path.exists(target_dir)) + # also test behaviour of copy_file under --dry-run build_options = { 'extended_dry_run': True, @@ -2015,7 +2359,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 83) + self.assertEqual(len(index), 89) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), @@ -2128,11 +2472,11 @@ def test_search_file(self): self.assertEqual(var_defs, []) self.assertEqual(len(hits), 5) self.assertTrue(all(os.path.exists(p) for p in hits)) - self.assertTrue(hits[0].endswith('/hwloc-1.11.8-GCC-4.6.4.eb')) - self.assertTrue(hits[1].endswith('/hwloc-1.11.8-GCC-6.4.0-2.28.eb')) - self.assertTrue(hits[2].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb')) - self.assertTrue(hits[3].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb')) - self.assertTrue(hits[4].endswith('/hwloc-1.8-gcccuda-2018a.eb')) + self.assertTrue(hits[0].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb')) + self.assertTrue(hits[1].endswith('/hwloc-1.8-gcccuda-2018a.eb')) + self.assertTrue(hits[2].endswith('/hwloc-1.11.8-GCC-4.6.4.eb')) + self.assertTrue(hits[3].endswith('/hwloc-1.11.8-GCC-6.4.0-2.28.eb')) + self.assertTrue(hits[4].endswith('/hwloc-1.11.8-GCC-7.3.0-2.30.eb')) # also test case-sensitive searching var_defs, hits_bis = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True) @@ -2146,9 +2490,12 @@ def test_search_file(self): # check filename-only mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True) self.assertEqual(var_defs, []) - self.assertEqual(hits, ['hwloc-1.11.8-GCC-4.6.4.eb', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', - 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb', - 'hwloc-1.8-gcccuda-2018a.eb']) + self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', + 'hwloc-1.8-gcccuda-2018a.eb', + 'hwloc-1.11.8-GCC-4.6.4.eb', + 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', + 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', + ]) # check specifying of ignored dirs var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, ignore_dirs=['hwloc']) @@ -2157,28 +2504,34 @@ def test_search_file(self): # check short mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, short=True) self.assertEqual(var_defs, [('CFGS1', os.path.join(test_ecs, 'h', 'hwloc'))]) - self.assertEqual(hits, ['$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb', '$CFGS1/hwloc-1.11.8-GCC-6.4.0-2.28.eb', - '$CFGS1/hwloc-1.11.8-GCC-7.3.0-2.30.eb', '$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb', - '$CFGS1/hwloc-1.8-gcccuda-2018a.eb']) + self.assertEqual(hits, ['$CFGS1/hwloc-1.6.2-GCC-4.9.3-2.26.eb', + '$CFGS1/hwloc-1.8-gcccuda-2018a.eb', + '$CFGS1/hwloc-1.11.8-GCC-4.6.4.eb', + '$CFGS1/hwloc-1.11.8-GCC-6.4.0-2.28.eb', + '$CFGS1/hwloc-1.11.8-GCC-7.3.0-2.30.eb' + ]) # check terse mode (implies 'silent', overrides 'short') var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, short=True) self.assertEqual(var_defs, []) expected = [ + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), ] self.assertEqual(hits, expected) # check combo of terse and filename-only var_defs, hits = ft.search_file([test_ecs], 'HWLOC', terse=True, filename_only=True) self.assertEqual(var_defs, []) - self.assertEqual(hits, ['hwloc-1.11.8-GCC-4.6.4.eb', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', - 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb', - 'hwloc-1.8-gcccuda-2018a.eb']) + self.assertEqual(hits, ['hwloc-1.6.2-GCC-4.9.3-2.26.eb', + 'hwloc-1.8-gcccuda-2018a.eb', + 'hwloc-1.11.8-GCC-4.6.4.eb', + 'hwloc-1.11.8-GCC-6.4.0-2.28.eb', + 'hwloc-1.11.8-GCC-7.3.0-2.30.eb', + ]) # patterns that include special characters + (or ++) shouldn't cause trouble # cfr. https://github.com/easybuilders/easybuild-framework/issues/2966 @@ -2208,21 +2561,26 @@ def makedirs_in_test(*paths): empty_dir = makedirs_in_test('empty_dir') self.assertFalse(ft.dir_contains_files(empty_dir)) + self.assertFalse(ft.dir_contains_files(empty_dir, recursive=False)) dir_w_subdir = makedirs_in_test('dir_w_subdir', 'sub_dir') self.assertFalse(ft.dir_contains_files(dir_w_subdir)) + self.assertFalse(ft.dir_contains_files(dir_w_subdir, recursive=False)) dir_subdir_file = makedirs_in_test('dir_subdir_file', 'sub_dir_w_file') ft.write_file(os.path.join(dir_subdir_file, 'sub_dir_w_file', 'file.h'), '') self.assertTrue(ft.dir_contains_files(dir_subdir_file)) + self.assertFalse(ft.dir_contains_files(dir_subdir_file, recursive=False)) dir_w_file = makedirs_in_test('dir_w_file') ft.write_file(os.path.join(dir_w_file, 'file.h'), '') self.assertTrue(ft.dir_contains_files(dir_w_file)) + self.assertTrue(ft.dir_contains_files(dir_w_file, recursive=False)) dir_w_dir_and_file = makedirs_in_test('dir_w_dir_and_file', 'sub_dir') ft.write_file(os.path.join(dir_w_dir_and_file, 'file.h'), '') self.assertTrue(ft.dir_contains_files(dir_w_dir_and_file)) + self.assertTrue(ft.dir_contains_files(dir_w_dir_and_file, recursive=False)) def test_find_eb_script(self): """Test find_eb_script function.""" @@ -2361,62 +2719,8 @@ def test_diff_files(self): def test_get_source_tarball_from_git(self): """Test get_source_tarball_from_git function.""" - git_config = { - 'repo_name': 'testrepository', - 'url': 'https://github.com/easybuilders', - 'tag': 'master', - } target_dir = os.path.join(self.test_prefix, 'target') - try: - ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) - # (only) tarball is created in specified target dir - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'test.tar.gz'))) - self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) - - del git_config['tag'] - git_config['commit'] = '8456f86' - ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'test2.tar.gz'))) - self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz']) - - except EasyBuildError as err: - if "Network is down" in str(err): - print("Ignoring download error in test_get_source_tarball_from_git, working offline?") - else: - raise err - - git_config = { - 'repo_name': 'testrepository', - 'url': 'git@github.com:easybuilders', - 'tag': 'master', - } - args = ['test.tar.gz', self.test_prefix, git_config] - - for key in ['repo_name', 'url', 'tag']: - orig_value = git_config.pop(key) - if key == 'tag': - error_pattern = "Neither tag nor commit found in git_config parameter" - else: - error_pattern = "%s not specified in git_config parameter" % key - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - git_config[key] = orig_value - - git_config['commit'] = '8456f86' - error_pattern = "Tag and commit are mutually exclusive in git_config parameter" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - del git_config['commit'] - - git_config['unknown'] = 'foobar' - error_pattern = "Found one or more unexpected keys in 'git_config' specification" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - del git_config['unknown'] - - args[0] = 'test.txt' - error_pattern = "git_config currently only supports filename ending in .tar.gz" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - args[0] = 'test.tar.gz' - # only test in dry run mode, i.e. check which commands would be executed without actually running them build_options = { 'extended_dry_run': True, @@ -2426,13 +2730,10 @@ def test_get_source_tarball_from_git(self): def run_check(): """Helper function to run get_source_tarball_from_git & check dry run output""" - self.mock_stdout(True) - self.mock_stderr(True) - res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) + with self.mocked_stdout_stderr(): + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + stdout = self.get_stdout() + stderr = self.get_stderr() self.assertEqual(stderr, '') regex = re.compile(expected) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) @@ -2443,58 +2744,158 @@ def run_check(): git_config = { 'repo_name': 'testrepository', 'url': 'git@github.com:easybuilders', - 'tag': 'master', + 'tag': 'tag_for_tests', } + git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter expected = '\n'.join([ - r' running command "git clone --branch master git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo + run_check() + + git_config['clone_into'] = 'test123' + expected = '\n'.join([ + r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', + r" \(in .*/tmp.*\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', + r" \(in .*/tmp.*\)", + ]) % git_repo run_check() + del git_config['clone_into'] git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --branch master --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch master --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() del git_config['keep_git_dir'] del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() del git_config['recursive'] expected = '\n'.join([ - r' running command "git clone git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86"', r" \(in testrepository\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() + # Test with real data. + init_config() + git_config = { + 'repo_name': 'testrepository', + 'url': 'https://github.com/easybuilders', + 'tag': 'branch_tag_for_test', + } + + try: + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + # (only) tarball is created in specified target dir + test_file = os.path.join(target_dir, 'test.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + test_tar_gzs = [os.path.basename(test_file)] + self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) + # Check that we indeed downloaded the right tag + extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') + extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-branch.txt'))) + os.remove(test_file) + + # use a tag that clashes with a branch name and make sure this is handled correctly + git_config['tag'] = 'tag_for_tests' + with self.mocked_stdout_stderr(): + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + stderr = self.get_stderr() + self.assertIn('Tag tag_for_tests was not downloaded in the first try', stderr) + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + # Check that we indeed downloaded the tag and not the branch + extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') + extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-tag.txt'))) + + del git_config['tag'] + git_config['commit'] = '90366ea' + res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) + test_file = os.path.join(target_dir, 'test2.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + test_tar_gzs.append(os.path.basename(test_file)) + self.assertEqual(sorted(os.listdir(target_dir)), test_tar_gzs) + + git_config['keep_git_dir'] = True + res = ft.get_source_tarball_from_git('test3.tar.gz', target_dir, git_config) + test_file = os.path.join(target_dir, 'test3.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + test_tar_gzs.append(os.path.basename(test_file)) + self.assertEqual(sorted(os.listdir(target_dir)), test_tar_gzs) + + except EasyBuildError as err: + if "Network is down" in str(err): + print("Ignoring download error in test_get_source_tarball_from_git, working offline?") + else: + raise err + + git_config = { + 'repo_name': 'testrepository', + 'url': 'git@github.com:easybuilders', + 'tag': 'tag_for_tests', + } + args = ['test.tar.gz', self.test_prefix, git_config] + + for key in ['repo_name', 'url', 'tag']: + orig_value = git_config.pop(key) + if key == 'tag': + error_pattern = "Neither tag nor commit found in git_config parameter" + else: + error_pattern = "%s not specified in git_config parameter" % key + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + git_config[key] = orig_value + + git_config['commit'] = '8456f86' + error_pattern = "Tag and commit are mutually exclusive in git_config parameter" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + del git_config['commit'] + + git_config['unknown'] = 'foobar' + error_pattern = "Found one or more unexpected keys in 'git_config' specification" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + del git_config['unknown'] + + args[0] = 'test.txt' + error_pattern = "git_config currently only supports filename ending in .tar.gz" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + args[0] = 'test.tar.gz' + def test_is_sha256_checksum(self): """Test for is_sha256_checksum function.""" a_sha256_checksum = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' @@ -2888,6 +3289,110 @@ def test_locate_files(self): error_pattern = r"One or more files not found: 2\.txt \(search paths: \)" self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['2.txt'], []) + def test_set_gid_sticky_bits(self): + """Test for set_gid_sticky_bits function.""" + test_dir = os.path.join(self.test_prefix, 'test_dir') + test_subdir = os.path.join(test_dir, 'subdir') + + ft.mkdir(test_subdir, parents=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + + # by default, GID & sticky bits are not set + ft.set_gid_sticky_bits(test_dir) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + + ft.set_gid_sticky_bits(test_dir, set_gid=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + ft.set_gid_sticky_bits(test_dir, sticky=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + ft.set_gid_sticky_bits(test_dir, set_gid=True, sticky=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + # no recursion by default + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0) + self.assertEqual(dir_perms & stat.S_ISVTX, 0) + + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + ft.set_gid_sticky_bits(test_dir, set_gid=True, sticky=True, recursive=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + + ft.remove_dir(test_dir) + ft.mkdir(test_subdir, parents=True) + + # set_gid_sticky_bits honors relevant build options + init_config(build_options={'set_gid_bit': True, 'sticky_bit': True}) + ft.set_gid_sticky_bits(test_dir, recursive=True) + dir_perms = os.lstat(test_dir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + dir_perms = os.lstat(test_subdir)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + + def test_create_unused_dir(self): + """Test create_unused_dir function.""" + path = ft.create_unused_dir(self.test_prefix, 'folder') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder')) + self.assertTrue(os.path.exists(path)) + + # Repeat with existing folder(s) should create new ones + for i in range(10): + path = ft.create_unused_dir(self.test_prefix, 'folder') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder_%s' % i)) + self.assertTrue(os.path.exists(path)) + + # Not influenced by similar folder + path = ft.create_unused_dir(self.test_prefix, 'folder2') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder2')) + self.assertTrue(os.path.exists(path)) + for i in range(10): + path = ft.create_unused_dir(self.test_prefix, 'folder2') + self.assertEqual(path, os.path.join(self.test_prefix, 'folder2_%s' % i)) + self.assertTrue(os.path.exists(path)) + + # Fail cleanly if passed a readonly folder + readonly_dir = os.path.join(self.test_prefix, 'ro_folder') + ft.mkdir(readonly_dir) + old_perms = os.lstat(readonly_dir)[stat.ST_MODE] + ft.adjust_permissions(readonly_dir, stat.S_IREAD | stat.S_IEXEC, relative=False) + try: + self.assertErrorRegex(EasyBuildError, 'Failed to create directory', + ft.create_unused_dir, readonly_dir, 'new_folder') + finally: + ft.adjust_permissions(readonly_dir, old_perms, relative=False) + + # Ignore files same as folders. So first just create a file with no contents + ft.write_file(os.path.join(self.test_prefix, 'file'), '') + path = ft.create_unused_dir(self.test_prefix, 'file') + self.assertEqual(path, os.path.join(self.test_prefix, 'file_0')) + self.assertTrue(os.path.exists(path)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py index 46d57bf1ac..1637c56768 100644 --- a/test/framework/format_convert.py +++ b/test/framework/format_convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/general.py b/test/framework/general.py index 92f082cebc..f1e231044c 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/github.py b/test/framework/github.py index fa3d7e5022..7a4fa44c85 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,17 +33,23 @@ import random import re import sys +import textwrap from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from time import gmtime from unittest import TextTestRunner +import easybuild.tools.testing from easybuild.base.rest import RestClient +from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.tools import categorize_files_by_type from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import build_option, module_classes +from easybuild.tools.config import build_option, module_classes, update_build_option from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file +from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_MERGEABLE_STATE_CLEAN from easybuild.tools.github import VALID_CLOSE_PR_REASONS -from easybuild.tools.testing import post_pr_test_report, session_state +from easybuild.tools.github import pick_default_branch +from easybuild.tools.testing import create_test_report, post_pr_test_report, session_state from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters import easybuild.tools.github as gh @@ -60,7 +66,7 @@ GITHUB_USER = "easybuilders" GITHUB_REPO = "testrepository" # branch to test -GITHUB_BRANCH = 'master' +GITHUB_BRANCH = 'main' class GithubTest(EnhancedTestCase): @@ -69,18 +75,35 @@ class GithubTest(EnhancedTestCase): for non authenticated users of 50""" def setUp(self): - """setup""" + """Test setup.""" super(GithubTest, self).setUp() + self.github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) + if self.github_token is None: - self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, None, None, None) + username, token = None, None else: - self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, GITHUB_TEST_ACCOUNT, - None, self.github_token) + username, token = GITHUB_TEST_ACCOUNT, self.github_token + + self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, username, None, token) self.skip_github_tests = self.github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None - def test_walk(self): + self.orig_testing_create_gist = easybuild.tools.testing.create_gist + + def tearDown(self): + """Cleanup after running test.""" + easybuild.tools.testing.create_gist = self.orig_testing_create_gist + + super(GithubTest, self).tearDown() + + def test_github_pick_default_branch(self): + """Test pick_default_branch function.""" + + self.assertEqual(pick_default_branch('easybuilders'), 'main') + self.assertEqual(pick_default_branch('foobar'), 'master') + + def test_github_walk(self): """test the gitubfs walk function""" if self.skip_github_tests: print("Skipping test_walk, no GitHub token available?") @@ -96,7 +119,7 @@ def test_walk(self): except IOError: pass - def test_read_api(self): + def test_github_read_api(self): """Test the githubfs read function""" if self.skip_github_tests: print("Skipping test_read_api, no GitHub token available?") @@ -107,7 +130,7 @@ def test_read_api(self): except IOError: pass - def test_read(self): + def test_github_read(self): """Test the githubfs read function without using the api""" if self.skip_github_tests: print("Skipping test_read, no GitHub token available?") @@ -115,12 +138,54 @@ def test_read(self): try: fp = self.ghfs.read("a_directory/a_file.txt", api=False) - self.assertEqual(open(fp, 'r').read().strip(), "this is a line of text") + self.assertEqual(read_file(fp).strip(), "this is a line of text") os.remove(fp) except (IOError, OSError): pass - def test_fetch_pr_data(self): + def test_github_add_pr_labels(self): + """Test add_pr_labels function.""" + if self.skip_github_tests: + print("Skipping test_add_pr_labels, no GitHub token available?") + return + + build_options = { + 'pr_target_account': GITHUB_USER, + 'pr_target_repo': GITHUB_EASYBLOCKS_REPO, + 'github_user': GITHUB_TEST_ACCOUNT, + 'dry_run': True, + } + init_config(build_options=build_options) + + self.mock_stdout(True) + error_pattern = "Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet" + self.assertErrorRegex(EasyBuildError, error_pattern, gh.add_pr_labels, 1) + self.mock_stdout(False) + + build_options['pr_target_repo'] = GITHUB_EASYCONFIGS_REPO + init_config(build_options=build_options) + + # PR #11262 includes easyconfigs that use 'dummy' toolchain, + # so we need to allow triggering deprecated behaviour + self.allow_deprecated_behaviour() + + self.mock_stdout(True) + self.mock_stderr(True) + gh.add_pr_labels(11262) + stdout = self.get_stdout() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertTrue("Could not determine any missing labels for PR #11262" in stdout) + + self.mock_stdout(True) + self.mock_stderr(True) + gh.add_pr_labels(8006) # closed, unmerged, unlabeled PR + stdout = self.get_stdout() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertTrue("PR #8006 should be labelled 'update'" in stdout) + + def test_github_fetch_pr_data(self): """Test fetch_pr_data function.""" if self.skip_github_tests: print("Skipping test_fetch_pr_data, no GitHub token available?") @@ -142,7 +207,7 @@ def test_fetch_pr_data(self): self.assertEqual(pr_data['reviews'][0]['user']['login'], 'boegel') self.assertEqual(pr_data['status_last_commit'], None) - def test_list_prs(self): + def test_github_list_prs(self): """Test list_prs function.""" if self.skip_github_tests: print("Skipping test_list_prs, no GitHub token available?") @@ -164,7 +229,7 @@ def test_list_prs(self): self.assertEqual(expected, output) - def test_reasons_for_closing(self): + def test_github_reasons_for_closing(self): """Test reasons_for_closing function.""" if self.skip_github_tests: print("Skipping test_reasons_for_closing, no GitHub token available?") @@ -204,7 +269,7 @@ def test_reasons_for_closing(self): for pattern in patterns: self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) - def test_close_pr(self): + def test_github_close_pr(self): """Test close_pr function.""" if self.skip_github_tests: print("Skipping test_close_pr, no GitHub token available?") @@ -249,7 +314,7 @@ def test_close_pr(self): for pattern in patterns: self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) - def test_fetch_easyblocks_from_pr(self): + def test_github_fetch_easyblocks_from_pr(self): """Test fetch_easyblocks_from_pr function.""" if self.skip_github_tests: print("Skipping test_fetch_easyblocks_from_pr, no GitHub token available?") @@ -276,7 +341,7 @@ def test_fetch_easyblocks_from_pr(self): except URLError as err: print("Ignoring URLError '%s' in test_fetch_easyblocks_from_pr" % err) - def test_fetch_easyconfigs_from_pr(self): + def test_github_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.skip_github_tests: print("Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?") @@ -327,7 +392,7 @@ def test_fetch_easyconfigs_from_pr(self): except URLError as err: print("Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err) - def test_fetch_files_from_pr_cache(self): + def test_github_fetch_files_from_pr_cache(self): """Test caching for fetch_files_from_pr.""" if self.skip_github_tests: print("Skipping test_fetch_files_from_pr_cache, no GitHub token available?") @@ -388,7 +453,7 @@ def test_fetch_files_from_pr_cache(self): res = gh.fetch_easyblocks_from_pr(12345, tmpdir) self.assertEqual(sorted(pr12345_files), sorted(res)) - def test_fetch_latest_commit_sha(self): + def test_github_fetch_latest_commit_sha(self): """Test fetch_latest_commit_sha function.""" if self.skip_github_tests: print("Skipping test_fetch_latest_commit_sha, no GitHub token available?") @@ -400,7 +465,7 @@ def test_fetch_latest_commit_sha(self): branch='develop') self.assertTrue(re.match('^[0-9a-f]{40}$', sha)) - def test_download_repo(self): + def test_github_download_repo(self): """Test download_repo function.""" if self.skip_github_tests: print("Skipping test_download_repo, no GitHub token available?") @@ -410,7 +475,7 @@ def test_download_repo(self): # default: download tarball for master branch of easybuilders/easybuild-easyconfigs repo path = gh.download_repo(path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) - repodir = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-master') + repodir = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-main') self.assertTrue(os.path.samefile(path, repodir)) self.assertTrue(os.path.exists(repodir)) shafile = os.path.join(repodir, 'latest-sha') @@ -494,7 +559,12 @@ def test_validate_github_token(self): self.assertTrue(gh.validate_github_token(self.github_token, GITHUB_TEST_ACCOUNT)) - def test_find_easybuild_easyconfig(self): + # if a token in the old format is available, test with that too + token_old_format = os.getenv('TEST_GITHUB_TOKEN_OLD_FORMAT') + if token_old_format: + self.assertTrue(gh.validate_github_token(token_old_format, GITHUB_TEST_ACCOUNT)) + + def test_github_find_easybuild_easyconfig(self): """Test for find_easybuild_easyconfig function""" if self.skip_github_tests: print("Skipping test_find_easybuild_easyconfig, no GitHub token available?") @@ -505,10 +575,10 @@ def test_find_easybuild_easyconfig(self): self.assertTrue(regex.search(path), "Pattern '%s' found in '%s'" % (regex.pattern, path)) self.assertTrue(os.path.exists(path), "Path %s exists" % path) - def test_find_patches(self): + def test_github_find_patches(self): """ Test for find_software_name_for_patch """ - testdir = os.path.dirname(os.path.abspath(__file__)) - ec_path = os.path.join(testdir, 'easyconfigs') + test_dir = os.path.dirname(os.path.abspath(__file__)) + ec_path = os.path.join(test_dir, 'easyconfigs') init_config(build_options={ 'allow_modules_tool_mismatch': True, 'minimal_toolchains': True, @@ -527,7 +597,17 @@ def test_find_patches(self): reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked') self.assertTrue(re.search(reg, txt)) - def test_det_commit_status(self): + self.assertEqual(gh.find_software_name_for_patch('test.patch', []), None) + + # check behaviour of find_software_name_for_patch when non-UTF8 patch files are present (only with Python 3) + if sys.version_info[0] >= 3: + non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch') + with open(non_utf8_patch, 'wb') as fp: + fp.write(bytes("+ ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1')) + + self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None) + + def test_github_det_commit_status(self): """Test det_commit_status function.""" if self.skip_github_tests: @@ -574,7 +654,7 @@ def test_det_commit_status(self): res = gh.det_commit_status('easybuilders', GITHUB_REPO, commit_sha, GITHUB_TEST_ACCOUNT) self.assertEqual(res, None) - def test_check_pr_eligible_to_merge(self): + def test_github_check_pr_eligible_to_merge(self): """Test check_pr_eligible_to_merge function""" def run_check(expected_result=False): """Helper function to check result of check_pr_eligible_to_merge""" @@ -592,7 +672,7 @@ def run_check(expected_result=False): pr_data = { 'base': { - 'ref': 'master', + 'ref': 'main', 'repo': { 'name': 'easybuild-easyconfigs', 'owner': {'login': 'easybuilders'}, @@ -602,7 +682,11 @@ def run_check(expected_result=False): 'issue_comments': [], 'milestone': None, 'number': '1234', - 'reviews': [], + 'merged': False, + 'mergeable_state': 'unknown', + 'reviews': [{'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}, + # to check that duplicates are filtered + {'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}], } test_result_warning_template = "* test suite passes: %s => not eligible for merging!" @@ -610,7 +694,7 @@ def run_check(expected_result=False): expected_stdout = "Checking eligibility of easybuilders/easybuild-easyconfigs PR #1234 for merging...\n" # target branch for PR must be develop - expected_warning = "* targets develop branch: FAILED; found 'master' => not eligible for merging!\n" + expected_warning = "* targets develop branch: FAILED; found 'main' => not eligible for merging!\n" run_check() pr_data['base']['ref'] = 'develop' @@ -662,11 +746,21 @@ def run_check(expected_result=False): pr_data['issue_comments'].insert(2, {'body': 'lgtm'}) run_check() - pr_data['reviews'].append({'state': 'CHANGES_REQUESTED', 'user': {'login': 'boegel'}}) + expected_warning = "* no pending change requests: FAILED (changes requested by boegel)" + expected_warning += " => not eligible for merging!" + run_check() + + # if PR is approved by a different user that requested changes and that request has not been dismissed, + # the PR is still not mergeable + pr_data['reviews'].append({'state': 'APPROVED', 'user': {'login': 'not_boegel'}}) + expected_stdout_saved = expected_stdout + expected_stdout += "* approved review: OK (by not_boegel)\n" run_check() + # if the user that requested changes approves the PR, it's mergeable pr_data['reviews'].append({'state': 'APPROVED', 'user': {'login': 'boegel'}}) - expected_stdout += "* approved review: OK (by boegel)\n" + expected_stdout = expected_stdout_saved + "* no pending change requests: OK\n" + expected_stdout += "* approved review: OK (by not_boegel, boegel)\n" expected_warning = '' run_check() @@ -677,63 +771,139 @@ def run_check(expected_result=False): pr_data['milestone'] = {'title': '3.3.1'} expected_stdout += "* milestone is set: OK (3.3.1)\n" + # mergeable state must be clean + expected_warning = "* mergeable state is clean: FAILED (mergeable state is 'unknown')" + run_check() + + pr_data['mergeable_state'] = GITHUB_MERGEABLE_STATE_CLEAN + expected_stdout += "* mergeable state is clean: OK\n" + # all checks pass, PR is eligible for merging expected_warning = '' self.assertEqual(run_check(True), '') - def test_det_patch_specs(self): + def test_github_det_pr_labels(self): + """Test for det_pr_labels function.""" + + file_info = {'new_folder': [False], 'new_file_in_existing_folder': [True]} + res = gh.det_pr_labels(file_info, GITHUB_EASYCONFIGS_REPO) + self.assertEqual(res, ['update']) + + file_info = {'new_folder': [True], 'new_file_in_existing_folder': [False]} + res = gh.det_pr_labels(file_info, GITHUB_EASYCONFIGS_REPO) + self.assertEqual(res, ['new']) + + file_info = {'new_folder': [True, False], 'new_file_in_existing_folder': [False, True]} + res = gh.det_pr_labels(file_info, GITHUB_EASYCONFIGS_REPO) + self.assertTrue(sorted(res), ['new', 'update']) + + file_info = {'new': [True]} + res = gh.det_pr_labels(file_info, GITHUB_EASYBLOCKS_REPO) + self.assertEqual(res, ['new']) + + def test_github_det_patch_specs(self): """Test for det_patch_specs function.""" patch_paths = [os.path.join(self.test_prefix, p) for p in ['1.patch', '2.patch', '3.patch']] - file_info = {'ecs': [ - {'name': 'A', 'patches': ['1.patch'], 'exts_list': []}, - {'name': 'B', 'patches': [], 'exts_list': []}, - ] - } + file_info = {'ecs': []} + + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'A' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + patches = ['1.patch'] + """) + file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt)) + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'B' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + """) + file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt)) + error_pattern = "Failed to determine software name to which patch file .*/2.patch relates" self.mock_stdout(True) self.assertErrorRegex(EasyBuildError, error_pattern, gh.det_patch_specs, patch_paths, file_info, []) self.mock_stdout(False) - file_info['ecs'].append({'name': 'C', 'patches': [('3.patch', 'subdir'), '2.patch'], 'exts_list': []}) + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'C' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + patches = [('3.patch', 'subdir'), '2.patch'] + """) + file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt)) self.mock_stdout(True) res = gh.det_patch_specs(patch_paths, file_info, []) self.mock_stdout(False) - self.assertEqual(len(res), 3) - self.assertEqual(os.path.basename(res[0][0]), '1.patch') - self.assertEqual(res[0][1], 'A') - self.assertEqual(os.path.basename(res[1][0]), '2.patch') - self.assertEqual(res[1][1], 'C') - self.assertEqual(os.path.basename(res[2][0]), '3.patch') - self.assertEqual(res[2][1], 'C') + self.assertEqual([i[0] for i in res], patch_paths) + self.assertEqual([i[1] for i in res], ['A', 'C', 'C']) # check if patches for extensions are found - file_info['ecs'][-1] = { - 'name': 'patched_ext', - 'patches': [], - 'exts_list': [ + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'patched_ext' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + exts_list = [ 'foo', ('bar', '1.2.3'), ('patched', '4.5.6', { - 'patches': [('2.patch', 1), '3.patch'], + 'patches': [('%(name)s-2.patch', 1), '%(name)s-3.patch'], }), - ], - } + ] + """) + patch_paths[1:3] = [os.path.join(self.test_prefix, p) for p in ['patched-2.patch', 'patched-3.patch']] + file_info['ecs'][-1] = EasyConfig(None, rawtxt=rawtxt) self.mock_stdout(True) res = gh.det_patch_specs(patch_paths, file_info, []) self.mock_stdout(False) - self.assertEqual(len(res), 3) - self.assertEqual(os.path.basename(res[0][0]), '1.patch') - self.assertEqual(res[0][1], 'A') - self.assertEqual(os.path.basename(res[1][0]), '2.patch') - self.assertEqual(res[1][1], 'patched_ext') - self.assertEqual(os.path.basename(res[2][0]), '3.patch') - self.assertEqual(res[2][1], 'patched_ext') + self.assertEqual([i[0] for i in res], patch_paths) + self.assertEqual([i[1] for i in res], ['A', 'patched_ext', 'patched_ext']) - def test_restclient(self): + # check if patches for components are found + rawtxt = textwrap.dedent(""" + easyblock = 'PythonBundle' + name = 'patched_bundle' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + components = [ + ('bar', '1.2.3'), + ('patched', '4.5.6', { + 'patches': [('%(name)s-2.patch', 1), '%(name)s-3.patch'], + }), + ] + """) + file_info['ecs'][-1] = EasyConfig(None, rawtxt=rawtxt) + + self.mock_stdout(True) + res = gh.det_patch_specs(patch_paths, file_info, []) + self.mock_stdout(False) + + self.assertEqual([i[0] for i in res], patch_paths) + self.assertEqual([i[1] for i in res], ['A', 'patched_bundle', 'patched_bundle']) + + def test_github_restclient(self): """Test use of RestClient.""" if self.skip_github_tests: print("Skipping test_restclient, no GitHub token available?") @@ -768,7 +938,7 @@ def test_restclient(self): httperror_hit = True self.assertTrue(httperror_hit, "expected HTTPError not encountered") - def test_create_delete_gist(self): + def test_github_create_delete_gist(self): """Test create_gist and delete_gist.""" if self.skip_github_tests: print("Skipping test_restclient, no GitHub token available?") @@ -780,7 +950,7 @@ def test_create_delete_gist(self): gist_id = gist_url.split('/')[-1] gh.delete_gist(gist_id, github_user=GITHUB_TEST_ACCOUNT, github_token=self.github_token) - def test_det_account_branch_for_pr(self): + def test_github_det_account_branch_for_pr(self): """Test det_account_branch_for_pr.""" if self.skip_github_tests: print("Skipping test_det_account_branch_for_pr, no GitHub token available?") @@ -810,7 +980,7 @@ def test_det_account_branch_for_pr(self): self.assertEqual(account, 'migueldiascosta') self.assertEqual(branch, 'fix_inject_checksums') - def test_det_pr_target_repo(self): + def test_github_det_pr_target_repo(self): """Test det_pr_target_repo.""" self.assertEqual(build_option('pr_target_repo'), None) @@ -818,15 +988,19 @@ def test_det_pr_target_repo(self): # no files => return default target repo (None) self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([])), None) + test_dir = os.path.dirname(os.path.abspath(__file__)) + # easyconfigs/patches (incl. files to delete) => easyconfigs repo - # this is solely based on filenames, actual files are not opened + # this is solely based on filenames, actual files are not opened, except for the patch file which must exist + toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' + toy_patch = os.path.join(test_dir, 'sandbox', 'sources', 'toy', toy_patch_fn) test_cases = [ ['toy.eb'], - ['toy.patch'], - ['toy.eb', 'toy.patch'], + [toy_patch], + ['toy.eb', toy_patch], [':toy.eb'], # deleting toy.eb ['one.eb', 'two.eb'], - ['one.eb', 'two.eb', 'toy.patch', ':todelete.eb'], + ['one.eb', 'two.eb', toy_patch, ':todelete.eb'], ] for test_case in test_cases: self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(test_case)), 'easybuild-easyconfigs') @@ -834,12 +1008,11 @@ def test_det_pr_target_repo(self): # if only Python files are involved, result is easyblocks or framework repo; # all Python files are easyblocks => easyblocks repo, otherwise => framework repo; # files are opened and inspected here to discriminate between easyblocks & other Python files, so must exist! - testdir = os.path.dirname(os.path.abspath(__file__)) - github_py = os.path.join(testdir, 'github.py') + github_py = os.path.join(test_dir, 'github.py') - configuremake = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') + configuremake = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') self.assertTrue(os.path.exists(configuremake)) - toy_eb = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + toy_eb = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') self.assertTrue(os.path.exists(toy_eb)) self.assertEqual(build_option('pr_target_repo'), None) @@ -853,14 +1026,14 @@ def test_det_pr_target_repo(self): self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files)), 'easybuild-framework') # as soon as an easyconfig file or patch files is involved => result is easybuild-easyconfigs repo - for fn in ['toy.eb', 'toy.patch']: + for fn in ['toy.eb', toy_patch]: self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files + [fn])), 'easybuild-easyconfigs') # if --pr-target-repo is specified, we always get this value (no guessing anymore) init_config(build_options={'pr_target_repo': 'thisisjustatest'}) self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([])), 'thisisjustatest') - self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(['toy.eb', 'toy.patch'])), 'thisisjustatest') + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(['toy.eb', toy_patch])), 'thisisjustatest') self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files)), 'thisisjustatest') self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([configuremake])), 'thisisjustatest') self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([toy_eb])), 'thisisjustatest') @@ -876,7 +1049,7 @@ def test_push_branch_to_github(self): self.mock_stderr(True) self.mock_stdout(True) - gh.setup_repo(git_repo, GITHUB_USER, GITHUB_REPO, 'master') + gh.setup_repo(git_repo, GITHUB_USER, GITHUB_REPO, 'main') git_repo.create_head(branch, force=True) gh.push_branch_to_github(git_repo, GITHUB_USER, GITHUB_REPO, branch) stderr = self.get_stderr() @@ -888,13 +1061,13 @@ def test_push_branch_to_github(self): github_path = '%s/%s.git' % (GITHUB_USER, GITHUB_REPO) pattern = r'^' + '\n'.join([ - r"== fetching branch 'master' from https://github.com/%s\.\.\." % github_path, + r"== fetching branch 'main' from https://github.com/%s\.\.\." % github_path, r"== pushing branch 'test123' to remote 'github_.*' \(git@github.com:%s\) \[DRY RUN\]" % github_path, ]) + r'$' regex = re.compile(pattern) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' doesn't match: %s" % (regex.pattern, stdout)) - def test_pr_test_report(self): + def test_github_pr_test_report(self): """Test for post_pr_test_report function.""" if self.skip_github_tests: print("Skipping test_post_pr_test_report, no GitHub token available?") @@ -943,6 +1116,80 @@ def test_pr_test_report(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + # also test combination of --from-pr and --include-easyblocks-from-pr + update_build_option('include_easyblocks_from_pr', ['6789']) + + self.mock_stderr(True) + self.mock_stdout(True) + post_pr_test_report('1234', gh.GITHUB_EASYCONFIGS_REPO, test_report, "OK!", init_session_state, True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertEqual(stderr, '') + + patterns = [ + r"^\[DRY RUN\] Adding comment to easybuild-easyconfigs issue #1234: 'Test report by @easybuild_test", + r"^See https://gist.github.com/DRY_RUN for a full test report.'", + r"Using easyblocks from PR\(s\) https://github.com/easybuilders/easybuild-easyblocks/pull/6789", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + def test_github_create_test_report(self): + """Test create_test_report function.""" + logfile = os.path.join(self.test_prefix, 'log.txt') + write_file(logfile, "Bazel failed with: error") + ecs_with_res = [ + ({'spec': 'test.eb'}, {'success': True}), + ({'spec': 'fail.eb'}, { + 'success': False, + 'err': EasyBuildError("error: bazel"), + 'traceback': "in bazel", + 'log_file': logfile, + }), + ] + init_session_state = { + 'easybuild_configuration': ['EASYBUILD_DEBUG=1'], + 'environment': {'USER': 'test'}, + 'module_list': [{'mod_name': 'test'}], + 'system_info': {'name': 'test'}, + 'time': gmtime(0), + } + res = create_test_report("just a test", ecs_with_res, init_session_state) + patterns = [ + "**SUCCESS** _test.eb_", + "**FAIL (build issue)** _fail.eb_", + "01 Jan 1970 00:00:00", + "EASYBUILD_DEBUG=1", + ] + for pattern in patterns: + self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['full'])) + + for pattern in patterns[:2]: + self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['overview'])) + + # mock create_gist function, we don't want to actually create a gist every time we run this test... + def fake_create_gist(*args, **kwargs): + return 'https://gist.github.com/test' + + easybuild.tools.testing.create_gist = fake_create_gist + + res = create_test_report("just a test", ecs_with_res, init_session_state, pr_nrs=[123], gist_log=True) + + patterns.insert(2, "https://gist.github.com/test") + patterns.extend([ + "https://github.com/easybuilders/easybuild-easyconfigs/pull/123", + ]) + for pattern in patterns: + self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['full'])) + + for pattern in patterns[:3]: + self.assertTrue(pattern in res['full'], "Pattern '%s' found in: %s" % (pattern, res['overview'])) + + self.assertTrue("**SUCCESS** _test.eb_" in res['overview']) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/hooks.py b/test/framework/hooks.py index f51cd0fc91..8826c4bf37 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2021 Ghent University +# Copyright 2017-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,7 +29,7 @@ """ import os import sys -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner import easybuild.tools.hooks @@ -124,6 +124,8 @@ def test_run_hook(self): hooks = load_hooks(self.test_hooks_pymod) + init_config(build_options={'debug': True}) + self.mock_stdout(True) self.mock_stderr(True) run_hook('start', hooks) diff --git a/test/framework/include.py b/test/framework/include.py index 4e25adac4c..fbc47ab10e 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/lib.py b/test/framework/lib.py index e4bbca3b15..3196ad63cf 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2021 Ghent University +# Copyright 2018-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/license.py b/test/framework/license.py index 609bffaa3e..4244835bc9 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 17911b8d05..4862f8ff79 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -616,7 +616,8 @@ def test_swap(self): else: expected = '\n'.join([ '', - 'swap("foo", "bar")', + 'unload("foo")', + 'load("bar")', '', ]) @@ -637,7 +638,8 @@ def test_swap(self): expected = '\n'.join([ '', 'if isloaded("foo") then', - ' swap("foo", "bar")', + ' unload("foo")', + ' load("bar")', 'else', ' load("bar")', 'end', @@ -668,6 +670,10 @@ def test_swap(self): def test_append_paths(self): """Test generating append-paths statements.""" # test append_paths + def append_paths(*args, **kwargs): + """Wrap this into start_module_creation which need to be called prior to append_paths""" + with self.modgen.start_module_creation(): + return self.modgen.append_paths(*args, **kwargs) if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: expected = ''.join([ @@ -676,17 +682,17 @@ def test_append_paths(self): "append-path\tkey\t\t$root\n", ]) paths = ['path1', 'path2', ''] - self.assertEqual(expected, self.modgen.append_paths("key", paths)) + self.assertEqual(expected, append_paths("key", paths)) # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! - self.assertEqual(expected, self.modgen.append_paths("key", paths)) + self.assertEqual(expected, append_paths("key", paths)) expected = "append-path\tbar\t\t$root/foo\n" - self.assertEqual(expected, self.modgen.append_paths("bar", "foo")) + self.assertEqual(expected, append_paths("bar", "foo")) - res = self.modgen.append_paths("key", ["/abs/path"], allow_abs=True) + res = append_paths("key", ["/abs/path"], allow_abs=True) self.assertEqual("append-path\tkey\t\t/abs/path\n", res) - res = self.modgen.append_paths('key', ['1234@example.com'], expand_relpaths=False) + res = append_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual("append-path\tkey\t\t1234@example.com\n", res) else: @@ -696,22 +702,33 @@ def test_append_paths(self): 'append_path("key", root)\n', ]) paths = ['path1', 'path2', ''] - self.assertEqual(expected, self.modgen.append_paths("key", paths)) + self.assertEqual(expected, append_paths("key", paths)) # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! - self.assertEqual(expected, self.modgen.append_paths("key", paths)) + self.assertEqual(expected, append_paths("key", paths)) expected = 'append_path("bar", pathJoin(root, "foo"))\n' - self.assertEqual(expected, self.modgen.append_paths("bar", "foo")) + self.assertEqual(expected, append_paths("bar", "foo")) expected = 'append_path("key", "/abs/path")\n' - self.assertEqual(expected, self.modgen.append_paths("key", ["/abs/path"], allow_abs=True)) + self.assertEqual(expected, append_paths("key", ["/abs/path"], allow_abs=True)) - res = self.modgen.append_paths('key', ['1234@example.com'], expand_relpaths=False) + res = append_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual('append_path("key", "1234@example.com")\n', res) self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths " "which only expects relative paths." % self.modgen.app.installdir, - self.modgen.append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + + # check for warning that is printed when same path is added multiple times + with self.modgen.start_module_creation(): + self.modgen.append_paths('TEST', 'path1') + self.mock_stderr(True) + self.modgen.append_paths('TEST', 'path1') + stderr = self.get_stderr() + self.mock_stderr(False) + expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module " + expected_warning += "as they were already added: path1\n\n" + self.assertEqual(stderr, expected_warning) def test_module_extensions(self): """test the extensions() for extensions""" @@ -743,6 +760,10 @@ def test_module_extensions(self): def test_prepend_paths(self): """Test generating prepend-paths statements.""" # test prepend_paths + def prepend_paths(*args, **kwargs): + """Wrap this into start_module_creation which need to be called prior to append_paths""" + with self.modgen.start_module_creation(): + return self.modgen.prepend_paths(*args, **kwargs) if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: expected = ''.join([ @@ -751,17 +772,17 @@ def test_prepend_paths(self): "prepend-path\tkey\t\t$root\n", ]) paths = ['path1', 'path2', ''] - self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + self.assertEqual(expected, prepend_paths("key", paths)) # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! - self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + self.assertEqual(expected, prepend_paths("key", paths)) expected = "prepend-path\tbar\t\t$root/foo\n" - self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) + self.assertEqual(expected, prepend_paths("bar", "foo")) - res = self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True) + res = prepend_paths("key", ["/abs/path"], allow_abs=True) self.assertEqual("prepend-path\tkey\t\t/abs/path\n", res) - res = self.modgen.prepend_paths('key', ['1234@example.com'], expand_relpaths=False) + res = prepend_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual("prepend-path\tkey\t\t1234@example.com\n", res) else: @@ -771,22 +792,33 @@ def test_prepend_paths(self): 'prepend_path("key", root)\n', ]) paths = ['path1', 'path2', ''] - self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + self.assertEqual(expected, prepend_paths("key", paths)) # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! - self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + self.assertEqual(expected, prepend_paths("key", paths)) expected = 'prepend_path("bar", pathJoin(root, "foo"))\n' - self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) + self.assertEqual(expected, prepend_paths("bar", "foo")) expected = 'prepend_path("key", "/abs/path")\n' - self.assertEqual(expected, self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True)) + self.assertEqual(expected, prepend_paths("key", ["/abs/path"], allow_abs=True)) - res = self.modgen.prepend_paths('key', ['1234@example.com'], expand_relpaths=False) + res = prepend_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual('prepend_path("key", "1234@example.com")\n', res) self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths " "which only expects relative paths." % self.modgen.app.installdir, - self.modgen.prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + + # check for warning that is printed when same path is added multiple times + with self.modgen.start_module_creation(): + self.modgen.prepend_paths('TEST', 'path1') + self.mock_stderr(True) + self.modgen.prepend_paths('TEST', 'path1') + stderr = self.get_stderr() + self.mock_stderr(False) + expected_warning = "\nWARNING: Suppressed adding the following path(s) to $TEST of the module " + expected_warning += "as they were already added: path1\n\n" + self.assertEqual(stderr, expected_warning) def test_det_user_modpath(self): """Test for generic det_user_modpath method.""" @@ -853,13 +885,44 @@ def test_env(self): def test_getenv_cmd(self): """Test getting value of environment variable.""" + + test_mod_file = os.path.join(self.test_prefix, 'test', '1.2.3') + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + # can't have $LMOD_QUIET set when testing with Tcl syntax, + # otherwise we won't get the output produced by the test module file... + if 'LMOD_QUIET' in os.environ: + del os.environ['LMOD_QUIET'] + self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME')) + + expected = '[if { [info exists ::env(TEST)] } { concat $::env(TEST) } else { concat "foobar" } ]' + getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') + self.assertEqual(getenv_txt, expected) + + write_file(test_mod_file, '#%%Module\nputs stderr %s' % getenv_txt) else: self.assertEqual('os.getenv("HOSTNAME")', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('os.getenv("HOME")', self.modgen.getenv_cmd('HOME')) + expected = 'os.getenv("TEST") or "foobar"' + getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') + self.assertEqual(getenv_txt, expected) + + test_mod_file += '.lua' + write_file(test_mod_file, "io.stderr:write(%s)" % getenv_txt) + + # only test loading of test module in Lua syntax when using Lmod + if isinstance(self.modtool, Lmod) or not test_mod_file.endswith('.lua'): + self.modtool.use(self.test_prefix) + out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) + self.assertEqual(out.strip(), 'foobar') + + os.environ['TEST'] = 'test_value_that_is_not_foobar' + out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) + self.assertEqual(out.strip(), 'test_value_that_is_not_foobar') + def test_alias(self): """Test setting of alias in modulefiles.""" if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: @@ -1268,6 +1331,14 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, ['Compiler/intel/2019.4.243'], ['Core']), 'imkl-2019.4.243-iimpi-2019.08.eb': ('imkl/2019.4.243', 'MPI/intel/2019.4.243/impi/2019.4.243', [], [], ['Core']), + 'intel-compilers-2021.2.0.eb': ('intel-compilers/2021.2.0', 'Core', + ['Compiler/intel/2021.2.0'], ['Compiler/intel/2021.2.0'], ['Core']), + 'impi-2021.2.0-intel-compilers-2021.2.0.eb': ('impi/2021.2.0', 'Compiler/intel/2021.2.0', + ['MPI/intel/2021.2.0/impi/2021.2.0'], + ['MPI/intel/2021.2.0/impi/2021.2.0'], + ['Core']), + 'imkl-2021.2.0-iimpi-2021a.eb': ('imkl/2021.2.0', 'MPI/intel/2021.2.0/impi/2021.2.0', + [], [], ['Core']), 'CUDA-9.1.85-GCC-6.4.0-2.28.eb': ('CUDA/9.1.85', 'Compiler/GCC/6.4.0-2.28', ['Compiler/GCC-CUDA/6.4.0-2.28-9.1.85'], ['Compiler/GCC-CUDA/6.4.0-2.28-9.1.85'], ['Core']), @@ -1295,6 +1366,10 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, ['Toolchain/CrayCCE/5.1.29'], ['Toolchain/CrayCCE/5.1.29'], ['Core']), + 'cpeGNU-21.04.eb': ('cpeGNU/21.04', 'Core', + ['Toolchain/cpeGNU/21.04'], + ['Toolchain/cpeGNU/21.04'], + ['Core']), 'HPL-2.1-CrayCCE-5.1.29.eb': ('HPL/2.1', 'Toolchain/CrayCCE/5.1.29', [], [], ['Core']), } for ecfile, mns_vals in test_ecs.items(): @@ -1485,6 +1560,48 @@ def test_det_installdir(self): self.assertEqual(self.modgen.det_installdir(test_modfile), expected) + def test_generated_module_file_swap(self): + """Test loading a generated module file that includes swap statements.""" + + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorLua: + mod_ext = '.lua' + + if not isinstance(self.modtool, Lmod): + # Lua module files are only supported by Lmod, + # so early exit if that's not the case in the test setup + return + + else: + mod_ext = '' + + # empty test modules + for modname in ('one/1.0', 'one/1.1'): + modfile = os.path.join(self.test_prefix, modname + mod_ext) + write_file(modfile, self.modgen.MODULE_SHEBANG) + + modulepath = os.getenv('MODULEPATH') + if modulepath: + self.modtool.unuse(modulepath) + + test_mod = os.path.join(self.test_prefix, 'test', '1.0' + mod_ext) + test_mod_txt = '\n'.join([ + self.modgen.MODULE_SHEBANG, + self.modgen.swap_module('one', 'one/1.1'), + ]) + write_file(test_mod, test_mod_txt) + + # prepare environment for loading test module + self.modtool.use(self.test_prefix) + self.modtool.load(['one/1.0']) + + self.modtool.load(['test/1.0']) + + # check whether resulting environment is correct + loaded_mods = self.modtool.list() + self.assertEqual(loaded_mods[-1]['mod_name'], 'test/1.0') + # one/1.0 module was swapped for one/1.1 + self.assertEqual(loaded_mods[-2]['mod_name'], 'one/1.1') + class TclModuleGeneratorTest(ModuleGeneratorTest): """Test for module_generator module for Tcl syntax.""" diff --git a/test/framework/modules.py b/test/framework/modules.py index f9b6ef6238..6950e372a5 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -54,7 +54,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 81 +TEST_MODULES_COUNT = 90 class ModulesTest(EnhancedTestCase): @@ -531,14 +531,15 @@ def test_check_module_path(self): os.environ['MODULEPATH'] = test2 modtool.check_module_path() self.assertEqual(modtool.mod_paths, [mod_install_dir, test1, test2]) - self.assertEqual(os.environ['MODULEPATH'], mod_install_dir + ':' + test1 + ':' + test2) + self.assertEqual(os.environ['MODULEPATH'], os.pathsep.join([mod_install_dir, test1, test2])) # check behaviour if non-existing directories are included in $MODULEPATH os.environ['MODULEPATH'] = '%s:/does/not/exist:%s' % (test3, test2) modtool.check_module_path() # non-existing dir is filtered from mod_paths, but stays in $MODULEPATH self.assertEqual(modtool.mod_paths, [mod_install_dir, test1, test3, test2]) - self.assertEqual(os.environ['MODULEPATH'], ':'.join([mod_install_dir, test1, test3, '/does/not/exist', test2])) + self.assertEqual(os.environ['MODULEPATH'], + os.pathsep.join([mod_install_dir, test1, test3, '/does/not/exist', test2])) def test_check_module_path_hmns(self): """Test behaviour of check_module_path with HierarchicalMNS.""" @@ -678,7 +679,7 @@ def test_get_software_root_version_libdir(self): self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64']) # only directories containing files in specified list should be retained - open(os.path.join(root, 'lib64', 'foo'), 'w').write('foo') + write_file(os.path.join(root, 'lib64', 'foo'), 'foo') self.assertEqual(get_software_libdir(name, fs=['foo']), 'lib64') # duplicate paths due to symlink get filtered @@ -1036,6 +1037,7 @@ def test_modules_tool_stateless(self): load_err_msg = "Unable to locate a modulefile" # GCC/4.6.3 is *not* an available Core module + os.environ['LC_ALL'] = 'C' self.assertErrorRegex(EasyBuildError, load_err_msg, self.modtool.load, ['GCC/4.6.3']) # GCC/6.4.0-2.28 is one of the available Core modules @@ -1089,11 +1091,12 @@ def test_module_caches(self): """Test module caches and invalidate_module_caches_for function.""" self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) - # purposely extending $MODULEPATH with non-existing path, should be handled fine + # purposely extending $MODULEPATH with an empty path, should be handled fine nonpath = os.path.join(self.test_prefix, 'nosuchfileordirectory') + mkdir(nonpath) self.modtool.use(nonpath) modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] - self.assertTrue(any([os.path.samefile(nonpath, mp) for mp in modulepaths])) + self.assertTrue(any(os.path.samefile(nonpath, mp) for mp in modulepaths)) shutil.rmtree(nonpath) # create symlink to entry in $MODULEPATH we're going to use, and add it to $MODULEPATH @@ -1133,33 +1136,162 @@ def test_module_caches(self): # invalidate caches with correct path modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p] - self.assertTrue(any([os.path.exists(mp) and os.path.samefile(test_mods_path, mp) for mp in modulepaths])) + self.assertTrue(any(os.path.exists(mp) and os.path.samefile(test_mods_path, mp) for mp in modulepaths)) paths_in_key = [p for p in avail_cache_key[0].split('=')[1].split(os.pathsep) if p] - self.assertTrue(any([os.path.exists(p) and os.path.samefile(test_mods_path, p) for p in paths_in_key])) + self.assertTrue(any(os.path.exists(p) and os.path.samefile(test_mods_path, p) for p in paths_in_key)) # verify cache invalidation, caches should be empty again invalidate_module_caches_for(test_mods_path) self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) self.assertEqual(mod.MODULE_SHOW_CACHE, {}) - def test_module_use(self): - """Test 'module use'.""" + def test_module_use_unuse(self): + """Test 'module use' and 'module unuse'.""" test_dir1 = os.path.join(self.test_prefix, 'one') test_dir2 = os.path.join(self.test_prefix, 'two') test_dir3 = os.path.join(self.test_prefix, 'three') + for subdir in ('one', 'two', 'three'): + modtxt = '\n'.join([ + '#%Module', + "setenv TEST123 %s" % subdir, + ]) + write_file(os.path.join(self.test_prefix, subdir, 'test'), modtxt) + self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) self.modtool.use(test_dir1) - self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir1)) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir1)) + self.modtool.use(test_dir2) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir2)) + self.modtool.use(test_dir3) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir3)) + + # Adding an empty modulepath is not possible + modulepath = os.environ.get('MODULEPATH', '') + self.assertErrorRegex(EasyBuildError, "Cannot add empty path", self.modtool.use, '') + self.assertEqual(os.environ.get('MODULEPATH', ''), modulepath) + + # make sure the right test module is loaded + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'three') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir3) + self.assertFalse(test_dir3 in os.environ.get('MODULEPATH', '')) + + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'two') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir2) + self.assertFalse(test_dir2 in os.environ.get('MODULEPATH', '')) + + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'one') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir1) + self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) # also test use with high priority self.modtool.use(test_dir2, priority=10000) self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir2)) - # check whether prepend with priority actually works (only for Lmod) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'two') + self.modtool.unload(['test']) + + # Tests for Lmod only if isinstance(self.modtool, Lmod): + # check whether prepend with priority actually works (priority is specific to Lmod) + self.modtool.use(test_dir1, priority=100) self.modtool.use(test_dir3) - self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:' % (test_dir2, test_dir3))) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:%s:' % (test_dir2, test_dir1, test_dir3))) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'two') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir2) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'one') + self.modtool.unload(['test']) + + self.modtool.unuse(test_dir1) + self.modtool.load(['test']) + self.assertEqual(os.getenv('TEST123'), 'three') + self.modtool.unload(['test']) + + # Check load and unload for a single path when it is the only one + # Only for Lmod as we have some shortcuts for avoiding the module call there + old_module_path = os.environ['MODULEPATH'] + del os.environ['MODULEPATH'] + self.modtool.use(test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir1) + self.modtool.unuse(test_dir1) + self.assertFalse('MODULEPATH' in os.environ) + os.environ['MODULEPATH'] = old_module_path # Restore + + def test_add_and_remove_module_path(self): + """Test add_module_path and whether remove_module_path undoes changes of add_module_path""" + test_dir1 = tempfile.mkdtemp(suffix="_dir1") + test_dir2 = tempfile.mkdtemp(suffix="_dir2") + old_module_path = os.environ.get('MODULEPATH') + del os.environ['MODULEPATH'] + self.modtool.add_module_path(test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir1) + self.modtool.add_module_path(test_dir2) + test_dir_2_and_1 = os.pathsep.join([test_dir2, test_dir1]) + self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1) + # Adding the same path does not change the path + self.modtool.add_module_path(test_dir1) + self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1) + self.modtool.add_module_path(test_dir2) + self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1) + # Even when a (meaningless) slash is added + # This occurs when using an empty modules directory name + self.modtool.add_module_path(os.path.join(test_dir1, '')) + self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1) + + # Similar tests for remove_module_path + self.modtool.remove_module_path(test_dir2) + self.assertEqual(os.environ['MODULEPATH'], test_dir1) + # Same again -> no-op + self.modtool.remove_module_path(test_dir2) + self.assertEqual(os.environ['MODULEPATH'], test_dir1) + # And with empty last part + self.modtool.remove_module_path(os.path.join(test_dir1, '')) + self.assertEqual(os.environ.get('MODULEPATH', ''), '') + + # And with some more trickery + # Lmod seems to remove empty paths: /foo//bar/. -> /foo/bar + # Environment-Modules 4.x seems to resolve relative paths: /foo/../foo -> /foo + # Hence we can only check the real paths + def get_resolved_module_path(): + return os.pathsep.join(os.path.realpath(p) for p in os.environ['MODULEPATH'].split(os.pathsep)) + + test_dir1_relative = os.path.join(test_dir1, '..', os.path.basename(test_dir1)) + test_dir2_dot = os.path.join(os.path.dirname(test_dir2), '.', os.path.basename(test_dir2)) + self.modtool.add_module_path(test_dir1_relative) + self.assertEqual(get_resolved_module_path(), test_dir1) + # Adding the same path, but in a different form may be possible, but may also be ignored, e.g. in EnvModules + self.modtool.add_module_path(test_dir1) + if get_resolved_module_path() != test_dir1: + self.assertEqual(get_resolved_module_path(), os.pathsep.join([test_dir1, test_dir1])) + self.modtool.remove_module_path(test_dir1) + self.assertEqual(get_resolved_module_path(), test_dir1) + self.modtool.add_module_path(test_dir2_dot) + self.assertEqual(get_resolved_module_path(), test_dir_2_and_1) + self.modtool.remove_module_path(test_dir2_dot) + self.assertEqual(get_resolved_module_path(), test_dir1) + # Force adding such a dot path which can be removed with either variant + os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative]) + self.modtool.remove_module_path(test_dir2_dot) + self.assertEqual(get_resolved_module_path(), test_dir1) + os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative]) + self.modtool.remove_module_path(test_dir2) + self.assertEqual(get_resolved_module_path(), test_dir1) + + os.environ['MODULEPATH'] = old_module_path # Restore def test_module_use_bash(self): """Test whether effect of 'module use' is preserved when a new bash session is started.""" diff --git a/test/framework/modules/FFTW.MPI/3.3.7 b/test/framework/modules/FFTW.MPI/3.3.7 new file mode 100644 index 0000000000..4c2b8db563 --- /dev/null +++ b/test/framework/modules/FFTW.MPI/3.3.7 @@ -0,0 +1,31 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org +} +} + +module-whatis {FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org} + +set root /home-2/khoste/.local/easybuild/software/FFTW.MPI/3.3.7 + +conflict FFTW.MPI + +if { ![is-loaded gompi/2018a] } { + module load gompi/2018a +} + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTFFTWMPI "$root" +setenv EBVERSIONFFTWMPI "3.3.7" +setenv EBDEVELFFTWMPI "$root/easybuild/FFTW.MPI-3.3.7-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/FFTW/3.3.7 b/test/framework/modules/FFTW/3.3.7 new file mode 100644 index 0000000000..350330b116 --- /dev/null +++ b/test/framework/modules/FFTW/3.3.7 @@ -0,0 +1,31 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org +} +} + +module-whatis {FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org} + +set root /home-2/khoste/.local/easybuild/software/FFTW/3.3.7 + +conflict FFTW + +if { ![is-loaded gompi/2018a] } { + module load gompi/2018a +} + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTFFTW "$root" +setenv EBVERSIONFFTW "3.3.7" +setenv EBDEVELFFTW "$root/easybuild/FFTW-3.3.7-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/foss/2018a-FFTW.MPI b/test/framework/modules/foss/2018a-FFTW.MPI new file mode 100644 index 0000000000..37d9ae0c14 --- /dev/null +++ b/test/framework/modules/foss/2018a-FFTW.MPI @@ -0,0 +1,46 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GCC based compiler toolchain including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) +} +} + +module-whatis {GCC based compiler toolchain including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none)} + +set root /prefix/software/foss/2018a + +conflict foss + +if { ![is-loaded GCC/6.4.0-2.28] } { + module load GCC/6.4.0-2.28 +} + +if { ![is-loaded OpenMPI/2.1.2-GCC-6.4.0-2.28] } { + module load OpenMPI/2.1.2-GCC-6.4.0-2.28 +} + +if { ![is-loaded OpenBLAS/0.2.20-GCC-6.4.0-2.28] } { + module load OpenBLAS/0.2.20-GCC-6.4.0-2.28 +} + +if { ![is-loaded FFTW/3.3.7] } { + module load FFTW/3.3.7 +} + +if { ![is-loaded FFTW.MPI/3.3.7 ] } { + module load FFTW.MPI/3.3.7 +} + +if { ![is-loaded ScaLAPACK/2.0.2-gompi-2018a-OpenBLAS-0.2.20] } { + module load ScaLAPACK/2.0.2-gompi-2018a-OpenBLAS-0.2.20 +} + + +setenv EBROOTFOSS "$root" +setenv EBVERSIONFOSS "2018a" +setenv EBDEVELFOSS "$root/easybuild/foss-2018a-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/gcccuda/2018a b/test/framework/modules/gcccuda/2018a new file mode 100644 index 0000000000..f9779f1be5 --- /dev/null +++ b/test/framework/modules/gcccuda/2018a @@ -0,0 +1,26 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GCC based compiler toolchain with CUDA support, and including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) +} +} + +module-whatis {GNU Compiler Collection (GCC) based compiler toolchain, along with CUDA toolkit. - Homepage: (none)} + +set root /prefix/software/gcccuda/2018a + +conflict gcccuda + +if { ![is-loaded GCC/6.4.0-2.28] } { + module load GCC/6.4.0-2.28 +} + +if { ![is-loaded CUDA/9.1.85] } { + module load CUDA/9.1.85 +} + + +setenv EBROOTGCCCUDA "$root" +setenv EBVERSIONGCCCUDA "2018a" +setenv EBDEVELGCCCUDA "$root/easybuild/gcccuda-2018a-easybuild-devel" diff --git a/test/framework/modules/imkl-FFTW/2021.4.0 b/test/framework/modules/imkl-FFTW/2021.4.0 new file mode 100644 index 0000000000..955bf68727 --- /dev/null +++ b/test/framework/modules/imkl-FFTW/2021.4.0 @@ -0,0 +1,31 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FFTW interfaces using Intel oneAPI Math Kernel Library + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html + } +} + +module-whatis {Description: FFTW interfaces using Intel oneAPI Math Kernel Library} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html} + +set root /tmp/imkl-FFTW/2021.4.0 + +conflict imkl-FFTW + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +setenv EBROOTIMKLMINFFTW "$root" +setenv EBVERSIONIMKLMINFFTW "2021.4.0" +setenv EBDEVELIMKLMINFFTW "$root/easybuild/imkl-FFTW-2021.4.0-easybuild-devel" + +# Built with EasyBuild version 4.5.0dev diff --git a/test/framework/modules/imkl/2021.4.0 b/test/framework/modules/imkl/2021.4.0 new file mode 100644 index 0000000000..f188251b48 --- /dev/null +++ b/test/framework/modules/imkl/2021.4.0 @@ -0,0 +1,37 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +Intel oneAPI Math Kernel Library + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html + } +} + +module-whatis {Description: Intel oneAPI Math Kernel Library} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html} + +set root /tmp/eb-bI0pBy/eb-DmuEpJ/eb-leoYDw/eb-UtJJqp/tmp8P3FOY + +conflict imkl + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/mkl/2021.4.0/include +prepend-path CPATH $root/mkl/2021.4.0/include/fftw +prepend-path LD_LIBRARY_PATH $root/compiler/2021.4.0/linux/compiler/lib/intel64_lin +prepend-path LD_LIBRARY_PATH $root/mkl/2021.4.0/lib/intel64 +prepend-path LIBRARY_PATH $root/compiler/2021.4.0/linux/compiler/lib/intel64_lin +prepend-path LIBRARY_PATH $root/mkl/2021.4.0/lib/intel64 +setenv EBROOTIMKL "$root" +setenv EBVERSIONIMKL "2021.4.0" +setenv EBDEVELIMKL "$root/easybuild/Core-imkl-2021.4.0-easybuild-devel" + +setenv MKL_EXAMPLES "$root/mkl/2021.4.0/examples" +setenv MKLROOT "$root/mkl/2021.4.0" +# Built with EasyBuild version 4.5.0dev diff --git a/test/framework/modules/impi/2021.4.0 b/test/framework/modules/impi/2021.4.0 new file mode 100644 index 0000000000..a8003ced0e --- /dev/null +++ b/test/framework/modules/impi/2021.4.0 @@ -0,0 +1,42 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +Intel MPI Library, compatible with MPICH ABI + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html + } +} + +module-whatis {Description: Intel MPI Library, compatible with MPICH ABI} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html} + +set root /tmp/impi/2021.4.0 + +conflict impi + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/mpi/2021.4.0/include +prepend-path FI_PROVIDER_PATH $root/mpi/2021.4.0/libfabric/lib/prov +prepend-path LD_LIBRARY_PATH $root/mpi/2021.4.0/lib +prepend-path LD_LIBRARY_PATH $root/mpi/2021.4.0/lib/release +prepend-path LD_LIBRARY_PATH $root/mpi/2021.4.0/libfabric/lib +prepend-path LIBRARY_PATH $root/mpi/2021.4.0/lib +prepend-path LIBRARY_PATH $root/mpi/2021.4.0/lib/release +prepend-path LIBRARY_PATH $root/mpi/2021.4.0/libfabric/lib +prepend-path MANPATH $root/mpi/2021.4.0/man +prepend-path PATH $root/mpi/2021.4.0/bin +prepend-path PATH $root/mpi/2021.4.0/libfabric/bin +setenv EBROOTIMPI "$root" +setenv EBVERSIONIMPI "2021.4.0" +setenv EBDEVELIMPI "$root/easybuild/impi-2021.4.0-easybuild-devel" + +setenv I_MPI_ROOT "$root/mpi/2021.4.0" +setenv UCX_TLS "all" +# Built with EasyBuild version 4.5.0dev diff --git a/test/framework/modules/intel-compilers/2021.4.0 b/test/framework/modules/intel-compilers/2021.4.0 new file mode 100644 index 0000000000..b9e93096d1 --- /dev/null +++ b/test/framework/modules/intel-compilers/2021.4.0 @@ -0,0 +1,41 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +Intel C, C++ & Fortran compilers (classic and oneAPI) + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html + } +} + +module-whatis {Description: Intel C, C++ & Fortran compilers (classic and oneAPI)} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html} + +set root /tmp/intel-compilers/2021.4.0 + +conflict intel-compilers + +prepend-path CPATH $root/tbb/2021.4.0/include +prepend-path LD_LIBRARY_PATH $root/compiler/2021.4.0/linux/lib +prepend-path LD_LIBRARY_PATH $root/compiler/2021.4.0/linux/lib/x64 +prepend-path LD_LIBRARY_PATH $root/compiler/2021.4.0/linux/compiler/lib/intel64_lin +prepend-path LD_LIBRARY_PATH $root/tbb/2021.4.0/lib/intel64/gcc4.8 +prepend-path LIBRARY_PATH $root/compiler/2021.4.0/linux/lib +prepend-path LIBRARY_PATH $root/compiler/2021.4.0/linux/lib/x64 +prepend-path LIBRARY_PATH $root/compiler/2021.4.0/linux/compiler/lib/intel64_lin +prepend-path LIBRARY_PATH $root/tbb/2021.4.0/lib/intel64/gcc4.8 +prepend-path OCL_ICD_FILENAMES $root/compiler/2021.4.0/linux/lib/x64/libintelocl.so +prepend-path PATH $root/compiler/2021.4.0/linux/bin +prepend-path PATH $root/compiler/2021.4.0/linux/bin/intel64 +prepend-path TBBROOT $root/tbb/2021.4.0 +setenv EBROOTINTELMINCOMPILERS "$root" +setenv EBVERSIONINTELMINCOMPILERS "2021.4.0" +setenv EBDEVELINTELMINCOMPILERS "$root/easybuild/Core-intel-compilers-2021.4.0-easybuild-devel" + +# Built with EasyBuild version 4.5.0dev diff --git a/test/framework/modules/intel/2021b b/test/framework/modules/intel/2021b new file mode 100644 index 0000000000..5695e2e0b3 --- /dev/null +++ b/test/framework/modules/intel/2021b @@ -0,0 +1,36 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ + } +} + +module-whatis {Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/} + +set root /tmp/intel/2021b + +conflict intel + +if { ![is-loaded intel-compilers/2021.4.0] } { + module load intel-compilers/2021.4.0 +} + +if { ![is-loaded impi/2021.4.0] } { + module load impi/2021.4.0 +} + +if { ![is-loaded imkl/2021.4.0] } { + module load imkl/2021.4.0 +} + +if { ![is-loaded imkl-FFTW/2021.4.0] } { + module load imkl-FFTW/2021.4.0 +} + + +setenv EBROOTINTEL "$root" +setenv EBVERSIONINTEL "2021b" +setenv EBDEVELINTEL "$root/easybuild/intel-2021b-easybuild-devel" + + +# built with EasyBuild version 4.5.0dev diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 9f9dc9ec48..7dbe10c9ac 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,7 +39,7 @@ from easybuild.base import fancylogger from easybuild.tools import modules from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import which, write_file +from easybuild.tools.filetools import read_file, which, write_file from easybuild.tools.modules import Lmod from test.framework.utilities import init_config @@ -123,9 +123,7 @@ def test_module_mismatch(self): fancylogger.logToFile(self.logfile) mt = MockModulesTool(testing=True) - f = open(self.logfile, 'r') - logtxt = f.read() - f.close() + logtxt = read_file(self.logfile) warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function") self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt)) @@ -137,9 +135,7 @@ def test_module_mismatch(self): # a warning should be logged if the 'module' function is undefined del os.environ['module'] mt = MockModulesTool(testing=True) - f = open(self.logfile, 'r') - logtxt = f.read() - f.close() + logtxt = read_file(self.logfile) warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*") self.assertTrue(warn_regex.search(logtxt), "Pattern %s found in %s" % (warn_regex.pattern, logtxt)) diff --git a/test/framework/options.py b/test/framework/options.py index cbdd57cc71..126c52931b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,8 +31,10 @@ import os import re import shutil +import stat import sys import tempfile +import textwrap from distutils.version import LooseVersion from unittest import TextTestRunner @@ -42,7 +44,6 @@ import easybuild.tools.toolchain from easybuild.base import fancylogger from easybuild.framework.easyblock import EasyBlock -from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, robot_find_easyconfig @@ -51,7 +52,8 @@ from easybuild.tools.config import DEFAULT_MODULECLASSES from easybuild.tools.config import find_last_log, get_build_log_path, get_module_syntax, module_classes from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, is_patch_file, mkdir +from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, download_file +from easybuild.tools.filetools import is_patch_file, mkdir, move_file, parse_http_header_fields_urlpat from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import URL_SEPARATOR, fetch_github_token @@ -394,6 +396,29 @@ def test_skip_test_step(self): found = re.search(test_run_msg, outtxt) self.assertFalse(found, "Test execution command is NOT present, outtxt: %s" % outtxt) + def test_ignore_test_failure(self): + """Test ignore failing tests (--ignore-test-failure).""" + + topdir = os.path.abspath(os.path.dirname(__file__)) + # This EC uses a `runtest` command which does not exist and hence will make the test step fail + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + + args = [toy_ec, '--ignore-test-failure', '--force'] + + with self.mocked_stdout_stderr() as (_, stderr): + outtxt = self.eb_main(args, do_build=True) + + msg = 'Test failure ignored' + self.assertTrue(re.search(msg, outtxt), + "Ignored test failure message in log should be found, outtxt: %s" % outtxt) + self.assertTrue(re.search(msg, stderr.getvalue()), + "Ignored test failure message in stderr should be found, stderr: %s" % stderr.getvalue()) + + # Passing skip and ignore options is disallowed + args.append('--skip-test-step') + error_pattern = 'Found both ignore-test-failure and skip-test-step enabled' + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + def test_job(self): """Test submitting build as a job.""" @@ -491,12 +516,16 @@ def run_test(fmt=None): if fmt == 'rst': pattern_lines = [ + r'^``ARCH``\s*``(aarch64|ppc64le|x86_64)``\s*CPU architecture .*', + r'^``EXTERNAL_MODULE``.*', r'^``HOME``.*', r'``OS_NAME``.*', r'``OS_PKG_IBVERBS_DEV``.*', ] else: pattern_lines = [ + r'^\s*ARCH: (aarch64|ppc64le|x86_64) \(CPU architecture .*\)', + r'^\s*EXTERNAL_MODULE:.*', r'^\s*HOME:.*', r'\s*OS_NAME: .*', r'\s*OS_PKG_IBVERBS_DEV: .*', @@ -508,6 +537,55 @@ def run_test(fmt=None): for fmt in [None, 'txt', 'rst']: run_test(fmt=fmt) + def test_avail_easyconfig_templates(self): + """Test listing available easyconfig file templates.""" + + def run_test(fmt=None): + """Helper function to test --avail-easyconfig-templates.""" + + args = ['--avail-easyconfig-templates'] + if fmt is not None: + args.append('--output-format=%s' % fmt) + + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(args, verbose=True, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + + if fmt == 'rst': + pattern_lines = [ + r'^``%\(version_major\)s``\s+Major version\s*$', + r'^``%\(cudaver\)s``\s+full version for CUDA\s*$', + r'^``%\(pyshortver\)s``\s+short version for Python \(.\)\s*$', + r'^\* ``%\(name\)s``$', + r'^``%\(namelower\)s``\s+lower case of value of name\s*$', + r'^``%\(arch\)s``\s+System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)\s*$', + r'^``%\(cuda_cc_space_sep\)s``\s+Space-separated list of CUDA compute capabilities\s*$', + r'^``SOURCE_TAR_GZ``\s+Source \.tar\.gz bundle\s+``%\(name\)s-%\(version\)s.tar.gz``\s*$', + ] + else: + pattern_lines = [ + r'^\s+%\(version_major\)s: Major version$', + r'^\s+%\(cudaver\)s: full version for CUDA$', + r'^\s+%\(pyshortver\)s: short version for Python \(.\)$', + r'^\s+%\(name\)s$', + r'^\s+%\(namelower\)s: lower case of value of name$', + r'^\s+%\(arch\)s: System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)$', + r'^\s+%\(cuda_cc_space_sep\)s: Space-separated list of CUDA compute capabilities$', + r'^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)$', + ] + + for pattern_line in pattern_lines: + regex = re.compile(pattern_line, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should match in: %s" % (regex.pattern, stdout)) + + for fmt in [None, 'txt', 'rst']: + run_test(fmt=fmt) + def test_avail_easyconfig_params(self): """Test listing available easyconfig parameters.""" @@ -730,7 +808,9 @@ def test_avail_cfgfile_constants(self): os.remove(dummylogfn) sys.path[:] = orig_sys_path - def test_list_easyblocks(self): + # use test_000_* to ensure this test is run *first*, + # before any tests that pick up additional easyblocks (which are difficult to clean up) + def test_000_list_easyblocks(self): """Test listing easyblock hierarchy.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -746,18 +826,21 @@ def test_list_easyblocks(self): list_arg, '--unittest-file=%s' % self.logfile, ] - self.eb_main(args, logfile=dummylogfn) + self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) expected = '\n'.join([ r'EasyBlock', r'\|-- bar', r'\|-- ConfigureMake', + r'\| \|-- MakeCp', + r'\|-- EB_EasyBuildMeta', r'\|-- EB_FFTW', r'\|-- EB_foo', r'\| \|-- EB_foofoo', r'\|-- EB_GCC', r'\|-- EB_HPL', + r'\|-- EB_libtoy', r'\|-- EB_OpenBLAS', r'\|-- EB_OpenMPI', r'\|-- EB_ScaLAPACK', @@ -769,6 +852,7 @@ def test_list_easyblocks(self): r'\| \| \|-- EB_toytoy', r'\| \|-- Toy_Extension', r'\|-- ModuleRC', + r'\|-- PythonBundle', r'\|-- Toolchain', r'Extension', r'\|-- ExtensionEasyBlock', @@ -880,7 +964,7 @@ def test_search(self): for search_arg in ['-S', '--search-short']: args = [ search_arg, - 'toy-0.0', + '^toy-0.0', '-r', test_easyconfigs_dir, ] @@ -936,7 +1020,7 @@ def test_ignore_index(self): toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') copy_file(toy_ec, self.test_prefix) - toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb'] + toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb', 'toy-11.5.6.eb'] # install index that list more files than are actually available, # so we can check whether it's used @@ -946,15 +1030,16 @@ def test_ignore_index(self): args = [ '--search=toy', '--robot-paths=%s' % self.test_prefix, + '--terse', ] self.mock_stdout(True) self.eb_main(args, testing=False, raise_error=True) stdout = self.get_stdout() self.mock_stdout(False) - for toy_ec_fn in toy_ec_list: - regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + # Also checks for ordering: 11.x comes last! + expected_output = '\n'.join(os.path.join(self.test_prefix, ec) for ec in toy_ec_list) + '\n' + self.assertEqual(stdout, expected_output) args.append('--ignore-index') self.mock_stdout(True) @@ -962,11 +1047,8 @@ def test_ignore_index(self): stdout = self.get_stdout() self.mock_stdout(False) - regex = re.compile(re.escape(os.path.join(self.test_prefix, 'toy-0.0.eb')), re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) - for toy_ec_fn in ['toy-1.2.3.eb', 'toy-4.5.6.eb']: - regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M) - self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout)) + # This should be the only EC found + self.assertEqual(stdout, os.path.join(self.test_prefix, 'toy-0.0.eb') + '\n') def test_search_archived(self): "Test searching for archived easyconfigs" @@ -976,6 +1058,7 @@ def test_search_archived(self): txt = self.get_stdout().rstrip() self.mock_stdout(False) expected = '\n'.join([ + ' * intel-compilers-2021.2.0.eb', ' * intel-2018a.eb', '', "Note: 1 matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them", @@ -988,6 +1071,7 @@ def test_search_archived(self): txt = self.get_stdout().rstrip() self.mock_stdout(False) expected = '\n'.join([ + ' * intel-compilers-2021.2.0.eb', ' * intel-2018a.eb', '', "Matching archived easyconfigs:", @@ -1025,11 +1109,14 @@ def test_show_ec(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - def mocked_main(self, args): + def mocked_main(self, args, **kwargs): """Run eb_main with mocked stdout/stderr.""" + if not kwargs: + kwargs = {'raise_error': True} + self.mock_stderr(True) self.mock_stdout(True) - self.eb_main(args, raise_error=True) + self.eb_main(args, **kwargs) stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) @@ -1141,7 +1228,7 @@ def check_copied_files(): error_pattern = "One or more files to copy should be specified!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) - def test_copy_ec_from_pr(self): + def test_github_copy_ec_from_pr(self): """Test combination of --copy-ec with --from-pr.""" if self.github_token is None: print("Skipping test_copy_ec_from_pr, no GitHub token available?") @@ -1341,7 +1428,7 @@ def test_dry_run_short(self): robot_decoy = os.path.join(self.test_prefix, 'robot_decoy') mkdir(robot_decoy) for dry_run_arg in ['-D', '--dry-run-short']: - open(self.logfile, 'w').write('') + write_file(self.logfile, '') args = [ os.path.join(tmpdir, 'easybuild', 'easyconfigs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb'), dry_run_arg, @@ -1629,7 +1716,7 @@ def test_dry_run_categorized(self): if os.path.exists(dummylogfn): os.remove(dummylogfn) - def test_from_pr(self): + def test_github_from_pr(self): """Test fetching easyconfigs from a PR.""" if self.github_token is None: print("Skipping test_from_pr, no GitHub token available?") @@ -1673,13 +1760,50 @@ def test_from_pr(self): self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr6424') - regex = re.compile("Appended list of robot search paths with %s:" % pr_tmpdir, re.M) + regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) shutil.rmtree(tmpdir) - def test_from_pr_token_log(self): + # test with multiple prs + tmpdir = tempfile.mkdtemp() + args = [ + # PRs for ReFrame 3.4.1 and 3.5.0 + '--from-pr=12150,12366', + '--dry-run', + # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed + '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % tmpdir, + ] + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (tmpdir, 'ReFrame/3.4.1'), + (tmpdir, 'ReFrame/3.5.0'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + path = '.*%s' % os.path.dirname(path_prefix) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + + for pr in ('12150', '12366'): + pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr%s' % pr) + regex = re.compile(r"Extended list of robot search paths with .*%s.*:" % pr_tmpdir, re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + except URLError as err: + print("Ignoring URLError '%s' in test_from_pr" % err) + shutil.rmtree(tmpdir) + + def test_github_from_pr_token_log(self): """Check that --from-pr doesn't leak GitHub token in log.""" if self.github_token is None: print("Skipping test_from_pr_token_log, no GitHub token available?") @@ -1712,7 +1836,7 @@ def test_from_pr_token_log(self): except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) - def test_from_pr_listed_ecs(self): + def test_github_from_pr_listed_ecs(self): """Test --from-pr in combination with specifying easyconfigs on the command line.""" if self.github_token is None: print("Skipping test_from_pr, no GitHub token available?") @@ -1767,7 +1891,7 @@ def test_from_pr_listed_ecs(self): print("Ignoring URLError '%s' in test_from_pr" % err) shutil.rmtree(tmpdir) - def test_from_pr_x(self): + def test_github_from_pr_x(self): """Test combination of --from-pr with --extended-dry-run.""" if self.github_token is None: print("Skipping test_from_pr_x, no GitHub token available?") @@ -1803,7 +1927,7 @@ def test_from_pr_x(self): re.compile(r"^\*\*\* DRY RUN using 'EB_FFTW' easyblock", re.M), re.compile(r"^== building and installing FFTW/3.3.8-gompi-2018b\.\.\.", re.M), re.compile(r"^building... \[DRY RUN\]", re.M), - re.compile(r"^== COMPLETED: Installation ended successfully \(took .* sec\)", re.M), + re.compile(r"^== COMPLETED: Installation ended successfully \(took .* secs?\)", re.M), ] for msg_regex in msg_regexs: @@ -1919,10 +2043,15 @@ def test_recursive_module_unload(self): self.eb_main(args, do_build=True, verbose=True) toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') + if get_module_syntax() == 'Lua': toy_module += '.lua' + is_loaded_regex = re.compile(r'if not \( isloaded\("gompi/2018a"\) \)', re.M) + else: + # Tcl syntax + is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/2018a\] }", re.M) + toy_module_txt = read_file(toy_module) - is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/2018a\] }", re.M) self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt) def test_tmpdir(self): @@ -2160,9 +2289,7 @@ def test_try(self): ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) - f = open(tweaked_toy_ec, 'a') - f.write("easyblock = 'ConfigureMake'") - f.close() + write_file(tweaked_toy_ec, "easyblock = 'ConfigureMake'", append=True) args = [ tweaked_toy_ec, @@ -2223,9 +2350,7 @@ def test_try_with_copy(self): ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) - f = open(tweaked_toy_ec, 'a') - f.write("easyblock = 'ConfigureMake'") - f.close() + write_file(tweaked_toy_ec, "easyblock = 'ConfigureMake'", append=True) args = [ tweaked_toy_ec, @@ -2292,9 +2417,7 @@ def test_recursive_try(self): ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) - f = open(tweaked_toy_ec, 'a') - f.write("dependencies = [('gzip', '1.4')]\n") # add fictious dependency - f.close() + write_file(tweaked_toy_ec, "dependencies = [('gzip', '1.4')]\n", append=True) # add fictious dependency sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') args = [ @@ -2348,9 +2471,7 @@ def test_recursive_try(self): self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) # clear fictitious dependency - f = open(tweaked_toy_ec, 'a') - f.write("dependencies = []\n") - f.close() + write_file(tweaked_toy_ec, "dependencies = []\n", append=True) # no recursive try if --disable-map-toolchains is involved for extra_args in [['--try-software-version=1.2.3'], ['--software-version=1.2.3']]: @@ -2410,7 +2531,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: zlib', outtxt)) # clear log file - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # filter deps (including a non-existing dep, i.e. zlib) args.extend(['--filter-deps', 'FFTW,ScaLAPACK,zlib']) @@ -2419,7 +2540,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # filter specific version of deps args[-1] = 'FFTW=3.2.3,zlib,ScaLAPACK=2.0.2' @@ -2428,7 +2549,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: ScaLAPACK', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') args[-1] = 'zlib,FFTW=3.3.7,ScaLAPACK=2.0.1' outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) @@ -2436,7 +2557,7 @@ def test_filter_deps(self): self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # filter deps with version range: only filter FFTW 3.x, ScaLAPACK 1.x args[-1] = 'zlib,ScaLAPACK=]1.0:2.0[,FFTW=[3.0:4.0[' @@ -2445,7 +2566,7 @@ def test_filter_deps(self): self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # also test open ended ranges args[-1] = 'zlib,ScaLAPACK=[1.0:,FFTW=:4.0[' @@ -2454,7 +2575,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: ScaLAPACK', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') args[-1] = 'zlib,ScaLAPACK=[2.1:,FFTW=:3.0[' outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) @@ -2469,7 +2590,7 @@ def test_filter_deps(self): self.assertFalse(re.search('module: ScaLAPACK', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # FFTW & ScaLAPACK versions are not included in range, so no filtering args[-1] = 'FFTW=]3.3.7:4.0],zlib,ScaLAPACK=[1.0:2.0.2[' @@ -2478,7 +2599,7 @@ def test_filter_deps(self): self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # also test mix of ranges & specific versions args[-1] = 'FFTW=3.3.7,zlib,ScaLAPACK=[1.0:2.0.2[' @@ -2487,7 +2608,7 @@ def test_filter_deps(self): self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - open(self.logfile, 'w').write('') + write_file(self.logfile, '') args[-1] = 'FFTW=]3.3.7:4.0],zlib,ScaLAPACK=2.0.2' outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.assertTrue(re.search('module: FFTW/3.3.7-gompi', outtxt)) @@ -2496,7 +2617,7 @@ def test_filter_deps(self): # This easyconfig contains a dependency of CMake for which no easyconfig exists. It should still # succeed when called with --filter-deps=CMake=:2.8.10] - open(self.logfile, 'w').write('') + write_file(self.logfile, '') ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb') args[0] = ec_file args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib' @@ -2506,7 +2627,7 @@ def test_filter_deps(self): self.assertTrue(re.search(regexp, outtxt)) # The test below fails without PR 2983 - open(self.logfile, 'w').write('') + write_file(self.logfile, '') ec_file = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a-broken.eb') args[0] = ec_file args[-1] = 'FFTW=3.3.7,CMake=:2.8.10],zlib' @@ -2535,7 +2656,7 @@ def test_hide_deps(self): self.assertFalse(re.search('module: zlib', outtxt)) # clear log file - open(self.logfile, 'w').write('') + write_file(self.logfile, '') # hide deps (including a non-existing dep, i.e. zlib) args.append('--hide-deps=FFTW,ScaLAPACK,zlib') @@ -2563,6 +2684,195 @@ def test_hide_toolchains(self): self.assertTrue(re.search(r'module: GCC/\.4\.9\.2', outtxt)) self.assertTrue(re.search(r'module: gzip/1\.6-GCC-4\.9\.2', outtxt)) + def test_parse_http_header_fields_urlpat(self): + """Test function parse_http_header_fields_urlpat""" + urlex = "example.com" + urlgnu = "gnu.org" + hdrauth = "Authorization" + valauth = "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" + hdragent = "User-Agent" + valagent = "James/0.0.7 (MI6)" + hdrrefer = "Referer" + valrefer = "http://www.example.com/" + filesub1 = os.path.join(self.test_prefix, "testhttpheaders1.txt") + filesub2 = os.path.join(self.test_prefix, "testhttpheaders2.txt") + filesub3 = os.path.join(self.test_prefix, "testhttpheaders3.txt") + filesub4 = os.path.join(self.test_prefix, "testhttpheaders4.txt") + fileauth = os.path.join(self.test_prefix, "testhttpheadersauth.txt") + write_file(filesub4, filesub3) + write_file(filesub3, filesub2) + write_file(filesub2, filesub1) + write_file(filesub1, "%s::%s:%s\n" % (urlgnu, hdrauth, valauth)) + write_file(filesub2, "%s::%s\n" % (urlex, filesub1)) + write_file(filesub3, "%s::%s:%s\n" % (urlex, hdragent, filesub2)) + write_file(fileauth, "%s\n" % (valauth)) + + # Case A: basic pattern + args = "%s::%s:%s" % (urlgnu, hdragent, valagent) + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: ["%s:%s" % (hdragent, valagent)]}, urlpat_headers) + + # Case B: urlpat has another urlpat: retain deepest level + args = "%s::%s::%s::%s:%s" % (urlgnu, urlgnu, urlex, hdragent, valagent) + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlex: ["%s:%s" % (hdragent, valagent)]}, urlpat_headers) + + # Case C: header value has a colon + args = "%s::%s:%s" % (urlex, hdrrefer, valrefer) + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlex: ["%s:%s" % (hdrrefer, valrefer)]}, urlpat_headers) + + # Case D: recurse into files + args = filesub3 + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: ["%s:%s" % (hdrauth, valauth)]}, urlpat_headers) + + # Case E: recurse into files as header + args = "%s::%s" % (urlex, filesub3) + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: ["%s:%s" % (hdrauth, valauth)]}, urlpat_headers) + + # Case F: recurse into files as value (header is replaced) + args = "%s::%s:%s" % (urlex, hdrrefer, filesub3) + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: ["%s:%s" % (hdrauth, valauth)]}, urlpat_headers) + + # Case G: recurse into files as value (header is retained) + args = "%s::%s:%s" % (urlgnu, hdrauth, fileauth) + urlpat_headers = parse_http_header_fields_urlpat(args) + self.assertEqual({urlgnu: ["%s:%s" % (hdrauth, valauth)]}, urlpat_headers) + + # Case H: recurse into files but hit limit + args = filesub4 + error_regex = r"Failed to parse_http_header_fields_urlpat \(recursion limit\)" + self.assertErrorRegex(EasyBuildError, error_regex, parse_http_header_fields_urlpat, args) + + # Case I: argument is not a string + args = list("foobar") + error_regex = r"Failed to parse_http_header_fields_urlpat \(argument not a string\)" + self.assertErrorRegex(EasyBuildError, error_regex, parse_http_header_fields_urlpat, args) + + def test_http_header_fields_urlpat(self): + """Test use of --http-header-fields-urlpat.""" + tmpdir = tempfile.mkdtemp() + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + gzip_ec = os.path.join(test_ecs_dir, 'g', 'gzip', 'gzip-1.6-GCC-4.9.2.eb') + gzip_ec_txt = read_file(gzip_ec) + regex = re.compile('^source_urls = .*', re.M) + test_ec_txt = regex.sub("source_urls = ['https://sources.easybuild.io/g/gzip']", gzip_ec_txt) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + common_args = [ + test_ec, + '--stop=fetch', + '--debug', + '--force', + '--force-download', + '--logtostdout', + '--sourcepath=%s' % tmpdir, + ] + + # define header fields:values that should (not) show up in the logs, either + # because they are secret or because they are not matched for the url + testdohdr = 'HeaderAPPLIED' + testdoval = 'SECRETvalue' + testdonthdr = 'HeaderIGNORED' + testdontval = 'BOGUSvalue' + + # header fields (or its values) could be files to be read instead of literals + testcmdfile = os.path.join(self.test_prefix, 'testhttpheaderscmdline.txt') + testincfile = os.path.join(self.test_prefix, 'testhttpheadersvalinc.txt') + testexcfile = os.path.join(self.test_prefix, 'testhttpheadersvalexc.txt') + testinchdrfile = os.path.join(self.test_prefix, 'testhttpheadershdrinc.txt') + testexchdrfile = os.path.join(self.test_prefix, 'testhttpheadershdrexc.txt') + testurlpatfile = os.path.join(self.test_prefix, 'testhttpheadersurlpat.txt') + + # log mention format upon header or file inclusion + mentionhdr = 'Custom HTTP header field set: %s' + mentionfile = 'File included in parse_http_header_fields_urlpat: %s' + + def run_and_assert(args, msg, words_expected=None, words_unexpected=None): + stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) + if words_expected is not None: + for thestring in words_expected: + self.assertTrue(re.compile(thestring).search(stdout), "Pattern '%s' missing from log (%s)" % + (thestring, msg)) + if words_unexpected is not None: + for thestring in words_unexpected: + self.assertFalse(re.compile(thestring).search(stdout), "Pattern '%s' leaked into log (%s)" % + (thestring, msg)) + + # A: simple direct case (all is logged because passed directly via EasyBuild configuration options) + args = list(common_args) + args.extend([ + '--http-header-fields-urlpat=easybuild.io::%s:%s' % (testdohdr, testdoval), + '--http-header-fields-urlpat=nomatch.com::%s:%s' % (testdonthdr, testdontval), + ]) + # expect to find everything passed on cmdline + expected = [mentionhdr % (testdohdr), testdoval, testdonthdr, testdontval] + run_and_assert(args, "case A", expected) + + # all subsequent tests share this argument list + args = common_args + args.append('--http-header-fields-urlpat=%s' % (testcmdfile)) + + # B: simple file case (secrets in file are not logged) + txt = '\n'.join([ + 'easybuild.io::%s: %s' % (testdohdr, testdoval), + 'nomatch.com::%s: %s' % (testdonthdr, testdontval), + '', + ]) + write_file(testcmdfile, txt) + # expect to find only the header key (not its value) and only for the appropriate url + expected = [mentionhdr % testdohdr, mentionfile % testcmdfile] + not_expected = [testdoval, testdonthdr, testdontval] + run_and_assert(args, "case B", expected, not_expected) + + # C: recursion one: header value is another file + txt = '\n'.join([ + 'easybuild.io::%s: %s' % (testdohdr, testincfile), + 'nomatch.com::%s: %s' % (testdonthdr, testexcfile), + '', + ]) + write_file(testcmdfile, txt) + write_file(testincfile, '%s\n' % (testdoval)) + write_file(testexcfile, '%s\n' % (testdontval)) + # expect to find only the header key (not its value and not the filename) and only for the appropriate url + expected = [mentionhdr % (testdohdr), mentionfile % (testcmdfile), + mentionfile % (testincfile), mentionfile % (testexcfile)] + not_expected = [testdoval, testdonthdr, testdontval] + run_and_assert(args, "case C", expected, not_expected) + + # D: recursion two: header field+value is another file, + write_file(testcmdfile, '\n'.join([ + 'easybuild.io::%s' % (testinchdrfile), + 'nomatch.com::%s' % (testexchdrfile), + '', + ])) + write_file(testinchdrfile, '%s: %s\n' % (testdohdr, testdoval)) + write_file(testexchdrfile, '%s: %s\n' % (testdonthdr, testdontval)) + # expect to find only the header key (and the literal filename) and only for the appropriate url + expected = [mentionhdr % (testdohdr), mentionfile % (testcmdfile), + mentionfile % (testinchdrfile), mentionfile % (testexchdrfile)] + not_expected = [testdoval, testdonthdr, testdontval] + run_and_assert(args, "case D", expected, not_expected) + + # E: recursion three: url pattern + header field + value in another file + write_file(testcmdfile, '%s\n' % (testurlpatfile)) + txt = '\n'.join([ + 'easybuild.io::%s: %s' % (testdohdr, testdoval), + 'nomatch.com::%s: %s' % (testdonthdr, testdontval), + '', + ]) + write_file(testurlpatfile, txt) + # expect to find only the header key (but not the literal filename) and only for the appropriate url + expected = [mentionhdr % (testdohdr), mentionfile % (testcmdfile), mentionfile % (testurlpatfile)] + not_expected = [testdoval, testdonthdr, testdontval] + run_and_assert(args, "case E", expected, not_expected) + + # cleanup downloads + shutil.rmtree(tmpdir) + def test_test_report_env_filter(self): """Test use of --test-report-env-filter.""" @@ -2583,9 +2893,7 @@ def toy(extra_args=None): software_path = os.path.join(self.test_installpath, 'software', 'toy', '0.0') test_report_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-0.0*test_report.md') - f = open(glob.glob(test_report_path_pattern)[0], 'r') - test_report_txt = f.read() - f.close() + test_report_txt = read_file(glob.glob(test_report_path_pattern)[0]) return test_report_txt # define environment variables that should (not) show up in the test report @@ -2925,14 +3233,38 @@ def test_xxx_include_easyblocks(self): import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) + # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic + # to avoid that easyblocks picked up from other places cause trouble + testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): + for path in sys.modules[pkg].__path__[:]: + if testdir_sandbox not in path: + sys.modules[pkg].__path__.remove(path) + # include extra test easyblocks - foo_txt = '\n'.join([ - 'from easybuild.framework.easyblock import EasyBlock', - 'class EB_foo(EasyBlock):', - ' pass', - '' - ]) + # Make them inherit from each other to trigger a known issue with changed imports, see #3779 + # Choose naming so that order of naming is different than inheritance order + afoo_txt = textwrap.dedent(""" + from easybuild.framework.easyblock import EasyBlock + class EB_afoo(EasyBlock): + def __init__(self, *args, **kwargs): + super(EB_afoo, self).__init__(*args, **kwargs) + """) + write_file(os.path.join(self.test_prefix, 'afoo.py'), afoo_txt) + foo_txt = textwrap.dedent(""" + from easybuild.easyblocks.zfoo import EB_zfoo + class EB_foo(EB_zfoo): + def __init__(self, *args, **kwargs): + super(EB_foo, self).__init__(*args, **kwargs) + """) write_file(os.path.join(self.test_prefix, 'foo.py'), foo_txt) + zfoo_txt = textwrap.dedent(""" + from easybuild.easyblocks.afoo import EB_afoo + class EB_zfoo(EB_afoo): + def __init__(self, *args, **kwargs): + super(EB_zfoo, self).__init__(*args, **kwargs) + """) + write_file(os.path.join(self.test_prefix, 'zfoo.py'), zfoo_txt) # clear log write_file(self.logfile, '') @@ -2950,12 +3282,27 @@ def test_xxx_include_easyblocks(self): foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) + ec_txt = '\n'.join([ + 'easyblock = "EB_foo"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + ec = EasyConfig(path=None, rawtxt=ec_txt) + # easyblock is found via get_easyblock_class - klass = get_easyblock_class('EB_foo') - self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) + for name in ('EB_afoo', 'EB_foo', 'EB_zfoo'): + klass = get_easyblock_class(name) + self.assertTrue(issubclass(klass, EasyBlock), "%s (%s) is an EasyBlock derivative class" % (klass, name)) - # 'undo' import of foo easyblock - del sys.modules['easybuild.easyblocks.foo'] + eb_inst = klass(ec) + self.assertTrue(eb_inst is not None, "Instantiating the injected class %s works" % name) + + # 'undo' import of the easyblocks + for name in ('afoo', 'foo', 'zfoo'): + del sys.modules['easybuild.easyblocks.' + name] # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... @@ -3006,6 +3353,14 @@ def test_xxx_include_generic_easyblocks(self): import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) + # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic + # to avoid that easyblocks picked up from other places cause trouble + testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): + for path in sys.modules[pkg].__path__[:]: + if testdir_sandbox not in path: + sys.modules[pkg].__path__.remove(path) + error_msg = "Failed to obtain class for FooBar easyblock" self.assertErrorRegex(EasyBuildError, error_msg, get_easyblock_class, 'FooBar') @@ -3040,7 +3395,7 @@ def test_xxx_include_generic_easyblocks(self): # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... - def test_xxx_include_easyblocks_from_pr(self): + def test_github_xxx_include_easyblocks_from_pr(self): """Test --include-easyblocks-from-pr.""" if self.github_token is None: print("Skipping test_preview_pr, no GitHub token available?") @@ -3147,6 +3502,14 @@ def test_xxx_include_easyblocks_from_pr(self): import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) + # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic, + # to avoid that easyblocks picked up from other places cause trouble + testdir_sandbox = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): + for path in sys.modules[pkg].__path__[:]: + if testdir_sandbox not in path: + sys.modules[pkg].__path__.remove(path) + # clear log write_file(self.logfile, '') @@ -3191,8 +3554,9 @@ def mk_eb_test_cmd(self, args): # make sure that location to 'easybuild.main' is included in $PYTHONPATH pythonpath = os.getenv('PYTHONPATH') + pythonpath = [pythonpath] if pythonpath else [] easybuild_loc = os.path.dirname(os.path.dirname(easybuild.main.__file__)) - os.environ['PYTHONPATH'] = ':'.join([easybuild_loc, pythonpath]) + os.environ['PYTHONPATH'] = ':'.join([easybuild_loc] + pythonpath) return '; '.join([ "cd %s" % self.test_prefix, @@ -3373,7 +3737,7 @@ def test_cleanup_tmpdir(self): tweaked_dir = os.path.join(tmpdir, tmpdir_files[0], 'tweaked_easyconfigs') self.assertTrue(os.path.exists(os.path.join(tweaked_dir, 'toy-1.0.eb'))) - def test_preview_pr(self): + def test_github_preview_pr(self): """Test --preview-pr.""" if self.github_token is None: print("Skipping test_preview_pr, no GitHub token available?") @@ -3395,7 +3759,7 @@ def test_preview_pr(self): regex = re.compile(r"^Comparing bzip2-1.0.6\S* with bzip2-1.0.6") self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt)) - def test_review_pr(self): + def test_github_review_pr(self): """Test --review-pr.""" if self.github_token is None: print("Skipping test_review_pr, no GitHub token available?") @@ -3416,6 +3780,51 @@ def test_review_pr(self): regex = re.compile(r"^Comparing gzip-1.10-\S* with gzip-1.10-") self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt)) + self.mock_stdout(True) + self.mock_stderr(True) + # closed PR for gzip 1.2.8 easyconfig, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/5365 + args = [ + '--color=never', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--review-pr=5365', + ] + self.eb_main(args, raise_error=True, testing=True) + txt = self.get_stdout() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertTrue("This PR should be labelled with 'update'" in txt) + + # test --review-pr-max + self.mock_stdout(True) + self.mock_stderr(True) + args = [ + '--color=never', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--review-pr=5365', + '--review-pr-max=1', + ] + self.eb_main(args, raise_error=True, testing=True) + txt = self.get_stdout() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertTrue("2016.04" not in txt) + + # test --review-pr-filter + self.mock_stdout(True) + self.mock_stderr(True) + args = [ + '--color=never', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--review-pr=5365', + '--review-pr-filter=2016a', + ] + self.eb_main(args, raise_error=True, testing=True) + txt = self.get_stdout() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertTrue("2016.04" not in txt) + def test_set_tmpdir(self): """Test set_tmpdir config function.""" self.purge_environment() @@ -3518,7 +3927,7 @@ def test_extended_dry_run(self): msg_regexs = [ re.compile(r"the actual build \& install procedure that will be performed may diverge", re.M), re.compile(r"^\*\*\* DRY RUN using 'EB_toy' easyblock", re.M), - re.compile(r"^== COMPLETED: Installation ended successfully \(took .* sec\)", re.M), + re.compile(r"^== COMPLETED: Installation ended successfully \(took .* secs?\)", re.M), re.compile(r"^\(no ignored errors during dry run\)", re.M), ] ignoring_error_regex = re.compile(r"WARNING: ignoring error", re.M) @@ -3650,7 +4059,7 @@ def test_new_branch_github(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git\.\.\.", r"^== copying files to .*/easybuild-easyconfigs\.\.\.", - r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + r"^== pushing branch '[0-9]{14}_new_pr_toy00' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) @@ -3671,7 +4080,7 @@ def test_new_branch_github(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyblocks.git\.\.\.", r"^== copying files to .*/easybuild-easyblocks\.\.\.", - r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + r"^== pushing branch '[0-9]{14}_new_pr_toy' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) @@ -3698,11 +4107,11 @@ def test_new_branch_github(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-framework.git\.\.\.", r"^== copying files to .*/easybuild-framework\.\.\.", - r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + r"^== pushing branch '[0-9]{14}_new_pr_[A-Za-z]{10}' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) - def test_new_pr_from_branch(self): + def test_github_new_pr_from_branch(self): """Test --new-pr-from-branch.""" if self.github_token is None: print("Skipping test_new_pr_from_branch, no GitHub token available?") @@ -3771,7 +4180,7 @@ def test_update_branch_github(self): ] self._assert_regexs(regexs, txt) - def test_new_update_pr(self): + def test_github_new_update_pr(self): """Test use of --new-pr (dry run only).""" if self.github_token is None: print("Skipping test_new_update_pr, no GitHub token available?") @@ -3871,7 +4280,7 @@ def test_new_update_pr(self): '--pr-branch-name=branch_name_for_new_pr_test', '--pr-commit-msg="this is a commit message. really!"', '--pr-descr="moar letters foar teh lettre box"', - '--pr-target-branch=master', + '--pr-target-branch=main', '--github-org=%s' % GITHUB_TEST_ORG, '--pr-target-account=boegel', # we need to be able to 'clone' from here (via https) '--pr-title=test-1-2-3', @@ -3879,9 +4288,9 @@ def test_new_update_pr(self): txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) regexs = [ - r"^== fetching branch 'master' from https://github.com/boegel/easybuild-easyconfigs.git...", + r"^== fetching branch 'main' from https://github.com/boegel/easybuild-easyconfigs.git...", r"^Opening pull request \[DRY RUN\]", - r"^\* target: boegel/easybuild-easyconfigs:master", + r"^\* target: boegel/easybuild-easyconfigs:main", r"^\* from: %s/easybuild-easyconfigs:branch_name_for_new_pr_test" % GITHUB_TEST_ORG, r"\(created using `eb --new-pr`\)", # description r"moar letters foar teh lettre box", # also description (see --pr-descr) @@ -3981,7 +4390,7 @@ def test_new_update_pr(self): ] self._assert_regexs(regexs, txt, assert_true=False) - def test_sync_pr_with_develop(self): + def test_github_sync_pr_with_develop(self): """Test use of --sync-pr-with-develop (dry run only).""" if self.github_token is None: print("Skipping test_sync_pr_with_develop, no GitHub token available?") @@ -4001,7 +4410,7 @@ def test_sync_pr_with_develop(self): github_path = r"boegel/easybuild-easyconfigs\.git" pattern = '\n'.join([ - r"== temporary log file in case of crash .*", + r"== Temporary log file in case of crash .*", r"== Determined branch name corresponding to easybuilders/easybuild-easyconfigs PR #9150: develop", r"== fetching branch 'develop' from https://github\.com/%s\.\.\." % github_path, r"== pulling latest version of 'develop' branch from easybuilders/easybuild-easyconfigs\.\.\.", @@ -4011,7 +4420,7 @@ def test_sync_pr_with_develop(self): regex = re.compile(pattern) self.assertTrue(regex.match(txt), "Pattern '%s' doesn't match: %s" % (regex.pattern, txt)) - def test_sync_branch_with_develop(self): + def test_github_sync_branch_with_develop(self): """Test use of --sync-branch-with-develop (dry run only).""" if self.github_token is None: print("Skipping test_sync_pr_with_develop, no GitHub token available?") @@ -4032,7 +4441,7 @@ def test_sync_branch_with_develop(self): github_path = r"boegel/easybuild-easyconfigs\.git" pattern = '\n'.join([ - r"== temporary log file in case of crash .*", + r"== Temporary log file in case of crash .*", r"== fetching branch '%s' from https://github\.com/%s\.\.\." % (test_branch, github_path), r"== pulling latest version of 'develop' branch from easybuilders/easybuild-easyconfigs\.\.\.", r"== merging 'develop' branch into PR branch '%s'\.\.\." % test_branch, @@ -4041,7 +4450,7 @@ def test_sync_branch_with_develop(self): regex = re.compile(pattern) self.assertTrue(regex.match(stdout), "Pattern '%s' doesn't match: %s" % (regex.pattern, stdout)) - def test_new_pr_python(self): + def test_github_new_pr_python(self): """Check generated PR title for --new-pr on easyconfig that includes Python dependency.""" if self.github_token is None: print("Skipping test_new_pr_python, no GitHub token available?") @@ -4086,7 +4495,7 @@ def test_new_pr_python(self): regex = re.compile(r"^\* title: \"\{tools\}\[system/system\] toy v0.0 w/ Python 2.7.15 \+ 3.7.2\"$", re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) - def test_new_pr_delete(self): + def test_github_new_pr_delete(self): """Test use of --new-pr to delete easyconfigs.""" if self.github_token is None: @@ -4111,7 +4520,7 @@ def test_new_pr_delete(self): ] self._assert_regexs(regexs, txt) - def test_new_pr_dependencies(self): + def test_github_new_pr_dependencies(self): """Test use of --new-pr with automatic dependency lookup.""" if self.github_token is None: @@ -4159,7 +4568,38 @@ def test_new_pr_dependencies(self): self._assert_regexs(regexs, txt) - def test_merge_pr(self): + def test_new_pr_easyblock(self): + """ + Test using --new-pr to open an easyblocks PR + """ + + if self.github_token is None: + print("Skipping test_new_pr_easyblock, no GitHub token available?") + return + + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + self.assertTrue(os.path.exists(toy_eb)) + + args = [ + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--new-pr', + toy_eb, + '-D', + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) + + patterns = [ + r'target: easybuilders/easybuild-easyblocks:develop', + r'from: easybuild_test/easybuild-easyblocks:[0-9]+_new_pr_toy', + r'title: "new easyblock for toy"', + r'easybuild/easyblocks/t/toy.py', + ] + for pattern in patterns: + regex = re.compile(pattern) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + def test_github_merge_pr(self): """ Test use of --merge-pr (dry run)""" if self.github_token is None: @@ -4203,7 +4643,9 @@ def test_merge_pr(self): "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4781 for merging...", "* test suite passes: OK", "* last test report is successful: OK", + "* no pending change requests: OK", "* milestone is set: OK (3.3.1)", + "* mergeable state is clean: PR is already merged", ]) expected_stderr = '\n'.join([ "* targets some_branch branch: FAILED; found 'develop' => not eligible for merging!", @@ -4225,8 +4667,10 @@ def test_merge_pr(self): "* targets develop branch: OK", "* test suite passes: OK", "* last test report is successful: OK", + "* no pending change requests: OK", "* approved review: OK (by wpoely86)", "* milestone is set: OK (3.3.1)", + "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", '', @@ -4251,14 +4695,16 @@ def test_merge_pr(self): "Checking eligibility of easybuilders/easybuild-easyblocks PR #1206 for merging...", "* targets develop branch: OK", "* test suite passes: OK", + "* no pending change requests: OK", "* approved review: OK (by migueldiascosta)", "* milestone is set: OK (3.3.1)", + "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", ]) self.assertTrue(expected_stdout in stdout) - def test_empty_pr(self): + def test_github_empty_pr(self): """Test use of --new-pr (dry run only) with no changes""" if self.github_token is None: print("Skipping test_empty_pr, no GitHub token available?") @@ -4430,7 +4876,7 @@ def test_modules_tool_vs_syntax_check(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) - def test_prefix(self): + def test_prefix_option(self): """Test which configuration settings are affected by --prefix.""" txt, _ = self._run_mock_eb(['--show-full-config', '--prefix=%s' % self.test_prefix], raise_error=True) @@ -4439,8 +4885,8 @@ def test_prefix(self): expected = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'prefix', 'repositorypath'] self.assertEqual(sorted(regex.findall(txt)), expected) - def test_dump_env_config(self): - """Test for --dump-env-config.""" + def test_dump_env_script(self): + """Test for --dump-env-script.""" fftw = 'FFTW-3.3.7-gompic-2018a' gcc = 'GCC-4.9.2' @@ -4496,7 +4942,7 @@ def test_stop(self): args = ['toy-0.0.eb', '--force', '--stop=configure'] txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) - regex = re.compile(r"COMPLETED: Installation STOPPED successfully \(took .* sec\)", re.M) + regex = re.compile(r"COMPLETED: Installation STOPPED successfully \(took .* secs?\)", re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) def test_fetch(self): @@ -4521,7 +4967,7 @@ def test_fetch(self): patterns = [ r"^== fetching files\.\.\.$", - r"^== COMPLETED: Installation STOPPED successfully \(took .* sec\)$", + r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$", ] for pattern in patterns: regex = re.compile(pattern, re.M) @@ -5469,6 +5915,33 @@ def test_show_system_info(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + def test_check_eb_deps(self): + """Test for --check-eb-deps.""" + txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True) + + # keep in mind that these patterns should match with both normal output and Rich output! + opt_dep_info_pattern = r'([0-9.]+|\(NOT FOUND\)|not found|\(unknown version\))' + tool_info_pattern = r'([0-9.]+|\(NOT FOUND\)|not found|\(found, UNKNOWN version\)|version\?\!)' + patterns = [ + r"Required dependencies", + r"Python.* [23][0-9.]+", + r"modules tool.* [A-Za-z0-9.\s-]+", + r"Optional dependencies", + r"archspec.* %s.*determining name" % opt_dep_info_pattern, + r"GitPython.* %s.*GitHub integration" % opt_dep_info_pattern, + r"Rich.* %s.*eb command rich terminal output" % opt_dep_info_pattern, + r"setuptools.* %s.*information on Python packages" % opt_dep_info_pattern, + r"System tools", + r"make.* %s" % tool_info_pattern, + r"patch.* %s" % tool_info_pattern, + r"sed.* %s" % tool_info_pattern, + r"Slurm.* %s" % tool_info_pattern, + ] + + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + def test_tmp_logdir(self): """Test use of --tmp-logdir.""" @@ -5500,6 +5973,180 @@ def test_tmp_logdir(self): logtxt = read_file(os.path.join(tmp_logdir, tmp_logs[0])) self.assertTrue("COMPLETED: Installation ended successfully" in logtxt) + def test_sanity_check_only(self): + """Test use of --sanity-check-only.""" + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.ec') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "sanity_check_commands = ['barbar', 'toy']", + "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + "exts_list = [", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " 'exts_filter': ('ls -l lib/lib%(ext_name)s.a', ''),", + " })", + "]", + ]) + write_file(test_ec, test_ec_txt) + + # sanity check fails if software was not installed yet + outtxt, error_thrown = self.eb_main([test_ec, '--sanity-check-only'], do_build=True, return_error=True) + self.assertTrue("Sanity check failed" in str(error_thrown)) + + # actually install, then try --sanity-check-only again; + # need to use --force to install toy because module already exists (but installation doesn't) + self.eb_main([test_ec, '--force'], do_build=True, raise_error=True) + + args = [test_ec, '--sanity-check-only'] + + stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) + + skipped = [ + "fetching files", + "creating build dir, resetting environment", + "unpacking", + "patching", + "preparing", + "configuring", + "building", + "testing", + "installing", + "taking care of extensions", + "restore after iterating", + "postprocessing", + "cleaning up", + "creating module", + "permissions", + "packaging" + ] + for skip in skipped: + self.assertTrue("== %s [skipped]" % skip) + + self.assertTrue("== sanity checking..." in stdout) + self.assertTrue("COMPLETED: Installation ended successfully" in stdout) + msgs = [ + " >> file 'bin/barbar' found: OK", + " >> file 'bin/toy' found: OK", + " >> (non-empty) directory 'bin' found: OK", + " >> loading modules: toy/0.0...", + " >> result for command 'toy': OK", + "ls -l lib/libbarbar.a", # sanity check for extension barbar (via exts_filter) + ] + for msg in msgs: + self.assertTrue(msg in stdout, "'%s' found in: %s" % (msg, stdout)) + + ebroottoy = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + + # check if sanity check for extension fails if a file provided by that extension, + # which is checked by the sanity check for that extension, is no longer there + libbarbar = os.path.join(ebroottoy, 'lib', 'libbarbar.a') + move_file(libbarbar, libbarbar + '.moved') + + outtxt, error_thrown = self.eb_main(args + ['--debug'], do_build=True, return_error=True) + error_msg = str(error_thrown) + error_patterns = [ + r"Sanity check failed", + r'command "ls -l lib/libbarbar\.a" failed', + ] + for error_pattern in error_patterns: + regex = re.compile(error_pattern) + self.assertTrue(regex.search(error_msg), "Pattern '%s' should be found in: %s" % (regex.pattern, error_msg)) + + # failing sanity check for extension can be bypassed via --skip-extensions + outtxt = self.eb_main(args + ['--skip-extensions'], do_build=True, raise_error=True) + self.assertTrue("Sanity check for toy successful" in outtxt) + + # restore fail, we want a passing sanity check for the next check + move_file(libbarbar + '.moved', libbarbar) + + # check use of --sanity-check-only when installation directory is read-only; + # cfr. https://github.com/easybuilders/easybuild-framework/issues/3757 + adjust_permissions(ebroottoy, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=True) + + stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False) + + # check whether %(builddir)s value is correct + # when buildininstalldir is enabled in easyconfig and --sanity-check-only is used + # (see https://github.com/easybuilders/easybuild-framework/issues/3895) + test_ec_txt += '\n' + '\n'.join([ + "buildininstalldir = True", + "sanity_check_commands = [", + # build and install directory should be the same path + " 'test %(builddir)s = %(installdir)s',", + # build/install directory must exist (even though step that creates build dir was never run) + " 'test -d %(builddir)s',", + "]", + ]) + write_file(test_ec, test_ec_txt) + self.eb_main(args, do_build=True, raise_error=True) + + # also check when using easyblock that enables build_in_installdir in its constructor + test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + toy_eb = os.path.join(test_ebs, 't', 'toy.py') + toy_eb_txt = read_file(toy_eb) + + self.assertFalse('self.build_in_installdir = True' in toy_eb_txt) + + regex = re.compile(r'^(\s+)(super\(EB_toy, self\).__init__.*)\n', re.M) + toy_eb_txt = regex.sub(r'\1\2\n\1self.build_in_installdir = True', toy_eb_txt) + self.assertTrue('self.build_in_installdir = True' in toy_eb_txt) + + toy_eb = os.path.join(self.test_prefix, 'toy.py') + write_file(toy_eb, toy_eb_txt) + + test_ec_txt = test_ec_txt.replace('buildininstalldir = True', '') + write_file(test_ec, test_ec_txt) + + orig_local_sys_path = sys.path[:] + args.append('--include-easyblocks=%s' % toy_eb) + self.eb_main(args, do_build=True, raise_error=True) + + # undo import of the toy easyblock, to avoid problems with other tests + del sys.modules['easybuild.easyblocks.toy'] + sys.path = orig_local_sys_path + import easybuild.easyblocks + reload(easybuild.easyblocks) + import easybuild.easyblocks.toy + reload(easybuild.easyblocks.toy) + # need to reload toy_extension, which imports EB_toy, to ensure right EB_toy is picked up in later tests + import easybuild.easyblocks.generic.toy_extension + reload(easybuild.easyblocks.generic.toy_extension) + + def test_skip_extensions(self): + """Test use of --skip-extensions.""" + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + # add extension, which should be skipped + test_ec = os.path.join(self.test_prefix, 'test.ec') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "exts_list = [", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " 'exts_filter': ('ls -l lib/lib%(ext_name)s.a', ''),", + " })", + "]", + ]) + write_file(test_ec, test_ec_txt) + + args = [test_ec, '--force', '--skip-extensions'] + self.eb_main(args, do_build=True, return_error=True) + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + + self.assertTrue(os.path.exists(toy_mod), "%s should exist" % toy_mod) + + toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + for path in (os.path.join('bin', 'barbar'), os.path.join('lib', 'libbarbar.a')): + path = os.path.join(toy_installdir, path) + self.assertFalse(os.path.exists(path), "Path %s should not exist" % path) + def test_fake_vsc_include(self): """Test whether fake 'vsc' namespace is triggered for modules included via --include-*.""" @@ -5658,7 +6305,7 @@ def test_sysroot(self): self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) def test_accept_eula_for(self): - """Test --accept-eula configuration option.""" + """Test --accept-eula-for configuration option.""" # use toy-0.0.eb easyconfig file that comes with the tests topdir = os.path.abspath(os.path.dirname(__file__)) @@ -5673,34 +6320,144 @@ def test_accept_eula_for(self): # by default, no EULAs are accepted at all args = [test_ec, '--force'] - error_pattern = r"The End User License Argreement \(EULA\) for toy is currently not accepted!" + error_pattern = r"The End User License Agreement \(EULA\) for toy is currently not accepted!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) - - # installation proceeds if EasyBuild is configured to accept EULA for specified software via --accept-eula - self.eb_main(args + ['--accept-eula=foo,toy,bar'], do_build=True, raise_error=True) - toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_modfile += '.lua' - self.assertTrue(os.path.exists(toy_modfile)) + + # installation proceeds if EasyBuild is configured to accept EULA for specified software via --accept-eula-for + for val in ('foo,toy,bar', '.*', 't.y'): + self.eb_main(args + ['--accept-eula-for=' + val], do_build=True, raise_error=True) + + self.assertTrue(os.path.exists(toy_modfile)) + + remove_dir(self.test_installpath) + self.assertFalse(os.path.exists(toy_modfile)) + + # also check use of $EASYBUILD_ACCEPT_EULA to accept EULA for specified software + os.environ['EASYBUILD_ACCEPT_EULA_FOR'] = val + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_modfile)) + + remove_dir(self.test_installpath) + self.assertFalse(os.path.exists(toy_modfile)) + + del os.environ['EASYBUILD_ACCEPT_EULA_FOR'] + + # also check deprecated --accept-eula configuration option + self.allow_deprecated_behaviour() + + self.mock_stderr(True) + self.eb_main(args + ['--accept-eula=foo,toy,bar'], do_build=True, raise_error=True) + stderr = self.get_stderr() + self.mock_stderr(False) + self.assertTrue("Use accept-eula-for configuration setting rather than accept-eula" in stderr) remove_dir(self.test_installpath) self.assertFalse(os.path.exists(toy_modfile)) - # also check use of $EASYBUILD_ACCEPT_EULA to accept EULA for specified software + # also via $EASYBUILD_ACCEPT_EULA + self.mock_stderr(True) os.environ['EASYBUILD_ACCEPT_EULA'] = 'toy' self.eb_main(args, do_build=True, raise_error=True) + stderr = self.get_stderr() + self.mock_stderr(False) + self.assertTrue(os.path.exists(toy_modfile)) + self.assertTrue("Use accept-eula-for configuration setting rather than accept-eula" in stderr) remove_dir(self.test_installpath) self.assertFalse(os.path.exists(toy_modfile)) # also check accepting EULA via 'accept_eula = True' in easyconfig file + self.disallow_deprecated_behaviour() del os.environ['EASYBUILD_ACCEPT_EULA'] write_file(test_ec, test_ec_txt + '\naccept_eula = True') self.eb_main(args, do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_modfile)) + def test_config_abs_path(self): + """Test ensuring of absolute path values for path configuration options.""" + + test_topdir = os.path.join(self.test_prefix, 'test_topdir') + test_subdir = os.path.join(test_topdir, 'test_middle_dir', 'test_subdir') + mkdir(test_subdir, parents=True) + change_dir(test_subdir) + + # a relative path specified in a configuration file is positively weird, but fine :) + cfgfile = os.path.join(self.test_prefix, 'test.cfg') + cfgtxt = '\n'.join([ + "[config]", + "containerpath = ..", + "repositorypath = /apps/easyconfigs_archive, somesubdir", + ]) + write_file(cfgfile, cfgtxt) + + # relative paths in environment variables is also weird, + # but OK for the sake of testing... + os.environ['EASYBUILD_INSTALLPATH'] = '../..' + os.environ['EASYBUILD_ROBOT_PATHS'] = '../..' + + args = [ + '--configfiles=%s' % cfgfile, + '--prefix=..', + '--sourcepath=.', + '--show-config', + ] + + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + patterns = [ + r"^containerpath\s+\(F\) = /.*/test_topdir/test_middle_dir$", + r"^installpath\s+\(E\) = /.*/test_topdir$", + r"^prefix\s+\(C\) = /.*/test_topdir/test_middle_dir$", + r"^repositorypath\s+\(F\) = /apps/easyconfigs_archive,\s+somesubdir$", + r"^sourcepath\s+\(C\) = /.*/test_topdir/test_middle_dir/test_subdir$", + r"^robot-paths\s+\(E\) = /.*/test_topdir$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + + # paths specified via --robot have precedence over those specified via $EASYBUILD_ROBOT_PATHS + change_dir(test_subdir) + args.append('--robot=..:.') + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + patterns.pop(-1) + robot_value_pattern = ', '.join([ + r'/.*/test_topdir/test_middle_dir', # via --robot (first path) + r'/.*/test_topdir/test_middle_dir/test_subdir', # via --robot (second path) + r'/.*/test_topdir', # via $EASYBUILD_ROBOT_PATHS + ]) + patterns.extend([ + r"^robot-paths\s+\(C\) = %s$" % robot_value_pattern, + r"^robot\s+\(C\) = %s$" % robot_value_pattern, + ]) + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + + def test_config_repositorypath(self): + """Test how special repositorypath values are handled.""" + + repositorypath = 'git@github.com:boegel/my_easyconfigs.git' + args = [ + '--repositorypath=%s' % repositorypath, + '--show-config', + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + regex = re.compile(r'repositorypath\s+\(C\) = %s' % repositorypath, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + args[0] = '--repositorypath=%s,some/subdir' % repositorypath + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + regex = re.compile(r"repositorypath\s+\(C\) = %s, some/subdir" % repositorypath, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + # end-to-end testing of unknown filename def test_easystack_wrong_read(self): """Test for --easystack when wrong name is provided""" @@ -5733,39 +6490,6 @@ def test_easystack_basic(self): regex = re.compile(pattern) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) - def test_easystack_wrong_structure(self): - """Test for --easystack when yaml easystack has wrong structure""" - easybuild.tools.build_log.EXPERIMENTAL = True - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') - - expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:" - expected_err += r"( 'float' object is not subscriptable[\S\s]*" - expected_err += r"| 'float' object is unsubscriptable" - expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*)" - self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) - - def test_easystack_asterisk(self): - """Test for --easystack when yaml easystack contains asterisk (wildcard)""" - easybuild.tools.build_log.EXPERIMENTAL = True - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml') - - expected_err = "EasyStack specifications of 'binutils' in .*/test_easystack_asterisk.yaml contain asterisk. " - expected_err += "Wildcard feature is not supported yet." - - self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) - - def test_easystack_labels(self): - """Test for --easystack when yaml easystack contains exclude-labels / include-labels""" - easybuild.tools.build_log.EXPERIMENTAL = True - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') - - error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. " - error_msg += "Labels aren't supported yet." - self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, toy_easystack) - def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/output.py b/test/framework/output.py new file mode 100644 index 0000000000..8e69ba68eb --- /dev/null +++ b/test/framework/output.py @@ -0,0 +1,193 @@ +# # +# Copyright 2021-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for functionality in easybuild.tools.output + +@author: Kenneth Hoste (Ghent University) +""" +import sys +from unittest import TextTestRunner +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered + +import easybuild.tools.output +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option, get_output_style, update_build_option +from easybuild.tools.output import PROGRESS_BAR_EXTENSIONS, PROGRESS_BAR_TYPES +from easybuild.tools.output import DummyRich, colorize, get_progress_bar, show_progress_bars +from easybuild.tools.output import start_progress_bar, status_bar, stop_progress_bar, update_progress_bar, use_rich + +try: + import rich.progress + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + + +class OutputTest(EnhancedTestCase): + """Tests for functions controlling terminal output.""" + + def test_status_bar(self): + """Test status_bar function.""" + + # restore default (was disabled in EnhancedTestCase.setUp to avoid messing up test output) + update_build_option('show_progress_bar', True) + + if HAVE_RICH: + expected_progress_bar_class = rich.progress.Progress + else: + expected_progress_bar_class = DummyRich + + progress_bar = status_bar(ignore_cache=True) + error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) + self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) + + update_build_option('output_style', 'basic') + progress_bar = status_bar(ignore_cache=True) + self.assertTrue(isinstance(progress_bar, DummyRich)) + + if HAVE_RICH: + update_build_option('output_style', 'rich') + progress_bar = status_bar(ignore_cache=True) + error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) + self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) + + update_build_option('show_progress_bar', False) + progress_bar = status_bar(ignore_cache=True) + self.assertTrue(isinstance(progress_bar, DummyRich)) + + def test_get_output_style(self): + """Test get_output_style function.""" + + self.assertEqual(build_option('output_style'), 'auto') + + for style in (None, 'auto'): + if style: + update_build_option('output_style', style) + + if HAVE_RICH: + self.assertEqual(get_output_style(), 'rich') + else: + self.assertEqual(get_output_style(), 'basic') + + test_styles = ['basic', 'no_color'] + if HAVE_RICH: + test_styles.append('rich') + + for style in test_styles: + update_build_option('output_style', style) + self.assertEqual(get_output_style(), style) + + if not HAVE_RICH: + update_build_option('output_style', 'rich') + error_pattern = "Can't use 'rich' output style, Rich Python package is not available!" + self.assertErrorRegex(EasyBuildError, error_pattern, get_output_style) + + def test_use_rich_show_progress_bars(self): + """Test use_rich and show_progress_bar functions.""" + + # restore default configuration to show progress bars (disabled to avoid mangled test output) + update_build_option('show_progress_bar', True) + + self.assertEqual(build_option('output_style'), 'auto') + + if HAVE_RICH: + self.assertTrue(use_rich()) + self.assertTrue(show_progress_bars()) + + update_build_option('output_style', 'rich') + self.assertTrue(use_rich()) + self.assertTrue(show_progress_bars()) + else: + self.assertFalse(use_rich()) + self.assertFalse(show_progress_bars()) + + update_build_option('output_style', 'basic') + self.assertFalse(use_rich()) + self.assertFalse(show_progress_bars()) + + def test_colorize(self): + """ + Test colorize function + """ + if HAVE_RICH: + for color in ('green', 'red', 'yellow'): + self.assertEqual(colorize('test', color), '[bold %s]test[/bold %s]' % (color, color)) + else: + self.assertEqual(colorize('test', 'green'), '\x1b[0;32mtest\x1b[0m') + self.assertEqual(colorize('test', 'red'), '\x1b[0;31mtest\x1b[0m') + self.assertEqual(colorize('test', 'yellow'), '\x1b[1;33mtest\x1b[0m') + + self.assertErrorRegex(EasyBuildError, "Unknown color: nosuchcolor", colorize, 'test', 'nosuchcolor') + + def test_get_progress_bar(self): + """ + Test get_progress_bar. + """ + # restore default configuration to show progress bars (disabled to avoid mangled test output), + # to ensure we'll get actual Progress instances when Rich is available + update_build_option('show_progress_bar', True) + + for pbar_type in PROGRESS_BAR_TYPES: + pbar = get_progress_bar(pbar_type, ignore_cache=True) + if HAVE_RICH: + self.assertTrue(isinstance(pbar, rich.progress.Progress)) + else: + self.assertTrue(isinstance(pbar, DummyRich)) + + def test_get_start_update_stop_progress_bar(self): + """ + Test starting/updating/stopping of progress bars. + """ + # clear progress bar cache first, this test assumes we start with a clean slate + easybuild.tools.output._progress_bar_cache.clear() + + # restore default configuration to show progress bars (disabled to avoid mangled test output) + update_build_option('show_progress_bar', True) + + # stopping a progress bar that never was started results in an error + error_pattern = "Failed to stop extensions progress bar, since it was never started" + self.assertErrorRegex(EasyBuildError, error_pattern, stop_progress_bar, PROGRESS_BAR_EXTENSIONS) + + # updating a progress bar that never was started is silently ignored on purpose + update_progress_bar(PROGRESS_BAR_EXTENSIONS) + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="foo") + update_progress_bar(PROGRESS_BAR_EXTENSIONS, progress_size=100) + + # also test normal cycle: start, update, stop + start_progress_bar(PROGRESS_BAR_EXTENSIONS, 100) + update_progress_bar(PROGRESS_BAR_EXTENSIONS) # single step progress + update_progress_bar(PROGRESS_BAR_EXTENSIONS, total=50) + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="test123", progress_size=5) + stop_progress_bar(PROGRESS_BAR_EXTENSIONS) + + +def suite(): + """ returns all the testcases in this module """ + return TestLoaderFiltered().loadTestsFromTestCase(OutputTest, sys.argv[1:]) + + +if __name__ == '__main__': + res = TextTestRunner(verbosity=1).run(suite()) + sys.exit(len(res.failures)) diff --git a/test/framework/package.py b/test/framework/package.py index e2a97ad0de..3e60fba515 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -49,9 +49,8 @@ import os, sys def verbose(msg): - fp = open('%(fpm_output_file)s', 'a') - fp.write(msg + '\\n') - fp.close() + with open('%(fpm_output_file)s', 'a') as fp: + fp.write(msg + '\\n') description, iteration, name, source, target, url, version, workdir = '', '', '', '', '', '', '', '' excludes = [] @@ -112,29 +111,26 @@ def verbose(msg): pkgfile = os.path.join(workdir, name + '-' + version + '.' + iteration + '.' + target) -fp = open(pkgfile, 'w') +with open(pkgfile, 'w') as fp: + fp.write('thisisan' + target + '\\n') + fp.write(' '.join(sys.argv[1:]) + '\\n') + fp.write("STARTCONTENTS of installdir " + installdir + ':\\n') -fp.write('thisisan' + target + '\\n') -fp.write(' '.join(sys.argv[1:]) + '\\n') -fp.write("STARTCONTENTS of installdir " + installdir + ':\\n') + find_cmd = 'find ' + installdir + ' ' + ''.join([" -not -path /" + x + ' ' for x in excludes]) + verbose("trying: " + find_cmd) + fp.write(find_cmd + '\\n') -find_cmd = 'find ' + installdir + ' ' + ''.join([" -not -path /" + x + ' ' for x in excludes]) -verbose("trying: " + find_cmd) -fp.write(find_cmd + '\\n') + fp.write('ENDCONTENTS\\n') -fp.write('ENDCONTENTS\\n') + fp.write("Contents of module file " + modulefile + ':') -fp.write("Contents of module file " + modulefile + ':') + fp.write('modulefile: ' + modulefile + '\\n') + #modtxt = open(modulefile).read() + #fp.write(modtxt + '\\n') -fp.write('modulefile: ' + modulefile + '\\n') -#modtxt = open(modulefile).read() -#fp.write(modtxt + '\\n') - -fp.write("I found excludes " + ' '.join(excludes) + '\\n') -fp.write("DESCRIPTION: " + description + '\\n') - -fp.close() + fp.write("I found excludes " + ' '.join(excludes) + '\\n') + fp.write("DESCRIPTION: " + description + '\\n') """ diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 9635e8eb00..1d4d2834be 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,7 @@ from easybuild.framework.easyconfig.tools import process_easyconfig from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import get_module_syntax +from easybuild.tools.config import get_module_syntax, update_build_option from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, remove_dir, which, write_file from easybuild.tools.job import pbs_python from easybuild.tools.job.pbs_python import PbsPython @@ -322,6 +322,12 @@ def test_submit_jobs(self): regex = re.compile(regex) self.assertFalse(regex.search(cmd), "Pattern '%s' should *not* be found in: %s" % (regex.pattern, cmd)) + # test again with custom EasyBuild command to use in jobs + update_build_option('job_eb_cmd', "/just/testing/bin/eb --debug") + cmd = submit_jobs([toy_ec], eb_go.generate_cmd_line(), testing=True) + regex = re.compile(r" && /just/testing/bin/eb --debug %\(spec\)s ") + self.assertTrue(regex.search(cmd), "Pattern '%s' found in: %s" % (regex.pattern, cmd)) + def test_build_easyconfigs_in_parallel_slurm(self): """Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job.""" diff --git a/test/framework/repository.py b/test/framework/repository.py index 05d3ede70f..528026142a 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/robot.py b/test/framework/robot.py index 9c29cb25cd..0647352935 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -670,6 +670,23 @@ def test_det_easyconfig_paths(self): regex = re.compile(r"^ \* \[.\] .*/__archive__/.*/intel-2012a.eb \(module: intel/2012a\)", re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + args = [ + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), + '--dry-run', + '--robot', + '--tmpdir=%s' % self.test_prefix, + '--filter-ecs=*oy-0.0.eb,*-test.eb', + ] + outtxt = self.eb_main(args, raise_error=True) + + regex = re.compile(r"^ \* \[.\] .*/toy-0.0-gompi-2018a.eb \(module: toy/0.0-gompi-2018a\)", re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + for ec in ('toy-0.0.eb', 'toy-0.0-gompi-2018a-test.eb'): + regex = re.compile(r"^ \* \[.\] .*/%s \(module:" % ec, re.M) + self.assertFalse(regex.search(outtxt), "%s should be fitered in %s" % (ec, outtxt)) + def test_search_paths(self): """Test search_paths command line argument.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -696,7 +713,7 @@ def test_search_paths(self): regex = re.compile(r"^ \* %s$" % os.path.join(self.test_prefix, test_ec), re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) - def test_det_easyconfig_paths_from_pr(self): + def test_github_det_easyconfig_paths_from_pr(self): """Test det_easyconfig_paths function, with --from-pr enabled as well.""" if self.github_token is None: print("Skipping test_from_pr, no GitHub token available?") @@ -1029,7 +1046,7 @@ def test_find_resolved_modules(self): ordered_ecs, new_easyconfigs, new_avail_modules = find_resolved_modules(ecs, mods, self.modtool) # all dependencies are resolved for easyconfigs included in ordered_ecs - self.assertFalse(any([ec['dependencies'] for ec in ordered_ecs])) + self.assertFalse(any(ec['dependencies'] for ec in ordered_ecs)) # nodeps/ondep easyconfigs have all dependencies resolved self.assertEqual(len(ordered_ecs), 2) @@ -1065,8 +1082,8 @@ def test_tweak_robotpath(self): test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # Create directories to store the tweaked easyconfigs - tweaked_ecs_paths, pr_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) - robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_path, auto_robot=True) + tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) + robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_paths, auto_robot=True) init_config(build_options={ 'valid_module_classes': module_classes(), @@ -1513,10 +1530,10 @@ def test_search_easyconfigs(self): paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False) ref_paths = [ + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), - os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), os.path.join(test_ecs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb') ] self.assertEqual(paths, ref_paths) diff --git a/test/framework/run.py b/test/framework/run.py index b6ceaf8127..dd7332e030 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1,6 +1,6 @@ # # # -*- coding: utf-8 -*- -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,6 +38,7 @@ import subprocess import sys import tempfile +import time from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from easybuild.base.fancylogger import setLogLevelDebug @@ -46,13 +47,8 @@ import easybuild.tools.utilities from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging from easybuild.tools.filetools import adjust_permissions, read_file, write_file -from easybuild.tools.run import ( - check_log_for_errors, - get_output_from_process, - run_cmd, - run_cmd_qa, - parse_log_for_error, -) +from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process +from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa from easybuild.tools.config import ERROR, IGNORE, WARN @@ -351,9 +347,27 @@ def test_run_cmd_qa_buffering(self): cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"' (out, ec) = run_cmd_qa(cmd, {'Pick a number: ': '42'}, log_all=True, maxhits=5) + self.assertEqual(ec, 0) regex = re.compile("Picked number: 42$") self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out)) + # also test with script run as interactive command that quickly exits with non-zero exit code; + # see https://github.com/easybuilders/easybuild-framework/issues/3593 + script_txt = '\n'.join([ + "#/bin/bash", + "echo 'Hello, I am about to exit'", + "echo 'ERROR: I failed' >&2", + "exit 1", + ]) + script = os.path.join(self.test_prefix, 'test.sh') + write_file(script, script_txt) + adjust_permissions(script, stat.S_IXUSR) + + out, ec = run_cmd_qa(script, {}, log_ok=False) + + self.assertEqual(ec, 1) + self.assertEqual(out, "Hello, I am about to exit\nERROR: I failed\n") + def test_run_cmd_qa_log_all(self): """Test run_cmd_qa with log_output enabled""" (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}, log_all=True) @@ -510,7 +524,10 @@ def test_dry_run(self): def test_run_cmd_list(self): """Test run_cmd with command specified as a list rather than a string""" - (out, ec) = run_cmd(['/bin/sh', '-c', "echo hello"], shell=False) + cmd = ['/bin/sh', '-c', "echo hello"] + self.assertErrorRegex(EasyBuildError, "When passing cmd as a list then `shell` must be set explictely!", + run_cmd, cmd) + (out, ec) = run_cmd(cmd, shell=False) self.assertEqual(out, "hello\n") # no reason echo hello could fail self.assertEqual(ec, 0) @@ -553,6 +570,115 @@ def test_run_cmd_stream(self): ]) self.assertEqual(stdout, expected) + def test_run_cmd_async(self): + """Test asynchronously running of a shell command via run_cmd + complete_cmd.""" + + os.environ['TEST'] = 'test123' + + test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" + cmd_info = run_cmd(test_cmd, asynchronous=True) + proc = cmd_info[0] + + # change value of $TEST to check that command is completed with correct environment + os.environ['TEST'] = 'some_other_value' + + # initial poll should result in None, since it takes a while for the command to complete + ec = proc.poll() + self.assertEqual(ec, None) + + # wait until command is done + while ec is None: + time.sleep(1) + ec = proc.poll() + + out, ec = complete_cmd(*cmd_info, simple=False) + self.assertEqual(ec, 0) + self.assertEqual(out, 'sleeping...\ntest123\n') + + # also test use of check_async_cmd function + os.environ['TEST'] = 'test123' + cmd_info = run_cmd(test_cmd, asynchronous=True) + + # first check, only read first 12 output characters + # (otherwise we'll be waiting until command is completed) + res = check_async_cmd(*cmd_info, output_read_size=12) + self.assertEqual(res, {'done': False, 'exit_code': None, 'output': 'sleeping...\n'}) + + # 2nd check with default output size (1024) gets full output + # (keep checking until command is fully done) + while not res['done']: + res = check_async_cmd(*cmd_info, output=res['output']) + self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'}) + + # check asynchronous running of failing command + error_test_cmd = "echo 'FAIL!' >&2; exit 123" + cmd_info = run_cmd(error_test_cmd, asynchronous=True) + time.sleep(1) + error_pattern = 'cmd ".*" exited with exit code 123' + self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info) + + cmd_info = run_cmd(error_test_cmd, asynchronous=True) + res = check_async_cmd(*cmd_info, fail_on_error=False) + # keep checking until command is fully done + while not res['done']: + res = check_async_cmd(*cmd_info, fail_on_error=False, output=res['output']) + self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"}) + + # also test with a command that produces a lot of output, + # since that tends to lock up things unless we frequently grab some output... + verbose_test_cmd = ';'.join([ + "echo start", + "for i in $(seq 1 50)", + "do sleep 0.1", + "for j in $(seq 1000)", + "do echo foo", + "done", + "done", + "echo done", + ]) + cmd_info = run_cmd(verbose_test_cmd, asynchronous=True) + proc = cmd_info[0] + + output = '' + ec = proc.poll() + self.assertEqual(ec, None) + + while ec is None: + time.sleep(1) + output += get_output_from_process(proc) + ec = proc.poll() + + out, ec = complete_cmd(*cmd_info, simple=False, output=output) + self.assertEqual(ec, 0) + self.assertTrue(out.startswith('start\n')) + self.assertTrue(out.endswith('\ndone\n')) + + # also test use of check_async_cmd on verbose test command + cmd_info = run_cmd(verbose_test_cmd, asynchronous=True) + + error_pattern = r"Number of output bytes to read should be a positive integer value \(or zero\)" + self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size=-1) + self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info, output_read_size='foo') + + # with output_read_size set to 0, no output is read yet, only status of command is checked + res = check_async_cmd(*cmd_info, output_read_size=0) + self.assertEqual(res['done'], False) + self.assertEqual(res['exit_code'], None) + self.assertEqual(res['output'], '') + + res = check_async_cmd(*cmd_info) + self.assertEqual(res['done'], False) + self.assertEqual(res['exit_code'], None) + self.assertTrue(res['output'].startswith('start\n')) + self.assertFalse(res['output'].endswith('\ndone\n')) + # keep checking until command is complete + while not res['done']: + res = check_async_cmd(*cmd_info, output=res['output']) + self.assertEqual(res['done'], True) + self.assertEqual(res['exit_code'], 0) + self.assertTrue(res['output'].startswith('start\n')) + self.assertTrue(res['output'].endswith('\ndone\n')) + def test_check_log_for_errors(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) diff --git a/test/framework/sandbox/easybuild/easyblocks/e/__init__.py b/test/framework/sandbox/easybuild/easyblocks/e/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py new file mode 100644 index 0000000000..e27c0c66d0 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py @@ -0,0 +1,34 @@ +## +# Copyright 2009-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for EasyBuildMeta + +@author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.framework.easyblock import EasyBlock + + +class EB_EasyBuildMeta(EasyBlock): + pass diff --git a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py index 7862d814a4..1df3706307 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foo.py b/test/framework/sandbox/easybuild/easyblocks/f/foo.py index 438d6dbd97..c3a52b1e4d 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py index 48991e160c..7b1324ee66 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py index cfad1ca661..ebb404cc79 100644 --- a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py +++ b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py index 2ed571fbdf..d0a6da6260 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py index 74d22be579..7e2da9b53c 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py index 18ccdb1902..da7017345c 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py new file mode 100644 index 0000000000..6b258c87d6 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py @@ -0,0 +1,49 @@ +## +# Copyright 2009-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for Makecp. + +@author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.framework.easyconfig import BUILD, MANDATORY + + +class MakeCp(ConfigureMake): + """Dummy support for software with no configure and no make install step.""" + + @staticmethod + def extra_options(extra_vars=None): + """ + Define list of files or directories to be copied after make + """ + extra = { + 'files_to_copy': [None, "List of files or dirs to copy", MANDATORY], + 'with_configure': [False, "Run configure script before building", BUILD], + } + if extra_vars is None: + extra_vars = {} + extra.update(extra_vars) + return ConfigureMake.extra_options(extra_vars=extra) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py index c80489f418..25708d9039 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py new file mode 100644 index 0000000000..0321602f3f --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py @@ -0,0 +1,44 @@ +## +# Copyright 2009-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for Makecp. + +@author: Miguel Dias Costa (National University of Singapore) +""" +from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import CUSTOM + + +class PythonBundle(EasyBlock): + """Dummy support for bundle of modules.""" + + @staticmethod + def extra_options(extra_vars=None): + if extra_vars is None: + extra_vars = {} + extra_vars.update({ + 'components': [(), "List of components to install: tuples w/ name, version and easyblock to use", CUSTOM], + }) + return EasyBlock.extra_options(extra_vars) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py index d07cd95300..09f330c1bf 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index bbb792e7ee..19eaf74338 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,7 +30,8 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock -from easybuild.easyblocks.toy import EB_toy +from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.run import run_cmd @@ -45,20 +46,59 @@ def extra_options(): } return ExtensionEasyBlock.extra_options(extra_vars=extra_vars) - def run(self): - """Build toy extension.""" + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + deps = { + 'bar': [], + 'barbar': ['bar'], + 'ls': [], + } + if self.name in deps: + return deps[self.name] + else: + raise EasyBuildError("Dependencies for %s are unknown!", self.name) + + def run(self, *args, **kwargs): + """ + Install toy extension. + """ if self.src: - super(Toy_Extension, self).run(unpack_src=True) - EB_toy.configure_step(self.master, name=self.name) EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts']) if self.cfg['toy_ext_param']: run_cmd(self.cfg['toy_ext_param']) - EB_toy.install_step(self.master, name=self.name) - return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name) + def prerun(self): + """ + Prepare installation of toy extension. + """ + super(Toy_Extension, self).prerun() + + if self.src: + super(Toy_Extension, self).run(unpack_src=True) + EB_toy.configure_step(self.master, name=self.name) + + def run_async(self): + """ + Install toy extension asynchronously. + """ + if self.src: + cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) + self.async_cmd_start(cmd) + else: + self.async_cmd_info = False + + def postrun(self): + """ + Wrap up installation of toy extension. + """ + super(Toy_Extension, self).postrun() + + EB_toy.install_step(self.master, name=self.name) + def sanity_check_step(self, *args, **kwargs): """Custom sanity check for toy extensions.""" self.log.info("Loaded modules: %s", self.modules_tool.list()) diff --git a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py index a7e6f4ba2e..0ee172e853 100644 --- a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py +++ b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/l/__init__.py b/test/framework/sandbox/easybuild/easyblocks/l/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py new file mode 100644 index 0000000000..ec76e9ae08 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -0,0 +1,60 @@ +## +# Copyright 2021-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing libtoy, implemented as an easyblock + +@author: Kenneth Hoste (Ghent University) +""" +import os + +from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext + +SHLIB_EXT = get_shared_lib_ext() + + +class EB_libtoy(EasyBlock): + """Support for building/installing libtoy.""" + + def banned_linked_shared_libs(self): + default = '/thiswillnotbethere,libtoytoytoy.%s,toytoytoy' % SHLIB_EXT + return os.getenv('EB_LIBTOY_BANNED_SHARED_LIBS', default).split(',') + + def required_linked_shared_libs(self): + default = '/lib,.*' + return os.getenv('EB_LIBTOY_REQUIRED_SHARED_LIBS', default).split(',') + + def configure_step(self, name=None): + """No configuration for libtoy.""" + pass + + def build_step(self, name=None, buildopts=None): + """Build libtoy.""" + run_cmd('make') + + def install_step(self, name=None): + """Install libtoy.""" + run_cmd('make install PREFIX="%s"' % self.installdir) diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py index f5ac2fd878..52ce315477 100644 --- a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py +++ b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py index fbc13cb4fc..5fabe4eb63 100644 --- a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py +++ b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py index 30e4cf8f7b..0e826c3f7c 100644 --- a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py +++ b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 5b57a6f32d..f1d39d0af2 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,11 +36,24 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.environment import setvar -from easybuild.tools.filetools import mkdir +from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd +def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts): + """ + Compose command to build toy. + """ + + cmd = "%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s" % { + 'name': name, + 'prebuildopts': prebuildopts, + 'buildopts': buildopts, + } + return cmd + + class EB_toy(ExtensionEasyBlock): """Support for building/installing toy.""" @@ -92,17 +105,13 @@ def configure_step(self, name=None): def build_step(self, name=None, buildopts=None): """Build toy.""" - if buildopts is None: buildopts = self.cfg['buildopts'] - if name is None: name = self.name - run_cmd('%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s' % { - 'name': name, - 'prebuildopts': self.cfg['prebuildopts'], - 'buildopts': buildopts, - }) + + cmd = compose_toy_build_cmd(self.cfg, name, self.cfg['prebuildopts'], buildopts) + run_cmd(cmd) def install_step(self, name=None): """Install toy.""" @@ -116,15 +125,40 @@ def install_step(self, name=None): # also install a dummy libtoy.a, to make the default sanity check happy libdir = os.path.join(self.installdir, 'lib') mkdir(libdir, parents=True) - f = open(os.path.join(libdir, 'lib%s.a' % name), 'w') - f.write(name.upper()) - f.close() + write_file(os.path.join(libdir, 'lib%s.a' % name), name.upper()) - def run(self): - """Install toy as extension.""" + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + if self.name == 'toy': + return ['bar', 'barbar'] + else: + raise EasyBuildError("Dependencies for %s are unknown!", self.name) + + def prerun(self): + """ + Prepare installation of toy as extension. + """ super(EB_toy, self).run(unpack_src=True) self.configure_step() + + def run(self): + """ + Install toy as extension. + """ self.build_step() + + def run_async(self): + """ + Asynchronous installation of toy as extension. + """ + cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) + self.async_cmd_start(cmd) + + def postrun(self): + """ + Wrap up installation of toy as extension. + """ self.install_step() def make_module_step(self, fake=False): diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py index 9d99eac45e..f730dbd8a3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py index 6f53997f8b..0bc400e29e 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2021 Ghent University +# Copyright 2020-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py index b303ecb849..269373e245 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/__init__.py b/test/framework/sandbox/easybuild/tools/__init__.py index 433662112e..3f77104b3a 100644 --- a/test/framework/sandbox/easybuild/tools/__init__.py +++ b/test/framework/sandbox/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2021 Ghent University +# Copyright 2009-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py index f447c49e35..03b33c11a4 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2021 Ghent University +# Copyright 2011-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py index 3d90a37485..62e9c033af 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index e6dc638af8..93e610b325 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py index 8c7412c5f3..5ab85d6968 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz b/test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz new file mode 100644 index 0000000000..a11efbdbdf Binary files /dev/null and b/test/framework/sandbox/sources/l/libtoy/libtoy-0.0.tar.gz differ diff --git a/test/framework/sandbox/sources/toy/toy-0.0_fix-README.patch b/test/framework/sandbox/sources/toy/toy-0.0_fix-README.patch new file mode 100644 index 0000000000..71adf829c4 --- /dev/null +++ b/test/framework/sandbox/sources/toy/toy-0.0_fix-README.patch @@ -0,0 +1,5 @@ +--- README.old 2022-03-01 14:28:43.000000000 +0100 ++++ README 2022-03-01 14:29:01.000000000 +0100 +@@ -1 +1 @@ +-TOY ++toy 0.0, a toy test program diff --git a/test/framework/style.py b/test/framework/style.py index 24646c5e1f..6a84b23b6a 100644 --- a/test/framework/style.py +++ b/test/framework/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2021 Ghent University +# Copyright 2016-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/suite.py b/test/framework/suite.py index 466401aa4e..d49d40bb6b 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -1,6 +1,6 @@ #!/usr/bin/python # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -49,8 +49,9 @@ import test.framework.easyconfig as e import test.framework.easyconfigparser as ep import test.framework.easyconfigformat as ef -import test.framework.ebconfigobj as ebco import test.framework.easyconfigversion as ev +import test.framework.easystack as es +import test.framework.ebconfigobj as ebco import test.framework.environment as env import test.framework.docs as d import test.framework.filetools as f @@ -65,6 +66,7 @@ import test.framework.modules as m import test.framework.modulestool as mt import test.framework.options as o +import test.framework.output as ou import test.framework.parallelbuild as p import test.framework.package as pkg import test.framework.repository as r @@ -77,6 +79,7 @@ import test.framework.toy_build as t import test.framework.type_checking as et import test.framework.tweak as tw +import test.framework.utilities_test as u import test.framework.variables as v import test.framework.yeb as y @@ -118,7 +121,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, - tw, p, i, pkg, d, env, et, y, st, h, ct, lib] + tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u, es, ou] SUITE = unittest.TestSuite([x.suite() for x in tests]) res = unittest.TextTestRunner().run(SUITE) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index eba1625a17..424ced9548 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,6 +28,7 @@ @author: Kenneth hoste (Ghent University) @author: Ward Poelmans (Ghent University) """ +import ctypes import re import os import sys @@ -38,18 +39,19 @@ import easybuild.tools.systemtools as st from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions, read_file, which, write_file +from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64 from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL from easybuild.tools.systemtools import MAX_FREQ_FP, PROC_CPUINFO_FP, PROC_MEMINFO_FP -from easybuild.tools.systemtools import check_os_dependency, check_python_version, pick_dep_version +from easybuild.tools.systemtools import check_linked_shared_libs, check_os_dependency, check_python_version from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_arch_name, get_cpu_architecture from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version from easybuild.tools.systemtools import get_platform_name, get_shared_lib_ext, get_system_info, get_total_memory +from easybuild.tools.systemtools import find_library_path, locate_solib, pick_dep_version PROC_CPUINFO_TXT = None @@ -364,10 +366,13 @@ def setUp(self): self.orig_is_readable = st.is_readable self.orig_read_file = st.read_file self.orig_run_cmd = st.run_cmd + self.orig_platform_dist = st.platform.dist if hasattr(st.platform, 'dist') else None self.orig_platform_uname = st.platform.uname self.orig_get_tool_version = st.get_tool_version self.orig_sys_version_info = st.sys.version_info self.orig_HAVE_ARCHSPEC = st.HAVE_ARCHSPEC + self.orig_HAVE_DISTRO = st.HAVE_DISTRO + self.orig_ETC_OS_RELEASE = st.ETC_OS_RELEASE if hasattr(st, 'archspec_cpu_host'): self.orig_archspec_cpu_host = st.archspec_cpu_host else: @@ -381,10 +386,14 @@ def tearDown(self): st.get_os_name = self.orig_get_os_name st.get_os_type = self.orig_get_os_type st.run_cmd = self.orig_run_cmd + if self.orig_platform_dist is not None: + st.platform.dist = self.orig_platform_dist st.platform.uname = self.orig_platform_uname st.get_tool_version = self.orig_get_tool_version st.sys.version_info = self.orig_sys_version_info st.HAVE_ARCHSPEC = self.orig_HAVE_ARCHSPEC + st.HAVE_DISTRO = self.orig_HAVE_DISTRO + st.ETC_OS_RELEASE = self.orig_ETC_OS_RELEASE if self.orig_archspec_cpu_host is not None: st.archspec_cpu_host = self.orig_archspec_cpu_host super(SystemToolsTest, self).tearDown() @@ -452,7 +461,7 @@ def test_cpu_speed_native(self): """Test getting CPU speed.""" cpu_speed = get_cpu_speed() self.assertTrue(isinstance(cpu_speed, float) or cpu_speed is None) - self.assertTrue(cpu_speed > 0.0 or cpu_speed is None) + self.assertTrue(cpu_speed is None or cpu_speed > 0.0) def test_cpu_speed_linux(self): """Test getting CPU speed (mocked for Linux).""" @@ -486,8 +495,8 @@ def test_cpu_features_native(self): """Test getting CPU features.""" cpu_feat = get_cpu_features() self.assertTrue(isinstance(cpu_feat, list)) - self.assertTrue(len(cpu_feat) > 0) - self.assertTrue(all([isinstance(x, string_type) for x in cpu_feat])) + self.assertTrue(len(cpu_feat) >= 0) + self.assertTrue(all(isinstance(x, string_type) for x in cpu_feat)) def test_cpu_features_linux(self): """Test getting CPU features (mocked for Linux).""" @@ -557,6 +566,7 @@ def test_cpu_architecture(self): machine_names = { 'aarch64': AARCH64, 'aarch64_be': AARCH64, + 'arm64': AARCH64, 'armv7l': AARCH32, 'ppc64': POWER, 'ppc64le': POWER, @@ -736,6 +746,21 @@ def test_os_version(self): os_version = get_os_version() self.assertTrue(isinstance(os_version, string_type) or os_version == UNKNOWN) + # make sure that bug fixed in https://github.com/easybuilders/easybuild-framework/issues/3952 + # does not surface again, by mocking what's needed to make get_os_version fall into SLES-specific path + + if hasattr(st.platform, 'dist'): + st.platform.dist = lambda: (None, None) + st.HAVE_DISTRO = False + + st.get_os_name = lambda: 'SLES' + fake_etc_os_release = os.path.join(self.test_prefix, 'os-release') + write_file(fake_etc_os_release, 'VERSION="15-SP1"') + st.ETC_OS_RELEASE = fake_etc_os_release + + os_version = get_os_version() + self.assertEqual(os_version, '15-SP1') + def test_gcc_version_native(self): """Test getting gcc version.""" gcc_version = get_gcc_version() @@ -848,8 +873,8 @@ def mock_python_ver(py_maj_ver, py_min_ver): error_pattern = r"EasyBuild is not compatible \(yet\) with Python 4.0" self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version) - mock_python_ver(2, 5) - error_pattern = r"Python 2.6 or higher is required when using Python 2, found Python 2.5" + mock_python_ver(2, 6) + error_pattern = r"Python 2.7 is required when using Python 2, found Python 2.6" self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version) # no problems when running with a supported Python version @@ -857,45 +882,19 @@ def mock_python_ver(py_maj_ver, py_min_ver): mock_python_ver(*pyver) self.assertEqual(check_python_version(), pyver) - mock_python_ver(2, 6) - # deprecation warning triggers an error in test environment - error_pattern = r"Running EasyBuild with Python 2.6 is deprecated" - self.assertErrorRegex(EasyBuildError, error_pattern, check_python_version) - - # we may trigger a deprecation warning below (when testing with Python 2.6) - py26_depr_warning = "\nWARNING: Deprecated functionality, will no longer work in v5.0: " - py26_depr_warning += "Running EasyBuild with Python 2.6 is deprecated" - - self.allow_deprecated_behaviour() - - # first test with mocked Python 2.6 - self.mock_stderr(True) - check_python_version() - stderr = self.get_stderr() - self.mock_stderr(False) - - # we should always get a deprecation warning here - self.assertTrue(stderr.startswith(py26_depr_warning)) - - # restore Python version info to check with Python version used to run tests - st.sys.version_info = self.orig_sys_version_info - # shouldn't raise any errors, since Python version used to run tests should be supported; self.mock_stderr(True) (py_maj_ver, py_min_ver) = check_python_version() stderr = self.get_stderr() self.mock_stderr(False) + self.assertFalse(stderr) self.assertTrue(py_maj_ver in [2, 3]) if py_maj_ver == 2: - self.assertTrue(py_min_ver in [6, 7]) + self.assertTrue(py_min_ver == 7) else: self.assertTrue(py_min_ver >= 5) - # only deprecation warning when actually testing with Python 2.6 - if sys.version_info[:2] == (2, 6): - self.assertTrue(stderr.startswith(py26_depr_warning)) - def test_pick_dep_version(self): """Test pick_dep_version function.""" @@ -916,6 +915,24 @@ def test_pick_dep_version(self): error_pattern = "Unknown value type for version" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, ('1.2.3', '4.5.6')) + # check support for using 'arch=*' as fallback key + dep_ver_dict = { + 'arch=*': '1.2.3', + 'arch=foo': '1.2.3-foo', + 'arch=POWER': '1.2.3-ppc64le', + } + self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le') + + del dep_ver_dict['arch=POWER'] + self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3') + + # check how faulty input is handled + self.assertErrorRegex(EasyBuildError, "Found empty dict as version!", pick_dep_version, {}) + error_pattern = r"Unexpected keys in version: bar,foo \(only 'arch=' keys are supported\)" + self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, {'foo': '1.2', 'bar': '2.3'}) + error_pattern = r"Unknown value type for version: .* \(1.23\), should be string value" + self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, 1.23) + def test_check_os_dependency(self): """Test check_os_dependency.""" @@ -959,6 +976,96 @@ def test_check_os_dependency(self): write_file(bash_profile, 'export LD_LIBRARY_PATH=%s' % self.test_prefix) self.assertTrue(check_os_dependency('bar')) + def test_check_linked_shared_libs(self): + """Test for check_linked_shared_libs function.""" + + txt_path = os.path.join(self.test_prefix, 'test.txt') + write_file(txt_path, "some text") + + broken_symlink_path = os.path.join(self.test_prefix, 'broken_symlink') + symlink('/doesnotexist', broken_symlink_path, use_abspath_source=False) + + # result is always None for anything other than dynamically linked binaries or shared libraries + self.assertEqual(check_linked_shared_libs(self.test_prefix), None) + self.assertEqual(check_linked_shared_libs(txt_path), None) + self.assertEqual(check_linked_shared_libs(broken_symlink_path), None) + + bin_ls_path = which('ls') + + os_type = get_os_type() + if os_type == LINUX: + out, _ = run_cmd("ldd %s" % bin_ls_path) + elif os_type == DARWIN: + out, _ = run_cmd("otool -L %s" % bin_ls_path) + else: + raise EasyBuildError("Unknown OS type: %s" % os_type) + + shlib_ext = get_shared_lib_ext() + lib_path_regex = re.compile(r'(?P[^\s]*/lib[^ ]+\.%s[^ ]*)' % shlib_ext, re.M) + lib_path = lib_path_regex.search(out).group(1) + + test_pattern_named_args = [ + # if no patterns are specified, result is always True + {}, + {'required_patterns': ['/lib', shlib_ext]}, + {'banned_patterns': ['this_pattern_should_not_match']}, + {'required_patterns': ['/lib', shlib_ext], 'banned_patterns': ['weirdstuff']}, + ] + for pattern_named_args in test_pattern_named_args: + # result is always None for anything other than dynamically linked binaries or shared libraries + self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) + for path in (bin_ls_path, lib_path): + # path may not exist, especially for library paths obtained via 'otool -L' on macOS + if os.path.exists(path): + error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args) + self.assertTrue(check_linked_shared_libs(path, **pattern_named_args), error_msg) + + # also test with input that should result in failing check + test_pattern_named_args = [ + {'required_patterns': ['this_pattern_will_not_match']}, + {'banned_patterns': ['/lib']}, + {'required_patterns': ['weirdstuff'], 'banned_patterns': ['/lib', shlib_ext]}, + ] + for pattern_named_args in test_pattern_named_args: + # result is always None for anything other than dynamically linked binaries or shared libraries + self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) + self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) + for path in (bin_ls_path, lib_path): + error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args) + self.assertFalse(check_linked_shared_libs(path, **pattern_named_args), error_msg) + + def test_locate_solib(self): + """Test locate_solib function (Linux only).""" + if get_os_type() == LINUX: + libname = 'libc.so.6' + libc_obj = None + try: + libc_obj = ctypes.cdll.LoadLibrary(libname) + except OSError: + pass + if libc_obj: + libc_path = locate_solib(libc_obj) + self.assertEqual(os.path.basename(libc_path), libname) + self.assertTrue(os.path.exists(libc_path), "%s should exist" % libname) + + def test_find_library_path(self): + """Test find_library_path function (Linux and Darwin only).""" + os_type = get_os_type() + if os_type == LINUX: + libname = 'libc.so.6' + elif os_type == DARWIN: + libname = 'libSystem.dylib' + else: + libname = None + + if libname: + lib_path = find_library_path(libname) + self.assertEqual(os.path.basename(lib_path), libname) + self.assertTrue(os.path.exists(lib_path) or os_type == DARWIN, "%s should exist" % libname) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index f1a02ec9ca..e20108dfdf 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -51,6 +51,7 @@ from easybuild.tools.filetools import read_file, symlink, write_file, which from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template from easybuild.tools.toolchain.toolchain import env_vars_external_module from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain @@ -631,6 +632,19 @@ def test_misc_flags_shared(self): self.assertTrue(tc.get_variable(var).endswith(' ' + value)) self.modtool.purge() + value = '--only-in-cxxflags' + flag_vars.remove('CXXFLAGS') + tc = self.get_toolchain('foss', version='2018a') + tc.set_options({'extra_cxxflags': value}) + tc.prepare() + self.assertTrue(tc.get_variable('CXXFLAGS').endswith(' ' + value)) + for var in flag_vars: + self.assertTrue(value not in tc.get_variable(var)) + # https://github.com/easybuilders/easybuild-framework/pull/3571 + # catch variable resued inside loop + self.assertTrue("-o -n -l -y" not in tc.get_variable(var)) + self.modtool.purge() + def test_misc_flags_unique(self): """Test whether unique compiler flags are set correctly.""" @@ -738,17 +752,28 @@ def test_compiler_dependent_optarch(self): intel_options = [('intelflag', 'intelflag'), ('GENERIC', 'xSSE2'), ('', '')] gcc_options = [('gccflag', 'gccflag'), ('march=nocona', 'march=nocona'), ('', '')] gcccore_options = [('gcccoreflag', 'gcccoreflag'), ('GENERIC', 'march=x86-64 -mtune=generic'), ('', '')] - toolchains = [ - ('iccifort', '2018.1.163'), - ('GCC', '6.4.0-2.28'), - ('GCCcore', '6.2.0'), - ('PGI', '16.7-GCC-5.4.0-2.26'), - ] + + tc_intel = ('iccifort', '2018.1.163') + tc_gcc = ('GCC', '6.4.0-2.28') + tc_gcccore = ('GCCcore', '6.2.0') + tc_pgi = ('PGI', '16.7-GCC-5.4.0-2.26') enabled = [True, False] - test_cases = product(intel_options, gcc_options, gcccore_options, toolchains, enabled) + test_cases = [] + for i, (tc, options) in enumerate(zip((tc_intel, tc_gcc, tc_gcccore), + (intel_options, gcc_options, gcccore_options))): + # Vary only the compiler specific option + for opt in options: + new_value = [intel_options[0], gcc_options[0], gcccore_options[0], tc] + new_value[i] = opt + test_cases.append(new_value) + # Add one case for PGI + test_cases.append((intel_options[0], gcc_options[0], gcccore_options[0], tc_pgi)) - for intel_flags, gcc_flags, gcccore_flags, (toolchain_name, toolchain_ver), enable in test_cases: + # Run each for enabled and disabled + test_cases = list(product(test_cases, enabled)) + + for (intel_flags, gcc_flags, gcccore_flags, (toolchain_name, toolchain_ver)), enable in test_cases: intel_flags, intel_flags_exp = intel_flags gcc_flags, gcc_flags_exp = gcc_flags @@ -992,9 +1017,64 @@ def test_fft_env_vars_foss(self): self.assertEqual(tc.get_variable('LIBFFT'), '-lfftw3_mpi -lfftw3') self.assertEqual(tc.get_variable('LIBFFT_MT'), '-lfftw3 -lpthread') + self.modtool.purge() + self.setup_sandbox_for_foss_fftw(self.test_prefix) + self.modtool.prepend_module_path(self.test_prefix) + + tc = self.get_toolchain('foss', version='2018a-FFTW.MPI') + tc.prepare() + + fft_static_libs = 'libfftw3.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + fft_static_libs_mt = 'libfftw3.a,libpthread.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + self.assertEqual(tc.get_variable('LIBFFT'), '-lfftw3') + self.assertEqual(tc.get_variable('LIBFFT_MT'), '-lfftw3 -lpthread') + + fft_lib_dir = os.path.join(modules.get_software_root('FFTW'), 'lib') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + + tc = self.get_toolchain('foss', version='2018a-FFTW.MPI') + tc.set_options({'openmp': True}) + tc.prepare() + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), 'libfftw3_omp.a,' + fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), 'libfftw3_omp.a,' + fft_static_libs_mt) + + self.assertEqual(tc.get_variable('LIBFFT'), '-lfftw3') + self.assertEqual(tc.get_variable('LIBFFT_MT'), '-lfftw3_omp -lfftw3 -lpthread') + + fft_lib_dir = os.path.join(modules.get_software_root('FFTW'), 'lib') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + + tc = self.get_toolchain('foss', version='2018a-FFTW.MPI') + tc.set_options({'usempi': True}) + tc.prepare() + + fft_static_libs = 'libfftw3_mpi.a,libfftw3.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + self.assertEqual(tc.get_variable('LIBFFT'), '-lfftw3_mpi -lfftw3') + self.assertEqual(tc.get_variable('LIBFFT_MT'), '-lfftw3 -lpthread') + + fft_lib_dir = os.path.join(modules.get_software_root('FFTW.MPI'), 'lib') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + def test_fft_env_vars_intel(self): """Test setting of $FFT* environment variables using intel toolchain.""" + self.modtool.purge() self.setup_sandbox_for_intel_fftw(self.test_prefix) self.modtool.prepend_module_path(self.test_prefix) @@ -1054,6 +1134,70 @@ def test_fft_env_vars_intel(self): libfft_mt += '-Wl,-Bdynamic -liomp5 -lpthread' self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + self.modtool.purge() + self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='2021.4.0') + tc = self.get_toolchain('intel', version='2021b') + tc.prepare() + + fft_static_libs = 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + fft_static_libs_mt = 'libmkl_intel_lp64.a,libmkl_intel_thread.a,libmkl_core.a,' + fft_static_libs_mt += 'libiomp5.a,libpthread.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + libfft = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_sequential -lmkl_core " + libfft += "-Wl,--end-group -Wl,-Bdynamic" + self.assertEqual(tc.get_variable('LIBFFT'), libfft) + + libfft_mt = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core " + libfft_mt += "-Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread" + self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + + tc = self.get_toolchain('intel', version='2021b') + tc.set_options({'openmp': True}) + tc.prepare() + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + self.assertEqual(tc.get_variable('LIBFFT'), libfft) + self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + + fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4.0/lib/intel64') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + + tc = self.get_toolchain('intel', version='2021b') + tc.set_options({'usempi': True}) + tc.prepare() + + fft_static_libs = 'libfftw3x_cdft_lp64.a,libmkl_cdft_core.a,libmkl_blacs_intelmpi_lp64.a,' + fft_static_libs += 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS'), fft_static_libs) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS'), fft_static_libs) + + fft_static_libs_mt = 'libfftw3x_cdft_lp64.a,libmkl_cdft_core.a,libmkl_blacs_intelmpi_lp64.a,' + fft_static_libs_mt += 'libmkl_intel_lp64.a,libmkl_intel_thread.a,libmkl_core.a,libiomp5.a,libpthread.a' + self.assertEqual(tc.get_variable('FFT_STATIC_LIBS_MT'), fft_static_libs_mt) + self.assertEqual(tc.get_variable('FFTW_STATIC_LIBS_MT'), fft_static_libs_mt) + + libfft = '-Wl,-Bstatic -Wl,--start-group -lfftw3x_cdft_lp64 -lmkl_cdft_core ' + libfft += '-lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -Wl,--end-group -Wl,-Bdynamic' + self.assertEqual(tc.get_variable('LIBFFT'), libfft) + + libfft_mt = '-Wl,-Bstatic -Wl,--start-group -lfftw3x_cdft_lp64 -lmkl_cdft_core ' + libfft_mt += '-lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -Wl,--end-group ' + libfft_mt += '-Wl,-Bdynamic -liomp5 -lpthread' + self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) + + fft_lib_dir = os.path.join(modules.get_software_root('imkl-FFTW'), 'lib') + self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) + def test_fosscuda(self): """Test whether fosscuda is handled properly.""" tc = self.get_toolchain("fosscuda", version="2018a") @@ -1084,6 +1228,33 @@ def test_fosscuda(self): # check CUDA runtime lib self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS')) + def setup_sandbox_for_foss_fftw(self, moddir, fftwver='3.3.7'): + """Set up sandbox for foss FFTW and FFTW.MPI""" + # hack to make foss FFTW lib check pass + # create dummy FFTW and FFTW.MPI modules + + fftw_module_path = os.path.join(moddir, 'FFTW', fftwver) + fftw_dir = os.path.join(self.test_prefix, 'software', 'FFTW', fftwver) + + fftw_mod_txt = '\n'.join([ + "#%Module", + "setenv EBROOTFFTW %s" % fftw_dir, + "setenv EBVERSIONFFTW %s" % fftwver, + ]) + write_file(fftw_module_path, fftw_mod_txt) + + fftw_mpi_module_path = os.path.join(moddir, 'FFTW.MPI', fftwver) + fftw_mpi_dir = os.path.join(self.test_prefix, 'software', 'FFTW.MPI', fftwver) + fftw_mpi_mod_txt = '\n'.join([ + "#%Module", + "setenv EBROOTFFTWMPI %s" % fftw_mpi_dir, + "setenv EBVERSIONFFTWMPI %s" % fftwver, + ]) + write_file(fftw_mpi_module_path, fftw_mpi_mod_txt) + + os.makedirs(os.path.join(fftw_dir, 'lib')) + os.makedirs(os.path.join(fftw_mpi_dir, 'lib')) + def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): """Set up sandbox for Intel FFTW""" # hack to make Intel FFTW lib check pass @@ -1099,17 +1270,38 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): ]) write_file(imkl_module_path, imkl_mod_txt) - fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] - fftw_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] + mkl_libs = ['mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] + mkl_libs += ['mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] + mkl_libs += ['mkl_intel_thread', 'mkl_pgi_thread'] + fftw_libs = ['fftw3xc_intel', 'fftw3xc_pgi'] if LooseVersion(imklver) >= LooseVersion('11'): fftw_libs.extend(['fftw3x_cdft_ilp64', 'fftw3x_cdft_lp64']) else: fftw_libs.append('fftw3x_cdft') - for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: + if LooseVersion(imklver) >= LooseVersion('2021.4.0'): + imkl_fftw_module_path = os.path.join(moddir, 'imkl-FFTW', imklver) + imkl_fftw_dir = os.path.join(self.test_prefix, 'software', 'imkl-FFTW', imklver) + imkl_fftw_mod_txt = '\n'.join([ + "#%Module", + "setenv EBROOTIMKLMINFFTW %s" % imkl_fftw_dir, + "setenv EBVERSIONIMKLMINFFTW %s" % imklver, + ]) + write_file(imkl_fftw_module_path, imkl_fftw_mod_txt) + + subdir = 'mkl/%s/lib/intel64' % imklver os.makedirs(os.path.join(imkl_dir, subdir)) - for fftlib in fftw_libs: + for fftlib in mkl_libs: write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo') + subdir = 'lib' + os.makedirs(os.path.join(imkl_fftw_dir, subdir)) + for fftlib in fftw_libs: + write_file(os.path.join(imkl_fftw_dir, subdir, 'lib%s.a' % fftlib), 'foo') + else: + for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: + os.makedirs(os.path.join(imkl_dir, subdir)) + for fftlib in mkl_libs + fftw_libs: + write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo') def test_intel_toolchain(self): """Test for intel toolchain.""" @@ -1409,17 +1601,41 @@ def test_prepare_deps_external(self): self.assertEqual(modules.get_software_root('foobar'), '/foo/bar') self.assertEqual(modules.get_software_version('toy'), '1.2.3') + def test_get_software_version(self): + """Test that get_software_version works""" + os.environ['EBROOTTOY'] = '/foo/bar' + os.environ['EBVERSIONTOY'] = '1.2.3' + os.environ['EBROOTFOOBAR'] = '/foo/bar' + os.environ['EBVERSIONFOOBAR'] = '4.5' + tc = self.get_toolchain('GCC', version='6.4.0-2.28') + self.assertEqual(tc.get_software_version('toy'), ['1.2.3']) + self.assertEqual(tc.get_software_version(['toy']), ['1.2.3']) + self.assertEqual(tc.get_software_version(['toy', 'foobar']), ['1.2.3', '4.5']) + # Non existing modules raise an error + self.assertErrorRegex(EasyBuildError, 'non-existing was not found', + tc.get_software_version, 'non-existing') + self.assertErrorRegex(EasyBuildError, 'non-existing was not found', + tc.get_software_version, ['toy', 'non-existing', 'foobar']) + # Can use required=False to avoid + self.assertEqual(tc.get_software_version('non-existing', required=False), [None]) + self.assertEqual(tc.get_software_version(['toy', 'non-existing', 'foobar'], required=False), + ['1.2.3', None, '4.5']) + def test_old_new_iccifort(self): """Test whether preparing for old/new Intel compilers works correctly.""" self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='2018.1.163') self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='10.2.6.038') self.modtool.prepend_module_path(self.test_prefix) + shlib_ext = get_shared_lib_ext() + # incl. -lguide libblas_mt_intel3 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" libblas_mt_intel3 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" # no -lguide + blas_static_libs_intel4 = 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a' + blas_shared_libs_intel4 = blas_static_libs_intel4.replace('.a', '.' + shlib_ext) libblas_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" libblas_intel4 += " -Wl,--end-group -Wl,-Bdynamic" libblas_mt_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" @@ -1436,18 +1652,83 @@ def test_old_new_iccifort(self): libscalack_intel4 = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential " libscalack_intel4 += "-lmkl_core" - libblas_mt_fosscuda = "-lopenblas -lgfortran" + blas_static_libs_fosscuda = "libopenblas.a,libgfortran.a" + blas_shared_libs_fosscuda = blas_static_libs_fosscuda.replace('.a', '.' + shlib_ext) + blas_mt_static_libs_fosscuda = blas_static_libs_fosscuda + ",libpthread.a" + blas_mt_shared_libs_fosscuda = blas_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) + libblas_fosscuda = "-lopenblas -lgfortran" + libblas_mt_fosscuda = libblas_fosscuda + " -lpthread" + + fft_static_libs_fosscuda = "libfftw3.a" + fft_shared_libs_fosscuda = fft_static_libs_fosscuda.replace('.a', '.' + shlib_ext) + fft_mt_static_libs_fosscuda = "libfftw3.a,libpthread.a" + fft_mt_shared_libs_fosscuda = fft_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) + fft_mt_static_libs_fosscuda_omp = "libfftw3_omp.a,libfftw3.a,libpthread.a" + fft_mt_shared_libs_fosscuda_omp = fft_mt_static_libs_fosscuda_omp.replace('.a', '.' + shlib_ext) + libfft_fosscuda = "-lfftw3" + libfft_mt_fosscuda = libfft_fosscuda + " -lpthread" + libfft_mt_fosscuda_omp = "-lfftw3_omp " + libfft_fosscuda + " -lpthread" + + lapack_static_libs_fosscuda = "libopenblas.a,libgfortran.a" + lapack_shared_libs_fosscuda = lapack_static_libs_fosscuda.replace('.a', '.' + shlib_ext) + lapack_mt_static_libs_fosscuda = lapack_static_libs_fosscuda + ",libpthread.a" + lapack_mt_shared_libs_fosscuda = lapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) + liblapack_fosscuda = "-lopenblas -lgfortran" + liblapack_mt_fosscuda = liblapack_fosscuda + " -lpthread" + libscalack_fosscuda = "-lscalapack -lopenblas -lgfortran" - libfft_mt_fosscuda = "-lfftw3_omp -lfftw3 -lpthread" + libscalack_mt_fosscuda = libscalack_fosscuda + " -lpthread" + scalapack_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a" + scalapack_shared_libs_fosscuda = scalapack_static_libs_fosscuda.replace('.a', '.' + shlib_ext) + scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" + scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) tc = self.get_toolchain('fosscuda', version='2018a') tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) self.modtool.purge() tc = self.get_toolchain('intel', version='2018a') tc.prepare() + self.assertEqual(os.environ.get('BLAS_SHARED_LIBS', "(not set)"), blas_shared_libs_intel4) + self.assertEqual(os.environ.get('BLAS_STATIC_LIBS', "(not set)"), blas_static_libs_intel4) + self.assertEqual(os.environ.get('LAPACK_SHARED_LIBS', "(not set)"), blas_shared_libs_intel4) + self.assertEqual(os.environ.get('LAPACK_STATIC_LIBS', "(not set)"), blas_static_libs_intel4) self.assertEqual(os.environ.get('LIBBLAS', "(not set)"), libblas_intel4) self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel4) self.assertEqual(os.environ.get('LIBFFT', "(not set)"), libfft_intel4) @@ -1484,9 +1765,42 @@ def test_old_new_iccifort(self): tc = self.get_toolchain('fosscuda', version='2018a') tc.set_options({'openmp': True}) tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) - self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda_omp) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda_omp) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda_omp) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda_omp) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda_omp) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) def test_standalone_iccifort(self): """Test whether standalone installation of iccifort matches the iccifort toolchain definition.""" @@ -1666,7 +1980,7 @@ def test_compiler_cache(self): ccache = which('ccache') if ccache is None: - msg = r"ccache binary not found in \$PATH, required by --use-compiler-cache" + msg = r"ccache binary not found in \$PATH, required by --use-ccache" self.assertErrorRegex(EasyBuildError, msg, self.eb_main, args, raise_error=True, do_build=True) # generate shell script to mock ccache/f90cache @@ -2290,12 +2604,16 @@ def test_toolchain_prepare_rpath(self): # check whether 'stubs' library directory are correctly filtered out paths = [ 'prefix/software/CUDA/1.2.3/lib/stubs/', # should be filtered (no -rpath) + 'prefix/software/CUDA/1.2.3/stubs/lib/', # should be filtered (no -rpath) 'tmp/foo/', 'prefix/software/stubs/1.2.3/lib', # should NOT be filtered 'prefix/software/CUDA/1.2.3/lib/stubs', # should be filtered (no -rpath) + 'prefix/software/CUDA/1.2.3/stubs/lib', # should be filtered (no -rpath) 'prefix/software/CUDA/1.2.3/lib64/stubs/', # should be filtered (no -rpath) + 'prefix/software/CUDA/1.2.3/stubs/lib64/', # should be filtered (no -rpath) 'prefix/software/foobar/4.5/notreallystubs', # should NOT be filtered 'prefix/software/CUDA/1.2.3/lib64/stubs', # should be filtered (no -rpath) + 'prefix/software/CUDA/1.2.3/stubs/lib64', # should be filtered (no -rpath) 'prefix/software/zlib/1.2.11/lib', 'prefix/software/bleh/0/lib/stubs', # should be filtered (no -rpath) 'prefix/software/foobar/4.5/stubsbutnotreally', # should NOT be filtered @@ -2318,12 +2636,16 @@ def test_toolchain_prepare_rpath(self): '-Wl,-rpath=%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix, '%(user)s.c', '-L%s/prefix/software/CUDA/1.2.3/lib/stubs/' % self.test_prefix, + '-L%s/prefix/software/CUDA/1.2.3/stubs/lib/' % self.test_prefix, '-L%s/tmp/foo/' % self.test_prefix, '-L%s/prefix/software/stubs/1.2.3/lib' % self.test_prefix, '-L%s/prefix/software/CUDA/1.2.3/lib/stubs' % self.test_prefix, + '-L%s/prefix/software/CUDA/1.2.3/stubs/lib' % self.test_prefix, '-L%s/prefix/software/CUDA/1.2.3/lib64/stubs/' % self.test_prefix, + '-L%s/prefix/software/CUDA/1.2.3/stubs/lib64/' % self.test_prefix, '-L%s/prefix/software/foobar/4.5/notreallystubs' % self.test_prefix, '-L%s/prefix/software/CUDA/1.2.3/lib64/stubs' % self.test_prefix, + '-L%s/prefix/software/CUDA/1.2.3/stubs/lib64' % self.test_prefix, '-L%s/prefix/software/zlib/1.2.11/lib' % self.test_prefix, '-L%s/prefix/software/bleh/0/lib/stubs' % self.test_prefix, '-L%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix, diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py index 740185d899..4547652466 100644 --- a/test/framework/toolchainvariables.py +++ b/test/framework/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 1282f7db57..8ea6b77d2a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,6 +39,7 @@ import stat import sys import tempfile +import textwrap from distutils.version import LooseVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from test.framework.package import mock_fpm @@ -51,12 +52,13 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax, get_repositorypath from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir +from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, move_file from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod from easybuild.tools.py2vs3 import reload, string_type from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -104,21 +106,26 @@ def tearDown(self): if os.path.exists(self.dummylogfn): os.remove(self.dummylogfn) - def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versionsuffix=''): + def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versionsuffix='', error=None): """Check whether toy build succeeded.""" full_version = ''.join([versionprefix, version, versionsuffix]) + if error is not None: + error_msg = '\nNote: Caught error: %s' % error + else: + error_msg = '' + # check for success - success = re.compile(r"COMPLETED: Installation ended successfully \(took .* sec\)") - self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s" % outtxt) + success = re.compile(r"COMPLETED: Installation (ended|STOPPED) successfully \(took .* secs?\)") + self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s'%s" % (outtxt, error_msg)) # if the module exists, it should be fine toy_module = os.path.join(installpath, 'modules', 'all', 'toy', full_version) msg = "module for toy build toy/%s found (path %s)" % (full_version, toy_module) if get_module_syntax() == 'Lua': toy_module += '.lua' - self.assertTrue(os.path.exists(toy_module), msg) + self.assertTrue(os.path.exists(toy_module), msg + error_msg) # module file is symlinked according to moduleclass toy_module_symlink = os.path.join(installpath, 'modules', 'tools', 'toy', full_version) @@ -130,11 +137,13 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio # make sure installation log file and easyconfig file are copied to install dir software_path = os.path.join(installpath, 'software', 'toy', full_version) install_log_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-%s*.log' % version) - self.assertTrue(len(glob.glob(install_log_path_pattern)) == 1, "Found 1 file at %s" % install_log_path_pattern) + self.assertTrue(len(glob.glob(install_log_path_pattern)) >= 1, + "Found at least 1 file at %s" % install_log_path_pattern) # make sure test report is available test_report_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-%s*test_report.md' % version) - self.assertTrue(len(glob.glob(test_report_path_pattern)) == 1, "Found 1 file at %s" % test_report_path_pattern) + self.assertTrue(len(glob.glob(test_report_path_pattern)) >= 1, + "Found at least 1 file at %s" % test_report_path_pattern) ec_file_path = os.path.join(software_path, 'easybuild', 'toy-%s.eb' % full_version) self.assertTrue(os.path.exists(ec_file_path)) @@ -144,7 +153,7 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True, raise_error=False, test_report=None, versionsuffix='', testing=True, - raise_systemexit=False): + raise_systemexit=False, force=True): """Perform a toy build.""" if extra_args is None: extra_args = [] @@ -160,9 +169,10 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True '--installpath=%s' % self.test_installpath, '--debug', '--unittest-file=%s' % self.logfile, - '--force', '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), ] + if force: + args.append('--force') if tmpdir is not None: args.append('--tmpdir=%s' % tmpdir) if test_report is not None: @@ -178,7 +188,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True raise myerr if verify: - self.check_toy(self.test_installpath, outtxt, versionsuffix=versionsuffix) + self.check_toy(self.test_installpath, outtxt, versionsuffix=versionsuffix, error=myerr) if test_readme: # make sure postinstallcmds were used @@ -270,6 +280,7 @@ def test_toy_tweaked(self): "modluafooter = 'io.stderr:write(\"oh hai!\")'", # ignored when module syntax is Tcl "usage = 'This toy is easy to use, 100%!'", "examples = 'No example available, 0% complete'", + "citing = 'If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863'", "docpaths = ['share/doc/toy/readme.txt', 'share/doc/toy/html/index.html']", "docurls = ['https://easybuilders.github.io/easybuild/toy/docs.html']", "upstream_contacts = 'support@toy.org'", @@ -573,13 +584,18 @@ def test_toy_permissions(self): def test_toy_permissions_installdir(self): """Test --read-only-installdir and --group-write-installdir.""" + # Avoid picking up the already prepared fake module + try: + del os.environ['MODULEPATH'] + except KeyError: + pass # set umask hard to verify default reliably orig_umask = os.umask(0o022) toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec_txt = read_file(toy_ec) # take away read permissions, to check whether they are correctly restored by EasyBuild after installation - test_ec_txt += "\npostinstallcmds = ['chmod -R og-r %(installdir)s']" + test_ec_txt += "\npostinstallcmds += ['chmod -R og-r %(installdir)s']" test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -599,19 +615,39 @@ def test_toy_permissions_installdir(self): shutil.rmtree(self.test_installpath) # check whether --read-only-installdir works as intended - self.test_toy_build(ec_file=test_ec, extra_args=['--read-only-installdir']) - installdir_perms = os.stat(toy_install_dir).st_mode & 0o777 - self.assertEqual(installdir_perms, 0o555, "%s has read-only permissions" % toy_install_dir) - installdir_perms = os.stat(os.path.dirname(toy_install_dir)).st_mode & 0o777 - self.assertEqual(installdir_perms, 0o755, "%s has default permissions" % os.path.dirname(toy_install_dir)) + # Tested 5 times: + # 1. Non existing build -> Install and set read-only + # 2. Existing build with --rebuild -> Reinstall and set read-only + # 3. Existing build with --force -> Reinstall and set read-only + # 4-5: Same as 2-3 but with --skip + # 6. Existing build with --fetch -> Test that logs can be written + test_cases = ( + [], + ['--rebuild'], + ['--force'], + ['--skip', '--rebuild'], + ['--skip', '--force'], + ['--rebuild', '--fetch'], + ) + for extra_args in test_cases: + self.mock_stdout(True) + self.test_toy_build(ec_file=test_ec, extra_args=['--read-only-installdir'] + extra_args, force=False) + self.mock_stdout(False) - # also log file copied into install dir should be read-only (not just the 'easybuild/' subdir itself) - log_path = glob.glob(os.path.join(toy_install_dir, 'easybuild', '*log'))[0] - log_perms = os.stat(log_path).st_mode & 0o777 - self.assertEqual(log_perms, 0o444, "%s has read-only permissions" % log_path) + installdir_perms = os.stat(os.path.dirname(toy_install_dir)).st_mode & 0o777 + self.assertEqual(installdir_perms, 0o755, "%s has default permissions" % os.path.dirname(toy_install_dir)) - toy_bin_perms = os.stat(toy_bin).st_mode & 0o777 - self.assertEqual(toy_bin_perms, 0o555, "%s has read-only permissions" % toy_bin_perms) + installdir_perms = os.stat(toy_install_dir).st_mode & 0o777 + self.assertEqual(installdir_perms, 0o555, "%s has read-only permissions" % toy_install_dir) + toy_bin_perms = os.stat(toy_bin).st_mode & 0o777 + self.assertEqual(toy_bin_perms, 0o555, "%s has read-only permissions" % toy_bin_perms) + toy_bin_perms = os.stat(os.path.join(toy_install_dir, 'README')).st_mode & 0o777 + self.assertEqual(toy_bin_perms, 0o444, "%s has read-only permissions" % toy_bin_perms) + + # also log file copied into install dir should be read-only (not just the 'easybuild/' subdir itself) + log_path = glob.glob(os.path.join(toy_install_dir, 'easybuild', '*log'))[0] + log_perms = os.stat(log_path).st_mode & 0o777 + self.assertEqual(log_perms, 0o444, "%s has read-only permissions" % log_path) adjust_permissions(toy_install_dir, stat.S_IWUSR, add=True) shutil.rmtree(self.test_installpath) @@ -1169,8 +1205,8 @@ def test_toy_patches(self): archived_patch_file = os.path.join(repositorypath, 'toy', 'toy-0.0_fix-silly-typo-in-printf-statement.patch') self.assertTrue(os.path.isfile(archived_patch_file)) - def test_toy_extension_patches(self): - """Test install toy that includes extensions with patches.""" + def test_toy_extension_patches_postinstallcmds(self): + """Test install toy that includes extensions with patches and postinstallcmds.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') toy_ec_txt = read_file(toy_ec) @@ -1189,6 +1225,7 @@ def test_toy_extension_patches(self): ' ("bar-0.0_fix-very-silly-typo-in-printf-statement.patch", 0),', # patch with patch level ' ("test.txt", "."),', # file to copy to build dir (not a real patch file) ' ],', + ' "postinstallcmds": ["touch %(installdir)s/created-via-postinstallcmds.txt"],', ' }),', ']', ]) @@ -1196,6 +1233,17 @@ def test_toy_extension_patches(self): self.test_toy_build(ec_file=test_ec) + installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + + # make sure that patches were actually applied (without them the message producded by 'bar' is different) + bar_bin = os.path.join(installdir, 'bin', 'bar') + out, _ = run_cmd(bar_bin) + self.assertEqual(out, "I'm a bar, and very very proud of it.\n") + + # verify that post-install command for 'bar' extension was executed + fn = 'created-via-postinstallcmds.txt' + self.assertTrue(os.path.exists(os.path.join(installdir, fn))) + def test_toy_extension_sources(self): """Test install toy that includes extensions with 'sources' spec (as single-item list).""" topdir = os.path.dirname(os.path.abspath(__file__)) @@ -1314,6 +1362,31 @@ def test_toy_extension_sources(self): write_file(test_ec, test_ec_txt) self.test_toy_build(ec_file=test_ec, raise_error=True) + def test_toy_extension_extract_cmd(self): + """Test for custom extract_cmd specified for an extension.""" + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + toy_ec_txt = read_file(toy_ec) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = '\n'.join([ + toy_ec_txt, + 'exts_list = [', + ' ("bar", "0.0", {', + # deliberately incorrect custom extract command, just to verify that it's picked up + ' "sources": [{', + ' "filename": "bar-%(version)s.tar.gz",', + ' "extract_cmd": "unzip %s",', + ' }],', + ' }),', + ']', + ]) + write_file(test_ec, test_ec_txt) + + error_pattern = "unzip .*/bar-0.0.tar.gz.* exited with exit code [1-9]" + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec, + raise_error=True, verbose=False) + def test_toy_extension_sources_git_config(self): """Test install toy that includes extensions with 'sources' spec including 'git_config'.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -1341,7 +1414,7 @@ def test_toy_extension_sources_git_config(self): ' "git_config": {', ' "repo_name": "testrepository",', ' "url": "https://github.com/easybuilders",', - ' "tag": "master",', + ' "tag": "main",', ' },', ' },', ' }),', @@ -1386,6 +1459,11 @@ def test_toy_module_fulltxt(self): r'No example available, 0% complete', r'', r'', + r'Citing', + r'======', + r'If you use this package, please cite our paper https://ieeexplore.ieee.org/document/6495863', + r'', + r'', r'More information', r'================', r' - Homepage: https://easybuilders.github.io/easybuild', @@ -1577,16 +1655,11 @@ def test_module_only(self): os.remove(toy_mod) - # --module-only --rebuild should be equivalent with --module-only --force - self.eb_main(args + ['--rebuild'], do_build=True, raise_error=True) - self.assertTrue(os.path.exists(toy_mod)) - - # make sure load statements for dependencies are included in additional module file generated with --module-only - modtxt = read_file(toy_mod) - self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module") - self.assertTrue(re.search('load.*GCC/6.4.0-2.28', modtxt), "load statement for GCC/6.4.0-2.28 found in module") - - os.remove(toy_mod) + # --module-only --rebuild should run sanity check + rebuild_args = args + ['--rebuild'] + err_msg = "Sanity check failed" + self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, rebuild_args, do_build=True, raise_error=True) + self.assertFalse(os.path.exists(toy_mod)) # installing another module under a different naming scheme and using Lua module syntax works fine @@ -1619,8 +1692,25 @@ def test_module_only(self): modtxt = read_file(toy_core_mod) self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module") - os.remove(toy_mod) + # Test we can create a module even for an installation where we don't have write permissions + os.remove(toy_core_mod) + # remove the write permissions on the installation + adjust_permissions(prefix, stat.S_IRUSR | stat.S_IXUSR, relative=False) + self.assertFalse(os.path.exists(toy_core_mod)) + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_core_mod)) + # existing install is reused + modtxt2 = read_file(toy_core_mod) + self.assertTrue(re.search("set root %s" % prefix, modtxt2)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 3) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + # make sure load statements for dependencies are included + modtxt = read_file(toy_core_mod) + self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module") + os.remove(toy_core_mod) + os.remove(toy_mod) # test installing (only) additional module in Lua syntax (if Lmod is available) lmod_abspath = os.environ.get('LMOD_CMD') or which('lmod') @@ -1644,6 +1734,164 @@ def test_module_only(self): modtxt = read_file(toy_mod + '.lua') self.assertTrue(re.search('load.*intel/2018a', modtxt), "load statement for intel/2018a found in module") + def test_module_only_extensions(self): + """ + Test use of --module-only with extensions involved. + Sanity check should catch problems with extensions, + extensions can be skipped using --skip-exts. + """ + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + + test_ec = os.path.join(self.test_prefix, 'test.ec') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "sanity_check_commands = ['barbar', 'toy']", + "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + "exts_list = [", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " 'exts_filter': ('ls -l lib/lib%(ext_name)s.a', ''),", + " })", + "]", + ]) + write_file(test_ec, test_ec_txt) + + # clean up $MODULEPATH so only modules in test prefix dir are found + self.reset_modulepath([os.path.join(self.test_installpath, 'modules', 'all')]) + self.assertEqual(self.modtool.available('toy'), []) + + # install toy/0.0 + self.eb_main([test_ec], do_build=True, raise_error=True) + + # remove module file so we can try --module-only + remove_file(toy_mod) + + # make sure that sources for extensions can't be found, + # they should not be needed when using --module-only + # (cfr. https://github.com/easybuilders/easybuild-framework/issues/3849) + del os.environ['EASYBUILD_SOURCEPATH'] + + # first try normal --module-only, should work fine + self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + remove_file(toy_mod) + + # rename file required for barbar extension, so we can check whether sanity check catches it + libbarbar = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'lib', 'libbarbar.a') + move_file(libbarbar, libbarbar + '.foobar') + + # check whether sanity check fails now when using --module-only + error_pattern = 'Sanity check failed: command "ls -l lib/libbarbar.a" failed' + for extra_args in (['--module-only'], ['--module-only', '--rebuild']): + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, [test_ec] + extra_args, + do_build=True, raise_error=True) + self.assertFalse(os.path.exists(toy_mod)) + + # failing sanity check for barbar extension is ignored when using --module-only --skip-extensions + for extra_args in (['--module-only'], ['--module-only', '--rebuild']): + self.eb_main([test_ec, '--skip-extensions'] + extra_args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + remove_file(toy_mod) + + # we can force module generation via --force (which skips sanity check entirely) + self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + + def test_toy_exts_parallel(self): + """ + Test parallel installation of extensions (--parallel-extensions-install) + """ + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "exts_list = [", + " ('ls'),", + " ('bar', '0.0'),", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " }),", + " ('toy', '0.0'),", + "]", + "sanity_check_commands = ['barbar', 'toy']", + "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + ]) + write_file(test_ec, test_ec_txt) + + args = ['--parallel-extensions-install', '--experimental', '--force', '--parallel=3'] + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + + # take into account that each of these lines may appear multiple times, + # in case no progress was made between checks + patterns = [ + r"== 0 out of 4 extensions installed \(2 queued, 2 running: ls, bar\)$", + r"== 2 out of 4 extensions installed \(1 queued, 1 running: barbar\)$", + r"== 3 out of 4 extensions installed \(0 queued, 1 running: toy\)$", + r"== 4 out of 4 extensions installed \(0 queued, 0 running: \)$", + '', + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) + self.assertTrue(regex.search(stdout), error_msg) + + # also test skipping of extensions in parallel + args.append('--skip') + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + + # order in which these patterns occur is not fixed, so check them one by one + patterns = [ + r"^== skipping installed extensions \(in parallel\)$", + r"^== skipping extension ls$", + r"^== skipping extension bar$", + r"^== skipping extension barbar$", + r"^== skipping extension toy$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) + self.assertTrue(regex.search(stdout), error_msg) + + # check behaviour when using Toy_Extension easyblock that doesn't implement required_deps method; + # framework should fall back to installing extensions sequentially + toy_ext_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py') + copy_file(toy_ext_eb, self.test_prefix) + toy_ext_eb = os.path.join(self.test_prefix, 'toy_extension.py') + toy_ext_eb_txt = read_file(toy_ext_eb) + toy_ext_eb_txt = toy_ext_eb_txt.replace('def required_deps', 'def xxx_required_deps') + write_file(toy_ext_eb, toy_ext_eb_txt) + + args[-1] = '--include-easyblocks=%s' % toy_ext_eb + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + # take into account that each of these lines may appear multiple times, + # in case no progress was made between checks + patterns = [ + r"^== 0 out of 4 extensions installed \(3 queued, 1 running: ls\)$", + r"^== 1 out of 4 extensions installed \(2 queued, 1 running: bar\)$", + r"^== 2 out of 4 extensions installed \(1 queued, 1 running: barbar\)$", + r"^== 3 out of 4 extensions installed \(0 queued, 1 running: toy\)$", + r"^== 4 out of 4 extensions installed \(0 queued, 0 running: \)$", + '', + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) + self.assertTrue(regex.search(stdout), error_msg) + def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" @@ -1864,8 +2112,8 @@ def test_minimal_toolchains(self): # this test doesn't check for anything specific to using minimal toolchains, only side-effects self.test_toy_build(extra_args=['--minimal-toolchains']) - def test_reproducability(self): - """Test toy build produces expected reproducability files""" + def test_reproducibility(self): + """Test toy build produces expected reproducibility files""" # We need hooks for a complete test hooks_filename = 'my_hooks.py' @@ -1893,6 +2141,23 @@ def test_reproducability(self): self.assertTrue(os.path.exists(reprod_ec)) + # Also check that the dumpenv script is placed alongside it + dumpenv_script = '%s.env' % os.path.splitext(reprod_ec)[0] + reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) + self.assertTrue(os.path.exists(reprod_dumpenv)) + + # Check contents of the dumpenv script + patterns = [ + """#!/bin/bash""", + """# usage: source toy-0.0.env""", + # defining build env + """# (no modules loaded)""", + """# (no build environment defined)""", + ] + env_file = open(reprod_dumpenv, "r").read() + for pattern in patterns: + self.assertTrue(pattern in env_file) + # Check that the toytoy easyblock is recorded in the reprod easyconfig ec = EasyConfig(reprod_ec) self.assertEqual(ec.parser.get_config_dict()['easyblock'], 'EB_toytoy') @@ -1917,8 +2182,8 @@ def test_reproducability(self): reprod_hooks = os.path.join(reprod_dir, 'hooks', hooks_filename) self.assertTrue(os.path.exists(reprod_hooks)) - def test_reproducability_ext_easyblocks(self): - """Test toy build produces expected reproducability files also when extensions are used""" + def test_reproducibility_ext_easyblocks(self): + """Test toy build produces expected reproducibility files also when extensions are used""" topdir = os.path.dirname(os.path.abspath(__file__)) toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') @@ -1991,6 +2256,24 @@ def test_toy_toy(self): load1_regex = re.compile('load.*toy/0.0-one', re.M) self.assertTrue(load1_regex.search(mod2_txt), "Pattern '%s' found in: %s" % (load1_regex.pattern, mod2_txt)) + # Check the contents of the dumped env in the reprod dir to ensure it contains the dependency load + reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0-two', 'easybuild', 'reprod') + dumpenv_script = os.path.join(reprod_dir, 'toy-0.0-two.env') + reprod_dumpenv = os.path.join(reprod_dir, dumpenv_script) + self.assertTrue(os.path.exists(reprod_dumpenv)) + + # Check contents of the dumpenv script + patterns = [ + """#!/bin/bash""", + """# usage: source toy-0.0-two.env""", + # defining build env + """module load toy/0.0-one""", + """# (no build environment defined)""", + ] + env_file = open(reprod_dumpenv, "r").read() + for pattern in patterns: + self.assertTrue(pattern in env_file) + def test_toy_sanity_check_commands(self): """Test toy build with extra sanity check commands.""" @@ -2267,7 +2550,7 @@ def test_toy_build_enhanced_sanity_check(self): test_ec_txt = test_ec_txt + '\nenhance_sanity_check = False' write_file(test_ec, test_ec_txt) - error_pattern = " Missing mandatory key 'dirs' in sanity_check_paths." + error_pattern = r" Missing mandatory key 'dirs' in sanity_check_paths." self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec, extra_args=eb_args, raise_error=True, verbose=False) @@ -2352,36 +2635,65 @@ def test_toy_rpath(self): top_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, top_path) - def grab_gcc_rpath_wrapper_filter_arg(): - """Helper function to grab filter argument from last RPATH wrapper for 'gcc'.""" + def grab_gcc_rpath_wrapper_args(): + """Helper function to grab arguments from last RPATH wrapper for 'gcc'.""" rpath_wrappers_dir = glob.glob(os.path.join(os.getenv('TMPDIR'), '*', '*', 'rpath_wrappers'))[0] gcc_rpath_wrapper_txt = read_file(glob.glob(os.path.join(rpath_wrappers_dir, '*', 'gcc'))[0]) + # First get the filter argument rpath_args_regex = re.compile(r"^rpath_args_out=.*rpath_args.py \$CMD '([^ ]*)'.*", re.M) - res = rpath_args_regex.search(gcc_rpath_wrapper_txt) - self.assertTrue(res, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) + res_filter = rpath_args_regex.search(gcc_rpath_wrapper_txt) + self.assertTrue(res_filter, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, gcc_rpath_wrapper_txt)) + + # Now get the include argument + rpath_args_regex = re.compile(r"^rpath_args_out=.*rpath_args.py \$CMD '.*' '([^ ]*)'.*", re.M) + res_include = rpath_args_regex.search(gcc_rpath_wrapper_txt) + self.assertTrue(res_include, "Pattern '%s' found in: %s" % (rpath_args_regex.pattern, + gcc_rpath_wrapper_txt)) shutil.rmtree(rpath_wrappers_dir) - return res.group(1) + return {'filter_paths': res_filter.group(1), 'include_paths': res_include.group(1)} args = ['--rpath', '--experimental'] self.test_toy_build(extra_args=args, raise_error=True) # by default, /lib and /usr are included in RPATH filter, # together with temporary directory and build directory - rpath_filter_paths = grab_gcc_rpath_wrapper_filter_arg().split(',') + rpath_filter_paths = grab_gcc_rpath_wrapper_args()['filter_paths'].split(',') self.assertTrue('/lib.*' in rpath_filter_paths) self.assertTrue('/usr.*' in rpath_filter_paths) self.assertTrue(any(p.startswith(os.getenv('TMPDIR')) for p in rpath_filter_paths)) self.assertTrue(any(p.startswith(self.test_buildpath) for p in rpath_filter_paths)) + # Check that we can use --rpath-override-dirs + args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib:/opt/eessi/lib'] + self.test_toy_build(extra_args=args, raise_error=True) + rpath_include_paths = grab_gcc_rpath_wrapper_args()['include_paths'].split(',') + # Make sure our directories appear in dirs to be included in the rpath (and in the right order) + self.assertEqual(rpath_include_paths[0], '/opt/eessi/2021.03/lib') + self.assertEqual(rpath_include_paths[1], '/opt/eessi/lib') + + # Check that when we use --rpath-override-dirs empty values are filtered + args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib::/opt/eessi/lib'] + self.test_toy_build(extra_args=args, raise_error=True) + rpath_include_paths = grab_gcc_rpath_wrapper_args()['include_paths'].split(',') + # Make sure our directories appear in dirs to be included in the rpath (and in the right order) + self.assertEqual(rpath_include_paths[0], '/opt/eessi/2021.03/lib') + self.assertEqual(rpath_include_paths[1], '/opt/eessi/lib') + + # Check that when we use --rpath-override-dirs we can only provide absolute paths + eb_args = ['--rpath', '--experimental', '--rpath-override-dirs=/opt/eessi/2021.03/lib:eessi/lib'] + error_pattern = r"Path used in rpath_override_dirs is not an absolute path: eessi/lib" + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=eb_args, raise_error=True, + verbose=False) + # also test use of --rpath-filter args.extend(['--rpath-filter=/test.*,/foo/bar.*', '--disable-cleanup-tmpdir']) self.test_toy_build(extra_args=args, raise_error=True) # check whether rpath filter was set correctly - rpath_filter_paths = grab_gcc_rpath_wrapper_filter_arg().split(',') + rpath_filter_paths = grab_gcc_rpath_wrapper_args()['filter_paths'].split(',') self.assertTrue('/test.*' in rpath_filter_paths) self.assertTrue('/foo/bar.*' in rpath_filter_paths) self.assertTrue(any(p.startswith(os.getenv('TMPDIR')) for p in rpath_filter_paths)) @@ -2488,6 +2800,7 @@ def test_toy_build_trace(self): r"== sanity checking\.\.\.", r" >> file 'bin/yot' or 'bin/toy' found: OK", r" >> \(non-empty\) directory 'bin' found: OK", + r" >> loading modules: toy/0.0\.\.\.", r" >> running command 'toy' \.\.\.", r" >> result for command 'toy': OK", ]) + r'$', @@ -2500,34 +2813,38 @@ def test_toy_build_trace(self): def test_toy_build_hooks(self): """Test use of --hooks.""" hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') - hooks_file_txt = '\n'.join([ - "import os", - '', - "def start_hook():", - " print('start hook triggered')", - '', - "def parse_hook(ec):", - " print('%s %s' % (ec.name, ec.version))", + hooks_file_txt = textwrap.dedent(""" + import os + + def start_hook(): + print('start hook triggered') + + def parse_hook(ec): + print('%s %s' % (ec.name, ec.version)) # print sources value to check that raw untemplated strings are exposed in parse_hook - " print(ec['sources'])", + print(ec['sources']) # try appending to postinstallcmd to see whether the modification is actually picked up # (required templating to be disabled before parse_hook is called) - " ec['postinstallcmds'].append('echo toy')", - " print(ec['postinstallcmds'][-1])", - '', - "def pre_configure_hook(self):", - " print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))", - '', - "def post_configure_hook(self):", - " print('post-configure: toy.source: %s' % os.path.exists('toy.source'))", - '', - "def post_install_hook(self):", - " print('in post-install hook for %s v%s' % (self.name, self.version))", - " print(', '.join(sorted(os.listdir(self.installdir))))", - '', - "def end_hook():", - " print('end hook triggered, all done!')", - ]) + ec['postinstallcmds'].append('echo toy') + print(ec['postinstallcmds'][-1]) + + def pre_configure_hook(self): + print('pre-configure: toy.source: %s' % os.path.exists('toy.source')) + + def post_configure_hook(self): + print('post-configure: toy.source: %s' % os.path.exists('toy.source')) + + def post_install_hook(self): + print('in post-install hook for %s v%s' % (self.name, self.version)) + print(', '.join(sorted(os.listdir(self.installdir)))) + + def module_write_hook(self, module_path, module_txt): + print('in module-write hook hook for %s' % os.path.basename(module_path)) + return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore') + + def end_hook(): + print('end hook triggered, all done!') + """) write_file(hooks_file, hooks_file_txt) self.mock_stderr(True) @@ -2538,26 +2855,44 @@ def test_toy_build_hooks(self): self.mock_stderr(False) self.mock_stdout(False) + test_mod_path = os.path.join(self.test_installpath, 'modules', 'all') + toy_mod_file = os.path.join(test_mod_path, 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod_file += '.lua' + self.assertEqual(stderr, '') - expected_output = '\n'.join([ - "== Running start hook...", - "start hook triggered", - "== Running parse hook for toy-0.0.eb...", - "toy 0.0", - "['%(name)s-%(version)s.tar.gz']", - "echo toy", - "== Running pre-configure hook...", - "pre-configure: toy.source: True", - "== Running post-configure hook...", - "post-configure: toy.source: False", - "== Running post-install hook...", - "in post-install hook for toy v0.0", - "bin, lib", - "== Running end hook...", - "end hook triggered, all done!", - ]) + # There are 4 modules written: + # Sanitycheck for extensions and main easyblock (1 each), main and devel module + expected_output = textwrap.dedent(""" + == Running start hook... + start hook triggered + == Running parse hook for toy-0.0.eb... + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + == Running pre-configure hook... + pre-configure: toy.source: True + == Running post-configure hook... + post-configure: toy.source: False + == Running post-install hook... + in post-install hook for toy v0.0 + bin, lib + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running end hook... + end hook triggered, all done! + """).strip().format(mod_name=os.path.basename(toy_mod_file)) self.assertEqual(stdout.strip(), expected_output) + toy_mod = read_file(toy_mod_file) + self.assertIn('Not a toy anymore', toy_mod) + def test_toy_multi_deps(self): """Test installation of toy easyconfig that uses multi_deps.""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -2727,13 +3062,16 @@ def check_toy_load(depends_on=False): modify_env(os.environ, self.orig_environ, verbose=False) self.modtool.use(test_mod_path) + # disable showing of progress bars (again), doesn't make sense when running tests + os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1' + write_file(test_ec, test_ec_txt) # also check behaviour when using 'depends_on' rather than 'load' statements (requires Lmod 7.6.1 or newer) if self.modtool.supports_depends_on: remove_file(toy_mod_file) - self.test_toy_build(ec_file=test_ec, extra_args=['--module-depends-on']) + self.test_toy_build(ec_file=test_ec, extra_args=['--module-depends-on'], raise_error=True) toy_mod_txt = read_file(toy_mod_file) @@ -2776,6 +3114,7 @@ def test_fix_shebang(self): # copy of bin/toy to use in fix_python_shebang_for and fix_perl_shebang_for " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.python',", " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.perl',", + " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.sh',", # hardcoded path to bin/python " 'echo \"#!/usr/bin/python\\n# test\" > %(installdir)s/bin/t1.py',", @@ -2812,9 +3151,26 @@ def test_fix_shebang(self): # shebang bash " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b2.sh',", + # tests for bash shebang + # hardcoded path to bin/bash + " 'echo \"#!/bin/bash\\n# test\" > %(installdir)s/bin/t1.sh',", + # hardcoded path to usr/bin/bash + " 'echo \"#!/usr/bin/bash\\n# test\" > %(installdir)s/bin/t2.sh',", + # already OK, should remain the same + " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/t3.sh',", + # shebang with space, should strip the space + " 'echo \"#! /usr/bin/env bash\\n# test\" > %(installdir)s/bin/t4.sh',", + # no shebang sh + " 'echo \"# test\" > %(installdir)s/bin/t5.sh',", + # shebang python + " 'echo \"#!/usr/bin/env python\\n# test\" > %(installdir)s/bin/b1.py',", + # shebang perl + " 'echo \"#!/usr/bin/env perl\\n# test\" > %(installdir)s/bin/b1.pl',", + "]", - "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']", - "fix_perl_shebang_for = ['bin/*.pl', 'bin/b2.sh', 'bin/toy.perl']", + "fix_python_shebang_for = ['bin/t1.py', 'bin/t*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']", + "fix_perl_shebang_for = ['bin/t*.pl', 'bin/b2.sh', 'bin/toy.perl']", + "fix_bash_shebang_for = ['bin/t*.sh', 'bin/b1.py', 'bin/b1.pl', 'bin/toy.sh']", ]) write_file(test_ec, test_ec_txt) self.test_toy_build(ec_file=test_ec, raise_error=True) @@ -2823,36 +3179,95 @@ def test_fix_shebang(self): # bin/toy and bin/toy2 should *not* be patched, since they're binary files toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') - for fn in ['toy.perl', 'toy.python']: + for fn in ['toy.sh', 'toy.perl', 'toy.python']: fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') # no shebang added self.assertFalse(fn_txt.startswith(b"#!/")) # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) + regexes = {} # no re.M, this should match at start of file! - py_shebang_regex = re.compile(r'^#!/usr/bin/env python\n# test$') - for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: - pybin_path = os.path.join(toy_bindir, pybin) - pybin_txt = read_file(pybin_path) - self.assertTrue(py_shebang_regex.match(pybin_txt), - "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt)) + regexes['py'] = re.compile(r'^#!/usr/bin/env python\n# test$') + regexes['pl'] = re.compile(r'^#!/usr/bin/env perl\n# test$') + regexes['sh'] = re.compile(r'^#!/usr/bin/env bash\n# test$') + + # all scripts should have a shebang that matches their extension + scripts = {} + scripts['py'] = ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py', 'b1.py'] + scripts['pl'] = ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl', 'b1.pl'] + scripts['sh'] = ['t1.sh', 't2.sh', 't3.sh', 't4.sh', 't5.sh', 'b1.sh', 'b2.sh'] + + for ext in ['sh', 'pl', 'py']: + for script in scripts[ext]: + bin_path = os.path.join(toy_bindir, script) + bin_txt = read_file(bin_path) + self.assertTrue(regexes[ext].match(bin_txt), + "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt)) + + # now test with a custom env command + extra_args = ['--env-for-shebang=/usr/bin/env -S'] + self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True) + + toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin') + + # bin/toy and bin/toy2 should *not* be patched, since they're binary files + toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') + for fn in ['toy.sh', 'toy.perl', 'toy.python']: + fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') + # no shebang added + self.assertFalse(fn_txt.startswith(b"#!/")) + # exact same file as original binary (untouched) + self.assertEqual(toy_txt, fn_txt) + + regexes_shebang = {} + + def check_shebangs(): + for ext in ['sh', 'pl', 'py']: + for script in scripts[ext]: + bin_path = os.path.join(toy_bindir, script) + bin_txt = read_file(bin_path) + # the scripts b1.py, b1.pl, b1.sh, b2.sh should keep their original shebang + if script.startswith('b'): + self.assertTrue(regexes[ext].match(bin_txt), + "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt)) + else: + regex_shebang = regexes_shebang[ext] + self.assertTrue(regex_shebang.match(bin_txt), + "Pattern '%s' found in %s: %s" % (regex_shebang.pattern, bin_path, bin_txt)) # no re.M, this should match at start of file! - perl_shebang_regex = re.compile(r'^#!/usr/bin/env perl\n# test$') - for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']: - perlbin_path = os.path.join(toy_bindir, perlbin) - perlbin_txt = read_file(perlbin_path) - self.assertTrue(perl_shebang_regex.match(perlbin_txt), - "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt)) - - # There are 2 bash files which shouldn't be influenced by fix_shebang - bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$') - for bashbin in ['b1.sh', 'b2.sh']: - bashbin_path = os.path.join(toy_bindir, bashbin) - bashbin_txt = read_file(bashbin_path) - self.assertTrue(bash_shebang_regex.match(bashbin_txt), - "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + regexes_shebang['py'] = re.compile(r'^#!/usr/bin/env -S python\n# test$') + regexes_shebang['pl'] = re.compile(r'^#!/usr/bin/env -S perl\n# test$') + regexes_shebang['sh'] = re.compile(r'^#!/usr/bin/env -S bash\n# test$') + + check_shebangs() + + # test again with EasyBuild configured with sysroot, which should be prepended + # automatically to env-for-shebang value (unless it's prefixed with sysroot already) + extra_args = [ + '--env-for-shebang=/usr/bin/env -S', + '--sysroot=/usr/../', # sysroot must exist, and so must /usr/bin/env when appended to it + ] + self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True) + + regexes_shebang['py'] = re.compile(r'^#!/usr/../usr/bin/env -S python\n# test$') + regexes_shebang['pl'] = re.compile(r'^#!/usr/../usr/bin/env -S perl\n# test$') + regexes_shebang['sh'] = re.compile(r'^#!/usr/../usr/bin/env -S bash\n# test$') + + check_shebangs() + + extra_args = [ + '--env-for-shebang=/usr/../usr/../usr/bin/env -S', + '--sysroot=/usr/../', # sysroot must exist, and so must /usr/bin/env when appended to it + ] + self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True) + + regexes_shebang['py'] = re.compile(r'^#!/usr/../usr/../usr/bin/env -S python\n# test$') + regexes_shebang['pl'] = re.compile(r'^#!/usr/../usr/../usr/bin/env -S perl\n# test$') + regexes_shebang['sh'] = re.compile(r'^#!/usr/../usr/../usr/bin/env -S bash\n# test$') + + check_shebangs() def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" @@ -2925,6 +3340,13 @@ def test_toy_build_lock(self): error_pattern = "Lock .*_software_toy_0.0.lock already exists, aborting!" self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) + # lock should still be there after it was hit + self.assertTrue(os.path.exists(toy_lock_path)) + + # trying again should give same result + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) + self.assertTrue(os.path.exists(toy_lock_path)) + locks_dir = os.path.join(self.test_prefix, 'locks') # no lock in place, so installation proceeds as normal @@ -2943,7 +3365,7 @@ def test_toy_build_lock(self): orig_sigalrm_handler = signal.getsignal(signal.SIGALRM) # define a context manager that remove a lock after a while, so we can check the use of --wait-for-lock - class remove_lock_after(object): + class RemoveLockAfter(object): def __init__(self, seconds, lock_fp): self.seconds = seconds self.lock_fp = lock_fp @@ -2991,7 +3413,7 @@ def __exit__(self, type, value, traceback): all_args = extra_args + opts # use context manager to remove lock after 3 seconds - with remove_lock_after(3, toy_lock_path): + with RemoveLockAfter(3, toy_lock_path): self.mock_stderr(True) self.mock_stdout(True) self.test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False) @@ -3059,7 +3481,7 @@ def test_toy_lock_cleanup_signals(self): orig_sigalrm_handler = signal.getsignal(signal.SIGALRM) # context manager which stops the function being called with the specified signal - class wait_and_signal(object): + class WaitAndSignal(object): def __init__(self, seconds, signum): self.seconds = seconds self.signum = signum @@ -3094,7 +3516,7 @@ def __exit__(self, type, value, traceback): # avoid recycling stderr of previous test stderr = '' - with wait_and_signal(1, signum): + with WaitAndSignal(1, signum): # change back to original working directory before each test change_dir(orig_wd) @@ -3133,25 +3555,31 @@ def test_toy_build_unicode_description(self): self.test_toy_build(ec_file=test_ec, raise_error=True) - def test_test_toy_build_lib64_symlink(self): + def test_toy_build_lib64_lib_symlink(self): """Check whether lib64 symlink to lib subdirectory is created.""" # this is done to ensure that /lib64 is considered before /lib64 by GCC linker, # see https://github.com/easybuilders/easybuild-easyconfigs/issues/5776 - # by default, lib64 symlink is created + # by default, lib64 -> lib symlink is created (--lib64-lib-symlink is enabled by default) self.test_toy_build() toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') lib_path = os.path.join(toy_installdir, 'lib') lib64_path = os.path.join(toy_installdir, 'lib64') + # lib64 subdir exists, is not a symlink self.assertTrue(os.path.exists(lib_path)) - self.assertTrue(os.path.exists(lib64_path)) self.assertTrue(os.path.isdir(lib_path)) self.assertFalse(os.path.islink(lib_path)) + + # lib64 subdir is a symlink to lib subdir + self.assertTrue(os.path.exists(lib64_path)) self.assertTrue(os.path.islink(lib64_path)) self.assertTrue(os.path.samefile(lib_path, lib64_path)) + # lib64 symlink should point to a relative path + self.assertFalse(os.path.isabs(os.readlink(lib64_path))) + # cleanup and try again with --disable-lib64-lib-symlink remove_dir(self.test_installpath) self.test_toy_build(extra_args=['--disable-lib64-lib-symlink']) @@ -3162,6 +3590,156 @@ def test_test_toy_build_lib64_symlink(self): self.assertTrue(os.path.isdir(lib_path)) self.assertFalse(os.path.islink(lib_path)) + def test_toy_build_lib_lib64_symlink(self): + """Check whether lib64 symlink to lib subdirectory is created.""" + + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += "\npostinstallcmds += ['mv %(installdir)s/lib %(installdir)s/lib64']" + + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + # by default, lib -> lib64 symlink is created (--lib-lib64-symlink is enabled by default) + self.test_toy_build(ec_file=test_ec) + + toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + lib_path = os.path.join(toy_installdir, 'lib') + lib64_path = os.path.join(toy_installdir, 'lib64') + + # lib64 subdir exists, is not a symlink + self.assertTrue(os.path.exists(lib64_path)) + self.assertTrue(os.path.isdir(lib64_path)) + self.assertFalse(os.path.islink(lib64_path)) + + # lib subdir is a symlink to lib64 subdir + self.assertTrue(os.path.exists(lib_path)) + self.assertTrue(os.path.isdir(lib_path)) + self.assertTrue(os.path.islink(lib_path)) + self.assertTrue(os.path.samefile(lib_path, lib64_path)) + + # lib symlink should point to a relative path + self.assertFalse(os.path.isabs(os.readlink(lib_path))) + + # cleanup and try again with --disable-lib-lib64-symlink + remove_dir(self.test_installpath) + self.test_toy_build(ec_file=test_ec, extra_args=['--disable-lib-lib64-symlink']) + + self.assertTrue(os.path.exists(lib64_path)) + self.assertFalse(os.path.exists(lib_path)) + self.assertFalse('lib' in os.listdir(toy_installdir)) + self.assertTrue(os.path.isdir(lib64_path)) + self.assertFalse(os.path.islink(lib64_path)) + + def test_toy_build_sanity_check_linked_libs(self): + """Test sanity checks for banned/requires libraries.""" + + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + libtoy_ec = os.path.join(test_ecs, 'l', 'libtoy', 'libtoy-0.0.eb') + + libtoy_modfile_path = os.path.join(self.test_installpath, 'modules', 'all', 'libtoy', '0.0') + if get_module_syntax() == 'Lua': + libtoy_modfile_path += '.lua' + + test_ec = os.path.join(self.test_prefix, 'test.eb') + + shlib_ext = get_shared_lib_ext() + + libtoy_fn = 'libtoy.%s' % shlib_ext + error_msg = "Check for banned/required shared libraries failed for" + + # default check is done via EB_libtoy easyblock, which specifies several banned/required libraries + self.test_toy_build(ec_file=libtoy_ec, raise_error=True, verbose=False, verify=False) + remove_file(libtoy_modfile_path) + + # we can make the check fail by defining environment variables picked up by the EB_libtoy easyblock + os.environ['EB_LIBTOY_BANNED_SHARED_LIBS'] = 'libtoy' + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=libtoy_ec, extra_args=['--module-only'], raise_error=True, verbose=False) + del os.environ['EB_LIBTOY_BANNED_SHARED_LIBS'] + + os.environ['EB_LIBTOY_REQUIRED_SHARED_LIBS'] = 'thisisnottheremostlikely' + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=libtoy_ec, extra_args=['--module-only'], raise_error=True, verbose=False) + del os.environ['EB_LIBTOY_REQUIRED_SHARED_LIBS'] + + # make sure default check passes (so we know better what triggered a failing test) + self.test_toy_build(ec_file=libtoy_ec, extra_args=['--module-only'], force=False, + raise_error=True, verbose=False, verify=False) + remove_file(libtoy_modfile_path) + + # check specifying banned/required libraries via EasyBuild configuration option + args = ['--banned-linked-shared-libs=%s,foobarbaz' % libtoy_fn, '--module-only'] + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=libtoy_ec, extra_args=args, raise_error=True, verbose=False) + + args = ['--required-linked-shared=libs=foobarbazisnotthereforsure', '--module-only'] + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=libtoy_ec, extra_args=args, raise_error=True, verbose=False) + + # check specifying banned/required libraries via easyconfig parameter + test_ec_txt = read_file(libtoy_ec) + test_ec_txt += "\nbanned_linked_shared_libs = ['toy']" + write_file(test_ec, test_ec_txt) + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False) + + test_ec_txt = read_file(libtoy_ec) + test_ec_txt += "\nrequired_linked_shared_libs = ['thereisnosuchlibraryyoudummy']" + write_file(test_ec, test_ec_txt) + self.assertErrorRegex(EasyBuildError, error_msg, self.test_toy_build, force=False, + ec_file=test_ec, extra_args=['--module-only'], raise_error=True, verbose=False) + + # check behaviour when alternate subdirectories are specified + test_ec_txt = read_file(libtoy_ec) + test_ec_txt += "\nbin_lib_subdirs = ['', 'lib', 'lib64']" + write_file(test_ec, test_ec_txt) + self.test_toy_build(ec_file=test_ec, extra_args=['--module-only'], force=False, + raise_error=True, verbose=False, verify=False) + + # one last time: supercombo (with patterns that should pass the check) + test_ec_txt = read_file(libtoy_ec) + test_ec_txt += "\nbanned_linked_shared_libs = ['yeahthisisjustatest', '/usr/lib/libssl.so']" + test_ec_txt += "\nrequired_linked_shared_libs = ['/lib']" + test_ec_txt += "\nbin_lib_subdirs = ['', 'lib', 'lib64']" + write_file(test_ec, test_ec_txt) + args = [ + '--banned-linked-shared-libs=the_forbidden_library', + '--required-linked-shared-libs=.*', + '--module-only', + ] + self.test_toy_build(ec_file=test_ec, extra_args=args, force=False, + raise_error=True, verbose=False, verify=False) + + def test_toy_ignore_test_failure(self): + """Check whether use of --ignore-test-failure is mentioned in build output.""" + args = ['--ignore-test-failure'] + stdout, stderr = self.run_test_toy_build_with_output(extra_args=args, verify=False, testing=False) + + self.assertTrue("Build succeeded (with --ignore-test-failure) for 1 out of 1" in stdout) + self.assertFalse(stderr) + + def test_toy_post_install_patches(self): + """ + Test use of post-install patches + """ + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += "\npostinstallpatches = ['toy-0.0_fix-README.patch']" + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + self.test_toy_build(ec_file=test_ec) + + toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + toy_readme_txt = read_file(os.path.join(toy_installdir, 'README')) + # verify whether patch indeed got applied + self.assertEqual(toy_readme_txt, "toy 0.0, a toy test program\n") + def suite(): """ return all the tests in this file """ diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 2570142c6d..314c964af3 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2021 Ghent University +# Copyright 2014-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index d416cb6b5e..bb26d386da 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 9d741c7477..c50f544e9e 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,6 +36,7 @@ import sys import tempfile import unittest +from contextlib import contextmanager from easybuild.base import fancylogger from easybuild.base.testing import TestCase @@ -47,10 +48,10 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.main import main from easybuild.tools import config -from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes +from easybuild.tools.config import GENERAL_CLASS, Singleton, module_classes, update_build_option from easybuild.tools.configobj import ConfigObj from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import copy_dir, mkdir, read_file +from easybuild.tools.filetools import copy_dir, mkdir, read_file, which from easybuild.tools.modules import curr_module_paths, modules_tool, reset_module_caches from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX, EasyBuildOptions, set_tmpdir from easybuild.tools.py2vs3 import reload @@ -124,12 +125,23 @@ def setUp(self): # make sure that the tests only pick up easyconfigs provided with the tests os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs') + # make sure that the EasyBuild installation is still known even if we purge an EB module + if os.getenv('EB_SCRIPT_PATH') is None: + eb_path = which('eb') + if eb_path is not None: + os.environ['EB_SCRIPT_PATH'] = eb_path + # make sure no deprecated behaviour is being triggered (unless intended by the test) self.orig_current_version = eb_build_log.CURRENT_VERSION self.disallow_deprecated_behaviour() init_config() + # disable progress bars when running the tests, + # since it messes with test suite progress output when test installations are performed + os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1' + update_build_option('show_progress_bar', False) + import easybuild # try to import easybuild.easyblocks(.generic) packages # it's OK if it fails here, but important to import first before fiddling with sys.path @@ -140,7 +152,8 @@ def setUp(self): pass # add sandbox to Python search path, update namespace packages - sys.path.append(os.path.join(testdir, 'sandbox')) + testdir_sandbox = os.path.join(testdir, 'sandbox') + sys.path.append(testdir_sandbox) # required to make sure the 'easybuild' dir in the sandbox is picked up; # this relates to the other 'reload' statements below @@ -153,14 +166,14 @@ def setUp(self): # remove any entries in Python search path that seem to provide easyblocks (except the sandbox) for path in sys.path[:]: if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')): - if not os.path.samefile(path, os.path.join(testdir, 'sandbox')): + if not os.path.samefile(path, testdir_sandbox): sys.path.remove(path) # hard inject location to (generic) test easyblocks into Python search path # only prepending to sys.path is not enough due to 'pkgutil.extend_path' in easybuild/easyblocks/__init__.py - easybuild.__path__.insert(0, os.path.join(testdir, 'sandbox', 'easybuild')) + easybuild.__path__.insert(0, os.path.join(testdir_sandbox, 'easybuild')) import easybuild.easyblocks - test_easyblocks_path = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks') + test_easyblocks_path = os.path.join(testdir_sandbox, 'easybuild', 'easyblocks') easybuild.easyblocks.__path__.insert(0, test_easyblocks_path) reload(easybuild.easyblocks) @@ -169,6 +182,13 @@ def setUp(self): easybuild.easyblocks.generic.__path__.insert(0, test_easyblocks_path) reload(easybuild.easyblocks.generic) + # kick out any paths that shouldn't be there for easybuild.easyblocks and easybuild.easyblocks.generic + # to avoid that easyblocks picked up from other places cause trouble + for pkg in ('easybuild.easyblocks', 'easybuild.easyblocks.generic'): + for path in sys.modules[pkg].__path__[:]: + if testdir_sandbox not in path: + sys.modules[pkg].__path__.remove(path) + # save values of $PATH & $PYTHONPATH, so they can be restored later # this is important in case EasyBuild was installed as a module, since that module may be unloaded, # for example due to changes to $MODULEPATH in case EasyBuild was installed in a module hierarchy @@ -191,6 +211,16 @@ def allow_deprecated_behaviour(self): del os.environ['EASYBUILD_DEPRECATED'] eb_build_log.CURRENT_VERSION = self.orig_current_version + @contextmanager + def log_to_testlogfile(self): + """Context manager class to capture log output in self.logfile for the scope used. Clears the file first""" + open(self.logfile, 'w').close() # Remove all contents + fancylogger.logToFile(self.logfile) + try: + yield self.logfile + finally: + fancylogger.logToFile(self.logfile, enable=False) + def tearDown(self): """Clean up after running testcase.""" super(EnhancedTestCase, self).tearDown() @@ -259,19 +289,28 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos """Helper method to call EasyBuild main function.""" cleanup() + # always run main in unit testing mode (which for example allows for using deprecated toolchains); + # note: don't change 'args' value, which is passed by reference! + main_args = args + ['--unit-testing-mode'] + myerr = False if logfile is None: logfile = self.logfile # clear log file if logfile: - f = open(logfile, 'w') - f.write('') - f.close() + with open(logfile, 'w') as fh: + fh.write('') env_before = copy.deepcopy(os.environ) try: - main(args=args, logfile=logfile, do_build=do_build, testing=testing, modtool=self.modtool) + if '--fetch' in args: + # The config sets modules_tool to None if --fetch is specified, + # so do the same here to keep the behavior consistent + modtool = None + else: + modtool = self.modtool + main(args=main_args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool) except SystemExit as err: if raise_systemexit: raise err @@ -441,6 +480,7 @@ def init_config(args=None, build_options=None, with_include=True): 'local_var_naming_check': 'error', 'silence_deprecation_warnings': eb_go.options.silence_deprecation_warnings, 'suffix_modules_path': GENERAL_CLASS, + 'unit_testing_mode': True, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py new file mode 100644 index 0000000000..2ef6aa6893 --- /dev/null +++ b/test/framework/utilities_test.py @@ -0,0 +1,111 @@ +## +# Copyright 2012-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Unit tests for utilities.py + +@author: Jens Timmerman (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Alexander Grund (TU Dresden) +""" +import os +import random +import sys +import tempfile +from datetime import datetime +from unittest import TextTestRunner + +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.utilities import time2str, natural_keys + + +class UtilitiesTest(EnhancedTestCase): + """Class for utilities testcases """ + + def setUp(self): + """ setup """ + super(UtilitiesTest, self).setUp() + + self.test_tmp_logdir = tempfile.mkdtemp() + os.environ['EASYBUILD_TMP_LOGDIR'] = self.test_tmp_logdir + + def test_time2str(self): + """Test time2str function.""" + + start = datetime(2019, 7, 30, 5, 14, 23) + + test_cases = [ + (start, "0 secs"), + (datetime(2019, 7, 30, 5, 14, 37), "14 secs"), + (datetime(2019, 7, 30, 5, 15, 22), "59 secs"), + (datetime(2019, 7, 30, 5, 15, 23), "1 min 0 secs"), + (datetime(2019, 7, 30, 5, 16, 22), "1 min 59 secs"), + (datetime(2019, 7, 30, 5, 16, 24), "2 mins 1 sec"), + (datetime(2019, 7, 30, 5, 37, 26), "23 mins 3 secs"), + (datetime(2019, 7, 30, 6, 14, 22), "59 mins 59 secs"), + (datetime(2019, 7, 30, 6, 14, 23), "1 hour 0 mins 0 secs"), + (datetime(2019, 7, 30, 6, 49, 14), "1 hour 34 mins 51 secs"), + (datetime(2019, 7, 30, 7, 14, 23), "2 hours 0 mins 0 secs"), + (datetime(2019, 7, 30, 8, 35, 59), "3 hours 21 mins 36 secs"), + (datetime(2019, 7, 30, 16, 29, 24), "11 hours 15 mins 1 sec"), + (datetime(2019, 7, 31, 5, 14, 22), "23 hours 59 mins 59 secs"), + (datetime(2019, 7, 31, 5, 14, 23), "24 hours 0 mins 0 secs"), + (datetime(2019, 7, 31, 5, 15, 24), "24 hours 1 min 1 sec"), + (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 mins 21 secs"), + ] + for end, expected in test_cases: + self.assertEqual(time2str(end - start), expected) + + error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>" + self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123) + + def test_natural_keys(self): + """Test the natural_keys function""" + sorted_items = [ + 'ACoolSw-1.0', + 'ACoolSw-2.1', + 'ACoolSw-11.0', + 'ACoolSw-23.0', + 'ACoolSw-30.0', + 'ACoolSw-30.1', + 'BigNumber-1234567890', + 'BigNumber-1234567891', + 'NoNumbers', + 'VeryLastEntry-10' + ] + shuffled_items = sorted_items[:] + random.shuffle(shuffled_items) + shuffled_items.sort(key=natural_keys) + self.assertEqual(shuffled_items, sorted_items) + + +def suite(): + """ return all the tests in this file """ + return TestLoaderFiltered().loadTestsFromTestCase(UtilitiesTest, sys.argv[1:]) + + +if __name__ == '__main__': + res = TextTestRunner(verbosity=1).run(suite()) + sys.exit(len(res.failures)) diff --git a/test/framework/variables.py b/test/framework/variables.py index 246ad212c2..e332e6cd20 100644 --- a/test/framework/variables.py +++ b/test/framework/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2021 Ghent University +# Copyright 2012-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/yeb.py b/test/framework/yeb.py index 70ea97bd75..80c447ddbc 100644 --- a/test/framework/yeb.py +++ b/test/framework/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2021 Ghent University +# Copyright 2015-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -110,6 +110,21 @@ def test_parse_yeb(self): self.assertEqual(yeb_val, eb_val) + def test_yeb_get_config_obj(self): + """Test get_config_dict method.""" + testdir = os.path.dirname(os.path.abspath(__file__)) + test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb') + ec = EasyConfig(os.path.join(test_yeb_easyconfigs, 'toy-0.0.yeb')) + ecdict = ec.parser.get_config_dict() + + # changes to this dict should not affect the return value of the next call to get_config_dict + fn = 'test.tar.gz' + ecdict['sources'].append(fn) + + ecdict_bis = ec.parser.get_config_dict() + self.assertTrue(fn in ecdict['sources']) + self.assertFalse(fn in ecdict_bis['sources']) + def test_is_yeb_format(self): """ Test is_yeb_format function """ testdir = os.path.dirname(os.path.abspath(__file__))