Merge pull request #4 from easybuilders/develop

Getting up to speed with main repo

.github/workflows/bootstrap_script.yml

@@ -0,0 +1,123 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: test EasyBuild bootstrap script
+on: [push, pull_request]
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+        lmod7: Lmod-7.8.22
+        lmod8: Lmod-8.4.27
+        modulesTcl: modules-tcl-1.147
+        modules3: modules-3.2.10
+        modules4: modules-4.1.4
+    steps:
+      - run: "true"
+  build:
+    needs: setup
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        # Don't run for Python 3.8, 3.9 , people should just use `pip install easybuild`
+        python: [2.7, 3.6, 3.7]
+        modules_tool:
+          # use variables defined by 'setup' job above, see also
+          # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
+          - ${{needs.setup.outputs.lmod7}}
+          - ${{needs.setup.outputs.lmod8}}
+        module_syntax: [Lua]
+        lc_all: [""]
+        include:
+          # also test with module tools other than Lmod (only Tcl syntax)
+          - modules_tool: ${{needs.setup.outputs.modulesTcl}}
+            module_syntax: Tcl
+            python: 2.7
+          - modules_tool: ${{needs.setup.outputs.modulesTcl}}
+            module_syntax: Tcl
+            python: 3.6
+          - modules_tool: ${{needs.setup.outputs.modules3}}
+            module_syntax: Tcl
+            python: 2.7
+          - modules_tool: ${{needs.setup.outputs.modules3}}
+            module_syntax: Tcl
+            python: 3.6
+          - modules_tool: ${{needs.setup.outputs.modules4}}
+            module_syntax: Tcl
+            python: 2.7
+          - modules_tool: ${{needs.setup.outputs.modules4}}
+            module_syntax: Tcl
+            python: 3.6
+          # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
+          # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
+          - python: 3.6
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            module_syntax: Lua
+            lc_all: C
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{matrix.python}}
+        architecture: x64
+
+    - name: install OS & Python packages
+      run: |
+        # disable apt-get update, we don't really need it,
+        # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
+        # sudo apt-get update
+        # for modules tool
+        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
+        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
+            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        fi
+
+    - name: install modules tool
+      run: |
+          # avoid downloading modules tool sources into easybuild-framework dir
+          cd $HOME
+          export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
+          # install Lmod
+          source $INSTALL_DEP ${{matrix.modules_tool}} $HOME
+          # changes in environment are not passed to other steps, so need to create files...
+          echo $MOD_INIT > mod_init
+          echo $PATH > path
+          if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
+
+    - name: test bootstrap script
+      run: |
+          # (re)initialize environment for modules tool
+          if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
+          source $(cat $HOME/mod_init); type module
+          # also pick up changes to $PATH set by sourcing $HOME/mod_init
+          export PATH=$(cat $HOME/path)
+
+          # define $EASYBUILD_MODULES_TOOL only for oldest module tools
+          # (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool)
+          if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
+            export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
+          elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
+            export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
+          fi
+
+          # version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents
+          EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
+          EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
+          EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
+          EB_BOOTSTRAP_EXPECTED="20210715.01 0ffdc17ed7eacf78369c9cd6743728f36e61bb8bf5c1bdc1e23cf2040b1ce301"
+          test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
+
+          # test bootstrap script
+          export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
+          export EASYBUILD_BOOTSTRAP_DEPRECATED=1
+          python easybuild/scripts/bootstrap_eb.py $PREFIX
+          unset EASYBUILD_BOOTSTRAP_DEPRECATED
+          # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
+          unset PYTHONPATH
+          # simple sanity check on bootstrapped EasyBuild module
+          module use $PREFIX/modules/all
+          module load EasyBuild
+          eb --version

.github/workflows/container_tests.yml

@@ -0,0 +1,95 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for container support
+on: [push, pull_request]
+jobs:
+  build:
+    # stick to Ubuntu 18.04, where we can still easily install yum via 'apt-get install'
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python: [2.7, 3.6]
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{matrix.python}}
+        architecture: x64
+
+    - name: install OS & Python packages
+      run: |
+        # ensure package list is up to date to avoid 404's for old packages
+        sudo apt-get update -yqq
+        # for building Singularity images
+        sudo apt-get install rpm
+        sudo apt-get install yum
+        # for modules tool
+        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
+        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
+            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        fi
+
+    - name: install Lmod
+      run: |
+          # avoid downloading modules tool sources into easybuild-framework dir
+          cd $HOME
+          export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
+          # install Lmod
+          source $INSTALL_DEP Lmod-8.4.27 $HOME
+          # changes in environment are not passed to other steps, so need to create files...
+          echo $MOD_INIT > mod_init
+          echo $PATH > path
+          if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
+
+    # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
+    - name: install Singularity
+      run: |
+        # install alien, which can be used to convert RPMs to Debian packages
+        sudo apt-get install alien
+        alien --version
+        # determine latest version of Singularity available in EPEL, and download RPM
+        singularity_rpm=$(curl -sL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*singularity/singularity/g' | sed 's/rpm.*/rpm/g')
+        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/${singularity_rpm}
+        # convert Singularity RPM to Debian package, and install it
+        sudo alien -d ${singularity_rpm}
+        sudo apt install ./singularity*.deb
+        singularity --version
+
+    - name: install sources
+      run: |
+          # install from source distribution tarball, to test release as published on PyPI
+          python setup.py sdist
+          ls dist
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz
+
+    - name: run test
+      run: |
+          # run tests *outside* of checked out easybuild-framework directory,
+          # to ensure we're testing installed version (see previous step)
+          cd $HOME
+          # initialize environment for modules tool
+          if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
+          source $(cat $HOME/mod_init); type module
+          # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
+          # also pick up changes to $PATH set by sourcing $MOD_INIT
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          export PATH=$PREFIX/bin:$(cat $HOME/path)
+          export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
+          eb --version
+          # create $HOME/.rpmmacros, see also https://github.com/apptainer/singularity/issues/241
+          echo '%_var /var' > $HOME/.rpmmacros
+          echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros
+          # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild;
+          # see https://docs.easybuild.io/en/latest/Containers.html
+          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
+          export EASYBUILD_CONTAINERPATH=$PWD
+          export EASYBUILD_CONTAINER_CONFIG='bootstrap=yum,osversion=7'
+          eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
+          singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1)
+          singularity exec bzip2-1.0.8.sif bzip2 --help

.github/workflows/eb_command.yml

@@ -0,0 +1,88 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for the 'eb' command
+on: [push, pull_request]
+jobs:
+  test-eb:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{matrix.python}}
+        architecture: x64
+
+    - name: install OS & Python packages
+      run: |
+        # check Python version
+        python -V
+        # update to latest pip, check version
+        pip install --upgrade pip
+        pip --version
+        # install packages required for modules tool
+        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
+        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
+            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        fi
+
+    - name: install modules tool
+      run: |
+          # avoid downloading modules tool sources into easybuild-framework dir
+          cd $HOME
+          export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
+          # install Lmod
+          source $INSTALL_DEP Lmod-8.4.26 $HOME
+          # changes in environment are not passed to other steps, so need to create files...
+          echo $MOD_INIT > mod_init
+          echo $PATH > path
+          if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
+
+    - name: install EasyBuild framework
+      run: |
+          # install from source distribution tarball, to test release as published on PyPI
+          python setup.py sdist
+          ls dist
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+
+    - name: run tests for 'eb' command
+      env:
+        EB_VERBOSE: 1
+      run: |
+          # run tests *outside* of checked out easybuild-framework directory,
+          # to ensure we're testing installed version (see previous step)
+          cd $HOME
+          # initialize environment for modules tool
+          if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
+          source $(cat $HOME/mod_init); type module
+          # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
+          # also pick up changes to $PATH set by sourcing $MOD_INIT
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          export PATH=$PREFIX/bin:$(cat $HOME/path)
+          export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
+          # run --version, capture (verbose) output
+          eb --version | tee eb_version.out 2>&1
+          # determine active Python version
+          pymajver=$(python -c 'import sys; print(sys.version_info[0])')
+          pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))')
+          # check patterns in verbose output
+          for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+              echo "Looking for pattern \"${pattern}\" in eb_version.out..."
+              grep "$pattern" eb_version.out
+          done
+          # also check when specifying Python command via $EB_PYTHON
+          for eb_python in "python${pymajver}" "python${pymajminver}"; do
+              export EB_PYTHON="${eb_python}"
+              eb --version | tee eb_version.out 2>&1
+              for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+                  echo "Looking for pattern \"${pattern}\" in eb_version.out..."
+                  grep "$pattern" eb_version.out
+              done
+          done

.github/workflows/linting.yml

@@ -3,13 +3,17 @@ on: [push, pull_request]
 jobs:
   python-linting:
     runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
+
     steps:
     - uses: actions/checkout@v2
 
     - name: set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: ${{ matrix.python-version }}
 
     - name: install Python packages
       run: |
@@ -18,4 +22,10 @@ jobs:
 
     - name: Run flake8 to verify PEP8-compliance of Python code
       run: |
-        flake8
+        # don't check py2vs3/py3.py when testing with Python 2, and vice versa
+        if [[ "${{ matrix.python-version }}" =~ "2." ]]; then
+            py_excl=py3
+        else
+            py_excl=py2
+        fi
+        flake8 --exclude ./easybuild/tools/py2vs3/${py_excl}.py