From f23e0d6b08b641d95010c2dd91ee22cad2cc3df4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Mar 2022 20:37:05 +0100 Subject: [PATCH 1/6] add separate workflow to test support for container images with Apptainer --- .../workflows/container_tests_apptainer.yml | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 .github/workflows/container_tests_apptainer.yml diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml new file mode 100644 index 0000000000..0cd2bca3f0 --- /dev/null +++ b/.github/workflows/container_tests_apptainer.yml @@ -0,0 +1,87 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for container support +on: [push, pull_request] +jobs: + build: + # stick to Ubuntu 18.04, where we can still easily install yum via 'apt-get install' + runs-on: ubuntu-18.04 + strategy: + matrix: + python: [2.7, 3.6] + apptainer: [1.0.0] + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: install OS & Python packages + run: | + # for modules tool + sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 + # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists + if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then + sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + fi + + - name: install Lmod + run: | + # avoid downloading modules tool sources into easybuild-framework dir + cd $HOME + export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh + # install Lmod + source $INSTALL_DEP Lmod-8.4.27 $HOME + # changes in environment are not passed to other steps, so need to create files... + echo $MOD_INIT > mod_init + echo $PATH > path + if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi + + - name: install Apptainer + run: | + curl -OL https://github.com/apptainer/apptainer/releases/download/v${{matrix.apptainer}}/apptainer_${{matrix.apptainer}}_amd64.deb + sudo apt install ./apptainer*.deb + # Apptainer provides both apptainer and singularity commands + apptainer --version + singularity --version + + - name: install sources + run: | + # install from source distribution tarball, to test release as published on PyPI + python setup.py sdist + ls dist + export PREFIX=/tmp/$USER/$GITHUB_SHA + pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz + + - name: run test + run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME + # initialize environment for modules tool + if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi + source $(cat $HOME/mod_init); type module + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); + # also pick up changes to $PATH set by sourcing $MOD_INIT + export PREFIX=/tmp/$USER/$GITHUB_SHA + export PATH=$PREFIX/bin:$(cat $HOME/path) + export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH + eb --version + # create $HOME/.rpmmacros, see also https://github.com/apptainer/singularity/issues/241 + echo '%_var /var' > $HOME/.rpmmacros + echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros + # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild; + # see https://docs.easybuild.io/en/latest/Containers.html + curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb + export EASYBUILD_CONTAINERPATH=$PWD + export EASYBUILD_CONTAINER_CONFIG='bootstrap=yum,osversion=7' + eb bzip2-1.0.8.eb --containerize --experimental --container-build-image + apptainer exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1) + apptainer exec bzip2-1.0.8.sif bzip2 --help + singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1) + singularity exec bzip2-1.0.8.sif bzip2 --help From 130128afd37540c872c047c67ec0ea18d3962064 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Mar 2022 21:08:05 +0100 Subject: [PATCH 2/6] add support for Apptainer container type --- .../workflows/container_tests_apptainer.yml | 3 +- easybuild/tools/config.py | 3 +- easybuild/tools/containers/apptainer.py | 112 ++++++++++++++++++ easybuild/tools/containers/common.py | 1 + 4 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 easybuild/tools/containers/apptainer.py diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 0cd2bca3f0..64b2b09a2c 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -80,8 +80,7 @@ jobs: curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb export EASYBUILD_CONTAINERPATH=$PWD export EASYBUILD_CONTAINER_CONFIG='bootstrap=yum,osversion=7' + export EASYBUILD_CONTAINER_TYPE='apptainer' eb bzip2-1.0.8.eb --containerize --experimental --container-build-image apptainer exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1) apptainer exec bzip2-1.0.8.sif bzip2 --help - singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1) - singularity exec bzip2-1.0.8.sif bzip2 --help diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index abcfb155be..799bb3364c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -79,9 +79,10 @@ CONT_IMAGE_FORMAT_SQUASHFS, ] +CONT_TYPE_APPTAINER = 'apptainer' CONT_TYPE_DOCKER = 'docker' CONT_TYPE_SINGULARITY = 'singularity' -CONT_TYPES = [CONT_TYPE_DOCKER, CONT_TYPE_SINGULARITY] +CONT_TYPES = [CONT_TYPE_DOCKER, CONT_TYPE_SINGULARITY, CONT_TYPE_APPTAINER] DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY DEFAULT_BRANCH = 'develop' diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py new file mode 100644 index 0000000000..9f1c607947 --- /dev/null +++ b/easybuild/tools/containers/apptainer.py @@ -0,0 +1,112 @@ +# Copyright 2022-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# +""" +Support for generating Apptainer container recipes and creating container images + +:author: Kenneth Hoste (HPC-UGent) +""" +import os +import re + +from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.containers.singularity import SingularityContainer +from easybuild.tools.config import CONT_IMAGE_FORMAT_EXT3, CONT_IMAGE_FORMAT_SANDBOX +from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS +from easybuild.tools.config import build_option, container_path +from easybuild.tools.filetools import remove_file, which +from easybuild.tools.run import run_cmd + + +class ApptainerContainer(SingularityContainer): + + TOOLS = {'apptainer': '1.0', 'sudo': None} + + RECIPE_FILE_NAME = 'Apptainer' + + @staticmethod + def apptainer_version(): + """Get Apptainer version.""" + version_cmd = "apptainer --version" + out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) + if ec: + raise EasyBuildError("Error running '%s': %s for tool {1} with output: {2}" % (version_cmd, out)) + + res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) + if not res: + raise EasyBuildError("Error parsing Apptainer version: %s" % out) + + return res.group(0) + + def build_image(self, recipe_path): + """Build container image by calling out to 'sudo singularity build'.""" + + cont_path = container_path() + def_file = os.path.basename(recipe_path) + + # use --imagename if specified, otherwise derive based on filename of recipe + img_name = self.img_name + if img_name is None: + # definition file Apptainer.--. + img_name = def_file.split('.', 1)[1] + + cmd_opts = '' + + image_format = self.image_format + + # singularity image format (default for Apptainer) + if image_format in [None, CONT_IMAGE_FORMAT_SQUASHFS, CONT_IMAGE_FORMAT_SIF]: + img_path = os.path.join(cont_path, img_name + '.sif') + + # ext3 image format, creating as writable container + elif image_format == CONT_IMAGE_FORMAT_EXT3: + raise EasyBuildError("ext3 image format is not supported with Apptainer") + + # sandbox image format, creates as a directory but acts like a container + elif image_format == CONT_IMAGE_FORMAT_SANDBOX: + img_path = os.path.join(cont_path, img_name) + cmd_opts = '--sandbox' + + else: + raise EasyBuildError("Unknown container image format specified for Apptainer: %s" % image_format) + + if os.path.exists(img_path): + if build_option('force'): + print_msg("WARNING: overwriting existing container image at %s due to --force" % img_path) + remove_file(img_path) + else: + raise EasyBuildError("Container image already exists at %s, not overwriting it without --force", + img_path) + + # resolve full path to 'singularity' binary, since it may not be available via $PATH under sudo... + singularity = which('apptainer') + cmd_env = '' + + singularity_tmpdir = self.tmpdir + if singularity_tmpdir: + cmd_env += 'APPTAINER_TMPDIR=%s' % singularity_tmpdir + + cmd = ' '.join(['sudo', cmd_env, singularity, 'build', cmd_opts, img_path, recipe_path]) + print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd) + run_cmd(cmd, stream_output=True) + print_msg("Apptainer image created at %s" % img_path, log=self.log) diff --git a/easybuild/tools/containers/common.py b/easybuild/tools/containers/common.py index b473d7fd73..a61cf23a1b 100644 --- a/easybuild/tools/containers/common.py +++ b/easybuild/tools/containers/common.py @@ -32,6 +32,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option +from easybuild.tools.containers.apptainer import ApptainerContainer # noqa from easybuild.tools.containers.docker import DockerContainer # noqa from easybuild.tools.containers.singularity import SingularityContainer # noqa From 84e185f7dbaebc614f90de73f8bdf04543398cde Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Mar 2022 21:16:03 +0100 Subject: [PATCH 3/6] add back rpm/yum commands in workflow for building Apptainer container images --- .github/workflows/container_tests.yml | 2 +- .github/workflows/container_tests_apptainer.yml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index cc8cafe298..605b8ed4e2 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -1,5 +1,5 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests for container support +name: Tests for Singularity container support on: [push, pull_request] jobs: build: diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 64b2b09a2c..46487c4917 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -1,5 +1,5 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests for container support +name: Tests for Apptainer container support on: [push, pull_request] jobs: build: @@ -21,6 +21,9 @@ jobs: - name: install OS & Python packages run: | + # for building CentOS 7 container images + sudo apt-get install rpm + sudo apt-get install yum # for modules tool sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 From f9f29841ba13c25d3e5646f1505359c64be3edc1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 8 Apr 2023 18:48:30 +0200 Subject: [PATCH 4/6] fix alphabetical order in CONT_TYPES constant --- easybuild/tools/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 799bb3364c..b82cabf245 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -82,7 +82,7 @@ CONT_TYPE_APPTAINER = 'apptainer' CONT_TYPE_DOCKER = 'docker' CONT_TYPE_SINGULARITY = 'singularity' -CONT_TYPES = [CONT_TYPE_DOCKER, CONT_TYPE_SINGULARITY, CONT_TYPE_APPTAINER] +CONT_TYPES = [CONT_TYPE_APPTAINER, CONT_TYPE_DOCKER, CONT_TYPE_SINGULARITY] DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY DEFAULT_BRANCH = 'develop' From b5414403b5ec1d15594d769131e729065f2e1543 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 8 Apr 2023 19:12:06 +0200 Subject: [PATCH 5/6] update workflow to run container tests with Apptainer to use Ubuntu 22.04 + EasyBuild CentOS 7.9 container --- .../workflows/container_tests_apptainer.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 46487c4917..6d8506e1e9 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -1,14 +1,21 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests for Apptainer container support on: [push, pull_request] + +permissions: + contents: read # to fetch code (actions/checkout) + +concurrency: + group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} + cancel-in-progress: true + jobs: build: - # stick to Ubuntu 18.04, where we can still easily install yum via 'apt-get install' - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 strategy: matrix: - python: [2.7, 3.6] - apptainer: [1.0.0] + python: [2.7, 3.7] + apptainer: [1.0.0, 1.1.7] fail-fast: false steps: - uses: actions/checkout@v2 @@ -23,7 +30,7 @@ jobs: run: | # for building CentOS 7 container images sudo apt-get install rpm - sudo apt-get install yum + sudo apt-get install dnf # for modules tool sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 @@ -82,7 +89,7 @@ jobs: # see https://docs.easybuild.io/en/latest/Containers.html curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb export EASYBUILD_CONTAINERPATH=$PWD - export EASYBUILD_CONTAINER_CONFIG='bootstrap=yum,osversion=7' + export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64' export EASYBUILD_CONTAINER_TYPE='apptainer' eb bzip2-1.0.8.eb --containerize --experimental --container-build-image apptainer exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1) From f7654c41d56eddbec7540e2ac6899ea03d84b1e6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 9 Apr 2023 12:57:43 +0200 Subject: [PATCH 6/6] replace 'singularity' occurences with 'apptainer' in container support for Apptainer + bump copyright to 2023 Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/tools/containers/apptainer.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py index 9f1c607947..67a6db5bc3 100644 --- a/easybuild/tools/containers/apptainer.py +++ b/easybuild/tools/containers/apptainer.py @@ -1,4 +1,4 @@ -# Copyright 2022-2022 Ghent University +# Copyright 2022-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -59,7 +59,7 @@ def apptainer_version(): return res.group(0) def build_image(self, recipe_path): - """Build container image by calling out to 'sudo singularity build'.""" + """Build container image by calling out to 'sudo apptainer build'.""" cont_path = container_path() def_file = os.path.basename(recipe_path) @@ -98,15 +98,15 @@ def build_image(self, recipe_path): raise EasyBuildError("Container image already exists at %s, not overwriting it without --force", img_path) - # resolve full path to 'singularity' binary, since it may not be available via $PATH under sudo... - singularity = which('apptainer') + # resolve full path to 'apptainer' binary, since it may not be available via $PATH under sudo... + apptainer = which('apptainer') cmd_env = '' - singularity_tmpdir = self.tmpdir - if singularity_tmpdir: - cmd_env += 'APPTAINER_TMPDIR=%s' % singularity_tmpdir + apptainer_tmpdir = self.tmpdir + if apptainer_tmpdir: + cmd_env += 'APPTAINER_TMPDIR=%s' % apptainer_tmpdir - cmd = ' '.join(['sudo', cmd_env, singularity, 'build', cmd_opts, img_path, recipe_path]) + cmd = ' '.join(['sudo', cmd_env, apptainer, 'build', cmd_opts, img_path, recipe_path]) print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd) run_cmd(cmd, stream_output=True) print_msg("Apptainer image created at %s" % img_path, log=self.log)