diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b80b22a4f..d8a6db1db 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,6 +61,7 @@ jobs: shell: bash env: module_sys: ${{ matrix.module[0] }} + container_tech: ${{ matrix.container_tech }} run: | case "${module_sys}" in lmod) @@ -101,6 +102,9 @@ jobs: cat test_output grep --quiet donuts test_output + python-container > container_output + cat container_output + python-run python --version >test_output cat test_output grep --quiet 'Python 3.9.5' test_output @@ -117,6 +121,7 @@ jobs: shell: tcsh -e {0} env: module_sys: ${{ matrix.module[0] }} + container_tech: ${{ matrix.container_tech }} run: | switch ("${module_sys}") case lmod: @@ -148,6 +153,9 @@ jobs: printf "\n\nmodule help ============================================\n" module help python/3.9.5-alpine + python-container > container_output + cat container_output + python-exec echo donuts >test_output cat test_output grep --quiet donuts test_output diff --git a/docs/getting_started/use-cases.rst b/docs/getting_started/use-cases.rst index 6ffbb962d..00bb38aa7 100644 --- a/docs/getting_started/use-cases.rst +++ b/docs/getting_started/use-cases.rst @@ -167,7 +167,7 @@ loaded, just do: This module is a singularity container wrapper for python v3.9.2-slim Commands include: - python-shell: - singularity shell -s /bin/bash /home/shpc-user/singularity-hpc/modules/python/3.9.2-slim/python-3.9.2-slim- sha256:85ed629e6ff79d0bf796339ea188c863048e9aedbf7f946171266671ee5c04ef.sif + singularity shell -s /bin/bash /home/shpc-user/singularity-hpc/modules/python/3.9.2-slim/python-3.9.2-slim-sha256:85ed629e6ff79d0bf796339ea188c863048e9aedbf7f946171266671ee5c04ef.sif - python: singularity exec /home/shpc-user/singularity-hpc/modules/python/3.9.2-slim/python-3.9.2-slim-sha256:85ed629e6ff79d0bf796339ea188c863048e9aedbf7f946171266671ee5c04ef.sif /usr/local/bin/pythonā€¯) @@ -192,6 +192,17 @@ singularity recipe) then you can do: python/3.9.2-slim/module : org.label-schema.usage.singularity.version : 3.7.1-1.el7 +If your workflow requires knowledge of the local path to the sif image, this information +can be output by using the "container" suffixed alias: + +.. code-block:: console + + $ python-container + /home/shpc-user/singularity-hpc/modules/python/3.9.2-slim/python-3.9.2-slim-sha256:85ed629e6ff79d0bf796339ea188c863048e9aedbf7f946171266671ee5c04ef.sif + +or equivalently by accessing the value of the **SINGULARITY_CONTAINER** environment variable (or **PODMAN_CONTAINER** for each of Podman and Docker). + + Adding Options -------------- @@ -200,8 +211,8 @@ a container intended for gpu will have a feature: gpu set to true, and this will it could be the case that you want to define custom options at the time of use. In this case, you can export the following custom environment variables to add them: -**SINGULARITY_OPTS**: will provide additional options to the base Singularity command, such as ``--debug`` -**SINGULARITY_COMMAND_OPTS**: will provide additional options to the command (e.g., exec), such as ``--cleanenv`` or ``--nv``. +**SINGULARITY_OPTS**: will provide additional options to the base Singularity command, such as ``--debug`` +**SINGULARITY_COMMAND_OPTS**: will provide additional options to the command (e.g., exec), such as ``--cleanenv`` or ``--nv``. Custom Images that are Added diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index 5da0e0b6b..0210acd16 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -397,8 +397,8 @@ Module Names The setting ``module_name`` is a format string in `Jinja2 `_ that is used to generate your module command names. For each module, in addition -to aliases that are custom to the module, a set of commands for run, inspect, exec, -and shell are generated. These commands will use the ``module_name`` format string +to aliases that are custom to the module, a set of commands for run, exec, +shell, inspect, and container are generated. These commands will use the ``module_name`` format string to determine their names. For example, for a python container with the default ``module_name`` of "{{ tool }}" we will derive the following aliases for a Singularity module: @@ -409,6 +409,7 @@ of "{{ tool }}" we will derive the following aliases for a Singularity module: python-exec python-inspect-deffile python-inspect-runscript + python-container A container identifier is parsed as follows: @@ -431,6 +432,7 @@ If you change the format string to ``{{ tool }}-{{ version }}`` you would see: python-3.9.5-alpine-exec python-3.9.5-alpine-deffile python-3.9.5-alpine-runscript + python-3.9.5-alpine-container And of course you are free to add any string that you wish, e.g., ``plab-{{ tool }}`` diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index c9a5da13b..4d03f31b2 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -23,6 +23,8 @@ Commands include: {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if settings.environment_file %}--env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "$@" - {|module_name|}-inspect: {{ command }} inspect + - {|module_name|}-container: + echo "$PODMAN_CONTAINER" {% if aliases %}{% for alias in aliases %} - {{ alias.name }}: {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ alias.entrypoint }} {% if settings.environment_file %}--env-file /{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options }} {% endif %} -v ${PWD} -w ${PWD} "{{ alias.args }}" "$@" @@ -32,6 +34,7 @@ For each of the above, you can export: - PODMAN_OPTS: to define custom options for {{ command }} - PODMAN_COMMAND_OPTS: to define custom options for the command + - PODMAN_CONTAINER: the container unique resource identifier ]]) {% include "includes/default_version.lua" %} @@ -48,6 +51,10 @@ local moduleDir = subprocess("realpath " .. myFileName()):match("(.*[/])") or ". -- interactive shell to any container, plus exec for aliases local containerPath = '{{ image }}' +-- service environment variable to access docker URI +setenv("PODMAN_CONTAINER", containerPath) +set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath) + local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath -- execCmd needs entrypoint to be the executor local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index 5ca483e98..936cb6b61 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -22,6 +22,8 @@ proc ModulesHelp { } { puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if settings.environment_file %} --env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\"" puts stderr " - {|module_name|}-inspect:" puts stderr " {{ command }} inspect " + puts stderr " - {|module_name|}-container:" + puts stderr " echo \"\$PODMAN_CONTAINER\"" puts stderr "" {% if aliases %}{% for alias in aliases %} puts stderr " - {{ alias.name }}:" puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} --rm -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint | replace("$", "\$") }} {% if settings.environment_file %}--settings.environment_file /{{ settings.environment_file }} {% endif %}{% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options | replace("$", "\$") }} {% endif %} -v . -w . {{ alias.args | replace("$", "\$") }} \"\$@\"" @@ -31,7 +33,7 @@ proc ModulesHelp { } { puts stderr "" puts stderr " - PODMAN_OPTS: to define custom options for {{ command }}" puts stderr " - PODMAN_COMMAND_OPTS: to define custom options for the command" - + puts stderr " - PODMAN_CONTAINER: to define the container unique resource identifier" } # Environment - only set if not already defined @@ -66,6 +68,10 @@ conflict {{ parsed_name.tool }} {% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %}conflict {{ alias.name }}{% endif %} {% endfor %}{% endif %} +# service environment variable to access full SIF image path +setenv PODMAN_CONTAINER "${containerPath}" +set-alias {|module_name|}-container "echo ${containerPath}" + # interactive shell to any container, plus exec for aliases set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if settings.enable_tty %}t{% endif %} --entrypoint {{ shell }} {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index 5239a165d..abef2e99a 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -9,7 +9,7 @@ help( This module is a singularity container wrapper for {{ name }} v{{ version }} {% if description %}{{ description }}{% endif %} -Container: +Container (available through variable SINGULARITY_CONTAINER): - {{ container_sif }} @@ -25,6 +25,8 @@ Commands include: singularity inspect -r - {|module_name|}-inspect-deffile: singularity inspect -d + - {|module_name|}-container: + echo "$SINGULARITY_CONTAINER" {% if aliases %}{% for alias in aliases %} - {{ alias.name }}: singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options }} {% endif %} {{ alias.command }} "$@" @@ -34,6 +36,7 @@ For each of the above, you can export: - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug) - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b) + - SINGULARITY_CONTAINER: full path to the Singularity Container ]]) {% include "includes/default_version.lua" %} @@ -50,8 +53,12 @@ setenv("SINGULARITY_SHELL", "{{ settings.singularity_shell }}") if not os.getenv("SINGULARITY_OPTS") then setenv ("SINGULARITY_OPTS", "") end if not os.getenv("SINGULARITY_COMMAND_OPTS") then setenv ("SINGULARITY_COMMAND_OPTS", "") end --- interactive shell to any container, plus exec for aliases local containerPath = '{{ container_sif }}' +-- service environment variable to access full SIF image path +setenv("SINGULARITY_CONTAINER", containerPath) +set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath) + +-- interactive shell to any container, plus exec for aliases local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index a631ed4df..aa86f6a14 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -11,7 +11,7 @@ proc ModulesHelp { } { puts stderr "This module is a singularity container wrapper for {{ name }} v{{ version }}" {% if description %}puts stderr "{{ description }}"{% endif %} puts stderr "" - puts stderr "Container:" + puts stderr "Container (available through variable SINGULARITY_CONTAINER):" puts stderr "" puts stderr " - {{ container_sif }}" puts stderr "" @@ -27,6 +27,8 @@ proc ModulesHelp { } { puts stderr " singularity inspect -r " puts stderr " - {|module_name|}-inspect-deffile:" puts stderr " singularity inspect -d " + puts stderr " - {|module_name|}-container:" + puts stderr " echo \"\$SINGULARITY_CONTAINER\"" puts stderr "" {% if aliases %}{% for alias in aliases %} puts stderr " - {{ alias.name }}:" puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options | replace("$", "\$") }} {% endif %} {{ alias.command | replace("$", "\$") }} \"\$@\"" @@ -36,6 +38,7 @@ proc ModulesHelp { } { puts stderr "" puts stderr " - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug)" puts stderr " - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b)" + puts stderr " - SINGULARITY_CONTAINER: The Singularity (sif) path" } @@ -73,6 +76,10 @@ conflict {{ parsed_name.tool }} # singularity environment variable to set shell setenv SINGULARITY_SHELL {{ settings.singularity_shell }} +# service environment variable to access full SIF image path +setenv SINGULARITY_CONTAINER "${containerPath}" +set-alias {|module_name|}-container "echo ${containerPath}" + # interactive shell to any container, plus exec for aliases set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}" set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " diff --git a/shpc/tests/test_container.py b/shpc/tests/test_container.py index 77e976807..357dca6de 100644 --- a/shpc/tests/test_container.py +++ b/shpc/tests/test_container.py @@ -15,7 +15,9 @@ def test_pull_gh(tmp_path): - """Test a singularity container command""" + """ + Test a singularity container command + """ cli = container.SingularityContainer() # Test default Singularity pull @@ -39,7 +41,9 @@ def test_pull_oras(tmp_path): def test_podman(tmp_path): - """Test a singularity container command""" + """ + Test a singularity container command + """ cli = container.PodmanContainer() # Test default Singularity pull @@ -50,7 +54,9 @@ def test_podman(tmp_path): def test_docker(tmp_path): - """Test a singularity container command""" + """ + Test a singularity container command + """ cli = container.DockerContainer() # Test default Singularity pull