diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b80b22a4f..d8a6db1db 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -61,6 +61,7 @@ jobs:
shell: bash
env:
module_sys: ${{ matrix.module[0] }}
+ container_tech: ${{ matrix.container_tech }}
run: |
case "${module_sys}" in
lmod)
@@ -101,6 +102,9 @@ jobs:
cat test_output
grep --quiet donuts test_output
+ python-container > container_output
+ cat container_output
+
python-run python --version >test_output
cat test_output
grep --quiet 'Python 3.9.5' test_output
@@ -117,6 +121,7 @@ jobs:
shell: tcsh -e {0}
env:
module_sys: ${{ matrix.module[0] }}
+ container_tech: ${{ matrix.container_tech }}
run: |
switch ("${module_sys}")
case lmod:
@@ -148,6 +153,9 @@ jobs:
printf "\n\nmodule help ============================================\n"
module help python/3.9.5-alpine
+ python-container > container_output
+ cat container_output
+
python-exec echo donuts >test_output
cat test_output
grep --quiet donuts test_output
diff --git a/docs/getting_started/use-cases.rst b/docs/getting_started/use-cases.rst
index 6ffbb962d..00bb38aa7 100644
--- a/docs/getting_started/use-cases.rst
+++ b/docs/getting_started/use-cases.rst
@@ -167,7 +167,7 @@ loaded, just do:
This module is a singularity container wrapper for python v3.9.2-slim
Commands include:
- python-shell:
- singularity shell -s /bin/bash /home/shpc-user/singularity-hpc/modules/python/3.9.2-slim/python-3.9.2-slim- sha256:85ed629e6ff79d0bf796339ea188c863048e9aedbf7f946171266671ee5c04ef.sif
+ singularity shell -s /bin/bash /home/shpc-user/singularity-hpc/modules/python/3.9.2-slim/python-3.9.2-slim-sha256:85ed629e6ff79d0bf796339ea188c863048e9aedbf7f946171266671ee5c04ef.sif
- python:
singularity exec /home/shpc-user/singularity-hpc/modules/python/3.9.2-slim/python-3.9.2-slim-sha256:85ed629e6ff79d0bf796339ea188c863048e9aedbf7f946171266671ee5c04ef.sif /usr/local/bin/pythonā€¯)
@@ -192,6 +192,17 @@ singularity recipe) then you can do:
python/3.9.2-slim/module : org.label-schema.usage.singularity.version : 3.7.1-1.el7
+If your workflow requires knowledge of the local path to the sif image, this information
+can be output by using the "container" suffixed alias:
+
+.. code-block:: console
+
+ $ python-container
+ /home/shpc-user/singularity-hpc/modules/python/3.9.2-slim/python-3.9.2-slim-sha256:85ed629e6ff79d0bf796339ea188c863048e9aedbf7f946171266671ee5c04ef.sif
+
+or equivalently by accessing the value of the **SINGULARITY_CONTAINER** environment variable (or **PODMAN_CONTAINER** for each of Podman and Docker).
+
+
Adding Options
--------------
@@ -200,8 +211,8 @@ a container intended for gpu will have a feature: gpu set to true, and this will
it could be the case that you want to define custom options at the time of use.
In this case, you can export the following custom environment variables to add them:
-**SINGULARITY_OPTS**: will provide additional options to the base Singularity command, such as ``--debug``
-**SINGULARITY_COMMAND_OPTS**: will provide additional options to the command (e.g., exec), such as ``--cleanenv`` or ``--nv``.
+**SINGULARITY_OPTS**: will provide additional options to the base Singularity command, such as ``--debug``
+**SINGULARITY_COMMAND_OPTS**: will provide additional options to the command (e.g., exec), such as ``--cleanenv`` or ``--nv``.
Custom Images that are Added
diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst
index 5da0e0b6b..0210acd16 100644
--- a/docs/getting_started/user-guide.rst
+++ b/docs/getting_started/user-guide.rst
@@ -397,8 +397,8 @@ Module Names
The setting ``module_name`` is a format string in `Jinja2 `_
that is used to generate your module command names. For each module, in addition
-to aliases that are custom to the module, a set of commands for run, inspect, exec,
-and shell are generated. These commands will use the ``module_name`` format string
+to aliases that are custom to the module, a set of commands for run, exec,
+shell, inspect, and container are generated. These commands will use the ``module_name`` format string
to determine their names. For example, for a python container with the default ``module_name``
of "{{ tool }}" we will derive the following aliases for a Singularity module:
@@ -409,6 +409,7 @@ of "{{ tool }}" we will derive the following aliases for a Singularity module:
python-exec
python-inspect-deffile
python-inspect-runscript
+ python-container
A container identifier is parsed as follows:
@@ -431,6 +432,7 @@ If you change the format string to ``{{ tool }}-{{ version }}`` you would see:
python-3.9.5-alpine-exec
python-3.9.5-alpine-deffile
python-3.9.5-alpine-runscript
+ python-3.9.5-alpine-container
And of course you are free to add any string that you wish, e.g., ``plab-{{ tool }}``
diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua
index c9a5da13b..4d03f31b2 100644
--- a/shpc/main/modules/templates/docker.lua
+++ b/shpc/main/modules/templates/docker.lua
@@ -23,6 +23,8 @@ Commands include:
{{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if settings.environment_file %}--env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "$@"
- {|module_name|}-inspect:
{{ command }} inspect
+ - {|module_name|}-container:
+ echo "$PODMAN_CONTAINER"
{% if aliases %}{% for alias in aliases %} - {{ alias.name }}:
{{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ alias.entrypoint }} {% if settings.environment_file %}--env-file /{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options }} {% endif %} -v ${PWD} -w ${PWD} "{{ alias.args }}" "$@"
@@ -32,6 +34,7 @@ For each of the above, you can export:
- PODMAN_OPTS: to define custom options for {{ command }}
- PODMAN_COMMAND_OPTS: to define custom options for the command
+ - PODMAN_CONTAINER: the container unique resource identifier
]])
{% include "includes/default_version.lua" %}
@@ -48,6 +51,10 @@ local moduleDir = subprocess("realpath " .. myFileName()):match("(.*[/])") or ".
-- interactive shell to any container, plus exec for aliases
local containerPath = '{{ image }}'
+-- service environment variable to access docker URI
+setenv("PODMAN_CONTAINER", containerPath)
+set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
+
local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath
-- execCmd needs entrypoint to be the executor
local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "
diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl
index 5ca483e98..936cb6b61 100644
--- a/shpc/main/modules/templates/docker.tcl
+++ b/shpc/main/modules/templates/docker.tcl
@@ -22,6 +22,8 @@ proc ModulesHelp { } {
puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if settings.environment_file %} --env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\""
puts stderr " - {|module_name|}-inspect:"
puts stderr " {{ command }} inspect "
+ puts stderr " - {|module_name|}-container:"
+ puts stderr " echo \"\$PODMAN_CONTAINER\""
puts stderr ""
{% if aliases %}{% for alias in aliases %} puts stderr " - {{ alias.name }}:"
puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} --rm -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint | replace("$", "\$") }} {% if settings.environment_file %}--settings.environment_file /{{ settings.environment_file }} {% endif %}{% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options | replace("$", "\$") }} {% endif %} -v . -w . {{ alias.args | replace("$", "\$") }} \"\$@\""
@@ -31,7 +33,7 @@ proc ModulesHelp { } {
puts stderr ""
puts stderr " - PODMAN_OPTS: to define custom options for {{ command }}"
puts stderr " - PODMAN_COMMAND_OPTS: to define custom options for the command"
-
+ puts stderr " - PODMAN_CONTAINER: to define the container unique resource identifier"
}
# Environment - only set if not already defined
@@ -66,6 +68,10 @@ conflict {{ parsed_name.tool }}
{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %}conflict {{ alias.name }}{% endif %}
{% endfor %}{% endif %}
+# service environment variable to access full SIF image path
+setenv PODMAN_CONTAINER "${containerPath}"
+set-alias {|module_name|}-container "echo ${containerPath}"
+
# interactive shell to any container, plus exec for aliases
set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if settings.enable_tty %}t{% endif %} --entrypoint {{ shell }} {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}"
diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua
index 5239a165d..abef2e99a 100644
--- a/shpc/main/modules/templates/singularity.lua
+++ b/shpc/main/modules/templates/singularity.lua
@@ -9,7 +9,7 @@ help(
This module is a singularity container wrapper for {{ name }} v{{ version }}
{% if description %}{{ description }}{% endif %}
-Container:
+Container (available through variable SINGULARITY_CONTAINER):
- {{ container_sif }}
@@ -25,6 +25,8 @@ Commands include:
singularity inspect -r
- {|module_name|}-inspect-deffile:
singularity inspect -d
+ - {|module_name|}-container:
+ echo "$SINGULARITY_CONTAINER"
{% if aliases %}{% for alias in aliases %} - {{ alias.name }}:
singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options }} {% endif %} {{ alias.command }} "$@"
@@ -34,6 +36,7 @@ For each of the above, you can export:
- SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug)
- SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b)
+ - SINGULARITY_CONTAINER: full path to the Singularity Container
]])
{% include "includes/default_version.lua" %}
@@ -50,8 +53,12 @@ setenv("SINGULARITY_SHELL", "{{ settings.singularity_shell }}")
if not os.getenv("SINGULARITY_OPTS") then setenv ("SINGULARITY_OPTS", "") end
if not os.getenv("SINGULARITY_COMMAND_OPTS") then setenv ("SINGULARITY_COMMAND_OPTS", "") end
--- interactive shell to any container, plus exec for aliases
local containerPath = '{{ container_sif }}'
+-- service environment variable to access full SIF image path
+setenv("SINGULARITY_CONTAINER", containerPath)
+set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
+
+-- interactive shell to any container, plus exec for aliases
local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath
local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} "
local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath
diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl
index a631ed4df..aa86f6a14 100644
--- a/shpc/main/modules/templates/singularity.tcl
+++ b/shpc/main/modules/templates/singularity.tcl
@@ -11,7 +11,7 @@ proc ModulesHelp { } {
puts stderr "This module is a singularity container wrapper for {{ name }} v{{ version }}"
{% if description %}puts stderr "{{ description }}"{% endif %}
puts stderr ""
- puts stderr "Container:"
+ puts stderr "Container (available through variable SINGULARITY_CONTAINER):"
puts stderr ""
puts stderr " - {{ container_sif }}"
puts stderr ""
@@ -27,6 +27,8 @@ proc ModulesHelp { } {
puts stderr " singularity inspect -r "
puts stderr " - {|module_name|}-inspect-deffile:"
puts stderr " singularity inspect -d "
+ puts stderr " - {|module_name|}-container:"
+ puts stderr " echo \"\$SINGULARITY_CONTAINER\""
puts stderr ""
{% if aliases %}{% for alias in aliases %} puts stderr " - {{ alias.name }}:"
puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options | replace("$", "\$") }} {% endif %} {{ alias.command | replace("$", "\$") }} \"\$@\""
@@ -36,6 +38,7 @@ proc ModulesHelp { } {
puts stderr ""
puts stderr " - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug)"
puts stderr " - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b)"
+ puts stderr " - SINGULARITY_CONTAINER: The Singularity (sif) path"
}
@@ -73,6 +76,10 @@ conflict {{ parsed_name.tool }}
# singularity environment variable to set shell
setenv SINGULARITY_SHELL {{ settings.singularity_shell }}
+# service environment variable to access full SIF image path
+setenv SINGULARITY_CONTAINER "${containerPath}"
+set-alias {|module_name|}-container "echo ${containerPath}"
+
# interactive shell to any container, plus exec for aliases
set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}"
set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} "
diff --git a/shpc/tests/test_container.py b/shpc/tests/test_container.py
index 77e976807..357dca6de 100644
--- a/shpc/tests/test_container.py
+++ b/shpc/tests/test_container.py
@@ -15,7 +15,9 @@
def test_pull_gh(tmp_path):
- """Test a singularity container command"""
+ """
+ Test a singularity container command
+ """
cli = container.SingularityContainer()
# Test default Singularity pull
@@ -39,7 +41,9 @@ def test_pull_oras(tmp_path):
def test_podman(tmp_path):
- """Test a singularity container command"""
+ """
+ Test a singularity container command
+ """
cli = container.PodmanContainer()
# Test default Singularity pull
@@ -50,7 +54,9 @@ def test_podman(tmp_path):
def test_docker(tmp_path):
- """Test a singularity container command"""
+ """
+ Test a singularity container command
+ """
cli = container.DockerContainer()
# Test default Singularity pull