Isolate singularity containers more thoroughly for better reproducibility.

lib/galaxy/config/sample/job_conf.xml.sample_advanced

@@ -604,6 +604,12 @@
         </destination>
         <destination id="singularity_local" runner="local">
           <param id="singularity_enabled">true</param>
+          <!--Galaxy requests an isolated /tmp directory from singularity, which means
+              it will be mounted as read-only. This can cause some problems with tools that
+              do not use the TMP, TEMP, TMPDIR variable family properly. Setting
+              tmp_dir to true ensures that the /tmp directory in the container points
+              to the job specific working tmp directory.-->
+          <param id="tmp_dir">true</param>
           <!-- See the above documentation for docker_volumes, singularity_volumes works
                almost the same way. The only difference is that $default will expand with
                rw directories that in Docker would expand as ro if any of subdirectories are rw.
@@ -629,6 +635,24 @@
                argument by default. You can turn this off by setting singularity_cleanenv to `false`.
           -->
           <!-- <param id="singularity_cleanenv">true</param> -->
+          <!-- Singularity by default inherits the PID namespace, this can give
+               issues with multiprocessing, hence galaxy passes the `pid` argument
+               by default to isolate the PID namespace. You can turn this of by setting
+               singularity_pid to `false`.
+          -->
+          <!-- <param id="singularity_pid">true</param> -->
+          <!-- Singularity by default inherits the IPC namespace, this can give
+               issues with multiprocessing, hence galaxy passes the `ipc` argument
+               by default to isolate the PID namespace. You can turn this of by setting
+               singularity_ipc to `false`.
+          -->
+          <!-- <param id="singularity_ipc">true</param> -->
+          <!-- Singularity mounts $HOME by default. This can be problematic if home
+               contains sensitive data, is shared across nodes or has configuration
+               files affecting reproducibility. Therefore Galaxy passes the `contain`
+               argument to isolate $HOME. If $HOME must be mounted, it can be
+               turned on by setting singularity_mount_home to `true`. -->
+          <!-- <param id="singularity_mount_home">false</param> -->
           <!-- Pass extra arguments to the singularity exec command not covered by the
                above options. -->
           <!-- <param id="singularity_run_extra_arguments"></param> -->

lib/galaxy/tool_util/deps/container_classes.py

@@ -573,6 +573,9 @@ def containerize_command(self, command: str) -> str:
             guest_ports=self.tool_info.guest_ports,
             container_name=self.container_name,
             cleanenv=asbool(self.prop("cleanenv", singularity_util.DEFAULT_CLEANENV)),
+            ipc=asbool(self.prop("ipc", singularity_util.DEFAULT_IPC)),
+            pid=asbool(self.prop("pid", singularity_util.DEFAULT_PID)),
+            mount_home=asbool(self.prop("mount_home", singularity_util.DEFAULT_MOUNT_HOME)),
             no_mount=self.prop("no_mount", singularity_util.DEFAULT_NO_MOUNT),
             **self.get_singularity_target_kwds(),
         )

lib/galaxy/tool_util/deps/singularity_util.py

@@ -13,7 +13,17 @@
 
 DEFAULT_WORKING_DIRECTORY = None
 DEFAULT_SINGULARITY_COMMAND = "singularity"
+# --cleanenv --pid --ipc --contain is equivalent to the --containall flag.
+# This isolates a singularity container in the same way as a Docker container
+# would be isolated. --contain ensures no "default mounts" are mounted, the
+# default mounts include $HOME and that might affect reproducibility due to
+# settings files in $HOME. --pid isolates the pid namespace. --ipc isolates
+# the ipc namespace, this fixes some issues with python multiprocessing.
+# --cleanenv makes sure the current environment is not inherited.
 DEFAULT_CLEANENV = True
+DEFAULT_IPC = True
+DEFAULT_PID = True
+DEFAULT_MOUNT_HOME = False
 DEFAULT_NO_MOUNT = ["tmp"]
 DEFAULT_SUDO = False
 DEFAULT_SUDO_COMMAND = "sudo"
@@ -71,6 +81,9 @@ def build_singularity_run_command(
     guest_ports: Union[bool, List[str]] = False,
     container_name: Optional[str] = None,
     cleanenv: bool = DEFAULT_CLEANENV,
+    ipc: bool = DEFAULT_IPC,
+    pid: bool = DEFAULT_PID,
+    mount_home: bool = DEFAULT_MOUNT_HOME,
     no_mount: Optional[List[str]] = DEFAULT_NO_MOUNT,
 ) -> str:
     volumes = volumes or []
@@ -89,13 +102,24 @@ def build_singularity_run_command(
     )
     command_parts.append("-s")
     command_parts.append("exec")
+    # Singularity mounts some directories, such as $HOME and $PWD by default.
+    # using --contain disables this behaviour and only allows explicitly
+    # requested volumes to be mounted. Since galaxy already mounts $PWD and
+    # mount_home can be activated, a switch for --contain is redundant.
+    command_parts.append("--contain")
+    if working_directory:
+        command_parts.extend(["--pwd", working_directory])
     if cleanenv:
         command_parts.append("--cleanenv")
+    if ipc:
+        command_parts.append("--ipc")
+    if pid:
+        command_parts.append("--pid")
     if no_mount:
         command_parts.extend(["--no-mount", ",".join(no_mount)])
     for volume in volumes:
         command_parts.extend(["-B", str(volume)])
-    if home is not None:
+    if mount_home and home is not None:
         command_parts.extend(["--home", f"{home}:{home}"])
     if run_extra_arguments:
         command_parts.append(run_extra_arguments)

test/unit/tool_util/test_singularity_util.py

@@ -6,13 +6,15 @@ def test_build_singularity_run_command_defaults():
         container_command="echo hi",
         image="busybox",
     )
-    assert cmd == "singularity -s exec --cleanenv --no-mount tmp busybox echo hi"
+    assert cmd == "singularity -s exec --contain --cleanenv --ipc --pid --no-mount tmp busybox echo hi"
 
 
-def test_build_singularity_run_command_no_cleanenv():
+def test_build_singularity_run_command_no_cleanenv_ipc_pid():
     cmd = build_singularity_run_command(
         container_command="echo hi",
         image="busybox",
         cleanenv=False,
+        pid=False,
+        ipc=False,
     )
-    assert cmd == "singularity -s exec --no-mount tmp busybox echo hi"
+    assert cmd == "singularity -s exec --contain --no-mount tmp busybox echo hi"