Isolate singularity containers more thoroughly for better reproducibility.

lib/galaxy/config/sample/job_conf.xml.sample_advanced

@@ -604,6 +604,12 @@
         </destination>
         <destination id="singularity_local" runner="local">
           <param id="singularity_enabled">true</param>
+          <!--Galaxy requests an isolated /tmp directory from singularity, which means
+              it will be mounted as read-only. This can cause some problems with tools that
+              do not use the TMP, TEMP, TMPDIR variable family properly. Setting
+              tmp_dir to true ensures that the /tmp directory in the container points
+              to the job specific working tmp directory.-->
+          <param id="tmp_dir">true</param>
           <!-- See the above documentation for docker_volumes, singularity_volumes works
                almost the same way. The only difference is that $default will expand with
                rw directories that in Docker would expand as ro if any of subdirectories are rw.
@@ -629,6 +635,18 @@
                argument by default. You can turn this off by setting singularity_cleanenv to `false`.
           -->
           <!-- <param id="singularity_cleanenv">true</param> -->
+          <!-- Singularity by default inherits the PID namespace, this can give
+               issues with multiprocessing, hence galaxy passes the `pid` argument
+               by default to isolate the PID namespace. You can turn this of by setting
+               singularity_pid to `false`.
+          -->
+          <!-- <param id="singularity_pid">true</param> -->
+          <!-- Singularity by default inherits the IPC namespace, this can give
+               issues with multiprocessing, hence galaxy passes the `ipc` argument
+               by default to isolate the PID namespace. You can turn this of by setting
+               singularity_ipc to `false`.
+          -->
+          <!-- <param id="singularity_ipc">true</param> -->
           <!-- Pass extra arguments to the singularity exec command not covered by the
                above options. -->
           <!-- <param id="singularity_run_extra_arguments"></param> -->

lib/galaxy/tool_util/deps/container_classes.py

@@ -573,6 +573,8 @@ def containerize_command(self, command: str) -> str:
             guest_ports=self.tool_info.guest_ports,
             container_name=self.container_name,
             cleanenv=asbool(self.prop("cleanenv", singularity_util.DEFAULT_CLEANENV)),
+            ipc=asbool(self.prop("ipc", singularity_util.DEFAULT_IPC)),
+            pid=asbool(self.prop("pid", singularity_util.DEFAULT_PID)),
             no_mount=self.prop("no_mount", singularity_util.DEFAULT_NO_MOUNT),
             **self.get_singularity_target_kwds(),
         )

lib/galaxy/tool_util/deps/singularity_util.py

@@ -13,7 +13,12 @@
 
 DEFAULT_WORKING_DIRECTORY = None
 DEFAULT_SINGULARITY_COMMAND = "singularity"
+# --pid isolates the pid namespace. --ipc isolates
+# the ipc namespace, this fixes some issues with python multiprocessing.
+# --cleanenv makes sure the current environment is not inherited.
 DEFAULT_CLEANENV = True
+DEFAULT_IPC = True
+DEFAULT_PID = True
 DEFAULT_NO_MOUNT = ["tmp"]
 DEFAULT_SUDO = False
 DEFAULT_SUDO_COMMAND = "sudo"
@@ -71,6 +76,8 @@ def build_singularity_run_command(
     guest_ports: Union[bool, List[str]] = False,
     container_name: Optional[str] = None,
     cleanenv: bool = DEFAULT_CLEANENV,
+    ipc: bool = DEFAULT_IPC,
+    pid: bool = DEFAULT_PID,
     no_mount: Optional[List[str]] = DEFAULT_NO_MOUNT,
 ) -> str:
     volumes = volumes or []
@@ -89,8 +96,19 @@ def build_singularity_run_command(
     )
     command_parts.append("-s")
     command_parts.append("exec")
+    # Singularity mounts some directories, such as $HOME and $PWD by default.
+    # using --contain disables this behaviour and only allows explicitly
+    # requested volumes to be mounted. This gives fully full-control over
+    # the mounting behavior.
+    command_parts.append("--contain")
+    if working_directory:
+        command_parts.extend(["--pwd", working_directory])
     if cleanenv:
         command_parts.append("--cleanenv")
+    if ipc:
+        command_parts.append("--ipc")
+    if pid:
+        command_parts.append("--pid")
     if no_mount:
         command_parts.extend(["--no-mount", ",".join(no_mount)])
     for volume in volumes:

test/unit/tool_util/test_singularity_util.py

@@ -6,13 +6,15 @@ def test_build_singularity_run_command_defaults():
         container_command="echo hi",
         image="busybox",
     )
-    assert cmd == "singularity -s exec --cleanenv --no-mount tmp busybox echo hi"
+    assert cmd == "singularity -s exec --contain --cleanenv --ipc --pid --no-mount tmp busybox echo hi"
 
 
-def test_build_singularity_run_command_no_cleanenv():
+def test_build_singularity_run_command_no_cleanenv_ipc_pid():
     cmd = build_singularity_run_command(
         container_command="echo hi",
         image="busybox",
         cleanenv=False,
+        pid=False,
+        ipc=False,
     )
-    assert cmd == "singularity -s exec --no-mount tmp busybox echo hi"
+    assert cmd == "singularity -s exec --contain --no-mount tmp busybox echo hi"