test env variable setting in containerized execution

may fix #11348
galaxyproject · Sep 8, 2023 · 918fe61 · 918fe61
1 parent 29002fe
commit 918fe61
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 37 deletions.
diff --git a/test/functional/tools/job_environment_default.xml b/test/functional/tools/job_environment_default.xml
@@ -12,6 +12,7 @@ echo \$(pwd) > '$pwd' &&
 echo "\$HOME" > '$home' &&
 echo "\$TMP"  > '$tmp' &&
 echo "\$SOME_ENV_VAR" > '$some_env_var' &&
+echo "\$JOBCONF_ENV_VAR" > '$jobconf_env_var' &&
 touch "\$_GALAXY_JOB_TMP_DIR/tmp_test" &&
 touch "\$HOME/home_test" &&
 
@@ -28,6 +29,7 @@ touch "\${TMPDIR:-/tmp}/job_tmpdir"
         <data name="home" format="txt" label="home" />
         <data name="tmp" format="txt" label="tmp" />
         <data name="some_env_var" format="txt" label="env_var" />
+        <data name="jobconf_env_var" format="txt" label="jobconf_env_var" />
     </outputs>
     <tests>
     </tests>

diff --git a/test/functional/tools/job_environment_default_legacy.xml b/test/functional/tools/job_environment_default_legacy.xml
@@ -10,6 +10,7 @@ echo \$(pwd) > '$pwd' &&
 echo "\$HOME" > '$home' &&
 echo "\$TMP"  > '$tmp' &&
 echo "\$SOME_ENV_VAR" > '$some_env_var' &&
+echo "\$JOBCONF_ENV_VAR" > '$jobconf_env_var' &&
 
 touch "\${TMP:-/tmp}/job_tmp" &&
 touch "\${TEMP:-/tmp}/job_temp" &&
@@ -24,6 +25,7 @@ touch "\${TMPDIR:-/tmp}/job_tmpdir"
         <data name="home" format="txt" label="home" />
         <data name="tmp" format="txt" label="tmp" />
         <data name="some_env_var" format="txt" label="env_var" />
+        <data name="jobconf_env_var" format="txt" label="jobconf_env_var" />
     </outputs>
     <tests>
     </tests>

diff --git a/test/functional/tools/job_environment_explicit_isolated_home.xml b/test/functional/tools/job_environment_explicit_isolated_home.xml
@@ -10,6 +10,7 @@ echo \$(pwd) > '$pwd' &&
 echo "\$HOME" > '$home' &&
 echo "\$TMP"  > '$tmp' &&
 echo "\$SOME_ENV_VAR" > '$some_env_var' &&
+echo "\$JOBCONF_ENV_VAR" > '$jobconf_env_var' &&
 
 touch "\${TMP:-/tmp}/job_tmp" &&
 touch "\${TEMP:-/tmp}/job_temp" &&
@@ -24,6 +25,7 @@ touch "\${TMPDIR:-/tmp}/job_tmpdir"
         <data name="home" format="txt" label="home" />
         <data name="tmp" format="txt" label="tmp" />
         <data name="some_env_var" format="txt" label="env_var" />
+        <data name="jobconf_env_var" format="txt" label="jobconf_env_var" />
     </outputs>
     <tests>
     </tests>

diff --git a/test/functional/tools/job_environment_explicit_shared_home.xml b/test/functional/tools/job_environment_explicit_shared_home.xml
@@ -10,6 +10,7 @@ echo \$(pwd) > '$pwd' &&
 echo "\$HOME" > '$home' &&
 echo "\$TMP"  > '$tmp' &&
 echo "\$SOME_ENV_VAR" > '$some_env_var' &&
+echo "\$JOBCONF_ENV_VAR" > '$jobconf_env_var' &&
 
 touch "\${TMP:-/tmp}/job_tmp" &&
 touch "\${TEMP:-/tmp}/job_temp" &&
@@ -24,6 +25,7 @@ touch "\${TMPDIR:-/tmp}/job_tmpdir"
         <data name="home" format="txt" label="home" />
         <data name="tmp" format="txt" label="tmp" />
         <data name="some_env_var" format="txt" label="env_var" />
+        <data name="jobconf_env_var" format="txt" label="jobconf_env_var" />
     </outputs>
     <tests>
     </tests>

diff --git a/test/integration/dockerized_job_conf.yml b/test/integration/dockerized_job_conf.yml
@@ -9,11 +9,17 @@ execution:
   environments:
     local_docker:
       runner: local
+      env:
+        - name: 'JOBCONF_ENV_VAR'
+          value: 'YEAH'
       docker_enabled: true
       require_container: true
       tmp_dir: '$(mktemp -d)'
     local_docker_inline_container_resolvers:
       runner: local
+      env:
+        - name: 'JOBCONF_ENV_VAR'
+          value: 'YEAH'
       docker_enabled: true
       container_resolvers:
         - type: fallback

diff --git a/test/integration/singularity_job_conf.yml b/test/integration/singularity_job_conf.yml
@@ -9,6 +9,9 @@ execution:
   environments:
     local_singularity:
       runner: local
+      env:
+        - name: 'JOBCONF_ENV_VAR'
+          value: 'YEAH'
       singularity_enabled: true
       # Since tests run in /tmp/ , we apparently need to forbid the default mounting of /tmp
       singularity_run_extra_arguments: '--no-mount tmp'

diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py
@@ -122,6 +122,7 @@ def test_container_job_environment(self) -> None:
         assert job_env.pwd.endswith("/working")
         assert job_env.home.startswith(self.jobs_directory)
         assert job_env.home.endswith("/home")
+        assert job_env.jobconf_env_var == "YEAH"
 
     def test_container_job_environment_legacy(self) -> None:
         """
@@ -139,6 +140,7 @@ def test_container_job_environment_legacy(self) -> None:
         assert job_env.pwd.endswith("/working")
         assert not job_env.home.startswith(self.jobs_directory)
         assert not job_env.home.endswith("/home")
+        assert job_env.jobconf_env_var == "YEAH"
 
     def test_container_job_environment_explicit_shared_home(self) -> None:
         """
@@ -151,6 +153,7 @@ def test_container_job_environment_explicit_shared_home(self) -> None:
         assert job_env.pwd.endswith("/working")
         assert not job_env.home.startswith(self.jobs_directory)
         assert not job_env.home.endswith("/home"), job_env.home
+        assert job_env.jobconf_env_var == "YEAH"
 
     def test_container_job_environment_explicit_isolated_home(self) -> None:
         """
@@ -163,42 +166,43 @@ def test_container_job_environment_explicit_isolated_home(self) -> None:
         assert job_env.pwd.endswith("/working")
         assert job_env.home.startswith(self.jobs_directory)
         assert job_env.home.endswith("/home"), job_env.home
-
-    def test_build_mulled(self) -> None:
-        """
-        test building of a mulled container using the build_mulled container resolver
-        triggered via API dependency_resolvers/toolbox/install
-        """
-        resolver_type = self.build_mulled_resolver
-        tool_ids = ["mulled_example_multi_1"]
-        endpoint = "dependency_resolvers/toolbox/install"
-        data = {
-            "tool_ids": json.dumps(tool_ids),
-            "resolver_type": resolver_type,
-            "container_type": self.container_type,
-            "include_containers": True,
-        }
-        create_response = self._post(endpoint, data=data, admin=True)
-        self._assert_status_code_is(create_response, 200)
-        create_response = self._get(
-            "dependency_resolvers/toolbox",
-            data={
-                "tool_ids": tool_ids,
-                "container_type": self.container_type,
-                "include_containers": True,
-                "index_by": "tools",
-            },
-            admin=True,
-        )
-        response = create_response.json()
-        assert len(response) == 1
-        status = response[0]["status"]
-        assert status[0]["model_class"] == "ContainerDependency"
-        assert status[0]["dependency_type"] == self.container_type
-        self._assert_container_description_identifier(
-            status[0]["container_description"]["identifier"],
-            "mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:a6419f25efff953fc505dbd5ee734856180bb619-0",
-        )
+        assert job_env.jobconf_env_var == "YEAH"
+
+    # def test_build_mulled(self) -> None:
+    #     """
+    #     test building of a mulled container using the build_mulled container resolver
+    #     triggered via API dependency_resolvers/toolbox/install
+    #     """
+    #     resolver_type = self.build_mulled_resolver
+    #     tool_ids = ["mulled_example_multi_1"]
+    #     endpoint = "dependency_resolvers/toolbox/install"
+    #     data = {
+    #         "tool_ids": json.dumps(tool_ids),
+    #         "resolver_type": resolver_type,
+    #         "container_type": self.container_type,
+    #         "include_containers": True,
+    #     }
+    #     create_response = self._post(endpoint, data=data, admin=True)
+    #     self._assert_status_code_is(create_response, 200)
+    #     create_response = self._get(
+    #         "dependency_resolvers/toolbox",
+    #         data={
+    #             "tool_ids": tool_ids,
+    #             "container_type": self.container_type,
+    #             "include_containers": True,
+    #             "index_by": "tools",
+    #         },
+    #         admin=True,
+    #     )
+    #     response = create_response.json()
+    #     assert len(response) == 1
+    #     status = response[0]["status"]
+    #     assert status[0]["model_class"] == "ContainerDependency"
+    #     assert status[0]["dependency_type"] == self.container_type
+    #     self._assert_container_description_identifier(
+    #         status[0]["container_description"]["identifier"],
+    #         "mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:a6419f25efff953fc505dbd5ee734856180bb619-0",
+    #     )
 
     def _assert_container_description_identifier(self, identifier: str, expected_hash: str):
         """

diff --git a/test/integration/test_job_environments.py b/test/integration/test_job_environments.py
@@ -26,6 +26,7 @@
         "home",
         "tmp",
         "some_env",
+        "jobconf_env_var",
     ],
 )
 
@@ -60,7 +61,8 @@ def _environment_properties(self, history_id):
         home = self.dataset_populator.get_history_dataset_content(history_id, hid=4).strip()
         tmp = self.dataset_populator.get_history_dataset_content(history_id, hid=5).strip()
         some_env = self.dataset_populator.get_history_dataset_content(history_id, hid=6).strip()
-        return JobEnvironmentProperties(user_id, group_id, pwd, home, tmp, some_env)
+        jobconf_env_var = self.dataset_populator.get_history_dataset_content(history_id, hid=7).strip()
+        return JobEnvironmentProperties(user_id, group_id, pwd, home, tmp, some_env, jobconf_env_var)
 
     def _check_completed_history(self, history_id):
         """Extension point that lets subclasses investigate the completed job."""