rename parse_requirements_and_containers to parse_requirements,

remove unused tests,
add credential yaml test,
refactor tool evaluator (needs more work)

lib/galaxy/structured_app.py

@@ -90,6 +90,7 @@ class MinimalToolApp(Protocol):
     tool_data_tables: "ToolDataTableManager"
     file_sources: ConfiguredFileSources
     security: IdEncodingHelper
+    vault: Vault
 
 
 class MinimalApp(BasicSharedApp):

lib/galaxy/tool_util/cwl/parser.py

@@ -78,6 +78,7 @@
     "SubworkflowFeatureRequirement",
     "StepInputExpressionRequirement",
     "MultipleInputFeatureRequirement",
+    "CredentialsRequirement",
 ]
 
 
@@ -219,8 +220,8 @@ def software_requirements(self) -> List:
     def resource_requirements(self) -> List:
         return self.hints_or_requirements_of_class("ResourceRequirement")
 
-    def credentials(self) -> List:
-        return self.hints_or_requirements_of_class("Credentials")
+    def credentials_requirements(self) -> List:
+        return self.hints_or_requirements_of_class("CredentialsRequirement")
 
 
 class CommandLineToolProxy(ToolProxy):

lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py

@@ -29,7 +29,7 @@
 
 def _mulled_build_tool(tool, args):
     tool_source = get_tool_source(tool)
-    requirements, *_ = tool_source.parse_requirements_and_containers()
+    requirements, *_ = tool_source.parse_requirements()
     targets = requirements_to_mulled_targets(requirements)
     kwds = args_to_mull_targets_kwds(args)
     mull_targets(targets, **kwds)

lib/galaxy/tool_util/linters/cwl.py

@@ -63,15 +63,15 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
 class CWLDockerMissing(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
-        _, containers, *_ = tool_source.parse_requirements_and_containers()
+        _, containers, *_ = tool_source.parse_requirements()
         if len(containers) == 0:
             lint_ctx.warn("Tool does not specify a DockerPull source.")
 
 
 class CWLDockerGood(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
-        _, containers, *_ = tool_source.parse_requirements_and_containers()
+        _, containers, *_ = tool_source.parse_requirements()
         if len(containers) > 0:
             identifier = containers[0].identifier
             lint_ctx.info(f"Tool will run in Docker image [{identifier}].")

lib/galaxy/tool_util/linters/general.py

@@ -183,7 +183,7 @@ class RequirementNameMissing(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
         _, tool_node = _tool_xml_and_root(tool_source)
-        requirements, *_ = tool_source.parse_requirements_and_containers()
+        requirements, *_ = tool_source.parse_requirements()
         for r in requirements:
             if r.type != "package":
                 continue
@@ -195,7 +195,7 @@ class RequirementVersionMissing(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
         _, tool_node = _tool_xml_and_root(tool_source)
-        requirements, *_ = tool_source.parse_requirements_and_containers()
+        requirements, *_ = tool_source.parse_requirements()
         for r in requirements:
             if r.type != "package":
                 continue
@@ -207,7 +207,7 @@ class RequirementVersionWhitespace(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
         _, tool_node = _tool_xml_and_root(tool_source)
-        requirements, *_ = tool_source.parse_requirements_and_containers()
+        requirements, *_ = tool_source.parse_requirements()
         for r in requirements:
             if r.type != "package":
                 continue
@@ -223,7 +223,7 @@ class ResourceRequirementExpression(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
         _, tool_node = _tool_xml_and_root(tool_source)
-        *_, resource_requirements, _ = tool_source.parse_requirements_and_containers()
+        *_, resource_requirements, _ = tool_source.parse_requirements()
         for rr in resource_requirements:
             if rr.runtime_required:
                 lint_ctx.warn(

lib/galaxy/tool_util/parser/cwl.py

@@ -154,15 +154,15 @@ def _parse_output(self, tool, output_instance):
         output.actions = ToolOutputActionGroup(output, None)
         return output
 
-    def parse_requirements_and_containers(self):
+    def parse_requirements(self):
         containers = []
         docker_identifier = self.tool_proxy.docker_identifier()
         if docker_identifier:
             containers.append({"type": "docker", "identifier": docker_identifier})
 
         software_requirements = self.tool_proxy.software_requirements()
         resource_requirements = self.tool_proxy.resource_requirements()
-        credentials = self.tool_proxy.credentials()
+        credentials = self.tool_proxy.credentials_requirements()
         return requirements.parse_requirements_from_lists(
             software_requirements=[{"name": r[0], "version": r[1], "type": "package"} for r in software_requirements],
             containers=containers,

lib/galaxy/tool_util/parser/interface.py

@@ -306,7 +306,7 @@ def parse_required_files(self) -> Optional["RequiredFiles"]:
         return None
 
     @abstractmethod
-    def parse_requirements_and_containers(
+    def parse_requirements(
         self,
     ) -> Tuple[
         "ToolRequirements", List["ContainerDescription"], List["ResourceRequirement"], List["CredentialsRequirement"]

lib/galaxy/tool_util/parser/xml.py

@@ -412,7 +412,7 @@ def parse_include_exclude_list(tag_name):
         as_dict["excludes"] = parse_include_exclude_list("exclude")
         return RequiredFiles.from_dict(as_dict)
 
-    def parse_requirements_and_containers(self):
+    def parse_requirements(self):
         return requirements.parse_requirements_from_xml(self.root, parse_resources_and_credentials=True)
 
     def parse_input_pages(self) -> "XmlPagesSource":

lib/galaxy/tool_util/parser/yaml.py

@@ -109,7 +109,7 @@ def parse_version_command(self):
     def parse_version_command_interpreter(self):
         return self.root_dict.get("runtime_version", {}).get("interpreter", None)
 
-    def parse_requirements_and_containers(self):
+    def parse_requirements(self):
         mixed_requirements = self.root_dict.get("requirements", [])
         return requirements.parse_requirements_from_lists(
             software_requirements=[r for r in mixed_requirements if r.get("type") != "resource"],

lib/galaxy/tools/__init__.py

@@ -1216,7 +1216,7 @@ def parse(self, tool_source: ToolSource, guid: Optional[str] = None, dynamic: bo
                     raise Exception(message)
 
         # Requirements (dependencies)
-        requirements, containers, resource_requirements, credentials = tool_source.parse_requirements_and_containers()
+        requirements, containers, resource_requirements, credentials = tool_source.parse_requirements()
         self.requirements = requirements
         self.containers = containers
         self.resource_requirements = resource_requirements

lib/galaxy/tools/evaluation.py

@@ -9,7 +9,6 @@
 from typing import (
     Any,
     Callable,
-    cast,
     Dict,
     List,
     Optional,
@@ -33,9 +32,9 @@
 from galaxy.structured_app import (
     BasicSharedApp,
     MinimalToolApp,
-    StructuredApp,
 )
 from galaxy.tool_util.data import TabularToolDataTable
+from galaxy.tool_util.deps.requirements import CredentialsRequirement
 from galaxy.tools.parameters import (
     visit_input_values,
     wrapped_json,
@@ -191,34 +190,21 @@ def set_compute_environment(self, compute_environment: ComputeEnvironment, get_s
         )
         self.execute_tool_hooks(inp_data=inp_data, out_data=out_data, incoming=incoming)
 
-        if self.tool.credentials:
-            app = cast(StructuredApp, self.app)
-            user_vault = UserVaultWrapper(app.vault, self._user)
-            for credentials in self.tool.credentials:
+        # TODO: provide all information needed (variable value, current group, etc) to this part...
+        if hasattr(self.tool, "credentials"):
+            user_vault = UserVaultWrapper(self.app.vault, self._user)
+            tool_credentials: List[CredentialsRequirement] = self.tool.credentials
+            for credentials in tool_credentials:
                 reference = credentials.reference
-                for secret in credentials.secret:
-                    vault_value = user_vault.read_secret(f"{reference}|{secret.name}") or ""
+                current_group = "default"
+                tool_id = self.tool.id
+                for secret in credentials.secrets:
+                    vault_ref = f"tool|{tool_id}|{reference}|{current_group}|{secret.name}"
+                    vault_value = user_vault.read_secret(vault_ref) or ""
                     self.environment_variables.append({"name": secret.inject_as_env, "value": vault_value})
-                for variable in credentials.variable:
-                    service_refrence = f"{reference}|{variable.name}"
-                    app_model = app.model
-                    query = (
-                        app_model.context.query(app_model.UserCredential)
-                        .filter_by(user_id=self._user.id, service_reference=service_refrence)
-                        .first()
-                    )
-                    if query:
-                        credential_id = query.id
-                        credential = (
-                            app_model.context.query(app_model.Credential)
-                            .filter_by(user_credential_id=credential_id, name=variable.name)
-                            .first()
-                        )
-                        if credential:
-                            variable_value = credential.value
-                            self.environment_variables.append({"name": variable.inject_as_env, "value": variable_value})
-                        else:
-                            log.warning(f"Variable {variable.name} not found in credentials")
+                for variable in credentials.variables:
+                    variable_value = "variable.value"
+                    self.environment_variables.append({"name": variable.inject_as_env, "value": variable_value})
 
     def execute_tool_hooks(self, inp_data, out_data, incoming):
         # Certain tools require tasks to be completed prior to job execution

test/integration/test_vault_extra_prefs.py

@@ -11,12 +11,6 @@
 )
 
 from galaxy.model.db.user import get_user_by_email
-
-# from galaxy_test.api.test_tools import TestsTools
-# from galaxy_test.base.populators import (
-#     DatasetPopulator,
-#     skip_without_tool,
-# )
 from galaxy_test.driver import integration_util
 
 TEST_USER_EMAIL = "[email protected]"
@@ -140,30 +134,3 @@ def __url(self, action, user):
 
     def _get_dbuser(self, app, user):
         return get_user_by_email(app.model.session, user["email"])
-
-
-# class TestSecretsInExtraUserPreferences(
-#     integration_util.IntegrationTestCase, integration_util.ConfiguresDatabaseVault, TestsTools
-# ):
-#     dataset_populator: DatasetPopulator
-
-#     @classmethod
-#     def handle_galaxy_config_kwds(cls, config):
-#         super().handle_galaxy_config_kwds(config)
-#         cls._configure_database_vault(config)
-#         config["user_preferences_extra_conf_path"] = os.path.join(
-#             os.path.dirname(__file__), "user_preferences_extra_conf.yml"
-#         )
-
-#     def setUp(self):
-#         super().setUp()
-#         self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
-
-#     @skip_without_tool("secret_tool")
-#     def test_secrets_tool(self, history_id):
-#         user = self._setup_user(TEST_USER_EMAIL)
-#         url = self._api_url(f"users/{user['id']}/information/inputs", params=dict(key=self.master_api_key))
-#         put(url, data=json.dumps({"secret_tool|api_key": "test"}))
-#         run_response = self._run("secret", history_id, assert_ok=True)
-#         outputs = run_response["outputs"]
-#         assert outputs[0]["extra_files"][0]["value"] == "test"

test/unit/tool_util/test_cwl.py

@@ -281,7 +281,7 @@ def test_load_proxy_simple():
     outputs, output_collections = tool_source.parse_outputs(None)
     assert len(outputs) == 1
 
-    software_requirements, containers, resource_requirements, _ = tool_source.parse_requirements_and_containers()
+    software_requirements, containers, resource_requirements, credentials = tool_source.parse_requirements()
     assert software_requirements.to_dict() == []
     assert len(containers) == 1
     assert containers[0].to_dict() == {
@@ -292,6 +292,7 @@ def test_load_proxy_simple():
     }
     assert len(resource_requirements) == 1
     assert resource_requirements[0].to_dict() == {"resource_type": "ram_min", "value_or_expression": 8}
+    assert len(credentials) == 0
 
 
 def test_representation_id():

test/unit/tool_util/test_parsing.py

@@ -164,6 +164,24 @@
 containers:
   - type: docker
     identifier: "awesome/bowtie"
+credentials:
+  - name: Apollo
+    reference: gmod.org/apollo
+    optional: true
+    secrets:
+      - name: username
+        label: Your Apollo username
+        description: Username for Apollo
+        inject_as_env: apollo_user
+      - name: password
+        label: Your Apollo password
+        description: Password for Apollo
+        inject_as_env: apollo_pass
+    variables:
+      - name: server
+        label: Your Apollo server
+        description: URL of your Apollo server
+        inject_as_env: apollo_url
 outputs:
   out1:
     format: bam
@@ -352,7 +370,7 @@ def test_action(self):
         assert self._tool_source.parse_action_module() is None
 
     def test_requirements(self):
-        requirements, containers, resource_requirements, _ = self._tool_source.parse_requirements_and_containers()
+        requirements, containers, resource_requirements, credentials = self._tool_source.parse_requirements()
         assert requirements[0].type == "package"
         assert list(containers)[0].identifier == "mycool/bwa"
         assert resource_requirements[0].resource_type == "cores_min"
@@ -363,9 +381,6 @@ def test_requirements(self):
         assert resource_requirements[5].resource_type == "cuda_device_count_max"
         assert resource_requirements[6].resource_type == "shm_size"
         assert not resource_requirements[0].runtime_required
-
-    def test_credentials(self):
-        *_, credentials = self._tool_source.parse_requirements_and_containers()
         assert credentials[0].name == "Apollo"
         assert credentials[0].reference == "gmod.org/apollo"
         assert credentials[0].optional
@@ -546,9 +561,7 @@ def test_action(self):
         assert self._tool_source.parse_action_module() is None
 
     def test_requirements(self):
-        software_requirements, containers, resource_requirements, _ = (
-            self._tool_source.parse_requirements_and_containers()
-        )
+        software_requirements, containers, resource_requirements, credentials = self._tool_source.parse_requirements()
         assert software_requirements.to_dict() == [{"name": "bwa", "type": "package", "version": "1.0.1", "specs": []}]
         assert len(containers) == 1
         assert containers[0].to_dict() == {
@@ -577,6 +590,9 @@ def test_requirements(self):
             "resource_type": "shm_size",
             "value_or_expression": 67108864,
         }
+        assert len(credentials) == 1
+        assert len(credentials[0].secrets) == 2
+        assert len(credentials[0].variables) == 1
 
     def test_outputs(self):
         outputs, output_collections = self._tool_source.parse_outputs(object())