diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index 1d1986dee1a8..473c11e00d4e 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -90,6 +90,7 @@ class MinimalToolApp(Protocol): tool_data_tables: "ToolDataTableManager" file_sources: ConfiguredFileSources security: IdEncodingHelper + vault: Vault class MinimalApp(BasicSharedApp): diff --git a/lib/galaxy/tool_util/cwl/parser.py b/lib/galaxy/tool_util/cwl/parser.py index cc0b9944483a..3a0a1fd8c805 100644 --- a/lib/galaxy/tool_util/cwl/parser.py +++ b/lib/galaxy/tool_util/cwl/parser.py @@ -78,6 +78,7 @@ "SubworkflowFeatureRequirement", "StepInputExpressionRequirement", "MultipleInputFeatureRequirement", + "CredentialsRequirement", ] @@ -219,8 +220,8 @@ def software_requirements(self) -> List: def resource_requirements(self) -> List: return self.hints_or_requirements_of_class("ResourceRequirement") - def credentials(self) -> List: - return self.hints_or_requirements_of_class("Credentials") + def credentials_requirements(self) -> List: + return self.hints_or_requirements_of_class("CredentialsRequirement") class CommandLineToolProxy(ToolProxy): diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py index a9dfea4d66d7..54fc6957d45d 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py @@ -29,7 +29,7 @@ def _mulled_build_tool(tool, args): tool_source = get_tool_source(tool) - requirements, *_ = tool_source.parse_requirements_and_containers() + requirements, *_ = tool_source.parse_requirements() targets = requirements_to_mulled_targets(requirements) kwds = args_to_mull_targets_kwds(args) mull_targets(targets, **kwds) diff --git a/lib/galaxy/tool_util/linters/cwl.py b/lib/galaxy/tool_util/linters/cwl.py index c72bd433f0aa..6e761dee2fa5 100644 --- a/lib/galaxy/tool_util/linters/cwl.py +++ b/lib/galaxy/tool_util/linters/cwl.py @@ -63,7 +63,7 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): class CWLDockerMissing(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): - _, containers, *_ = tool_source.parse_requirements_and_containers() + _, containers, *_ = tool_source.parse_requirements() if len(containers) == 0: lint_ctx.warn("Tool does not specify a DockerPull source.") @@ -71,7 +71,7 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): class CWLDockerGood(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): - _, containers, *_ = tool_source.parse_requirements_and_containers() + _, containers, *_ = tool_source.parse_requirements() if len(containers) > 0: identifier = containers[0].identifier lint_ctx.info(f"Tool will run in Docker image [{identifier}].") diff --git a/lib/galaxy/tool_util/linters/general.py b/lib/galaxy/tool_util/linters/general.py index ddc2c4e6621c..3676dc6a651f 100644 --- a/lib/galaxy/tool_util/linters/general.py +++ b/lib/galaxy/tool_util/linters/general.py @@ -183,7 +183,7 @@ class RequirementNameMissing(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): _, tool_node = _tool_xml_and_root(tool_source) - requirements, *_ = tool_source.parse_requirements_and_containers() + requirements, *_ = tool_source.parse_requirements() for r in requirements: if r.type != "package": continue @@ -195,7 +195,7 @@ class RequirementVersionMissing(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): _, tool_node = _tool_xml_and_root(tool_source) - requirements, *_ = tool_source.parse_requirements_and_containers() + requirements, *_ = tool_source.parse_requirements() for r in requirements: if r.type != "package": continue @@ -207,7 +207,7 @@ class RequirementVersionWhitespace(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): _, tool_node = _tool_xml_and_root(tool_source) - requirements, *_ = tool_source.parse_requirements_and_containers() + requirements, *_ = tool_source.parse_requirements() for r in requirements: if r.type != "package": continue @@ -223,7 +223,7 @@ class ResourceRequirementExpression(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): _, tool_node = _tool_xml_and_root(tool_source) - *_, resource_requirements, _ = tool_source.parse_requirements_and_containers() + *_, resource_requirements, _ = tool_source.parse_requirements() for rr in resource_requirements: if rr.runtime_required: lint_ctx.warn( diff --git a/lib/galaxy/tool_util/parser/cwl.py b/lib/galaxy/tool_util/parser/cwl.py index 540f8edcfbdd..4d9d6d22ae47 100644 --- a/lib/galaxy/tool_util/parser/cwl.py +++ b/lib/galaxy/tool_util/parser/cwl.py @@ -154,7 +154,7 @@ def _parse_output(self, tool, output_instance): output.actions = ToolOutputActionGroup(output, None) return output - def parse_requirements_and_containers(self): + def parse_requirements(self): containers = [] docker_identifier = self.tool_proxy.docker_identifier() if docker_identifier: @@ -162,7 +162,7 @@ def parse_requirements_and_containers(self): software_requirements = self.tool_proxy.software_requirements() resource_requirements = self.tool_proxy.resource_requirements() - credentials = self.tool_proxy.credentials() + credentials = self.tool_proxy.credentials_requirements() return requirements.parse_requirements_from_lists( software_requirements=[{"name": r[0], "version": r[1], "type": "package"} for r in software_requirements], containers=containers, diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py index 0a8f3c2978c7..10337e9d7b86 100644 --- a/lib/galaxy/tool_util/parser/interface.py +++ b/lib/galaxy/tool_util/parser/interface.py @@ -306,7 +306,7 @@ def parse_required_files(self) -> Optional["RequiredFiles"]: return None @abstractmethod - def parse_requirements_and_containers( + def parse_requirements( self, ) -> Tuple[ "ToolRequirements", List["ContainerDescription"], List["ResourceRequirement"], List["CredentialsRequirement"] diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 255fe9482519..7465ff7f1325 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -412,7 +412,7 @@ def parse_include_exclude_list(tag_name): as_dict["excludes"] = parse_include_exclude_list("exclude") return RequiredFiles.from_dict(as_dict) - def parse_requirements_and_containers(self): + def parse_requirements(self): return requirements.parse_requirements_from_xml(self.root, parse_resources_and_credentials=True) def parse_input_pages(self) -> "XmlPagesSource": diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index 673e5a2cb418..a64a14e2bc2a 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -109,7 +109,7 @@ def parse_version_command(self): def parse_version_command_interpreter(self): return self.root_dict.get("runtime_version", {}).get("interpreter", None) - def parse_requirements_and_containers(self): + def parse_requirements(self): mixed_requirements = self.root_dict.get("requirements", []) return requirements.parse_requirements_from_lists( software_requirements=[r for r in mixed_requirements if r.get("type") != "resource"], diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 839b32d963c6..995cfe639294 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1216,7 +1216,7 @@ def parse(self, tool_source: ToolSource, guid: Optional[str] = None, dynamic: bo raise Exception(message) # Requirements (dependencies) - requirements, containers, resource_requirements, credentials = tool_source.parse_requirements_and_containers() + requirements, containers, resource_requirements, credentials = tool_source.parse_requirements() self.requirements = requirements self.containers = containers self.resource_requirements = resource_requirements diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index e41a42b18b41..5c13b48e7349 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -9,7 +9,6 @@ from typing import ( Any, Callable, - cast, Dict, List, Optional, @@ -33,9 +32,9 @@ from galaxy.structured_app import ( BasicSharedApp, MinimalToolApp, - StructuredApp, ) from galaxy.tool_util.data import TabularToolDataTable +from galaxy.tool_util.deps.requirements import CredentialsRequirement from galaxy.tools.parameters import ( visit_input_values, wrapped_json, @@ -191,34 +190,21 @@ def set_compute_environment(self, compute_environment: ComputeEnvironment, get_s ) self.execute_tool_hooks(inp_data=inp_data, out_data=out_data, incoming=incoming) - if self.tool.credentials: - app = cast(StructuredApp, self.app) - user_vault = UserVaultWrapper(app.vault, self._user) - for credentials in self.tool.credentials: + # TODO: provide all information needed (variable value, current group, etc) to this part... + if hasattr(self.tool, "credentials"): + user_vault = UserVaultWrapper(self.app.vault, self._user) + tool_credentials: List[CredentialsRequirement] = self.tool.credentials + for credentials in tool_credentials: reference = credentials.reference - for secret in credentials.secret: - vault_value = user_vault.read_secret(f"{reference}|{secret.name}") or "" + current_group = "default" + tool_id = self.tool.id + for secret in credentials.secrets: + vault_ref = f"tool|{tool_id}|{reference}|{current_group}|{secret.name}" + vault_value = user_vault.read_secret(vault_ref) or "" self.environment_variables.append({"name": secret.inject_as_env, "value": vault_value}) - for variable in credentials.variable: - service_refrence = f"{reference}|{variable.name}" - app_model = app.model - query = ( - app_model.context.query(app_model.UserCredential) - .filter_by(user_id=self._user.id, service_reference=service_refrence) - .first() - ) - if query: - credential_id = query.id - credential = ( - app_model.context.query(app_model.Credential) - .filter_by(user_credential_id=credential_id, name=variable.name) - .first() - ) - if credential: - variable_value = credential.value - self.environment_variables.append({"name": variable.inject_as_env, "value": variable_value}) - else: - log.warning(f"Variable {variable.name} not found in credentials") + for variable in credentials.variables: + variable_value = "variable.value" + self.environment_variables.append({"name": variable.inject_as_env, "value": variable_value}) def execute_tool_hooks(self, inp_data, out_data, incoming): # Certain tools require tasks to be completed prior to job execution diff --git a/test/integration/test_vault_extra_prefs.py b/test/integration/test_vault_extra_prefs.py index d9166a553b8a..5072ac69d270 100644 --- a/test/integration/test_vault_extra_prefs.py +++ b/test/integration/test_vault_extra_prefs.py @@ -11,12 +11,6 @@ ) from galaxy.model.db.user import get_user_by_email - -# from galaxy_test.api.test_tools import TestsTools -# from galaxy_test.base.populators import ( -# DatasetPopulator, -# skip_without_tool, -# ) from galaxy_test.driver import integration_util TEST_USER_EMAIL = "vault_test_user@bx.psu.edu" @@ -140,30 +134,3 @@ def __url(self, action, user): def _get_dbuser(self, app, user): return get_user_by_email(app.model.session, user["email"]) - - -# class TestSecretsInExtraUserPreferences( -# integration_util.IntegrationTestCase, integration_util.ConfiguresDatabaseVault, TestsTools -# ): -# dataset_populator: DatasetPopulator - -# @classmethod -# def handle_galaxy_config_kwds(cls, config): -# super().handle_galaxy_config_kwds(config) -# cls._configure_database_vault(config) -# config["user_preferences_extra_conf_path"] = os.path.join( -# os.path.dirname(__file__), "user_preferences_extra_conf.yml" -# ) - -# def setUp(self): -# super().setUp() -# self.dataset_populator = DatasetPopulator(self.galaxy_interactor) - -# @skip_without_tool("secret_tool") -# def test_secrets_tool(self, history_id): -# user = self._setup_user(TEST_USER_EMAIL) -# url = self._api_url(f"users/{user['id']}/information/inputs", params=dict(key=self.master_api_key)) -# put(url, data=json.dumps({"secret_tool|api_key": "test"})) -# run_response = self._run("secret", history_id, assert_ok=True) -# outputs = run_response["outputs"] -# assert outputs[0]["extra_files"][0]["value"] == "test" diff --git a/test/unit/tool_util/test_cwl.py b/test/unit/tool_util/test_cwl.py index 8e95793cc8a9..f9d48cd503b3 100644 --- a/test/unit/tool_util/test_cwl.py +++ b/test/unit/tool_util/test_cwl.py @@ -281,7 +281,7 @@ def test_load_proxy_simple(): outputs, output_collections = tool_source.parse_outputs(None) assert len(outputs) == 1 - software_requirements, containers, resource_requirements, _ = tool_source.parse_requirements_and_containers() + software_requirements, containers, resource_requirements, credentials = tool_source.parse_requirements() assert software_requirements.to_dict() == [] assert len(containers) == 1 assert containers[0].to_dict() == { @@ -292,6 +292,7 @@ def test_load_proxy_simple(): } assert len(resource_requirements) == 1 assert resource_requirements[0].to_dict() == {"resource_type": "ram_min", "value_or_expression": 8} + assert len(credentials) == 0 def test_representation_id(): diff --git a/test/unit/tool_util/test_parsing.py b/test/unit/tool_util/test_parsing.py index ea4ddb1f334f..16991d3129d7 100644 --- a/test/unit/tool_util/test_parsing.py +++ b/test/unit/tool_util/test_parsing.py @@ -164,6 +164,24 @@ containers: - type: docker identifier: "awesome/bowtie" +credentials: + - name: Apollo + reference: gmod.org/apollo + optional: true + secrets: + - name: username + label: Your Apollo username + description: Username for Apollo + inject_as_env: apollo_user + - name: password + label: Your Apollo password + description: Password for Apollo + inject_as_env: apollo_pass + variables: + - name: server + label: Your Apollo server + description: URL of your Apollo server + inject_as_env: apollo_url outputs: out1: format: bam @@ -352,7 +370,7 @@ def test_action(self): assert self._tool_source.parse_action_module() is None def test_requirements(self): - requirements, containers, resource_requirements, _ = self._tool_source.parse_requirements_and_containers() + requirements, containers, resource_requirements, credentials = self._tool_source.parse_requirements() assert requirements[0].type == "package" assert list(containers)[0].identifier == "mycool/bwa" assert resource_requirements[0].resource_type == "cores_min" @@ -363,9 +381,6 @@ def test_requirements(self): assert resource_requirements[5].resource_type == "cuda_device_count_max" assert resource_requirements[6].resource_type == "shm_size" assert not resource_requirements[0].runtime_required - - def test_credentials(self): - *_, credentials = self._tool_source.parse_requirements_and_containers() assert credentials[0].name == "Apollo" assert credentials[0].reference == "gmod.org/apollo" assert credentials[0].optional @@ -546,9 +561,7 @@ def test_action(self): assert self._tool_source.parse_action_module() is None def test_requirements(self): - software_requirements, containers, resource_requirements, _ = ( - self._tool_source.parse_requirements_and_containers() - ) + software_requirements, containers, resource_requirements, credentials = self._tool_source.parse_requirements() assert software_requirements.to_dict() == [{"name": "bwa", "type": "package", "version": "1.0.1", "specs": []}] assert len(containers) == 1 assert containers[0].to_dict() == { @@ -577,6 +590,9 @@ def test_requirements(self): "resource_type": "shm_size", "value_or_expression": 67108864, } + assert len(credentials) == 1 + assert len(credentials[0].secrets) == 2 + assert len(credentials[0].variables) == 1 def test_outputs(self): outputs, output_collections = self._tool_source.parse_outputs(object())