From 7a0e66e96c11e85afff7dedbd6664848a4540feb Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 7 Aug 2023 17:29:45 -0400 Subject: [PATCH 1/6] Move get_user_by_username function to managers.users --- lib/galaxy/managers/users.py | 10 ++++++++++ lib/galaxy/util/tool_shed/common_util.py | 11 ----------- lib/tool_shed/util/common_util.py | 2 -- lib/tool_shed/webapp/controllers/repository.py | 5 +++-- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index bd739a169ffe..34304fe4162e 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -20,6 +20,7 @@ and_, exc, func, + select, true, ) from sqlalchemy.orm.exc import NoResultFound @@ -837,3 +838,12 @@ def _add_parsers(self): ) self.fn_filter_parsers.update({}) + + +def get_user_by_username(app, username): + """Get a user from the database by username.""" + try: + stmt = select(app.model.User).filter(app.model.User.username == username) + return app.model.session.execute(stmt).scalar_one() + except Exception: + return None diff --git a/lib/galaxy/util/tool_shed/common_util.py b/lib/galaxy/util/tool_shed/common_util.py index 49f17a7f8f31..6636897ceea8 100644 --- a/lib/galaxy/util/tool_shed/common_util.py +++ b/lib/galaxy/util/tool_shed/common_util.py @@ -163,16 +163,6 @@ def get_tool_shed_repository_url(app: HasToolShedRegistry, tool_shed: str, owner return tool_shed_url -def get_user_by_username(app, username): - """Get a user from the database by username.""" - sa_session = app.model.session - try: - user = sa_session.query(app.model.User).filter(app.model.User.table.c.username == username).one() - return user - except Exception: - return None - - def handle_galaxy_url(trans, **kwd): galaxy_url = kwd.get("galaxy_url", None) if galaxy_url: @@ -303,7 +293,6 @@ def remove_protocol_from_tool_shed_url(tool_shed_url: str) -> str: "get_tool_shed_repository_ids", "get_tool_shed_url_from_tool_shed_registry", "get_tool_shed_repository_url", - "get_user_by_username", "handle_galaxy_url", "handle_tool_shed_url_protocol", "parse_repository_dependency_tuple", diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index f500f9f7fa6a..00ce0d153ff0 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -9,7 +9,6 @@ get_tool_shed_repository_ids, get_tool_shed_repository_url, get_tool_shed_url_from_tool_shed_registry, - get_user_by_username, handle_galaxy_url, handle_tool_shed_url_protocol, parse_repository_dependency_tuple, @@ -30,7 +29,6 @@ "get_tool_shed_repository_ids", "get_tool_shed_url_from_tool_shed_registry", "get_tool_shed_repository_url", - "get_user_by_username", "handle_galaxy_url", "handle_tool_shed_url_protocol", "parse_repository_dependency_tuple", diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py index d82ac50f5a6d..899574f653b9 100644 --- a/lib/tool_shed/webapp/controllers/repository.py +++ b/lib/tool_shed/webapp/controllers/repository.py @@ -24,6 +24,7 @@ util, web, ) +from galaxy.managers.users import get_user_by_username from galaxy.model.base import transaction from galaxy.tool_shed.util import dependency_display from galaxy.tools.repositories import ValidationContext @@ -2292,7 +2293,7 @@ def set_malicious(self, trans, id, ctx_str, **kwd): def sharable_owner(self, trans, owner): """Support for sharable URL for each repository owner's tools, e.g. http://example.org/view/owner.""" try: - user = common_util.get_user_by_username(trans, owner) + user = get_user_by_username(trans, owner) except Exception: user = None if user: @@ -2320,7 +2321,7 @@ def sharable_repository(self, trans, owner, name): else: # If the owner is valid, then show all of their repositories. try: - user = common_util.get_user_by_username(trans, owner) + user = get_user_by_username(trans, owner) except Exception: user = None if user: From e43761b3555039a284c36efcc671fe4e95fddb96 Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 7 Aug 2023 19:03:41 -0400 Subject: [PATCH 2/6] Move dbkeys to managers --- lib/galaxy/app.py | 2 +- lib/galaxy/app_unittest_utils/galaxy_mock.py | 2 +- lib/galaxy/{util => managers}/dbkeys.py | 12 ++++++++---- lib/galaxy/structured_app.py | 2 +- lib/galaxy/tools/parameters/basic.py | 2 +- lib/galaxy/tools/remote_tool_eval.py | 2 +- lib/galaxy/tools/repositories.py | 2 +- lib/tool_shed/webapp/app.py | 2 +- 8 files changed, 15 insertions(+), 11 deletions(-) rename lib/galaxy/{util => managers}/dbkeys.py (97%) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 5af6173eb86a..c33207390d23 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -41,6 +41,7 @@ from galaxy.managers.api_keys import ApiKeyManager from galaxy.managers.citations import CitationsManager from galaxy.managers.collections import DatasetCollectionManager +from galaxy.managers.dbkeys import GenomeBuilds from galaxy.managers.folders import FolderManager from galaxy.managers.hdas import HDAManager from galaxy.managers.histories import HistoryManager @@ -126,7 +127,6 @@ listify, StructuredExecutionTimer, ) -from galaxy.util.dbkeys import GenomeBuilds from galaxy.util.task import IntervalTask from galaxy.util.tool_shed import tool_shed_registry from galaxy.visualization.data_providers.registry import DataProviderRegistry diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index 049cad5908ca..fd01cebc8e73 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -22,6 +22,7 @@ from galaxy.job_metrics import JobMetrics from galaxy.jobs.manager import NoopManager from galaxy.managers.collections import DatasetCollectionManager +from galaxy.managers.dbkeys import GenomeBuilds from galaxy.managers.hdas import HDAManager from galaxy.managers.histories import HistoryManager from galaxy.managers.jobs import JobSearch @@ -51,7 +52,6 @@ from galaxy.tools.data import ToolDataTableManager from galaxy.util import StructuredExecutionTimer from galaxy.util.bunch import Bunch -from galaxy.util.dbkeys import GenomeBuilds from galaxy.web.short_term_storage import ( ShortTermStorageAllocator, ShortTermStorageConfiguration, diff --git a/lib/galaxy/util/dbkeys.py b/lib/galaxy/managers/dbkeys.py similarity index 97% rename from lib/galaxy/util/dbkeys.py rename to lib/galaxy/managers/dbkeys.py index a9da14f4d4a3..736f9952d337 100644 --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/managers/dbkeys.py @@ -5,6 +5,10 @@ import os.path import re from json import loads +from typing import ( + Dict, + List, +) from galaxy.util import ( galaxy_directory, @@ -17,9 +21,9 @@ def read_dbnames(filename): """Read build names from file""" - db_names = [] + db_names: List = [] try: - ucsc_builds = {} + ucsc_builds: Dict = {} man_builds = [] # assume these are integers name_to_db_base = {} if filename is None: @@ -43,9 +47,9 @@ def read_dbnames(filename): ucsc_builds[db_base] = [] name_to_db_base[fields[1]] = db_base # we want to sort within a species numerically by revision number - build_rev = re.compile(r"\d+$") + pattern = re.compile(r"\d+$") try: - build_rev = int(build_rev.findall(fields[0])[0]) + build_rev = int(pattern.findall(fields[0])[0]) except Exception: build_rev = 0 ucsc_builds[db_base].append((build_rev, fields[0], fields[1])) diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index 5878e0e0d147..f21de41d4aca 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -14,6 +14,7 @@ from galaxy.di import Container from galaxy.files import ConfiguredFileSources from galaxy.job_metrics import JobMetrics +from galaxy.managers.dbkeys import GenomeBuilds from galaxy.model.base import ( ModelMapping, SharedModelMapping, @@ -32,7 +33,6 @@ from galaxy.tool_util.deps.containers import ContainerFinder from galaxy.tool_util.deps.views import DependencyResolversView from galaxy.tool_util.verify import test_data -from galaxy.util.dbkeys import GenomeBuilds from galaxy.util.tool_shed.tool_shed_registry import Registry as ToolShedRegistry from galaxy.web_stack import ApplicationStack from galaxy.webhooks import WebhooksRegistry diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index be1ed9002c0d..bde8e351959b 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -21,6 +21,7 @@ from galaxy import util from galaxy.files import ProvidesUserFileSourcesUserContext +from galaxy.managers import dbkeys from galaxy.model import ( cached_id, Dataset, @@ -33,7 +34,6 @@ from galaxy.schema.fetch_data import FilesPayload from galaxy.tool_util.parser import get_input_source as ensure_input_source from galaxy.util import ( - dbkeys, sanitize_param, string_as_bool, string_as_bool_or_none, diff --git a/lib/galaxy/tools/remote_tool_eval.py b/lib/galaxy/tools/remote_tool_eval.py index 8dd158f822d1..074a99f17e97 100644 --- a/lib/galaxy/tools/remote_tool_eval.py +++ b/lib/galaxy/tools/remote_tool_eval.py @@ -12,6 +12,7 @@ from galaxy.files import ConfiguredFileSources from galaxy.job_execution.compute_environment import SharedComputeEnvironment from galaxy.job_execution.setup import JobIO +from galaxy.managers.dbkeys import GenomeBuilds from galaxy.metadata.set_metadata import ( get_metadata_params, get_object_store, @@ -30,7 +31,6 @@ ToolDataTableManager, ) from galaxy.util.bunch import Bunch -from galaxy.util.dbkeys import GenomeBuilds class ToolAppConfig(NamedTuple): diff --git a/lib/galaxy/tools/repositories.py b/lib/galaxy/tools/repositories.py index bb4a3f436352..f8cfc8f43532 100644 --- a/lib/galaxy/tools/repositories.py +++ b/lib/galaxy/tools/repositories.py @@ -4,9 +4,9 @@ import tempfile from contextlib import contextmanager +from galaxy.managers.dbkeys import GenomeBuilds from galaxy.tools.data import ToolDataTableManager from galaxy.util.bunch import Bunch -from galaxy.util.dbkeys import GenomeBuilds class ValidationContext: diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index f9a9d5632df1..bdd32ef95d86 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -18,6 +18,7 @@ from galaxy.config import configure_logging from galaxy.managers.api_keys import ApiKeyManager from galaxy.managers.citations import CitationsManager +from galaxy.managers.dbkeys import GenomeBuilds from galaxy.managers.users import UserManager from galaxy.model.base import SharedModelMapping from galaxy.model.tags import CommunityTagHandler @@ -27,7 +28,6 @@ ) from galaxy.security import idencoding from galaxy.structured_app import BasicSharedApp -from galaxy.util.dbkeys import GenomeBuilds from galaxy.web_stack import application_stack_instance from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager from tool_shed.structured_app import ToolShedApp From bc60013c30a23298a833e674bdde288bff3fa9d9 Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 8 Aug 2023 13:29:10 -0400 Subject: [PATCH 3/6] Only pass the objects that are used by the function --- lib/galaxy/managers/users.py | 11 +++++++---- lib/tool_shed/webapp/controllers/repository.py | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 34304fe4162e..4489a00689e6 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -840,10 +840,13 @@ def _add_parsers(self): self.fn_filter_parsers.update({}) -def get_user_by_username(app, username): - """Get a user from the database by username.""" +def get_user_by_username(session, user_class, username): + """ + Get a user from the database by username. + (We pass the session and the user_class to accommodate usage from the tool_shed app.) + """ try: - stmt = select(app.model.User).filter(app.model.User.username == username) - return app.model.session.execute(stmt).scalar_one() + stmt = select(user_class).filter(user_class.username == username) + return session.execute(stmt).scalar_one() except Exception: return None diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py index 899574f653b9..e7627292f426 100644 --- a/lib/tool_shed/webapp/controllers/repository.py +++ b/lib/tool_shed/webapp/controllers/repository.py @@ -2293,7 +2293,7 @@ def set_malicious(self, trans, id, ctx_str, **kwd): def sharable_owner(self, trans, owner): """Support for sharable URL for each repository owner's tools, e.g. http://example.org/view/owner.""" try: - user = get_user_by_username(trans, owner) + user = get_user_by_username(trans.model.session, trans.model.User, owner) except Exception: user = None if user: @@ -2321,7 +2321,7 @@ def sharable_repository(self, trans, owner, name): else: # If the owner is valid, then show all of their repositories. try: - user = get_user_by_username(trans, owner) + user = get_user_by_username(trans.model.session, trans.model.User, owner) except Exception: user = None if user: From 7ff212c3fcac9fc2db10ed9866b2842a82bbbb79 Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 9 Aug 2023 14:04:40 -0400 Subject: [PATCH 4/6] Update mypy.ini --- mypy.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/mypy.ini b/mypy.ini index 591927279feb..99ab2d8a9073 100644 --- a/mypy.ini +++ b/mypy.ini @@ -167,8 +167,6 @@ check_untyped_defs = False check_untyped_defs = False [mypy-galaxy.util.sanitize_html] check_untyped_defs = False -[mypy-galaxy.util.dbkeys] -check_untyped_defs = False [mypy-galaxy.util.commands] check_untyped_defs = False [mypy-galaxy.util.tool_shed.xml_util] From 1168d30659972415d991b8d069029a18bf1856f8 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 9 Aug 2023 19:03:48 +0100 Subject: [PATCH 5/6] Improve type annotation of ``read_dbnames()`` --- lib/galaxy/managers/dbkeys.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/managers/dbkeys.py b/lib/galaxy/managers/dbkeys.py index 736f9952d337..ead48c9ee756 100644 --- a/lib/galaxy/managers/dbkeys.py +++ b/lib/galaxy/managers/dbkeys.py @@ -8,6 +8,8 @@ from typing import ( Dict, List, + Optional, + Tuple, ) from galaxy.util import ( @@ -19,11 +21,11 @@ log = logging.getLogger(__name__) -def read_dbnames(filename): +def read_dbnames(filename: Optional[str]) -> List[Tuple[str, str]]: """Read build names from file""" - db_names: List = [] + db_names: List[Tuple[str, str]] = [] try: - ucsc_builds: Dict = {} + ucsc_builds: Dict[str, List[Tuple[int, str, str]]] = {} man_builds = [] # assume these are integers name_to_db_base = {} if filename is None: @@ -60,11 +62,10 @@ def read_dbnames(filename): db_base = name_to_db_base[name] ucsc_builds[db_base].sort() ucsc_builds[db_base].reverse() - ucsc_builds[db_base] = [(build, name) for _, build, name in ucsc_builds[db_base]] - db_names = list(db_names + ucsc_builds[db_base]) + db_names += [(build, name) for _, build, name in ucsc_builds[db_base]] man_builds.sort() man_builds = [(build, name) for name, build in man_builds] - db_names = list(db_names + man_builds) + db_names += man_builds except Exception as e: log.error("ERROR: Unable to read builds file: %s", unicodify(e)) return db_names From 0d0d5308c59d143ff77f98fc2ee4ff41ed3a5b32 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 9 Aug 2023 19:04:50 +0100 Subject: [PATCH 6/6] More specific import --- lib/galaxy/tools/parameters/basic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index bde8e351959b..6de845c6fd32 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -21,7 +21,7 @@ from galaxy import util from galaxy.files import ProvidesUserFileSourcesUserContext -from galaxy.managers import dbkeys +from galaxy.managers.dbkeys import read_dbnames from galaxy.model import ( cached_id, Dataset, @@ -1164,7 +1164,7 @@ class GenomeBuildParameter(SelectToolParameter): >>> # Create a mock transaction with 'hg17' as the current build >>> from galaxy.util.bunch import Bunch - >>> trans = Bunch(app=None, history=Bunch(genome_build='hg17'), db_builds=dbkeys.read_dbnames(None)) + >>> trans = Bunch(app=None, history=Bunch(genome_build='hg17'), db_builds=read_dbnames(None)) >>> p = GenomeBuildParameter(None, XML('')) >>> print(p.name) _name @@ -1220,7 +1220,7 @@ def to_dict(self, trans, other_values=None): def _get_dbkey_names(self, trans=None): if not self.tool: # Hack for unit tests, since we have no tool - return dbkeys.read_dbnames(None) + return read_dbnames(None) return self.tool.app.genome_builds.get_genome_build_names(trans=trans) @@ -1546,7 +1546,7 @@ class DrillDownSelectToolParameter(SelectToolParameter): Creating a hierarchical select menu, which allows users to 'drill down' a tree-like set of options. >>> from galaxy.util.bunch import Bunch - >>> trans = Bunch(app=None, history=Bunch(genome_build='hg17'), db_builds=dbkeys.read_dbnames(None)) + >>> trans = Bunch(app=None, history=Bunch(genome_build='hg17'), db_builds=read_dbnames(None)) >>> p = DrillDownSelectToolParameter(None, XML( ... ''' ...