Merge pull request #16526 from jdavcs/dev_util_no_db

Move database access code out of ``galaxy.util``

lib/galaxy/app.py

@@ -41,6 +41,7 @@
 from galaxy.managers.api_keys import ApiKeyManager
 from galaxy.managers.citations import CitationsManager
 from galaxy.managers.collections import DatasetCollectionManager
+from galaxy.managers.dbkeys import GenomeBuilds
 from galaxy.managers.folders import FolderManager
 from galaxy.managers.hdas import HDAManager
 from galaxy.managers.histories import HistoryManager
@@ -126,7 +127,6 @@
     listify,
     StructuredExecutionTimer,
 )
-from galaxy.util.dbkeys import GenomeBuilds
 from galaxy.util.task import IntervalTask
 from galaxy.util.tool_shed import tool_shed_registry
 from galaxy.visualization.data_providers.registry import DataProviderRegistry

lib/galaxy/app_unittest_utils/galaxy_mock.py

@@ -22,6 +22,7 @@
 from galaxy.job_metrics import JobMetrics
 from galaxy.jobs.manager import NoopManager
 from galaxy.managers.collections import DatasetCollectionManager
+from galaxy.managers.dbkeys import GenomeBuilds
 from galaxy.managers.hdas import HDAManager
 from galaxy.managers.histories import HistoryManager
 from galaxy.managers.jobs import JobSearch
@@ -51,7 +52,6 @@
 from galaxy.tools.data import ToolDataTableManager
 from galaxy.util import StructuredExecutionTimer
 from galaxy.util.bunch import Bunch
-from galaxy.util.dbkeys import GenomeBuilds
 from galaxy.web.short_term_storage import (
     ShortTermStorageAllocator,
     ShortTermStorageConfiguration,

lib/galaxy/util/dbkeys.py → lib/galaxy/managers/dbkeys.py

@@ -5,6 +5,12 @@
 import os.path
 import re
 from json import loads
+from typing import (
+    Dict,
+    List,
+    Optional,
+    Tuple,
+)
 
 from galaxy.util import (
     galaxy_directory,
@@ -15,11 +21,11 @@
 log = logging.getLogger(__name__)
 
 
-def read_dbnames(filename):
+def read_dbnames(filename: Optional[str]) -> List[Tuple[str, str]]:
     """Read build names from file"""
-    db_names = []
+    db_names: List[Tuple[str, str]] = []
     try:
-        ucsc_builds = {}
+        ucsc_builds: Dict[str, List[Tuple[int, str, str]]] = {}
         man_builds = []  # assume these are integers
         name_to_db_base = {}
         if filename is None:
@@ -43,9 +49,9 @@ def read_dbnames(filename):
                         ucsc_builds[db_base] = []
                         name_to_db_base[fields[1]] = db_base
                     # we want to sort within a species numerically by revision number
-                    build_rev = re.compile(r"\d+$")
+                    pattern = re.compile(r"\d+$")
                     try:
-                        build_rev = int(build_rev.findall(fields[0])[0])
+                        build_rev = int(pattern.findall(fields[0])[0])
                     except Exception:
                         build_rev = 0
                     ucsc_builds[db_base].append((build_rev, fields[0], fields[1]))
@@ -56,11 +62,10 @@ def read_dbnames(filename):
             db_base = name_to_db_base[name]
             ucsc_builds[db_base].sort()
             ucsc_builds[db_base].reverse()
-            ucsc_builds[db_base] = [(build, name) for _, build, name in ucsc_builds[db_base]]
-            db_names = list(db_names + ucsc_builds[db_base])
+            db_names += [(build, name) for _, build, name in ucsc_builds[db_base]]
         man_builds.sort()
         man_builds = [(build, name) for name, build in man_builds]
-        db_names = list(db_names + man_builds)
+        db_names += man_builds
     except Exception as e:
         log.error("ERROR: Unable to read builds file: %s", unicodify(e))
     return db_names

lib/galaxy/managers/users.py

@@ -20,6 +20,7 @@
     and_,
     exc,
     func,
+    select,
     true,
 )
 from sqlalchemy.orm.exc import NoResultFound
@@ -837,3 +838,15 @@ def _add_parsers(self):
         )
 
         self.fn_filter_parsers.update({})
+
+
+def get_user_by_username(session, user_class, username):
+    """
+    Get a user from the database by username.
+    (We pass the session and the user_class to accommodate usage from the tool_shed app.)
+    """
+    try:
+        stmt = select(user_class).filter(user_class.username == username)
+        return session.execute(stmt).scalar_one()
+    except Exception:
+        return None

lib/galaxy/structured_app.py

@@ -14,6 +14,7 @@
 from galaxy.di import Container
 from galaxy.files import ConfiguredFileSources
 from galaxy.job_metrics import JobMetrics
+from galaxy.managers.dbkeys import GenomeBuilds
 from galaxy.model.base import (
     ModelMapping,
     SharedModelMapping,
@@ -32,7 +33,6 @@
 from galaxy.tool_util.deps.containers import ContainerFinder
 from galaxy.tool_util.deps.views import DependencyResolversView
 from galaxy.tool_util.verify import test_data
-from galaxy.util.dbkeys import GenomeBuilds
 from galaxy.util.tool_shed.tool_shed_registry import Registry as ToolShedRegistry
 from galaxy.web_stack import ApplicationStack
 from galaxy.webhooks import WebhooksRegistry

lib/galaxy/tools/parameters/basic.py

@@ -21,6 +21,7 @@
 
 from galaxy import util
 from galaxy.files import ProvidesUserFileSourcesUserContext
+from galaxy.managers.dbkeys import read_dbnames
 from galaxy.model import (
     cached_id,
     Dataset,
@@ -33,7 +34,6 @@
 from galaxy.schema.fetch_data import FilesPayload
 from galaxy.tool_util.parser import get_input_source as ensure_input_source
 from galaxy.util import (
-    dbkeys,
     sanitize_param,
     string_as_bool,
     string_as_bool_or_none,
@@ -1164,7 +1164,7 @@ class GenomeBuildParameter(SelectToolParameter):
 
     >>> # Create a mock transaction with 'hg17' as the current build
     >>> from galaxy.util.bunch import Bunch
-    >>> trans = Bunch(app=None, history=Bunch(genome_build='hg17'), db_builds=dbkeys.read_dbnames(None))
+    >>> trans = Bunch(app=None, history=Bunch(genome_build='hg17'), db_builds=read_dbnames(None))
     >>> p = GenomeBuildParameter(None, XML('<param name="_name" type="genomebuild" value="hg17" />'))
     >>> print(p.name)
     _name
@@ -1220,7 +1220,7 @@ def to_dict(self, trans, other_values=None):
     def _get_dbkey_names(self, trans=None):
         if not self.tool:
             # Hack for unit tests, since we have no tool
-            return dbkeys.read_dbnames(None)
+            return read_dbnames(None)
         return self.tool.app.genome_builds.get_genome_build_names(trans=trans)
 
 
@@ -1546,7 +1546,7 @@ class DrillDownSelectToolParameter(SelectToolParameter):
     Creating a hierarchical select menu, which allows users to 'drill down' a tree-like set of options.
 
     >>> from galaxy.util.bunch import Bunch
-    >>> trans = Bunch(app=None, history=Bunch(genome_build='hg17'), db_builds=dbkeys.read_dbnames(None))
+    >>> trans = Bunch(app=None, history=Bunch(genome_build='hg17'), db_builds=read_dbnames(None))
     >>> p = DrillDownSelectToolParameter(None, XML(
     ... '''
     ... <param name="_name" type="drill_down" display="checkbox" hierarchy="recurse" multiple="true">

lib/galaxy/tools/remote_tool_eval.py

@@ -12,6 +12,7 @@
 from galaxy.files import ConfiguredFileSources
 from galaxy.job_execution.compute_environment import SharedComputeEnvironment
 from galaxy.job_execution.setup import JobIO
+from galaxy.managers.dbkeys import GenomeBuilds
 from galaxy.metadata.set_metadata import (
     get_metadata_params,
     get_object_store,
@@ -30,7 +31,6 @@
     ToolDataTableManager,
 )
 from galaxy.util.bunch import Bunch
-from galaxy.util.dbkeys import GenomeBuilds
 
 
 class ToolAppConfig(NamedTuple):

lib/galaxy/tools/repositories.py

@@ -4,9 +4,9 @@
 import tempfile
 from contextlib import contextmanager
 
+from galaxy.managers.dbkeys import GenomeBuilds
 from galaxy.tools.data import ToolDataTableManager
 from galaxy.util.bunch import Bunch
-from galaxy.util.dbkeys import GenomeBuilds
 
 
 class ValidationContext:

lib/galaxy/util/tool_shed/common_util.py

@@ -163,16 +163,6 @@ def get_tool_shed_repository_url(app: HasToolShedRegistry, tool_shed: str, owner
     return tool_shed_url
 
 
-def get_user_by_username(app, username):
-    """Get a user from the database by username."""
-    sa_session = app.model.session
-    try:
-        user = sa_session.query(app.model.User).filter(app.model.User.table.c.username == username).one()
-        return user
-    except Exception:
-        return None
-
-
 def handle_galaxy_url(trans, **kwd):
     galaxy_url = kwd.get("galaxy_url", None)
     if galaxy_url:
@@ -303,7 +293,6 @@ def remove_protocol_from_tool_shed_url(tool_shed_url: str) -> str:
     "get_tool_shed_repository_ids",
     "get_tool_shed_url_from_tool_shed_registry",
     "get_tool_shed_repository_url",
-    "get_user_by_username",
     "handle_galaxy_url",
     "handle_tool_shed_url_protocol",
     "parse_repository_dependency_tuple",

lib/tool_shed/util/common_util.py

@@ -9,7 +9,6 @@
     get_tool_shed_repository_ids,
     get_tool_shed_repository_url,
     get_tool_shed_url_from_tool_shed_registry,
-    get_user_by_username,
     handle_galaxy_url,
     handle_tool_shed_url_protocol,
     parse_repository_dependency_tuple,
@@ -30,7 +29,6 @@
     "get_tool_shed_repository_ids",
     "get_tool_shed_url_from_tool_shed_registry",
     "get_tool_shed_repository_url",
-    "get_user_by_username",
     "handle_galaxy_url",
     "handle_tool_shed_url_protocol",
     "parse_repository_dependency_tuple",

lib/tool_shed/webapp/app.py

@@ -18,6 +18,7 @@
 from galaxy.config import configure_logging
 from galaxy.managers.api_keys import ApiKeyManager
 from galaxy.managers.citations import CitationsManager
+from galaxy.managers.dbkeys import GenomeBuilds
 from galaxy.managers.users import UserManager
 from galaxy.model.base import SharedModelMapping
 from galaxy.model.tags import CommunityTagHandler
@@ -27,7 +28,6 @@
 )
 from galaxy.security import idencoding
 from galaxy.structured_app import BasicSharedApp
-from galaxy.util.dbkeys import GenomeBuilds
 from galaxy.web_stack import application_stack_instance
 from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager
 from tool_shed.structured_app import ToolShedApp

lib/tool_shed/webapp/controllers/repository.py

@@ -24,6 +24,7 @@
     util,
     web,
 )
+from galaxy.managers.users import get_user_by_username
 from galaxy.model.base import transaction
 from galaxy.tool_shed.util import dependency_display
 from galaxy.tools.repositories import ValidationContext
@@ -2292,7 +2293,7 @@ def set_malicious(self, trans, id, ctx_str, **kwd):
     def sharable_owner(self, trans, owner):
         """Support for sharable URL for each repository owner's tools, e.g. http://example.org/view/owner."""
         try:
-            user = common_util.get_user_by_username(trans, owner)
+            user = get_user_by_username(trans.model.session, trans.model.User, owner)
         except Exception:
             user = None
         if user:
@@ -2320,7 +2321,7 @@ def sharable_repository(self, trans, owner, name):
         else:
             # If the owner is valid, then show all of their repositories.
             try:
-                user = common_util.get_user_by_username(trans, owner)
+                user = get_user_by_username(trans.model.session, trans.model.User, owner)
             except Exception:
                 user = None
             if user:

mypy.ini

@@ -167,8 +167,6 @@ check_untyped_defs = False
 check_untyped_defs = False
 [mypy-galaxy.util.sanitize_html]
 check_untyped_defs = False
-[mypy-galaxy.util.dbkeys]
-check_untyped_defs = False
 [mypy-galaxy.util.commands]
 check_untyped_defs = False
 [mypy-galaxy.util.tool_shed.xml_util]