From 06b8db47ba02dd65c6da87618fb0c8753704c4b4 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 29 May 2024 18:56:30 +0200 Subject: [PATCH 01/16] Change hardcoded repo location in more place --- lib/tool_shed/util/hgweb_config.py | 1 + lib/tool_shed/util/repository_util.py | 4 ++-- lib/tool_shed/util/shed_index.py | 8 ++++---- lib/tool_shed/webapp/api/tools.py | 1 + lib/tool_shed/webapp/app.py | 1 + lib/tool_shed/webapp/model/__init__.py | 4 +++- scripts/tool_shed/build_ts_whoosh_index.py | 1 + 7 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/tool_shed/util/hgweb_config.py b/lib/tool_shed/util/hgweb_config.py index 8c1656406b0f..7bf230106bbe 100644 --- a/lib/tool_shed/util/hgweb_config.py +++ b/lib/tool_shed/util/hgweb_config.py @@ -20,6 +20,7 @@ def __init__(self): self.hgweb_config_dir = None self.in_memory_config = None self.lock = threading.Lock() + self.hgweb_repo_prefix = None def add_entry(self, lhs, rhs): """Add an entry in the hgweb.config file for a new repository.""" diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index c84b182ae993..2d52bfd6d65c 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -505,8 +505,8 @@ def update_repository(trans: "ProvidesUserContext", id: str, **kwds) -> Tuple[Op repo_dir = repository.repo_path(app) # Change the entry in the hgweb.config file for the repository. - old_lhs = f"repos/{repository.user.username}/{repository.name}" - new_lhs = f"repos/{repository.user.username}/{kwds['name']}" + old_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" + new_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{kwds['name']}" trans.app.hgweb_config_manager.change_entry(old_lhs, new_lhs, repo_dir) # Change the entry in the repository's hgrc file. diff --git a/lib/tool_shed/util/shed_index.py b/lib/tool_shed/util/shed_index.py index 4035a8c5c5c3..7a610ddcabdf 100644 --- a/lib/tool_shed/util/shed_index.py +++ b/lib/tool_shed/util/shed_index.py @@ -33,7 +33,7 @@ def _get_or_create_index(whoosh_index_dir): return get_or_create_index(whoosh_index_dir, repo_schema), get_or_create_index(tool_index_dir, tool_schema) -def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): +def build_index(whoosh_index_dir, file_path, hgweb_config_dir, hgweb_repo_prefix, dburi, **kwargs): """ Build two search indexes simultaneously One is for repositories and the other for tools. @@ -51,7 +51,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): execution_timer = ExecutionTimer() with repo_index.searcher() as searcher: - for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): + for repo in get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs): tools_list = repo.pop("tools_list") repo_id = repo["id"] indexed_document = searcher.document(id=repo_id) @@ -85,7 +85,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): return repos_indexed, tools_indexed -def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): +def get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs): """ Load repos from DB and included tools from .xml configs. """ @@ -126,7 +126,7 @@ def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): # Load all changesets of the repo for lineage. repo_path = os.path.join( - hgweb_config_dir, hgwcm.get_entry(os.path.join("repos", repo.user.username, repo.name)) + hgweb_config_dir, hgwcm.get_entry(os.path.join(hgweb_repo_prefix, repo.user.username, repo.name)) ) hg_repo = hg.repository(ui.ui(), repo_path.encode("utf-8")) lineage = [] diff --git a/lib/tool_shed/webapp/api/tools.py b/lib/tool_shed/webapp/api/tools.py index fdbdc1043a54..78f5db195ab6 100644 --- a/lib/tool_shed/webapp/api/tools.py +++ b/lib/tool_shed/webapp/api/tools.py @@ -35,6 +35,7 @@ def build_search_index(self, trans, **kwd): trans.app.config.whoosh_index_dir, trans.app.config.file_path, trans.app.config.hgweb_config_dir, + trans.app.config.hgweb_repo_prefix, trans.app.config.database_connection, ) return { diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index 3a9e0fe27389..e551d7c2862d 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -101,6 +101,7 @@ def __init__(self, **kwd) -> None: # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir + self.hgweb_config_manager.hgweb_repo_prefix = self.config.hgweb_repo_prefix # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry(self) # Configure Sentry client if configured diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index a62bb52e39f7..ace84e5b6577 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -508,7 +508,9 @@ def is_new(self): def repo_path(self, app=None): # Keep app argument for compatibility with tool_shed_install Repository model - return hgweb_config_manager.get_entry(os.path.join("repos", self.user.username, self.name)) + return hgweb_config_manager.get_entry( + os.path.join(hgweb_config_manager.hgweb_repo_prefix, self.user.username, self.name) + ) def revision(self): repo = self.hg_repo diff --git a/scripts/tool_shed/build_ts_whoosh_index.py b/scripts/tool_shed/build_ts_whoosh_index.py index d74cd99d80cc..d74420f17ad7 100644 --- a/scripts/tool_shed/build_ts_whoosh_index.py +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -40,6 +40,7 @@ def parse_arguments(): config = ts_config.ToolShedAppConfiguration(**app_properties) args.dburi = config.database_connection args.hgweb_config_dir = config.hgweb_config_dir + args.hgweb_repo_prefix = config.hgweb_repo_prefix args.whoosh_index_dir = config.whoosh_index_dir args.file_path = config.file_path if args.debug: From 01e9f0243f8274c62a77218012101933beb4eb85 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 31 May 2024 11:18:27 +0200 Subject: [PATCH 02/16] Log exceptions regardless of error log middleware outcome --- lib/galaxy/web/framework/middleware/error.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/web/framework/middleware/error.py b/lib/galaxy/web/framework/middleware/error.py index e94a73894461..2f55d172fcdc 100644 --- a/lib/galaxy/web/framework/middleware/error.py +++ b/lib/galaxy/web/framework/middleware/error.py @@ -9,6 +9,7 @@ When an exception is thrown from the wrapper application, this logs the exception and displays an error page. """ +import logging import sys import traceback from io import StringIO @@ -25,6 +26,8 @@ reporter, ) +log = logging.getLogger(__name__) + __all__ = ("ErrorMiddleware", "handle_exception") @@ -170,6 +173,7 @@ def __call__(self, environ, start_response): for expect in environ.get("paste.expected_exceptions", []): if isinstance(exc_info[1], expect): raise + log.exception("Uncaught Exception") start_response("500 Internal Server Error", [("content-type", "text/html")], exc_info) # @@: it would be nice to deal with bad content types here response = self.exception_handler(exc_info, environ) From 7a29691dca0c06d0d348fb431a20faae2f515a83 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 31 May 2024 11:19:18 +0200 Subject: [PATCH 03/16] Return byte strings for error html --- lib/galaxy/web/framework/middleware/error.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/web/framework/middleware/error.py b/lib/galaxy/web/framework/middleware/error.py index 2f55d172fcdc..cfa97acf8e0a 100644 --- a/lib/galaxy/web/framework/middleware/error.py +++ b/lib/galaxy/web/framework/middleware/error.py @@ -177,7 +177,7 @@ def __call__(self, environ, start_response): start_response("500 Internal Server Error", [("content-type", "text/html")], exc_info) # @@: it would be nice to deal with bad content types here response = self.exception_handler(exc_info, environ) - return [response] + return [response.encode(errors="ignore")] finally: # clean up locals... exc_info = None @@ -188,7 +188,7 @@ def make_catching_iter(self, app_iter, environ, sr_checker): return app_iter return CatchingIter(app_iter, environ, sr_checker, self) - def exception_handler(self, exc_info, environ): + def exception_handler(self, exc_info, environ) -> str: simple_html_error = False if self.xmlhttp_key: get_vars = wsgilib.parse_querystring(environ) @@ -362,7 +362,7 @@ def handle_exception( error_message=None, simple_html_error=False, environ=None, -): +) -> str: """ For exception handling outside of a web context From 36b69e125dc1adc23fa0fde955dc360e68aae3e1 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 31 May 2024 11:20:07 +0200 Subject: [PATCH 04/16] Fix TS admin panel --- lib/tool_shed/webapp/model/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index ace84e5b6577..1475ce0a3df9 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -143,6 +143,11 @@ def __init__(self, email=None, password=None): self.purged = False self.new_repo_alert = False + @property + def current_galaxy_session(self): + if self.galaxy_sessions: + return self.galaxy_sessions[0] + def all_roles(self): roles = [ura.role for ura in self.roles] for group in [uga.group for uga in self.groups]: From 1b985e74c13479936373bc094f176f7c7e9dfeac Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Fri, 31 May 2024 15:48:47 +0200 Subject: [PATCH 05/16] Fix exception handler type hints Thank you! Co-authored-by: Nicola Soranzo --- lib/galaxy/web/framework/middleware/error.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/web/framework/middleware/error.py b/lib/galaxy/web/framework/middleware/error.py index cfa97acf8e0a..0bfaf764b4c0 100644 --- a/lib/galaxy/web/framework/middleware/error.py +++ b/lib/galaxy/web/framework/middleware/error.py @@ -13,7 +13,10 @@ import sys import traceback from io import StringIO -from typing import cast +from typing import ( + cast, + Optional, +) import markupsafe from paste import ( @@ -188,7 +191,7 @@ def make_catching_iter(self, app_iter, environ, sr_checker): return app_iter return CatchingIter(app_iter, environ, sr_checker, self) - def exception_handler(self, exc_info, environ) -> str: + def exception_handler(self, exc_info, environ) -> Optional[str]: simple_html_error = False if self.xmlhttp_key: get_vars = wsgilib.parse_querystring(environ) @@ -362,7 +365,7 @@ def handle_exception( error_message=None, simple_html_error=False, environ=None, -) -> str: +) -> Optional[str]: """ For exception handling outside of a web context From ea6751a1498d3a94e27f09f4112f6fe05591f4e7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 31 May 2024 17:42:30 +0200 Subject: [PATCH 06/16] Make repository creation slightly safer --- lib/tool_shed/util/hg_util.py | 3 +- lib/tool_shed/util/hgweb_config.py | 10 ++-- lib/tool_shed/util/repository_util.py | 60 ++++++++----------- lib/tool_shed/webapp/api/repositories.py | 2 +- .../webapp/controllers/repository.py | 2 +- lib/tool_shed/webapp/model/__init__.py | 3 +- 6 files changed, 35 insertions(+), 45 deletions(-) diff --git a/lib/tool_shed/util/hg_util.py b/lib/tool_shed/util/hg_util.py index f2c81c34d285..8e8400327453 100644 --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -71,14 +71,13 @@ def get_hgrc_path(repo_path): return os.path.join(repo_path, ".hg", "hgrc") -def create_hgrc_file(app, repository): +def create_hgrc_file(app, repository, repo_path): # Since we support both http and https, we set `push_ssl` to False to # override the default (which is True) in the Mercurial API. # The hg purge extension purges all files and directories not being tracked # by Mercurial in the current repository. It will remove unknown files and # empty directories. This is not currently used because it is not supported # in the Mercurial API. - repo_path = repository.repo_path(app) hgrc_path = get_hgrc_path(repo_path) with open(hgrc_path, "w") as fp: fp.write("[paths]\n") diff --git a/lib/tool_shed/util/hgweb_config.py b/lib/tool_shed/util/hgweb_config.py index 7bf230106bbe..2efc1a06422e 100644 --- a/lib/tool_shed/util/hgweb_config.py +++ b/lib/tool_shed/util/hgweb_config.py @@ -5,8 +5,6 @@ import threading from datetime import date -from galaxy.util import unicodify - log = logging.getLogger(__name__) new_hgweb_config_template = """ @@ -36,8 +34,8 @@ def add_entry(self, lhs, rhs): self.in_memory_config.set("paths", lhs, rhs) # Persist our in-memory configuration. self.write_config() - except Exception as e: - log.debug("Exception in HgWebConfigManager.add_entry(): %s", unicodify(e)) + except Exception: + log.exception("Exception in HgWebConfigManager.add_entry()") finally: self.lock.release() @@ -52,8 +50,8 @@ def change_entry(self, old_lhs, new_lhs, new_rhs): self.in_memory_config.set("paths", new_lhs, new_rhs) # Persist our in-memory configuration. self.write_config() - except Exception as e: - log.debug("Exception in HgWebConfigManager.change_entry(): %s", unicodify(e)) + except Exception: + log.exception("Exception in HgWebConfigManager.change_entry()") finally: self.lock.release() diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 2d52bfd6d65c..0d09ffa1249e 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -2,6 +2,7 @@ import logging import os import re +import tempfile from typing import ( Optional, Tuple, @@ -154,23 +155,18 @@ def create_repo_info_dict( def create_repository_admin_role(app: "ToolShedApp", repository: "Repository"): """ Create a new role with name-spaced name based on the repository name and its owner's public user - name. This will ensure that the tole name is unique. + name. This will ensure that the role name is unique. """ sa_session = app.model.session name = get_repository_admin_role_name(str(repository.name), str(repository.user.username)) description = "A user or group member with this role can administer this repository." role = app.model.Role(name=name, description=description, type=app.model.Role.types.SYSTEM) sa_session.add(role) - session = sa_session() - with transaction(session): - session.commit() # Associate the role with the repository owner. app.model.UserRoleAssociation(repository.user, role) # Associate the role with the repository. rra = app.model.RepositoryRoleAssociation(repository, role) sa_session.add(rra) - with transaction(session): - session.commit() return role @@ -180,7 +176,7 @@ def create_repository( type: str, description, long_description, - user_id, + user, category_ids=None, remote_repository_url=None, homepage_url=None, @@ -196,43 +192,39 @@ def create_repository( homepage_url=homepage_url, description=description, long_description=long_description, - user_id=user_id, + user=user, ) - # Flush to get the id. sa_session.add(repository) - session = sa_session() - with transaction(session): - session.commit() - # Create an admin role for the repository. - create_repository_admin_role(app, repository) - # Determine the repository's repo_path on disk. - dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id)) - # Create directory if it does not exist. - if not os.path.exists(dir): - os.makedirs(dir) - # Define repo name inside hashed directory. - repository_path = os.path.join(dir, "repo_%d" % repository.id) - # Create local repository directory. - if not os.path.exists(repository_path): - os.makedirs(repository_path) - # Create the local repository. - init_repository(repo_path=repository_path) - # Add an entry in the hgweb.config file for the local repository. - lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" - app.hgweb_config_manager.add_entry(lhs, repository_path) - # Create a .hg/hgrc file for the local repository. - create_hgrc_file(app, repository) - flush_needed = False if category_ids: # Create category associations for category_id in category_ids: category = sa_session.query(app.model.Category).get(app.security.decode_id(category_id)) rca = app.model.RepositoryCategoryAssociation(repository, category) sa_session.add(rca) - flush_needed = True - if flush_needed: + # Create an admin role for the repository. + create_repository_admin_role(app, repository) + # Create a temporary repo_path on disk. + with tempfile.TemporaryDirectory( + dir=app.config.file_path, prefix="f{repository.user.username}-{repository.name}" + ) as repository_path: + # Create the local repository. + init_repository(repo_path=repository_path) + # Create a .hg/hgrc file for the local repository. + create_hgrc_file(app, repository, repo_path=repository_path) + # Add an entry in the hgweb.config file for the local repository. + lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" + # Flush to get the id. + session = sa_session() with transaction(session): session.commit() + dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id)) + # Define repo name inside hashed directory. + final_repository_path = os.path.join(dir, "repo_%d" % repository.id) + # Create final repository directory. + if not os.path.exists(final_repository_path): + os.makedirs(final_repository_path) + os.rename(repository_path, final_repository_path) + app.hgweb_config_manager.add_entry(lhs, final_repository_path) # Update the repository registry. app.repository_registry.add_entry(repository) message = f"Repository {escape(str(repository.name))} has been created." diff --git a/lib/tool_shed/webapp/api/repositories.py b/lib/tool_shed/webapp/api/repositories.py index 21c0249e4d5b..8fb9fc1b4fee 100644 --- a/lib/tool_shed/webapp/api/repositories.py +++ b/lib/tool_shed/webapp/api/repositories.py @@ -986,7 +986,7 @@ def create(self, trans, **kwd): type=repo_type, description=synopsis, long_description=description, - user_id=trans.user.id, + user=trans.user, category_ids=category_ids, remote_repository_url=remote_repository_url, homepage_url=homepage_url, diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py index d82ac50f5a6d..4c5b6ccc1bbc 100644 --- a/lib/tool_shed/webapp/controllers/repository.py +++ b/lib/tool_shed/webapp/controllers/repository.py @@ -759,7 +759,7 @@ def create_repository(self, trans, **kwd): repository_type, description, long_description, - user_id=trans.user.id, + user=trans.user, category_ids=category_ids, remote_repository_url=remote_repository_url, homepage_url=homepage_url, diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index 1475ce0a3df9..bb52cfafca48 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -422,12 +422,13 @@ class Repository(Base, Dictifiable): ] file_states = Bunch(NORMAL="n", NEEDS_MERGING="m", MARKED_FOR_REMOVAL="r", MARKED_FOR_ADDITION="a", NOT_TRACKED="?") - def __init__(self, private=False, times_downloaded=0, deprecated=False, **kwd): + def __init__(self, private=False, times_downloaded=0, deprecated=False, user=None, **kwd): super().__init__(**kwd) self.private = private self.times_downloaded = times_downloaded self.deprecated = deprecated self.name = self.name or "Unnamed repository" + self.user = user @property def hg_repo(self): From 91610ddbb06210ad61ea9083e55e8c7b9dbad287 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 3 Jun 2024 17:05:48 +0200 Subject: [PATCH 07/16] Don't serialize display application links for deleted datasets Doing that often requires datasets on disk, which we shouldn't guarantee for deleted datasets. Fixes https://sentry.galaxyproject.org/share/issue/0ae7a9d9ad564054bd11c9a43aa6994c/: ``` Message Unexpected error Stack Trace Newest FileNotFoundError: [Errno 2] No such file or directory: '' File "galaxy/datatypes/interval.py", line 914, in get_estimated_display_viewport with compression_utils.get_fileobj(dataset.get_file_name()) as fh: File "galaxy/util/compression_utils.py", line 79, in get_fileobj return get_fileobj_raw(filename, mode, compressed_formats)[1] File "galaxy/util/compression_utils.py", line 139, in get_fileobj_raw return compressed_format, open(filename, mode, encoding="utf-8") ``` --- lib/galaxy/managers/hdas.py | 67 +++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index f92a75b56f24..5dc7da8eb348 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -617,18 +617,19 @@ def serialize_display_apps(self, item, key, trans=None, **context): """ hda = item display_apps: List[Dict[str, Any]] = [] - for display_app in hda.get_display_applications(trans).values(): - app_links = [] - for link_app in display_app.links.values(): - app_links.append( - { - "target": link_app.url.get("target_frame", "_blank"), - "href": link_app.get_display_url(hda, trans), - "text": gettext.gettext(link_app.name), - } - ) - if app_links: - display_apps.append(dict(label=display_app.name, links=app_links)) + if hda.state == model.HistoryDatasetAssociation.states.OK and not hda.deleted: + for display_app in hda.get_display_applications(trans).values(): + app_links = [] + for link_app in display_app.links.values(): + app_links.append( + { + "target": link_app.url.get("target_frame", "_blank"), + "href": link_app.get_display_url(hda, trans), + "text": gettext.gettext(link_app.name), + } + ) + if app_links: + display_apps.append(dict(label=display_app.name, links=app_links)) return display_apps @@ -638,28 +639,30 @@ def serialize_old_display_applications(self, item, key, trans=None, **context): """ hda = item display_apps: List[Dict[str, Any]] = [] - if not self.app.config.enable_old_display_applications: - return display_apps - - display_link_fn = hda.datatype.get_display_links - for display_app in hda.datatype.get_display_types(): - target_frame, display_links = display_link_fn( - hda, - display_app, - self.app, - trans.request.base, - ) + if ( + self.app.config.enable_old_display_applications + and hda.state == model.HistoryDatasetAssociation.states.OK + and not hda.deleted + ): + display_link_fn = hda.datatype.get_display_links + for display_app in hda.datatype.get_display_types(): + target_frame, display_links = display_link_fn( + hda, + display_app, + self.app, + trans.request.base, + ) - if len(display_links) > 0: - display_label = hda.datatype.get_display_label(display_app) + if len(display_links) > 0: + display_label = hda.datatype.get_display_label(display_app) - app_links = [] - for display_name, display_link in display_links: - app_links.append( - {"target": target_frame, "href": display_link, "text": gettext.gettext(display_name)} - ) - if app_links: - display_apps.append(dict(label=display_label, links=app_links)) + app_links = [] + for display_name, display_link in display_links: + app_links.append( + {"target": target_frame, "href": display_link, "text": gettext.gettext(display_name)} + ) + if app_links: + display_apps.append(dict(label=display_label, links=app_links)) return display_apps From e2678914a7ec423c87d9738596a99afc4adf2d3e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 31 May 2024 17:56:15 +0200 Subject: [PATCH 08/16] Cast html=True to string I tried type overloads but they are incredibly verbose. --- lib/galaxy/web/framework/middleware/error.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/web/framework/middleware/error.py b/lib/galaxy/web/framework/middleware/error.py index 0bfaf764b4c0..316cfae336fe 100644 --- a/lib/galaxy/web/framework/middleware/error.py +++ b/lib/galaxy/web/framework/middleware/error.py @@ -191,13 +191,13 @@ def make_catching_iter(self, app_iter, environ, sr_checker): return app_iter return CatchingIter(app_iter, environ, sr_checker, self) - def exception_handler(self, exc_info, environ) -> Optional[str]: + def exception_handler(self, exc_info, environ) -> str: simple_html_error = False if self.xmlhttp_key: get_vars = wsgilib.parse_querystring(environ) if dict(get_vars).get(self.xmlhttp_key): simple_html_error = True - return handle_exception( + rval = handle_exception( exc_info, environ["wsgi.errors"], html=True, @@ -215,6 +215,8 @@ def exception_handler(self, exc_info, environ) -> Optional[str]: simple_html_error=simple_html_error, environ=environ, ) + rval = cast(str, rval) # we know handle_exception returns string because html=True + return rval class ResponseStartChecker: @@ -351,7 +353,7 @@ def extraData(self): def handle_exception( exc_info, error_stream, - html=True, + html: bool = True, debug_mode=False, error_email=None, error_log=None, From 4978d647045890b7b1fde346ed60dfb06176f03b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 3 Jun 2024 15:45:41 +0200 Subject: [PATCH 09/16] Don't delete tempdir --- lib/tool_shed/util/repository_util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 0d09ffa1249e..f39572c89462 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -205,7 +205,9 @@ def create_repository( create_repository_admin_role(app, repository) # Create a temporary repo_path on disk. with tempfile.TemporaryDirectory( - dir=app.config.file_path, prefix="f{repository.user.username}-{repository.name}" + dir=app.config.file_path, + prefix="f{repository.user.username}-{repository.name}", + delete=False, ) as repository_path: # Create the local repository. init_repository(repo_path=repository_path) From 2e8c0df88bd764ea9793231707a45c3b27849a9d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 3 Jun 2024 17:52:27 +0200 Subject: [PATCH 10/16] Test and lint fixes --- lib/tool_shed/util/repository_util.py | 41 +++++++++++++------------- test/unit/shed_unit/test_shed_index.py | 6 +++- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index f39572c89462..7d4ead78a148 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -204,28 +204,27 @@ def create_repository( # Create an admin role for the repository. create_repository_admin_role(app, repository) # Create a temporary repo_path on disk. - with tempfile.TemporaryDirectory( + repository_path = tempfile.mkdtemp( dir=app.config.file_path, - prefix="f{repository.user.username}-{repository.name}", - delete=False, - ) as repository_path: - # Create the local repository. - init_repository(repo_path=repository_path) - # Create a .hg/hgrc file for the local repository. - create_hgrc_file(app, repository, repo_path=repository_path) - # Add an entry in the hgweb.config file for the local repository. - lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" - # Flush to get the id. - session = sa_session() - with transaction(session): - session.commit() - dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id)) - # Define repo name inside hashed directory. - final_repository_path = os.path.join(dir, "repo_%d" % repository.id) - # Create final repository directory. - if not os.path.exists(final_repository_path): - os.makedirs(final_repository_path) - os.rename(repository_path, final_repository_path) + prefix=f"{repository.user.username}-{repository.name}", + ) + # Create the local repository. + init_repository(repo_path=repository_path) + # Create a .hg/hgrc file for the local repository. + create_hgrc_file(app, repository, repo_path=repository_path) + # Add an entry in the hgweb.config file for the local repository. + lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" + # Flush to get the id. + session = sa_session() + with transaction(session): + session.commit() + dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id)) + # Define repo name inside hashed directory. + final_repository_path = os.path.join(dir, "repo_%d" % repository.id) + # Create final repository directory. + if not os.path.exists(final_repository_path): + os.makedirs(final_repository_path) + os.rename(repository_path, final_repository_path) app.hgweb_config_manager.add_entry(lhs, final_repository_path) # Update the repository registry. app.repository_registry.add_entry(repository) diff --git a/test/unit/shed_unit/test_shed_index.py b/test/unit/shed_unit/test_shed_index.py index b589dc74fd83..4da7c08205bc 100644 --- a/test/unit/shed_unit/test_shed_index.py +++ b/test/unit/shed_unit/test_shed_index.py @@ -36,10 +36,11 @@ def community_file_dir(): @pytest.fixture() def community_file_structure(community_file_dir): - community = namedtuple("community", "file_path hgweb_config_dir dburi") + community = namedtuple("community", "file_path hgweb_config_dir hgweb_repos_prefix dburi") return community( file_path=os.path.join(community_file_dir, "database", "community_files"), hgweb_config_dir=community_file_dir, + hgweb_repos_prefix="repos/", dburi="sqlite:///%s" % os.path.join(community_file_dir, "database", "community.sqlite"), ) @@ -49,6 +50,7 @@ def test_build_index(whoosh_index_dir, community_file_structure): whoosh_index_dir, community_file_structure.file_path, community_file_structure.hgweb_config_dir, + community_file_structure.hgweb_repo_prefix, community_file_structure.dburi, ) assert repos_indexed == 1 @@ -59,6 +61,7 @@ def test_build_index(whoosh_index_dir, community_file_structure): whoosh_index_dir, community_file_structure.file_path, community_file_structure.hgweb_config_dir, + community_file_structure.hgweb_repo_prefix, community_file_structure.dburi, ) assert repos_indexed == 0 @@ -74,6 +77,7 @@ def test_build_index(whoosh_index_dir, community_file_structure): whoosh_index_dir, community_file_structure.file_path, community_file_structure.hgweb_config_dir, + community_file_structure.hgweb_repo_prefix, community_file_structure.dburi, ) assert repos_indexed == 1 From 80e15b62e6231a6387c13c3dd461a6df6b6f3cd3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 3 Jun 2024 19:42:37 +0200 Subject: [PATCH 11/16] Remove html parameter from handle_exception --- lib/galaxy/web/framework/middleware/error.py | 68 +++++++++----------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/lib/galaxy/web/framework/middleware/error.py b/lib/galaxy/web/framework/middleware/error.py index 316cfae336fe..06ee98430e14 100644 --- a/lib/galaxy/web/framework/middleware/error.py +++ b/lib/galaxy/web/framework/middleware/error.py @@ -13,10 +13,7 @@ import sys import traceback from io import StringIO -from typing import ( - cast, - Optional, -) +from typing import cast import markupsafe from paste import ( @@ -197,10 +194,9 @@ def exception_handler(self, exc_info, environ) -> str: get_vars = wsgilib.parse_querystring(environ) if dict(get_vars).get(self.xmlhttp_key): simple_html_error = True - rval = handle_exception( + return handle_exception( exc_info, environ["wsgi.errors"], - html=True, debug_mode=self.debug_mode, error_email=self.error_email, error_log=self.error_log, @@ -215,8 +211,6 @@ def exception_handler(self, exc_info, environ) -> str: simple_html_error=simple_html_error, environ=environ, ) - rval = cast(str, rval) # we know handle_exception returns string because html=True - return rval class ResponseStartChecker: @@ -353,7 +347,6 @@ def extraData(self): def handle_exception( exc_info, error_stream, - html: bool = True, debug_mode=False, error_email=None, error_log=None, @@ -367,7 +360,7 @@ def handle_exception( error_message=None, simple_html_error=False, environ=None, -) -> Optional[str]: +) -> str: """ For exception handling outside of a web context @@ -398,54 +391,51 @@ def handle_exception( smtp_use_tls=smtp_use_tls, subject_prefix=error_subject_prefix, ) - rep_err = send_report(rep, exc_data, html=html) + rep_err = send_report(rep, exc_data, html=True) if rep_err: extra_data += rep_err else: reported = True if error_log: rep = reporter.LogReporter(filename=error_log) - rep_err = send_report(rep, exc_data, html=html) + rep_err = send_report(rep, exc_data, html=True) if rep_err: extra_data += rep_err else: reported = True if show_exceptions_in_wsgi_errors: rep = reporter.FileReporter(file=error_stream) - rep_err = send_report(rep, exc_data, html=html) + rep_err = send_report(rep, exc_data, html=True) if rep_err: extra_data += rep_err else: reported = True else: error_stream.write(f"Error - {exc_data.exception_type}: {exc_data.exception_value}\n") - if html: - if debug_mode and simple_html_error: - return_error = formatter.format_html( - exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False - ) - reported = True - elif debug_mode and not simple_html_error: - error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False) - head_html = formatter.error_css + formatter.hide_display_js - return_error = error_template(head_html, error_html, extra_data) - extra_data = "" - reported = True - else: - msg = ( - error_message - or """ - An error occurred. - """ - ) - extra = "

The error has been logged to our team." - if "sentry_event_id" in environ: - extra += " If you want to contact us about this error, please reference the following

" - extra += f"GURU MEDITATION: #{environ['sentry_event_id']}" - extra += "

" - return_error = error_template("", msg, extra) + if debug_mode and simple_html_error: + return_error = formatter.format_html( + exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False + ) + reported = True + elif debug_mode and not simple_html_error: + error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False) + head_html = formatter.error_css + formatter.hide_display_js + return_error = error_template(head_html, error_html, extra_data) + extra_data = "" + reported = True else: - return_error = None + msg = ( + error_message + or """ + An error occurred. + """ + ) + extra = "

The error has been logged to our team." + if "sentry_event_id" in environ: + extra += " If you want to contact us about this error, please reference the following

" + extra += f"GURU MEDITATION: #{environ['sentry_event_id']}" + extra += "

" + return_error = error_template("", msg, extra) if not reported and error_stream: err_report = formatter.format_text(exc_data, show_hidden_frames=True) err_report += f"\n{'-' * 60}\n" From 992777edeebac9a083004148e61c8f1750bc03ed Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 4 Jun 2024 16:58:26 +0200 Subject: [PATCH 12/16] Fix typo --- test/unit/shed_unit/test_shed_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/shed_unit/test_shed_index.py b/test/unit/shed_unit/test_shed_index.py index 4da7c08205bc..e76cfee5bb1c 100644 --- a/test/unit/shed_unit/test_shed_index.py +++ b/test/unit/shed_unit/test_shed_index.py @@ -36,11 +36,11 @@ def community_file_dir(): @pytest.fixture() def community_file_structure(community_file_dir): - community = namedtuple("community", "file_path hgweb_config_dir hgweb_repos_prefix dburi") + community = namedtuple("community", "file_path hgweb_config_dir hgweb_repo_prefix dburi") return community( file_path=os.path.join(community_file_dir, "database", "community_files"), hgweb_config_dir=community_file_dir, - hgweb_repos_prefix="repos/", + hgweb_repo_prefix="repos/", dburi="sqlite:///%s" % os.path.join(community_file_dir, "database", "community.sqlite"), ) From 0eaf1dc131d0bbf4d62217fc6fcec78423bce481 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 24 Apr 2024 17:11:53 +0200 Subject: [PATCH 13/16] Update k8s-version --- .github/workflows/integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index d3660517fb72..06aefec85cc8 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -62,7 +62,7 @@ jobs: id: minikube uses: CodingNagger/minikube-setup-action@v1.0.6 with: - k8s-version: '1.19.16' + k8s-version: '1.23.0' - name: Launch Minikube run: eval ${{ steps.minikube.outputs.launcher }} - name: Check pods From 043d99c0bbf24be2c9b10ea3620e7fdfab9d3443 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 21 May 2024 16:47:03 +0100 Subject: [PATCH 14/16] Use official minikube GitHub action Cannot used the new default driver (`docker`) as it crashes the PostgreSQL Docker container started via job services. --- .github/workflows/integration.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 06aefec85cc8..aa3025ca6968 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -59,15 +59,12 @@ jobs: # ffmpeg: ffprobe needed by media datatypes run: sudo apt-get update && sudo apt-get -y install conntrack ffmpeg - name: Setup Minikube - id: minikube - uses: CodingNagger/minikube-setup-action@v1.0.6 + uses: medyagh/setup-minikube@latest with: - k8s-version: '1.23.0' - - name: Launch Minikube - run: eval ${{ steps.minikube.outputs.launcher }} + driver: none + kubernetes-version: '1.23.0' - name: Check pods - run: | - kubectl get pods + run: kubectl get pods -A - uses: actions/checkout@v3 with: path: 'galaxy root' From 54a27a19e64262c32bc8a3b6908126db1ae75187 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 5 Jun 2024 12:15:46 +0200 Subject: [PATCH 15/16] Fix tool_shed package unit tests --- test/unit/tool_shed/_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/tool_shed/_util.py b/test/unit/tool_shed/_util.py index cf4f82255d0c..4ef21ab5f322 100644 --- a/test/unit/tool_shed/_util.py +++ b/test/unit/tool_shed/_util.py @@ -133,7 +133,7 @@ def repository_fixture(app: ToolShedApp, user: User, name: str, category: Option type, description, long_description, - user.id, + user, category_ids=category_ids, remote_repository_url=None, homepage_url=None, From 9681d8647f4542efabc32901cd2b8dd4378f578e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 5 Jun 2024 14:44:04 +0200 Subject: [PATCH 16/16] Fix one more tool shed unit test --- test/unit/tool_shed/_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/tool_shed/_util.py b/test/unit/tool_shed/_util.py index 4ef21ab5f322..df50c5270ef2 100644 --- a/test/unit/tool_shed/_util.py +++ b/test/unit/tool_shed/_util.py @@ -76,6 +76,7 @@ def __init__(self, temp_directory=None): hgweb_config_dir = os.path.join(temp_directory, "hgweb") safe_makedirs(hgweb_config_dir) self.hgweb_config_manager.hgweb_config_dir = hgweb_config_dir + self.hgweb_config_manager.hgweb_repo_prefix = "repos/" self.config = TestToolShedConfig(temp_directory) self.security = IdEncodingHelper(id_secret=self.config.id_secret) self.repository_registry = tool_shed.repository_registry.Registry(self)