Merge pull request #18320 from mvdbeek/release_23.2

Merge 23.1 into 23.2

lib/galaxy/web/framework/middleware/error.py

@@ -9,6 +9,7 @@
 When an exception is thrown from the wrapper application, this logs
 the exception and displays an error page.
 """
+import logging
 import sys
 import traceback
 from io import StringIO
@@ -25,6 +26,8 @@
     reporter,
 )
 
+log = logging.getLogger(__name__)
+
 __all__ = ("ErrorMiddleware", "handle_exception")
 
 
@@ -170,10 +173,11 @@ def __call__(self, environ, start_response):
                 for expect in environ.get("paste.expected_exceptions", []):
                     if isinstance(exc_info[1], expect):
                         raise
+                log.exception("Uncaught Exception")
                 start_response("500 Internal Server Error", [("content-type", "text/html")], exc_info)
                 # @@: it would be nice to deal with bad content types here
                 response = self.exception_handler(exc_info, environ)
-                return [response]
+                return [response.encode(errors="ignore")]
             finally:
                 # clean up locals...
                 exc_info = None
@@ -184,7 +188,7 @@ def make_catching_iter(self, app_iter, environ, sr_checker):
             return app_iter
         return CatchingIter(app_iter, environ, sr_checker, self)
 
-    def exception_handler(self, exc_info, environ):
+    def exception_handler(self, exc_info, environ) -> str:
         simple_html_error = False
         if self.xmlhttp_key:
             get_vars = wsgilib.parse_querystring(environ)
@@ -193,7 +197,6 @@ def exception_handler(self, exc_info, environ):
         return handle_exception(
             exc_info,
             environ["wsgi.errors"],
-            html=True,
             debug_mode=self.debug_mode,
             error_email=self.error_email,
             error_log=self.error_log,
@@ -344,7 +347,6 @@ def extraData(self):
 def handle_exception(
     exc_info,
     error_stream,
-    html=True,
     debug_mode=False,
     error_email=None,
     error_log=None,
@@ -358,7 +360,7 @@ def handle_exception(
     error_message=None,
     simple_html_error=False,
     environ=None,
-):
+) -> str:
     """
     For exception handling outside of a web context
 
@@ -389,54 +391,51 @@ def handle_exception(
             smtp_use_tls=smtp_use_tls,
             subject_prefix=error_subject_prefix,
         )
-        rep_err = send_report(rep, exc_data, html=html)
+        rep_err = send_report(rep, exc_data, html=True)
         if rep_err:
             extra_data += rep_err
         else:
             reported = True
     if error_log:
         rep = reporter.LogReporter(filename=error_log)
-        rep_err = send_report(rep, exc_data, html=html)
+        rep_err = send_report(rep, exc_data, html=True)
         if rep_err:
             extra_data += rep_err
         else:
             reported = True
     if show_exceptions_in_wsgi_errors:
         rep = reporter.FileReporter(file=error_stream)
-        rep_err = send_report(rep, exc_data, html=html)
+        rep_err = send_report(rep, exc_data, html=True)
         if rep_err:
             extra_data += rep_err
         else:
             reported = True
     else:
         error_stream.write(f"Error - {exc_data.exception_type}: {exc_data.exception_value}\n")
-    if html:
-        if debug_mode and simple_html_error:
-            return_error = formatter.format_html(
-                exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False
-            )
-            reported = True
-        elif debug_mode and not simple_html_error:
-            error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False)
-            head_html = formatter.error_css + formatter.hide_display_js
-            return_error = error_template(head_html, error_html, extra_data)
-            extra_data = ""
-            reported = True
-        else:
-            msg = (
-                error_message
-                or """
-            An error occurred.
-            """
-            )
-            extra = "<p><b>The error has been logged to our team.</b>"
-            if "sentry_event_id" in environ:
-                extra += " If you want to contact us about this error, please reference the following<br><br>"
-                extra += f"<b><large>GURU MEDITATION: #{environ['sentry_event_id']}</large></b>"
-            extra += "</p>"
-            return_error = error_template("", msg, extra)
+    if debug_mode and simple_html_error:
+        return_error = formatter.format_html(
+            exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False
+        )
+        reported = True
+    elif debug_mode and not simple_html_error:
+        error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False)
+        head_html = formatter.error_css + formatter.hide_display_js
+        return_error = error_template(head_html, error_html, extra_data)
+        extra_data = ""
+        reported = True
     else:
-        return_error = None
+        msg = (
+            error_message
+            or """
+        An error occurred.
+        """
+        )
+        extra = "<p><b>The error has been logged to our team.</b>"
+        if "sentry_event_id" in environ:
+            extra += " If you want to contact us about this error, please reference the following<br><br>"
+            extra += f"<b><large>GURU MEDITATION: #{environ['sentry_event_id']}</large></b>"
+        extra += "</p>"
+        return_error = error_template("", msg, extra)
     if not reported and error_stream:
         err_report = formatter.format_text(exc_data, show_hidden_frames=True)
         err_report += f"\n{'-' * 60}\n"

lib/tool_shed/managers/repositories.py

@@ -468,7 +468,7 @@ def create_repository(trans: ProvidesUserContext, request: CreateRepositoryReque
         type=request.type_,
         description=request.synopsis,
         long_description=request.description,
-        user_id=user.id,
+        user=user,
         category_ids=category_ids,
         remote_repository_url=request.remote_repository_url,
         homepage_url=request.homepage_url,

lib/tool_shed/util/hg_util.py

@@ -71,14 +71,13 @@ def get_hgrc_path(repo_path):
     return os.path.join(repo_path, ".hg", "hgrc")
 
 
-def create_hgrc_file(app, repository):
+def create_hgrc_file(app, repository, repo_path):
     # Since we support both http and https, we set `push_ssl` to False to
     # override the default (which is True) in the Mercurial API.
     # The hg purge extension purges all files and directories not being tracked
     # by Mercurial in the current repository. It will remove unknown files and
     # empty directories. This is not currently used because it is not supported
     # in the Mercurial API.
-    repo_path = repository.repo_path(app)
     hgrc_path = get_hgrc_path(repo_path)
     with open(hgrc_path, "w") as fp:
         fp.write("[paths]\n")

lib/tool_shed/util/hgweb_config.py

@@ -5,8 +5,6 @@
 import threading
 from datetime import date
 
-from galaxy.util import unicodify
-
 log = logging.getLogger(__name__)
 
 new_hgweb_config_template = """
@@ -20,6 +18,7 @@ def __init__(self):
         self.hgweb_config_dir = None
         self.in_memory_config = None
         self.lock = threading.Lock()
+        self.hgweb_repo_prefix = None
 
     def add_entry(self, lhs, rhs):
         """Add an entry in the hgweb.config file for a new repository."""
@@ -35,8 +34,8 @@ def add_entry(self, lhs, rhs):
             self.in_memory_config.set("paths", lhs, rhs)
             # Persist our in-memory configuration.
             self.write_config()
-        except Exception as e:
-            log.debug("Exception in HgWebConfigManager.add_entry(): %s", unicodify(e))
+        except Exception:
+            log.exception("Exception in HgWebConfigManager.add_entry()")
         finally:
             self.lock.release()
 
@@ -51,8 +50,8 @@ def change_entry(self, old_lhs, new_lhs, new_rhs):
             self.in_memory_config.set("paths", new_lhs, new_rhs)
             # Persist our in-memory configuration.
             self.write_config()
-        except Exception as e:
-            log.debug("Exception in HgWebConfigManager.change_entry(): %s", unicodify(e))
+        except Exception:
+            log.exception("Exception in HgWebConfigManager.change_entry()")
         finally:
             self.lock.release()
 

lib/tool_shed/util/repository_util.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import re
+import tempfile
 from typing import (
     List,
     Optional,
@@ -164,23 +165,18 @@ def create_repo_info_dict(
 def create_repository_admin_role(app: "ToolShedApp", repository: "Repository"):
     """
     Create a new role with name-spaced name based on the repository name and its owner's public user
-    name.  This will ensure that the tole name is unique.
+    name.  This will ensure that the role name is unique.
     """
     sa_session = app.model.session
     name = get_repository_admin_role_name(str(repository.name), str(repository.user.username))
     description = "A user or group member with this role can administer this repository."
     role = app.model.Role(name=name, description=description, type=app.model.Role.types.SYSTEM)
     sa_session.add(role)
-    session = sa_session()
-    with transaction(session):
-        session.commit()
     # Associate the role with the repository owner.
     app.model.UserRoleAssociation(repository.user, role)
     # Associate the role with the repository.
     rra = app.model.RepositoryRoleAssociation(repository, role)
     sa_session.add(rra)
-    with transaction(session):
-        session.commit()
     return role
 
 
@@ -190,7 +186,7 @@ def create_repository(
     type: str,
     description,
     long_description,
-    user_id,
+    user,
     category_ids: Optional[List[str]] = None,
     remote_repository_url=None,
     homepage_url=None,
@@ -206,43 +202,40 @@ def create_repository(
         homepage_url=homepage_url,
         description=description,
         long_description=long_description,
-        user_id=user_id,
+        user=user,
     )
-    # Flush to get the id.
     sa_session.add(repository)
-    session = sa_session()
-    with transaction(session):
-        session.commit()
-    # Create an admin role for the repository.
-    create_repository_admin_role(app, repository)
-    # Determine the repository's repo_path on disk.
-    dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id))
-    # Create directory if it does not exist.
-    if not os.path.exists(dir):
-        os.makedirs(dir)
-    # Define repo name inside hashed directory.
-    repository_path = os.path.join(dir, "repo_%d" % repository.id)
-    # Create local repository directory.
-    if not os.path.exists(repository_path):
-        os.makedirs(repository_path)
-    # Create the local repository.
-    init_repository(repo_path=repository_path)
-    # Add an entry in the hgweb.config file for the local repository.
-    lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}"
-    app.hgweb_config_manager.add_entry(lhs, repository_path)
-    # Create a .hg/hgrc file for the local repository.
-    create_hgrc_file(app, repository)
-    flush_needed = False
     if category_ids:
         # Create category associations
         for category_id in category_ids:
             category = sa_session.get(app.model.Category, app.security.decode_id(category_id))
             rca = app.model.RepositoryCategoryAssociation(repository, category)
             sa_session.add(rca)
-            flush_needed = True
-    if flush_needed:
-        with transaction(session):
-            session.commit()
+    # Create an admin role for the repository.
+    create_repository_admin_role(app, repository)
+    # Create a temporary repo_path on disk.
+    repository_path = tempfile.mkdtemp(
+        dir=app.config.file_path,
+        prefix=f"{repository.user.username}-{repository.name}",
+    )
+    # Create the local repository.
+    init_repository(repo_path=repository_path)
+    # Create a .hg/hgrc file for the local repository.
+    create_hgrc_file(app, repository, repo_path=repository_path)
+    # Add an entry in the hgweb.config file for the local repository.
+    lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}"
+    # Flush to get the id.
+    session = sa_session()
+    with transaction(session):
+        session.commit()
+    dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id))
+    # Define repo name inside hashed directory.
+    final_repository_path = os.path.join(dir, "repo_%d" % repository.id)
+    # Create final repository directory.
+    if not os.path.exists(final_repository_path):
+        os.makedirs(final_repository_path)
+    os.rename(repository_path, final_repository_path)
+    app.hgweb_config_manager.add_entry(lhs, final_repository_path)
     # Update the repository registry.
     app.repository_registry.add_entry(repository)
     message = f"Repository <b>{escape(str(repository.name))}</b> has been created."
@@ -486,8 +479,8 @@ def update_repository(trans: "ProvidesUserContext", id: str, **kwds) -> Tuple[Op
 
         repo_dir = repository.repo_path(app)
         # Change the entry in the hgweb.config file for the repository.
-        old_lhs = f"repos/{repository.user.username}/{repository.name}"
-        new_lhs = f"repos/{repository.user.username}/{kwds['name']}"
+        old_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}"
+        new_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{kwds['name']}"
         trans.app.hgweb_config_manager.change_entry(old_lhs, new_lhs, repo_dir)
 
         # Change the entry in the repository's hgrc file.

lib/tool_shed/util/shed_index.py

@@ -37,7 +37,7 @@ def _get_or_create_index(whoosh_index_dir):
     return get_or_create_index(whoosh_index_dir, repo_schema), get_or_create_index(tool_index_dir, tool_schema)
 
 
-def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs):
+def build_index(whoosh_index_dir, file_path, hgweb_config_dir, hgweb_repo_prefix, dburi, **kwargs):
     """
     Build two search indexes simultaneously
     One is for repositories and the other for tools.
@@ -55,7 +55,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs):
 
     execution_timer = ExecutionTimer()
     with repo_index.searcher() as searcher:
-        for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs):
+        for repo in get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs):
             tools_list = repo.pop("tools_list")
             repo_id = repo["id"]
             indexed_document = searcher.document(id=repo_id)
@@ -89,7 +89,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs):
     return repos_indexed, tools_indexed
 
 
-def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs):
+def get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs):
     """
     Load repos from DB and included tools from .xml configs.
     """
@@ -120,7 +120,7 @@ def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs):
 
         # Load all changesets of the repo for lineage.
         repo_path = os.path.join(
-            hgweb_config_dir, hgwcm.get_entry(os.path.join("repos", repo.user.username, repo.name))
+            hgweb_config_dir, hgwcm.get_entry(os.path.join(hgweb_repo_prefix, repo.user.username, repo.name))
         )
         hg_repo = hg.repository(ui.ui(), repo_path.encode("utf-8"))
         lineage = []

lib/tool_shed/webapp/api/tools.py

@@ -33,6 +33,7 @@ def build_search_index(self, trans, **kwd):
             trans.app.config.whoosh_index_dir,
             trans.app.config.file_path,
             trans.app.config.hgweb_config_dir,
+            trans.app.config.hgweb_repo_prefix,
             trans.app.config.database_connection,
         )
         return {

lib/tool_shed/webapp/api2/tools.py

@@ -74,6 +74,7 @@ def build_search_index(self) -> BuildSearchIndexResponse:
             config.whoosh_index_dir,
             config.file_path,
             config.hgweb_config_dir,
+            config.hgweb_repo_prefix,
             config.database_connection,
         )
         return BuildSearchIndexResponse(

lib/tool_shed/webapp/app.py

@@ -104,6 +104,7 @@ def __init__(self, **kwd) -> None:
         # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located.
         self.hgweb_config_manager = hgweb_config_manager
         self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
+        self.hgweb_config_manager.hgweb_repo_prefix = self.config.hgweb_repo_prefix
         # Initialize the repository registry.
         self.repository_registry = tool_shed.repository_registry.Registry(self)
         # Configure Sentry client if configured