diff --git a/lib/galaxy/web/framework/middleware/error.py b/lib/galaxy/web/framework/middleware/error.py index 99b4c8654c24..e28fc049824d 100644 --- a/lib/galaxy/web/framework/middleware/error.py +++ b/lib/galaxy/web/framework/middleware/error.py @@ -9,6 +9,7 @@ When an exception is thrown from the wrapper application, this logs the exception and displays an error page. """ +import logging import sys import traceback from io import StringIO @@ -25,6 +26,8 @@ reporter, ) +log = logging.getLogger(__name__) + __all__ = ("ErrorMiddleware", "handle_exception") @@ -170,10 +173,11 @@ def __call__(self, environ, start_response): for expect in environ.get("paste.expected_exceptions", []): if isinstance(exc_info[1], expect): raise + log.exception("Uncaught Exception") start_response("500 Internal Server Error", [("content-type", "text/html")], exc_info) # @@: it would be nice to deal with bad content types here response = self.exception_handler(exc_info, environ) - return [response] + return [response.encode(errors="ignore")] finally: # clean up locals... exc_info = None @@ -184,7 +188,7 @@ def make_catching_iter(self, app_iter, environ, sr_checker): return app_iter return CatchingIter(app_iter, environ, sr_checker, self) - def exception_handler(self, exc_info, environ): + def exception_handler(self, exc_info, environ) -> str: simple_html_error = False if self.xmlhttp_key: get_vars = wsgilib.parse_querystring(environ) @@ -193,7 +197,6 @@ def exception_handler(self, exc_info, environ): return handle_exception( exc_info, environ["wsgi.errors"], - html=True, debug_mode=self.debug_mode, error_email=self.error_email, error_log=self.error_log, @@ -344,7 +347,6 @@ def extraData(self): def handle_exception( exc_info, error_stream, - html=True, debug_mode=False, error_email=None, error_log=None, @@ -358,7 +360,7 @@ def handle_exception( error_message=None, simple_html_error=False, environ=None, -): +) -> str: """ For exception handling outside of a web context @@ -389,54 +391,51 @@ def handle_exception( smtp_use_tls=smtp_use_tls, subject_prefix=error_subject_prefix, ) - rep_err = send_report(rep, exc_data, html=html) + rep_err = send_report(rep, exc_data, html=True) if rep_err: extra_data += rep_err else: reported = True if error_log: rep = reporter.LogReporter(filename=error_log) - rep_err = send_report(rep, exc_data, html=html) + rep_err = send_report(rep, exc_data, html=True) if rep_err: extra_data += rep_err else: reported = True if show_exceptions_in_wsgi_errors: rep = reporter.FileReporter(file=error_stream) - rep_err = send_report(rep, exc_data, html=html) + rep_err = send_report(rep, exc_data, html=True) if rep_err: extra_data += rep_err else: reported = True else: error_stream.write(f"Error - {exc_data.exception_type}: {exc_data.exception_value}\n") - if html: - if debug_mode and simple_html_error: - return_error = formatter.format_html( - exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False - ) - reported = True - elif debug_mode and not simple_html_error: - error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False) - head_html = formatter.error_css + formatter.hide_display_js - return_error = error_template(head_html, error_html, extra_data) - extra_data = "" - reported = True - else: - msg = ( - error_message - or """ - An error occurred. - """ - ) - extra = "

The error has been logged to our team." - if "sentry_event_id" in environ: - extra += " If you want to contact us about this error, please reference the following

" - extra += f"GURU MEDITATION: #{environ['sentry_event_id']}" - extra += "

" - return_error = error_template("", msg, extra) + if debug_mode and simple_html_error: + return_error = formatter.format_html( + exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False + ) + reported = True + elif debug_mode and not simple_html_error: + error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False) + head_html = formatter.error_css + formatter.hide_display_js + return_error = error_template(head_html, error_html, extra_data) + extra_data = "" + reported = True else: - return_error = None + msg = ( + error_message + or """ + An error occurred. + """ + ) + extra = "

The error has been logged to our team." + if "sentry_event_id" in environ: + extra += " If you want to contact us about this error, please reference the following

" + extra += f"GURU MEDITATION: #{environ['sentry_event_id']}" + extra += "

" + return_error = error_template("", msg, extra) if not reported and error_stream: err_report = formatter.format_text(exc_data, show_hidden_frames=True) err_report += f"\n{'-' * 60}\n" diff --git a/lib/tool_shed/managers/repositories.py b/lib/tool_shed/managers/repositories.py index c25918547b98..115a88d9a745 100644 --- a/lib/tool_shed/managers/repositories.py +++ b/lib/tool_shed/managers/repositories.py @@ -468,7 +468,7 @@ def create_repository(trans: ProvidesUserContext, request: CreateRepositoryReque type=request.type_, description=request.synopsis, long_description=request.description, - user_id=user.id, + user=user, category_ids=category_ids, remote_repository_url=request.remote_repository_url, homepage_url=request.homepage_url, diff --git a/lib/tool_shed/util/hg_util.py b/lib/tool_shed/util/hg_util.py index 965013846c4f..10ed45f99bc3 100644 --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -71,14 +71,13 @@ def get_hgrc_path(repo_path): return os.path.join(repo_path, ".hg", "hgrc") -def create_hgrc_file(app, repository): +def create_hgrc_file(app, repository, repo_path): # Since we support both http and https, we set `push_ssl` to False to # override the default (which is True) in the Mercurial API. # The hg purge extension purges all files and directories not being tracked # by Mercurial in the current repository. It will remove unknown files and # empty directories. This is not currently used because it is not supported # in the Mercurial API. - repo_path = repository.repo_path(app) hgrc_path = get_hgrc_path(repo_path) with open(hgrc_path, "w") as fp: fp.write("[paths]\n") diff --git a/lib/tool_shed/util/hgweb_config.py b/lib/tool_shed/util/hgweb_config.py index 8c1656406b0f..2efc1a06422e 100644 --- a/lib/tool_shed/util/hgweb_config.py +++ b/lib/tool_shed/util/hgweb_config.py @@ -5,8 +5,6 @@ import threading from datetime import date -from galaxy.util import unicodify - log = logging.getLogger(__name__) new_hgweb_config_template = """ @@ -20,6 +18,7 @@ def __init__(self): self.hgweb_config_dir = None self.in_memory_config = None self.lock = threading.Lock() + self.hgweb_repo_prefix = None def add_entry(self, lhs, rhs): """Add an entry in the hgweb.config file for a new repository.""" @@ -35,8 +34,8 @@ def add_entry(self, lhs, rhs): self.in_memory_config.set("paths", lhs, rhs) # Persist our in-memory configuration. self.write_config() - except Exception as e: - log.debug("Exception in HgWebConfigManager.add_entry(): %s", unicodify(e)) + except Exception: + log.exception("Exception in HgWebConfigManager.add_entry()") finally: self.lock.release() @@ -51,8 +50,8 @@ def change_entry(self, old_lhs, new_lhs, new_rhs): self.in_memory_config.set("paths", new_lhs, new_rhs) # Persist our in-memory configuration. self.write_config() - except Exception as e: - log.debug("Exception in HgWebConfigManager.change_entry(): %s", unicodify(e)) + except Exception: + log.exception("Exception in HgWebConfigManager.change_entry()") finally: self.lock.release() diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 4acfd2bb7330..11c739a70f8d 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -2,6 +2,7 @@ import logging import os import re +import tempfile from typing import ( List, Optional, @@ -164,23 +165,18 @@ def create_repo_info_dict( def create_repository_admin_role(app: "ToolShedApp", repository: "Repository"): """ Create a new role with name-spaced name based on the repository name and its owner's public user - name. This will ensure that the tole name is unique. + name. This will ensure that the role name is unique. """ sa_session = app.model.session name = get_repository_admin_role_name(str(repository.name), str(repository.user.username)) description = "A user or group member with this role can administer this repository." role = app.model.Role(name=name, description=description, type=app.model.Role.types.SYSTEM) sa_session.add(role) - session = sa_session() - with transaction(session): - session.commit() # Associate the role with the repository owner. app.model.UserRoleAssociation(repository.user, role) # Associate the role with the repository. rra = app.model.RepositoryRoleAssociation(repository, role) sa_session.add(rra) - with transaction(session): - session.commit() return role @@ -190,7 +186,7 @@ def create_repository( type: str, description, long_description, - user_id, + user, category_ids: Optional[List[str]] = None, remote_repository_url=None, homepage_url=None, @@ -206,43 +202,40 @@ def create_repository( homepage_url=homepage_url, description=description, long_description=long_description, - user_id=user_id, + user=user, ) - # Flush to get the id. sa_session.add(repository) - session = sa_session() - with transaction(session): - session.commit() - # Create an admin role for the repository. - create_repository_admin_role(app, repository) - # Determine the repository's repo_path on disk. - dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id)) - # Create directory if it does not exist. - if not os.path.exists(dir): - os.makedirs(dir) - # Define repo name inside hashed directory. - repository_path = os.path.join(dir, "repo_%d" % repository.id) - # Create local repository directory. - if not os.path.exists(repository_path): - os.makedirs(repository_path) - # Create the local repository. - init_repository(repo_path=repository_path) - # Add an entry in the hgweb.config file for the local repository. - lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" - app.hgweb_config_manager.add_entry(lhs, repository_path) - # Create a .hg/hgrc file for the local repository. - create_hgrc_file(app, repository) - flush_needed = False if category_ids: # Create category associations for category_id in category_ids: category = sa_session.get(app.model.Category, app.security.decode_id(category_id)) rca = app.model.RepositoryCategoryAssociation(repository, category) sa_session.add(rca) - flush_needed = True - if flush_needed: - with transaction(session): - session.commit() + # Create an admin role for the repository. + create_repository_admin_role(app, repository) + # Create a temporary repo_path on disk. + repository_path = tempfile.mkdtemp( + dir=app.config.file_path, + prefix=f"{repository.user.username}-{repository.name}", + ) + # Create the local repository. + init_repository(repo_path=repository_path) + # Create a .hg/hgrc file for the local repository. + create_hgrc_file(app, repository, repo_path=repository_path) + # Add an entry in the hgweb.config file for the local repository. + lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" + # Flush to get the id. + session = sa_session() + with transaction(session): + session.commit() + dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id)) + # Define repo name inside hashed directory. + final_repository_path = os.path.join(dir, "repo_%d" % repository.id) + # Create final repository directory. + if not os.path.exists(final_repository_path): + os.makedirs(final_repository_path) + os.rename(repository_path, final_repository_path) + app.hgweb_config_manager.add_entry(lhs, final_repository_path) # Update the repository registry. app.repository_registry.add_entry(repository) message = f"Repository {escape(str(repository.name))} has been created." @@ -486,8 +479,8 @@ def update_repository(trans: "ProvidesUserContext", id: str, **kwds) -> Tuple[Op repo_dir = repository.repo_path(app) # Change the entry in the hgweb.config file for the repository. - old_lhs = f"repos/{repository.user.username}/{repository.name}" - new_lhs = f"repos/{repository.user.username}/{kwds['name']}" + old_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" + new_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{kwds['name']}" trans.app.hgweb_config_manager.change_entry(old_lhs, new_lhs, repo_dir) # Change the entry in the repository's hgrc file. diff --git a/lib/tool_shed/util/shed_index.py b/lib/tool_shed/util/shed_index.py index bd823b6cb094..ee94d4c1a785 100644 --- a/lib/tool_shed/util/shed_index.py +++ b/lib/tool_shed/util/shed_index.py @@ -37,7 +37,7 @@ def _get_or_create_index(whoosh_index_dir): return get_or_create_index(whoosh_index_dir, repo_schema), get_or_create_index(tool_index_dir, tool_schema) -def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): +def build_index(whoosh_index_dir, file_path, hgweb_config_dir, hgweb_repo_prefix, dburi, **kwargs): """ Build two search indexes simultaneously One is for repositories and the other for tools. @@ -55,7 +55,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): execution_timer = ExecutionTimer() with repo_index.searcher() as searcher: - for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): + for repo in get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs): tools_list = repo.pop("tools_list") repo_id = repo["id"] indexed_document = searcher.document(id=repo_id) @@ -89,7 +89,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): return repos_indexed, tools_indexed -def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): +def get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs): """ Load repos from DB and included tools from .xml configs. """ @@ -120,7 +120,7 @@ def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): # Load all changesets of the repo for lineage. repo_path = os.path.join( - hgweb_config_dir, hgwcm.get_entry(os.path.join("repos", repo.user.username, repo.name)) + hgweb_config_dir, hgwcm.get_entry(os.path.join(hgweb_repo_prefix, repo.user.username, repo.name)) ) hg_repo = hg.repository(ui.ui(), repo_path.encode("utf-8")) lineage = [] diff --git a/lib/tool_shed/webapp/api/tools.py b/lib/tool_shed/webapp/api/tools.py index 33099f5e2ada..99b676bbdc32 100644 --- a/lib/tool_shed/webapp/api/tools.py +++ b/lib/tool_shed/webapp/api/tools.py @@ -33,6 +33,7 @@ def build_search_index(self, trans, **kwd): trans.app.config.whoosh_index_dir, trans.app.config.file_path, trans.app.config.hgweb_config_dir, + trans.app.config.hgweb_repo_prefix, trans.app.config.database_connection, ) return { diff --git a/lib/tool_shed/webapp/api2/tools.py b/lib/tool_shed/webapp/api2/tools.py index 0d8c2f2d5524..bd48db71ce01 100644 --- a/lib/tool_shed/webapp/api2/tools.py +++ b/lib/tool_shed/webapp/api2/tools.py @@ -74,6 +74,7 @@ def build_search_index(self) -> BuildSearchIndexResponse: config.whoosh_index_dir, config.file_path, config.hgweb_config_dir, + config.hgweb_repo_prefix, config.database_connection, ) return BuildSearchIndexResponse( diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index c71ad3938c68..58ccf206596c 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -104,6 +104,7 @@ def __init__(self, **kwd) -> None: # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir + self.hgweb_config_manager.hgweb_repo_prefix = self.config.hgweb_repo_prefix # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry(self) # Configure Sentry client if configured diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py index 688e2397525b..161e1237620e 100644 --- a/lib/tool_shed/webapp/controllers/repository.py +++ b/lib/tool_shed/webapp/controllers/repository.py @@ -747,7 +747,7 @@ def create_repository(self, trans, **kwd): repository_type, description, long_description, - user_id=trans.user.id, + user=trans.user, category_ids=category_ids, remote_repository_url=remote_repository_url, homepage_url=homepage_url, diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index 2c99c9fbd0c4..db1b2476222c 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -143,6 +143,11 @@ def __init__(self, email=None, password=None): self.purged = False self.new_repo_alert = False + @property + def current_galaxy_session(self): + if self.galaxy_sessions: + return self.galaxy_sessions[0] + def all_roles(self): roles = [ura.role for ura in self.roles] for group in [uga.group for uga in self.groups]: @@ -419,12 +424,13 @@ class Repository(Base, Dictifiable): ] file_states = Bunch(NORMAL="n", NEEDS_MERGING="m", MARKED_FOR_REMOVAL="r", MARKED_FOR_ADDITION="a", NOT_TRACKED="?") - def __init__(self, private=False, times_downloaded=0, deprecated=False, **kwd): + def __init__(self, private=False, times_downloaded=0, deprecated=False, user=None, **kwd): super().__init__(**kwd) self.private = private self.times_downloaded = times_downloaded self.deprecated = deprecated self.name = self.name or "Unnamed repository" + self.user = user @property def hg_repo(self): @@ -510,7 +516,9 @@ def is_new(self): def repo_path(self, app=None): # Keep app argument for compatibility with tool_shed_install Repository model - return hgweb_config_manager.get_entry(os.path.join("repos", self.user.username, self.name)) + return hgweb_config_manager.get_entry( + os.path.join(hgweb_config_manager.hgweb_repo_prefix, self.user.username, self.name) + ) def revision(self): repo = self.hg_repo diff --git a/scripts/tool_shed/build_ts_whoosh_index.py b/scripts/tool_shed/build_ts_whoosh_index.py index d74cd99d80cc..d74420f17ad7 100644 --- a/scripts/tool_shed/build_ts_whoosh_index.py +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -40,6 +40,7 @@ def parse_arguments(): config = ts_config.ToolShedAppConfiguration(**app_properties) args.dburi = config.database_connection args.hgweb_config_dir = config.hgweb_config_dir + args.hgweb_repo_prefix = config.hgweb_repo_prefix args.whoosh_index_dir = config.whoosh_index_dir args.file_path = config.file_path if args.debug: diff --git a/test/unit/tool_shed/test_shed_index.py b/test/unit/tool_shed/test_shed_index.py index b589dc74fd83..e76cfee5bb1c 100644 --- a/test/unit/tool_shed/test_shed_index.py +++ b/test/unit/tool_shed/test_shed_index.py @@ -36,10 +36,11 @@ def community_file_dir(): @pytest.fixture() def community_file_structure(community_file_dir): - community = namedtuple("community", "file_path hgweb_config_dir dburi") + community = namedtuple("community", "file_path hgweb_config_dir hgweb_repo_prefix dburi") return community( file_path=os.path.join(community_file_dir, "database", "community_files"), hgweb_config_dir=community_file_dir, + hgweb_repo_prefix="repos/", dburi="sqlite:///%s" % os.path.join(community_file_dir, "database", "community.sqlite"), ) @@ -49,6 +50,7 @@ def test_build_index(whoosh_index_dir, community_file_structure): whoosh_index_dir, community_file_structure.file_path, community_file_structure.hgweb_config_dir, + community_file_structure.hgweb_repo_prefix, community_file_structure.dburi, ) assert repos_indexed == 1 @@ -59,6 +61,7 @@ def test_build_index(whoosh_index_dir, community_file_structure): whoosh_index_dir, community_file_structure.file_path, community_file_structure.hgweb_config_dir, + community_file_structure.hgweb_repo_prefix, community_file_structure.dburi, ) assert repos_indexed == 0 @@ -74,6 +77,7 @@ def test_build_index(whoosh_index_dir, community_file_structure): whoosh_index_dir, community_file_structure.file_path, community_file_structure.hgweb_config_dir, + community_file_structure.hgweb_repo_prefix, community_file_structure.dburi, ) assert repos_indexed == 1