From e8a3b8c0cdbfa89351aa5cbf5f6892f352c98ea4 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 28 Mar 2024 12:55:31 +0100 Subject: [PATCH] Create wrapper for requests with default user agent --- cron/build_chrom_db.py | 3 +- cron/parse_builds.py | 2 +- cron/parse_builds_3_sites.py | 2 +- lib/galaxy/authnz/custos_authnz.py | 2 +- lib/galaxy/authnz/managers.py | 2 +- lib/galaxy/authnz/psa_authnz.py | 6 ++- lib/galaxy/files/sources/invenio.py | 2 +- lib/galaxy/files/sources/util.py | 7 +-- lib/galaxy/job_execution/container_monitor.py | 7 +-- lib/galaxy/jobs/runners/godocker.py | 3 +- lib/galaxy/managers/citations.py | 6 ++- lib/galaxy/model/store/load_objects.py | 3 +- lib/galaxy/selenium/has_driver.py | 2 +- lib/galaxy/selenium/navigates_galaxy.py | 6 ++- lib/galaxy/tool_util/biotools/source.py | 7 +-- lib/galaxy/tool_util/cwl/cwltool_deps.py | 2 +- lib/galaxy/tool_util/data/__init__.py | 2 +- lib/galaxy/tool_util/deps/mulled/get_tests.py | 16 +++++-- .../deps/mulled/mulled_build_channel.py | 3 +- .../tool_util/deps/mulled/mulled_list.py | 3 +- .../tool_util/deps/mulled/mulled_search.py | 3 +- lib/galaxy/tool_util/deps/mulled/util.py | 2 +- lib/galaxy/tool_util/locations/dockstore.py | 6 ++- lib/galaxy/tool_util/verify/interactor.py | 2 +- .../tools/error_reports/plugins/base_git.py | 3 +- .../tools/error_reports/plugins/gitlab.py | 2 +- .../tools/error_reports/plugins/slack.py | 7 +-- lib/galaxy/tools/recommendations.py | 6 ++- lib/galaxy/util/__init__.py | 3 +- lib/galaxy/util/requests.py | 44 +++++++++++++++++++ lib/galaxy/util/unittest_utils/__init__.py | 2 +- lib/galaxy/util/user_agent.py | 33 +------------- lib/galaxy/web/proxy/__init__.py | 3 +- .../galaxy/api/container_resolution.py | 3 +- .../webapps/galaxy/controllers/async.py | 3 +- lib/galaxy/webapps/galaxy/services/help.py | 3 +- lib/galaxy/workflow/trs_proxy.py | 2 +- lib/galaxy_test/api/test_drs.py | 3 +- lib/galaxy_test/api/test_jobs.py | 2 +- lib/galaxy_test/base/api.py | 2 +- lib/galaxy_test/base/json_schema_utils.py | 2 +- lib/galaxy_test/base/populators.py | 2 +- lib/galaxy_test/conftest.py | 6 ++- lib/galaxy_test/selenium/framework.py | 2 +- lib/tool_shed/test/base/api_util.py | 2 +- lib/tool_shed/test/base/populators.py | 2 +- lib/tool_shed/test/base/twilltestcase.py | 2 +- scripts/api/fetch_to_library.py | 3 +- scripts/api/search.py | 2 +- scripts/api/upload_to_history.py | 2 +- scripts/bootstrap_test_shed.py | 2 +- scripts/edam_mapping.py | 2 +- scripts/grt/upload.py | 3 +- scripts/microbes/harvest_bacteria.py | 2 +- scripts/resumable_upload.py | 3 +- scripts/tool_shed/api/common.py | 2 +- test/integration/oidc/test_auth_oidc.py | 3 +- test/integration/test_interactivetools_api.py | 2 +- test/integration/test_job_files.py | 2 +- test/unit/app/authnz/test_custos_authnz.py | 6 ++- test/unit/tool_shed/test_shed_index.py | 2 +- test/unit/util/test_get_url.py | 6 ++- test/unit/webapps/test_client_disconnect.py | 6 ++- 63 files changed, 163 insertions(+), 123 deletions(-) create mode 100644 lib/galaxy/util/requests.py diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py index 615e791e5c29..de83836aae68 100644 --- a/cron/build_chrom_db.py +++ b/cron/build_chrom_db.py @@ -18,7 +18,8 @@ from urllib.parse import urlencode import parse_builds # noqa: I100,I202 -import requests + +from galaxy.util import requests def getchrominfo(url, db): diff --git a/cron/parse_builds.py b/cron/parse_builds.py index dcd2005c537f..4477625747f9 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -8,7 +8,7 @@ import sys import xml.etree.ElementTree as ElementTree -import requests +from galaxy.util import requests def getbuilds(url): diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py index 52ad1bd6acdb..65fef089db9a 100644 --- a/cron/parse_builds_3_sites.py +++ b/cron/parse_builds_3_sites.py @@ -5,7 +5,7 @@ import xml.etree.ElementTree as ElementTree -import requests +from galaxy.util import requests sites = [ "http://genome.ucsc.edu/cgi-bin/", diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index a216b3239c14..806f6e1cafba 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -16,7 +16,6 @@ from urllib.parse import quote import jwt -import requests from oauthlib.common import generate_nonce from requests_oauthlib import OAuth2Session @@ -30,6 +29,7 @@ ) from galaxy.model.base import transaction from galaxy.model.orm.util import add_object_to_object_session +from galaxy.util import requests from . import IdentityProvider try: diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 35211f9fe57e..bb791d30bf79 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -6,7 +6,6 @@ import random import string -import requests from cloudauthz import CloudAuthz from cloudauthz.exceptions import CloudAuthzBaseException @@ -19,6 +18,7 @@ etree, listify, parse_xml, + requests, string_as_bool, unicodify, ) diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index a1193f02868c..b8a34b68e710 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -3,7 +3,6 @@ import time import jwt -import requests from msal import ConfidentialClientApplication from social_core.actions import ( do_auth, @@ -27,7 +26,10 @@ UserAuthnzToken, ) from galaxy.model.base import transaction -from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + requests, +) from . import IdentityProvider log = logging.getLogger(__name__) diff --git a/lib/galaxy/files/sources/invenio.py b/lib/galaxy/files/sources/invenio.py index 12fcbd761b19..b725a3a08d07 100644 --- a/lib/galaxy/files/sources/invenio.py +++ b/lib/galaxy/files/sources/invenio.py @@ -9,7 +9,6 @@ ) from urllib.parse import quote -import requests from typing_extensions import ( Literal, TypedDict, @@ -30,6 +29,7 @@ from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, get_charset_from_http_headers, + requests, stream_to_open_named_file, ) diff --git a/lib/galaxy/files/sources/util.py b/lib/galaxy/files/sources/util.py index 0616910b76ff..868e139a8101 100644 --- a/lib/galaxy/files/sources/util.py +++ b/lib/galaxy/files/sources/util.py @@ -7,8 +7,6 @@ Union, ) -import requests - from galaxy import exceptions from galaxy.files import ( ConfiguredFileSources, @@ -17,7 +15,10 @@ from galaxy.files.sources import FilesSourceOptions from galaxy.files.sources.http import HTTPFilesSourceProperties from galaxy.files.uris import stream_url_to_file -from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + requests, +) from galaxy.util.config_parsers import IpAllowedListEntryT TargetPathT = Union[str, PathLike] diff --git a/lib/galaxy/job_execution/container_monitor.py b/lib/galaxy/job_execution/container_monitor.py index ebc9be73677e..dfed37daf8d0 100644 --- a/lib/galaxy/job_execution/container_monitor.py +++ b/lib/galaxy/job_execution/container_monitor.py @@ -9,10 +9,11 @@ from functools import partial from typing import Callable -import requests - from galaxy.tool_util.deps import docker_util -from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + requests, +) from galaxy.util.sockets import get_ip GetIpCallable = Callable[[], str] diff --git a/lib/galaxy/jobs/runners/godocker.py b/lib/galaxy/jobs/runners/godocker.py index 4ba4b6c67ab5..01fa737f3bc4 100644 --- a/lib/galaxy/jobs/runners/godocker.py +++ b/lib/galaxy/jobs/runners/godocker.py @@ -3,8 +3,6 @@ import time from datetime import datetime -import requests - from galaxy import model from galaxy.jobs.runners import ( AsynchronousJobRunner, @@ -12,6 +10,7 @@ ) from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, + requests, unicodify, ) diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index 09a4e137cbee..768c6e829bcf 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -1,12 +1,14 @@ import functools import logging -import requests from beaker.cache import CacheManager from beaker.util import parse_cache_config_options from galaxy.structured_app import BasicSharedApp -from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + requests, +) log = logging.getLogger(__name__) diff --git a/lib/galaxy/model/store/load_objects.py b/lib/galaxy/model/store/load_objects.py index 7666187764ca..8b6bcdb08dfc 100644 --- a/lib/galaxy/model/store/load_objects.py +++ b/lib/galaxy/model/store/load_objects.py @@ -8,9 +8,10 @@ Dict, ) -import requests import yaml +from galaxy.util import requests + DESCRIPTION = """Load a Galaxy model store into a running Galaxy instance. See the corresponding galaxy-build-objects script for one possible way to diff --git a/lib/galaxy/selenium/has_driver.py b/lib/galaxy/selenium/has_driver.py index ed11cba91182..9fae270ca7b8 100644 --- a/lib/galaxy/selenium/has_driver.py +++ b/lib/galaxy/selenium/has_driver.py @@ -14,7 +14,6 @@ Union, ) -import requests from axe_selenium_python import Axe from selenium.common.exceptions import ( NoSuchElementException, @@ -29,6 +28,7 @@ from selenium.webdriver.support.wait import WebDriverWait from galaxy.navigation.components import Target +from galaxy.util import requests from .axe_results import ( AxeResults, NullAxeResults, diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py index de765d0375fe..555158a6419a 100644 --- a/lib/galaxy/selenium/navigates_galaxy.py +++ b/lib/galaxy/selenium/navigates_galaxy.py @@ -24,7 +24,6 @@ Union, ) -import requests import yaml from selenium.webdriver.common.by import By from selenium.webdriver.remote.webdriver import WebDriver @@ -36,7 +35,10 @@ HasText, ) from galaxy.navigation.data import load_root_component -from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + requests, +) from . import sizzle from .has_driver import ( exception_indicates_click_intercepted, diff --git a/lib/galaxy/tool_util/biotools/source.py b/lib/galaxy/tool_util/biotools/source.py index 92c8b08a69b7..6f101abcbee5 100644 --- a/lib/galaxy/tool_util/biotools/source.py +++ b/lib/galaxy/tool_util/biotools/source.py @@ -8,9 +8,10 @@ Optional, ) -import requests - -from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + requests, +) from .interface import BiotoolsEntry diff --git a/lib/galaxy/tool_util/cwl/cwltool_deps.py b/lib/galaxy/tool_util/cwl/cwltool_deps.py index bd40a6c3092a..414c516caf8b 100644 --- a/lib/galaxy/tool_util/cwl/cwltool_deps.py +++ b/lib/galaxy/tool_util/cwl/cwltool_deps.py @@ -10,7 +10,7 @@ warnings.filterwarnings("ignore", message=r"[\n.]DEPRECATION: Python 2", module="cwltool") -import requests +from galaxy.util import requests try: from cwltool import ( diff --git a/lib/galaxy/tool_util/data/__init__.py b/lib/galaxy/tool_util/data/__init__.py index 04786715be8e..8e7629192837 100644 --- a/lib/galaxy/tool_util/data/__init__.py +++ b/lib/galaxy/tool_util/data/__init__.py @@ -34,7 +34,6 @@ Union, ) -import requests from typing_extensions import ( Protocol, TypedDict, @@ -44,6 +43,7 @@ from galaxy.exceptions import MessageException from galaxy.util import ( Element, + requests, RW_R__R__, ) from galaxy.util.compression_utils import decompress_path_to_directory diff --git a/lib/galaxy/tool_util/deps/mulled/get_tests.py b/lib/galaxy/tool_util/deps/mulled/get_tests.py index 847f50d255c5..33a83cc67c4a 100644 --- a/lib/galaxy/tool_util/deps/mulled/get_tests.py +++ b/lib/galaxy/tool_util/deps/mulled/get_tests.py @@ -15,9 +15,10 @@ Optional, ) -import requests import yaml +from galaxy.util import requests + try: from jinja2 import Template from jinja2.exceptions import UndefinedError @@ -130,7 +131,10 @@ def find_anaconda_versions(name, anaconda_channel="bioconda"): """ Find a list of available anaconda versions for a given container name """ - r = requests.get(f"https://anaconda.org/{anaconda_channel}/{name}/files", timeout=MULLED_SOCKET_TIMEOUT) + r = requests.get( + f"https://anaconda.org/{anaconda_channel}/{name}/files", + timeout=MULLED_SOCKET_TIMEOUT, + ) r.raise_for_status() urls = [] for line in r.text.splitlines(): @@ -147,7 +151,8 @@ def open_recipe_file(file, recipes_path=None, github_repo="bioconda/bioconda-rec return open(f"{recipes_path}/{file}").read() else: # if no clone of the repo is available locally, download from GitHub r = requests.get( - f"https://raw.githubusercontent.com/{github_repo}/master/{file}", timeout=MULLED_SOCKET_TIMEOUT + f"https://raw.githubusercontent.com/{github_repo}/master/{file}", + timeout=MULLED_SOCKET_TIMEOUT, ) if r.status_code == 404: raise OSError @@ -163,7 +168,10 @@ def get_alternative_versions(filepath, filename, recipes_path=None, github_repo= return [n.replace(f"{recipes_path}/", "") for n in glob(f"{recipes_path}/{filepath}/*/{filename}")] # else use the GitHub API: versions = [] - r = requests.get(f"https://api.github.com/repos/{github_repo}/contents/{filepath}", timeout=MULLED_SOCKET_TIMEOUT) + r = requests.get( + f"https://api.github.com/repos/{github_repo}/contents/{filepath}", + timeout=MULLED_SOCKET_TIMEOUT, + ) check_github_api_response_rate_limit(r) r.raise_for_status() for subfile in json.loads(r.text): diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py index 56a567dcf9f8..8dd6b5702225 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py @@ -22,8 +22,7 @@ import sys import time -import requests - +from galaxy.util import requests from ._cli import arg_parser from .mulled_build import ( add_build_arguments, diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_list.py b/lib/galaxy/tool_util/deps/mulled/mulled_list.py index 00e892fbd643..6d2e41de7201 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_list.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_list.py @@ -5,8 +5,7 @@ from glob import glob from html.parser import HTMLParser -import requests - +from galaxy.util import requests from .util import MULLED_SOCKET_TIMEOUT QUAY_API_ENDPOINT = "https://quay.io/api/v1/repository" diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_search.py b/lib/galaxy/tool_util/deps/mulled/mulled_search.py index a2553ddf770c..d936802a0840 100755 --- a/lib/galaxy/tool_util/deps/mulled/mulled_search.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_search.py @@ -6,11 +6,10 @@ import sys import tempfile -import requests - from galaxy.tool_util.deps.conda_util import CondaContext from galaxy.util import ( check_github_api_response_rate_limit, + requests, which, ) from .mulled_list import get_singularity_containers diff --git a/lib/galaxy/tool_util/deps/mulled/util.py b/lib/galaxy/tool_util/deps/mulled/util.py index 17aa33ba8fa2..a7eacb33806c 100644 --- a/lib/galaxy/tool_util/deps/mulled/util.py +++ b/lib/galaxy/tool_util/deps/mulled/util.py @@ -18,7 +18,6 @@ Union, ) -import requests from conda_package_streaming.package_streaming import stream_conda_info from conda_package_streaming.url import stream_conda_info as stream_conda_info_from_url from packaging.version import Version @@ -29,6 +28,7 @@ LegacyVersion, parse_version, ) +from galaxy.util import requests if TYPE_CHECKING: from galaxy.tool_util.deps.container_resolvers import ResolutionCache diff --git a/lib/galaxy/tool_util/locations/dockstore.py b/lib/galaxy/tool_util/locations/dockstore.py index fe00e2847b04..fb5cb84e65a5 100644 --- a/lib/galaxy/tool_util/locations/dockstore.py +++ b/lib/galaxy/tool_util/locations/dockstore.py @@ -1,9 +1,11 @@ from urllib.parse import quote -import requests import yaml -from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + requests, +) from . import ToolLocationResolver diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index be9d1e57e442..8116882e7496 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -24,7 +24,6 @@ Union, ) -import requests from packaging.version import Version from requests import Response from requests.cookies import RequestsCookieJar @@ -41,6 +40,7 @@ TestCollectionDef, TestCollectionOutputDef, ) +from galaxy.util import requests from galaxy.util.bunch import Bunch from galaxy.util.hash_util import ( memory_bound_hexdigest, diff --git a/lib/galaxy/tools/error_reports/plugins/base_git.py b/lib/galaxy/tools/error_reports/plugins/base_git.py index e1487642c6f9..07ee050b7d13 100644 --- a/lib/galaxy/tools/error_reports/plugins/base_git.py +++ b/lib/galaxy/tools/error_reports/plugins/base_git.py @@ -8,11 +8,10 @@ ) from typing import Dict -import requests - from galaxy.tools.errors import EmailErrorReporter from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, + requests, unicodify, ) from . import ErrorPlugin diff --git a/lib/galaxy/tools/error_reports/plugins/gitlab.py b/lib/galaxy/tools/error_reports/plugins/gitlab.py index b8f2d923bade..349e76b207ae 100644 --- a/lib/galaxy/tools/error_reports/plugins/gitlab.py +++ b/lib/galaxy/tools/error_reports/plugins/gitlab.py @@ -4,7 +4,7 @@ import os import urllib.parse -import requests +from galaxy.util import requests try: import gitlab diff --git a/lib/galaxy/tools/error_reports/plugins/slack.py b/lib/galaxy/tools/error_reports/plugins/slack.py index 6af1431ba86d..53affb80bc1a 100644 --- a/lib/galaxy/tools/error_reports/plugins/slack.py +++ b/lib/galaxy/tools/error_reports/plugins/slack.py @@ -7,9 +7,10 @@ Dict, ) -import requests - -from galaxy.util import string_as_bool +from galaxy.util import ( + requests, + string_as_bool, +) from .base_git import BaseGitPlugin log = logging.getLogger(__name__) diff --git a/lib/galaxy/tools/recommendations.py b/lib/galaxy/tools/recommendations.py index bcfbe89b0a8c..00d8490a9737 100644 --- a/lib/galaxy/tools/recommendations.py +++ b/lib/galaxy/tools/recommendations.py @@ -6,12 +6,14 @@ import h5py import numpy as np -import requests import yaml from galaxy.tools.parameters import populate_state from galaxy.tools.parameters.workflow_utils import workflow_building_modes -from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + requests, +) from galaxy.workflow.modules import module_factory log = logging.getLogger(__name__) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 814e9fdd9c6b..96e7edff7c23 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -56,7 +56,6 @@ urlunsplit, ) -import requests from boltons.iterutils import ( default_enter, remap, @@ -68,6 +67,8 @@ Self, ) +from galaxy.util import requests + try: import grp except ImportError: diff --git a/lib/galaxy/util/requests.py b/lib/galaxy/util/requests.py new file mode 100644 index 000000000000..8183c5371557 --- /dev/null +++ b/lib/galaxy/util/requests.py @@ -0,0 +1,44 @@ +from typing import ( + Callable, + cast, + TypeVar, +) + +import requests +from requests import ( # noqa: F401 + codes as codes, + exceptions as exceptions, + Response as Response, +) +from typing_extensions import ParamSpec + +from .user_agent import get_default_headers + +Param = ParamSpec("Param") +RetType = TypeVar("RetType") + + +def default_user_agent_decorator(f: Callable[Param, RetType]) -> Callable[Param, RetType]: + + def wrapper(*args: Param.args, **kwargs: Param.kwargs) -> RetType: + headers = cast(dict, kwargs.pop("headers", None) or {}) + headers.update(get_default_headers()) + is_session = f in (requests.session, requests.Session) + if not is_session: + kwargs["headers"] = headers + rval = f(*args, **kwargs) + if is_session: + rval.headers = headers # type: ignore[attr-defined] + return rval + + return wrapper + + +delete = default_user_agent_decorator(requests.delete) +get = default_user_agent_decorator(requests.get) +head = default_user_agent_decorator(requests.head) +patch = default_user_agent_decorator(requests.patch) +post = default_user_agent_decorator(requests.post) +put = default_user_agent_decorator(requests.put) +session = default_user_agent_decorator(requests.session) +Session = default_user_agent_decorator(requests.Session) diff --git a/lib/galaxy/util/unittest_utils/__init__.py b/lib/galaxy/util/unittest_utils/__init__.py index 1a3d229eae21..4397a1593ecc 100644 --- a/lib/galaxy/util/unittest_utils/__init__.py +++ b/lib/galaxy/util/unittest_utils/__init__.py @@ -6,8 +6,8 @@ from unittest import SkipTest import pytest -import requests +from galaxy.util import requests from galaxy.util.commands import which diff --git a/lib/galaxy/util/user_agent.py b/lib/galaxy/util/user_agent.py index 4e914d816ccb..8894d022771b 100644 --- a/lib/galaxy/util/user_agent.py +++ b/lib/galaxy/util/user_agent.py @@ -1,34 +1,5 @@ -import urllib - -import requests - from galaxy.version import VERSION -def __append_word_to_user_agent(word): - - # set requests User-Agent - old_default_user_agent = requests.utils.default_user_agent - - def new_default_user_agent(*args): - return f"{old_default_user_agent(*args)} {word}" - - requests.utils.default_user_agent = new_default_user_agent - - # set urllib User-Agent - old_build_opener = urllib.request.build_opener - - def modify_user_agent_header(header): - if header[0].lower() == "user-agent": - return (header[0], f"{header[1]} {word}") - return header - - def new_build_opener(*handlers): - opener = old_build_opener(*handlers) - opener.addheaders = [modify_user_agent_header(header) for header in opener.addheaders] - return opener - - urllib.request.build_opener = new_build_opener - - -__append_word_to_user_agent(f"galaxy/{VERSION}") +def get_default_headers(): + return {"user-agent": f"galaxy/{VERSION}"} diff --git a/lib/galaxy/web/proxy/__init__.py b/lib/galaxy/web/proxy/__init__.py index 617516a1bee9..d0212756fa4f 100644 --- a/lib/galaxy/web/proxy/__init__.py +++ b/lib/galaxy/web/proxy/__init__.py @@ -4,9 +4,8 @@ import time from collections import namedtuple -import requests - from galaxy.util import ( + requests, sockets, sqlite, unique_id, diff --git a/lib/galaxy/webapps/galaxy/api/container_resolution.py b/lib/galaxy/webapps/galaxy/api/container_resolution.py index 5431888a9830..3b670bed638f 100644 --- a/lib/galaxy/webapps/galaxy/api/container_resolution.py +++ b/lib/galaxy/webapps/galaxy/api/container_resolution.py @@ -4,10 +4,9 @@ import logging -import requests - from galaxy.structured_app import StructuredApp from galaxy.tool_util.deps import views +from galaxy.util import requests from galaxy.web import ( expose_api, require_admin, diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index 4a9615fc700d..7a8c45a7badf 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -5,13 +5,12 @@ import logging from urllib.parse import urlencode -import requests - from galaxy import web from galaxy.model.base import transaction from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, Params, + requests, unicodify, ) from galaxy.util.hash_util import hmac_new diff --git a/lib/galaxy/webapps/galaxy/services/help.py b/lib/galaxy/webapps/galaxy/services/help.py index 1f20d12286df..e75bf087e685 100644 --- a/lib/galaxy/webapps/galaxy/services/help.py +++ b/lib/galaxy/webapps/galaxy/services/help.py @@ -1,11 +1,10 @@ import logging -import requests - from galaxy.config import GalaxyAppConfiguration from galaxy.exceptions import ServerNotConfiguredForRequest from galaxy.schema.help import HelpForumSearchResponse from galaxy.security.idencoding import IdEncodingHelper +from galaxy.util import requests from galaxy.webapps.galaxy.services.base import ServiceBase log = logging.getLogger(__name__) diff --git a/lib/galaxy/workflow/trs_proxy.py b/lib/galaxy/workflow/trs_proxy.py index 7bf65631b46a..e465127a99ad 100644 --- a/lib/galaxy/workflow/trs_proxy.py +++ b/lib/galaxy/workflow/trs_proxy.py @@ -3,13 +3,13 @@ import re import urllib.parse -import requests import yaml from galaxy.exceptions import MessageException from galaxy.util import ( asbool, DEFAULT_SOCKET_TIMEOUT, + requests, ) from galaxy.util.search import parse_filters diff --git a/lib/galaxy_test/api/test_drs.py b/lib/galaxy_test/api/test_drs.py index b1c77c00e1e1..732a83e4996d 100644 --- a/lib/galaxy_test/api/test_drs.py +++ b/lib/galaxy_test/api/test_drs.py @@ -9,8 +9,6 @@ urlparse, ) -import requests - from galaxy.files import ( ConfiguredFileSources, ConfiguredFileSourcesConfig, @@ -20,6 +18,7 @@ fetch_drs_to_file, RetryOptions, ) +from galaxy.util import requests from galaxy.util.config_parsers import parse_allowlist_ips from galaxy_test.base.populators import DatasetPopulator from ._framework import ApiTestCase diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 69a5bb520ccd..4212311bcddf 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -7,9 +7,9 @@ from unittest import SkipTest import pytest -import requests from dateutil.parser import isoparse +from galaxy.util import requests from galaxy_test.api.test_tools import TestsTools from galaxy_test.base.api_asserts import assert_status_code_is_ok from galaxy_test.base.populators import ( diff --git a/lib/galaxy_test/base/api.py b/lib/galaxy_test/base/api.py index 63e33dd5e90a..a25411873f08 100644 --- a/lib/galaxy_test/base/api.py +++ b/lib/galaxy_test/base/api.py @@ -11,9 +11,9 @@ ) import pytest -import requests from typing_extensions import Protocol +from galaxy.util import requests from galaxy.util.properties import get_from_env from .api_asserts import ( assert_error_code_is, diff --git a/lib/galaxy_test/base/json_schema_utils.py b/lib/galaxy_test/base/json_schema_utils.py index befe5105804d..a1571ba8feb4 100644 --- a/lib/galaxy_test/base/json_schema_utils.py +++ b/lib/galaxy_test/base/json_schema_utils.py @@ -4,8 +4,8 @@ ) import jsonschema -import requests +from galaxy.util import requests from galaxy_test.base import api_asserts schema_store: Dict[str, Any] = {} diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index e4fd0026245f..59f42a6b9061 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -69,7 +69,6 @@ from uuid import UUID import cwltest.compare -import requests import yaml from bioblend.galaxyclient import GalaxyClient from gxformat2 import ( @@ -99,6 +98,7 @@ from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, galaxy_root_path, + requests, UNKNOWN, ) from galaxy.util.resources import resource_string diff --git a/lib/galaxy_test/conftest.py b/lib/galaxy_test/conftest.py index 64f69aee6f57..f6125d2df83d 100644 --- a/lib/galaxy_test/conftest.py +++ b/lib/galaxy_test/conftest.py @@ -4,9 +4,11 @@ from urllib.parse import urlencode import pytest -import requests -from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + requests, +) from galaxy.web import statsd_client as statsd from galaxy_test.api._framework import ApiTestCase from galaxy_test.driver.driver_util import GalaxyTestDriver diff --git a/lib/galaxy_test/selenium/framework.py b/lib/galaxy_test/selenium/framework.py index 310c88c88053..e551cc6badb1 100644 --- a/lib/galaxy_test/selenium/framework.py +++ b/lib/galaxy_test/selenium/framework.py @@ -18,7 +18,6 @@ TYPE_CHECKING, ) -import requests import yaml from gxformat2 import ( convert_and_import_workflow, @@ -39,6 +38,7 @@ asbool, classproperty, DEFAULT_SOCKET_TIMEOUT, + requests, ) from galaxy.util.unittest_utils import skip_if_github_down from galaxy_test.base import populators diff --git a/lib/tool_shed/test/base/api_util.py b/lib/tool_shed/test/base/api_util.py index e90a495aa69b..436fc4839c43 100644 --- a/lib/tool_shed/test/base/api_util.py +++ b/lib/tool_shed/test/base/api_util.py @@ -9,9 +9,9 @@ ) from urllib.parse import urljoin -import requests from typing_extensions import Literal +from galaxy.util import requests from galaxy_test.base.api_asserts import ( assert_has_keys, assert_status_code_is, diff --git a/lib/tool_shed/test/base/populators.py b/lib/tool_shed/test/base/populators.py index 460addd27e17..5b7c9c18c3d5 100644 --- a/lib/tool_shed/test/base/populators.py +++ b/lib/tool_shed/test/base/populators.py @@ -8,9 +8,9 @@ Union, ) -import requests from typing_extensions import Protocol +from galaxy.util import requests from galaxy.util.resources import ( as_file, resource_path, diff --git a/lib/tool_shed/test/base/twilltestcase.py b/lib/tool_shed/test/base/twilltestcase.py index 30d6525ec192..cb941f3e3a0d 100644 --- a/lib/tool_shed/test/base/twilltestcase.py +++ b/lib/tool_shed/test/base/twilltestcase.py @@ -25,7 +25,6 @@ ) import pytest -import requests from mercurial import ( commands, hg, @@ -53,6 +52,7 @@ from galaxy.tool_shed.util.repository_util import check_for_updates from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, + requests, smart_str, ) from galaxy_test.base.api_asserts import assert_status_code_is_ok diff --git a/scripts/api/fetch_to_library.py b/scripts/api/fetch_to_library.py index c291762edfa8..89b6b3e43866 100644 --- a/scripts/api/fetch_to_library.py +++ b/scripts/api/fetch_to_library.py @@ -1,9 +1,10 @@ import argparse import json -import requests import yaml +from galaxy.util import requests + def main(): parser = argparse.ArgumentParser(description="Upload a directory into a data library") diff --git a/scripts/api/search.py b/scripts/api/search.py index 14e30f1866a5..0f9f77c702b6 100644 --- a/scripts/api/search.py +++ b/scripts/api/search.py @@ -5,7 +5,7 @@ import json import sys -import requests +from galaxy.util import requests class RemoteGalaxy: diff --git a/scripts/api/upload_to_history.py b/scripts/api/upload_to_history.py index e3c71e091444..c5746495b76d 100755 --- a/scripts/api/upload_to_history.py +++ b/scripts/api/upload_to_history.py @@ -8,7 +8,7 @@ import sys try: - import requests + from galaxy.util import requests except ImportError: print( "Could not import the requests module. See http://docs.python-requests.org/en/latest/" diff --git a/scripts/bootstrap_test_shed.py b/scripts/bootstrap_test_shed.py index 58c7c475b033..aa657476bace 100644 --- a/scripts/bootstrap_test_shed.py +++ b/scripts/bootstrap_test_shed.py @@ -17,7 +17,7 @@ Optional, ) -import requests +from galaxy.util import requests sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) diff --git a/scripts/edam_mapping.py b/scripts/edam_mapping.py index 817765a6957f..75364178d673 100644 --- a/scripts/edam_mapping.py +++ b/scripts/edam_mapping.py @@ -16,7 +16,7 @@ import sys from xml import etree -import requests +from galaxy.util import requests sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) diff --git a/scripts/grt/upload.py b/scripts/grt/upload.py index 7f732cf1a971..5f227b99cec2 100644 --- a/scripts/grt/upload.py +++ b/scripts/grt/upload.py @@ -8,9 +8,10 @@ import os import sys -import requests import yaml +from galaxy.util import requests + sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), "grt.yml.sample")) default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), "grt.yml")) diff --git a/scripts/microbes/harvest_bacteria.py b/scripts/microbes/harvest_bacteria.py index a94ae088dbae..fce70ee445ad 100644 --- a/scripts/microbes/harvest_bacteria.py +++ b/scripts/microbes/harvest_bacteria.py @@ -11,7 +11,7 @@ from ftplib import FTP from urllib.request import urlretrieve -import requests +from galaxy.util import requests try: from bs4 import BeautifulSoup diff --git a/scripts/resumable_upload.py b/scripts/resumable_upload.py index 32fd67ff2fa8..18b58c1ef7f9 100644 --- a/scripts/resumable_upload.py +++ b/scripts/resumable_upload.py @@ -2,10 +2,11 @@ import os import click -import requests from tusclient import client from tusclient.storage import filestorage +from galaxy.util import requests + UPLOAD_ENDPOINT = "/api/upload/resumable_upload" SUBMISSION_ENDPOINT = "/api/tools/fetch" CHUNK_SIZE = 10**7 diff --git a/scripts/tool_shed/api/common.py b/scripts/tool_shed/api/common.py index 16ad847ec524..2fcd86138d3e 100644 --- a/scripts/tool_shed/api/common.py +++ b/scripts/tool_shed/api/common.py @@ -1,4 +1,4 @@ -import requests +from galaxy.util import requests def delete(api_key, url, data, return_formatted=True): diff --git a/test/integration/oidc/test_auth_oidc.py b/test/integration/oidc/test_auth_oidc.py index 29ff1c18b2b6..07169ca6fc78 100644 --- a/test/integration/oidc/test_auth_oidc.py +++ b/test/integration/oidc/test_auth_oidc.py @@ -10,9 +10,8 @@ from typing import ClassVar from urllib import parse -import requests - from galaxy.model.base import transaction +from galaxy.util import requests from galaxy_test.base.api import ApiTestInteractor from galaxy_test.driver import integration_util diff --git a/test/integration/test_interactivetools_api.py b/test/integration/test_interactivetools_api.py index 7058070be0bc..a6b00f5da2b5 100644 --- a/test/integration/test_interactivetools_api.py +++ b/test/integration/test_interactivetools_api.py @@ -10,8 +10,8 @@ ) import pytest -import requests +from galaxy.util import requests from galaxy_test.base import api_asserts from galaxy_test.base.populators import ( DatasetPopulator, diff --git a/test/integration/test_job_files.py b/test/integration/test_job_files.py index 36babdcebf6e..fb21fbd98ede 100644 --- a/test/integration/test_job_files.py +++ b/test/integration/test_job_files.py @@ -20,7 +20,6 @@ import tempfile from typing import Dict -import requests from sqlalchemy import select from tusclient import client @@ -29,6 +28,7 @@ ensure_object_added_to_session, transaction, ) +from galaxy.util import requests from galaxy_test.base import api_asserts from galaxy_test.base.populators import DatasetPopulator from galaxy_test.driver import integration_util diff --git a/test/unit/app/authnz/test_custos_authnz.py b/test/unit/app/authnz/test_custos_authnz.py index f7beb2d3351d..0a940f5695d7 100644 --- a/test/unit/app/authnz/test_custos_authnz.py +++ b/test/unit/app/authnz/test_custos_authnz.py @@ -15,7 +15,6 @@ ) import jwt -import requests from galaxy.app_unittest_utils.galaxy_mock import MockTrans from galaxy.authnz import custos_authnz @@ -23,7 +22,10 @@ CustosAuthnzToken, User, ) -from galaxy.util import unicodify +from galaxy.util import ( + requests, + unicodify, +) from galaxy.util.unittest import TestCase diff --git a/test/unit/tool_shed/test_shed_index.py b/test/unit/tool_shed/test_shed_index.py index b589dc74fd83..a53096b5cc36 100644 --- a/test/unit/tool_shed/test_shed_index.py +++ b/test/unit/tool_shed/test_shed_index.py @@ -6,9 +6,9 @@ from io import BytesIO import pytest -import requests from whoosh import index +from galaxy.util import requests from tool_shed.util.shed_index import build_index URL = "https://github.com/mvdbeek/toolshed-test-data/blob/master/toolshed_community_files.tgz?raw=true" diff --git a/test/unit/util/test_get_url.py b/test/unit/util/test_get_url.py index 499bbbddd2d8..f1cb2bd3c049 100644 --- a/test/unit/util/test_get_url.py +++ b/test/unit/util/test_get_url.py @@ -1,9 +1,11 @@ import pytest -import requests import responses from werkzeug.wrappers.response import Response -from galaxy.util import url_get +from galaxy.util import ( + requests, + url_get, +) @responses.activate diff --git a/test/unit/webapps/test_client_disconnect.py b/test/unit/webapps/test_client_disconnect.py index 0381d40cebc0..89adb590136a 100644 --- a/test/unit/webapps/test_client_disconnect.py +++ b/test/unit/webapps/test_client_disconnect.py @@ -5,14 +5,16 @@ from typing import Optional import pytest -import requests import uvicorn from fastapi import status from fastapi.applications import FastAPI from requests import ReadTimeout from starlette.middleware.base import BaseHTTPMiddleware -from galaxy.util import sockets +from galaxy.util import ( + requests, + sockets, +) error_encountered: Optional[str] = None