diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 49120a4fdcec..842418458a30 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -115,7 +115,6 @@ def get_app_properties(): def get_config(): kwargs = get_app_properties() or {} kwargs["override_tempdir"] = False - kwargs["fetch_url_allowlist"] = ["127.0.0.0/24"] return Configuration(**kwargs) diff --git a/lib/galaxy/files/sources/drs.py b/lib/galaxy/files/sources/drs.py index 6586ade4c306..b4bf4e3e89e9 100644 --- a/lib/galaxy/files/sources/drs.py +++ b/lib/galaxy/files/sources/drs.py @@ -42,10 +42,21 @@ def __init__(self, **kwd: Unpack[FilesSourceProperties]): self._force_http = props.pop("force_http", False) self._props = props + @property + def _allowlist(self): + return self._file_sources_config.fetch_url_allowlist + def _realize_to(self, source_path, native_path, user_context=None, opts: Optional[FilesSourceOptions] = None): props = self._serialization_props(user_context) headers = props.pop("http_headers", {}) or {} - fetch_drs_to_file(source_path, native_path, user_context, headers=headers, force_http=self._force_http) + fetch_drs_to_file( + source_path, + native_path, + user_context, + fetch_url_allowlist=self._allowlist, + headers=headers, + force_http=self._force_http, + ) def _write_from(self, target_path, native_path, user_context=None, opts: Optional[FilesSourceOptions] = None): raise NotImplementedError() diff --git a/lib/galaxy/files/sources/http.py b/lib/galaxy/files/sources/http.py index e3f2bcc8a0f4..1db1d54bc8b7 100644 --- a/lib/galaxy/files/sources/http.py +++ b/lib/galaxy/files/sources/http.py @@ -4,6 +4,7 @@ from typing import ( cast, Dict, + List, Optional, ) @@ -15,6 +16,7 @@ get_charset_from_http_headers, stream_to_open_named_file, ) +from galaxy.util.config_parsers import IpAllowedListEntryT from . import ( BaseFilesSource, FilesSourceOptions, @@ -27,6 +29,7 @@ class HTTPFilesSourceProperties(FilesSourceProperties, total=False): url_regex: str http_headers: Dict[str, str] + fetch_url_allowlist: List[IpAllowedListEntryT] class HTTPFilesSource(BaseFilesSource): @@ -61,7 +64,7 @@ def _realize_to( with urllib.request.urlopen(req, timeout=DEFAULT_SOCKET_TIMEOUT) as page: # Verify url post-redirects is still allowlisted - validate_non_local(page.geturl(), self._allowlist) + validate_non_local(page.geturl(), self._allowlist or extra_props.get("fetch_url_allowlist") or []) f = open(native_path, "wb") # fd will be .close()ed in stream_to_open_named_file return stream_to_open_named_file( page, f.fileno(), native_path, source_encoding=get_charset_from_http_headers(page.headers) diff --git a/lib/galaxy/util/drs.py b/lib/galaxy/util/drs.py index 689bb621b263..0616910b76ff 100644 --- a/lib/galaxy/util/drs.py +++ b/lib/galaxy/util/drs.py @@ -1,6 +1,7 @@ import time from os import PathLike from typing import ( + List, Optional, Tuple, Union, @@ -17,6 +18,7 @@ from galaxy.files.sources.http import HTTPFilesSourceProperties from galaxy.files.uris import stream_url_to_file from galaxy.util import DEFAULT_SOCKET_TIMEOUT +from galaxy.util.config_parsers import IpAllowedListEntryT TargetPathT = Union[str, PathLike] @@ -81,6 +83,7 @@ def fetch_drs_to_file( force_http=False, retry_options: Optional[RetryOptions] = None, headers: Optional[dict] = None, + fetch_url_allowlist: Optional[List[IpAllowedListEntryT]] = None, ): """Fetch contents of drs:// URI to a target path.""" if not drs_uri.startswith("drs://"): @@ -107,7 +110,10 @@ def fetch_drs_to_file( access_url, access_headers = _get_access_info(get_url, access_method, headers=headers) opts = FilesSourceOptions() if access_method["type"] == "https": - extra_props: HTTPFilesSourceProperties = {"http_headers": access_headers or {}} + extra_props: HTTPFilesSourceProperties = { + "http_headers": access_headers or {}, + "fetch_url_allowlist": fetch_url_allowlist or [], + } opts.extra_props = extra_props else: opts.extra_props = {}