From b884f4f8d314eec97823a29dc3156fd5df430a03 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Fri, 29 Nov 2024 09:46:32 +0100 Subject: [PATCH] Split `geneve.utils.download` out of `geneve.utils.resource` --- geneve/utils/__init__.py | 45 ++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/geneve/utils/__init__.py b/geneve/utils/__init__.py index 24498f16..732238ad 100644 --- a/geneve/utils/__init__.py +++ b/geneve/utils/__init__.py @@ -54,6 +54,30 @@ def expand_wildcards(s, alphabet, min_star_len, max_star_len): return "".join(chars) +def download(uri, destdir, *, basedir=None, cachedir=None, cachefile=None, validate=None): + import requests + + uri_parts = urlparse(str(uri)) + if uri_parts.scheme.startswith("http"): + if cachedir and cachefile: + local_file = cachedir / cachefile + else: + local_file = Path(cachedir or destdir) / Path(uri_parts.path).name + if local_file.exists() and validate and not validate(local_file): + local_file.unlink() + if not local_file.exists(): + local_file.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with open(local_file, "wb") as f: + f.write(requests.get(uri).content) + elif uri_parts.scheme == "file": + local_file = Path(basedir or Path.cwd()) / (uri_parts.netloc + uri_parts.path) + elif uri_parts.scheme == "": + local_file = Path(basedir or Path.cwd()) / uri_parts.path + else: + raise ValueError(f"uri scheme not supported: {uri_parts.scheme}") + return local_file + + @contextmanager def tempdir(): tmpdir = mkdtemp() @@ -65,28 +89,9 @@ def tempdir(): @contextmanager def resource(uri, basedir=None, cachedir=None, cachefile=None, validate=None): - import requests with tempdir() as tmpdir: - uri_parts = urlparse(str(uri)) - if uri_parts.scheme.startswith("http"): - download_dir = Path(cachedir or tmpdir) - if cachedir and cachefile: - local_file = download_dir / cachefile - else: - local_file = download_dir / Path(uri_parts.path).name - if local_file.exists() and validate and not validate(local_file): - local_file.unlink() - if not local_file.exists(): - download_dir.mkdir(parents=True, exist_ok=True, mode=0o700) - with open(local_file, "wb") as f: - f.write(requests.get(uri).content) - elif uri_parts.scheme == "file": - local_file = Path(basedir or Path.cwd()) / (uri_parts.netloc + uri_parts.path) - elif uri_parts.scheme == "": - local_file = Path(basedir or Path.cwd()) / uri_parts.path - else: - raise ValueError(f"uri scheme not supported: {uri_parts.scheme}") + local_file = download(uri, tmpdir, basedir=basedir, cachedir=cachedir, cachefile=cachefile, validate=validate) if local_file.is_dir(): tmpdir = local_file