From b0133570fedfff2a38488ca974a36bd20e929999 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 10 Apr 2024 14:52:33 -0400 Subject: [PATCH 1/4] fix: write temp file uploads (during upload) to ingest tmp dir --- chord_drs/app.py | 2 ++ chord_drs/request.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 chord_drs/request.py diff --git a/chord_drs/app.py b/chord_drs/app.py index e41bad7..1e2c112 100644 --- a/chord_drs/app.py +++ b/chord_drs/app.py @@ -13,6 +13,7 @@ from .constants import SERVICE_NAME from .db import db from .metrics import metrics +from .request import DrsRequest from .routes import drs_service @@ -20,6 +21,7 @@ application = Flask(__name__) application.config.from_object(Config) +application.request_class = DrsRequest # Set up CORS CORS( diff --git a/chord_drs/request.py b/chord_drs/request.py new file mode 100644 index 0000000..e6e7cc4 --- /dev/null +++ b/chord_drs/request.py @@ -0,0 +1,18 @@ +from flask import Request, current_app +from tempfile import SpooledTemporaryFile +from typing import Optional + +__all__ = ["DrsRequest"] + +MAX_IN_MEMORY_SIZE = 1024 * 1024 # 1 MB + + +class DrsRequest(Request): + def _get_file_stream( + self, + total_content_length: Optional[int], + content_type: Optional[str], + filename: Optional[str] = None, + content_length: Optional[int] = None, + ): + return SpooledTemporaryFile(max_size=MAX_IN_MEMORY_SIZE, dir=current_app.config["DRS_INGEST_TMP_DIR"]) From d3e16d5bf17ed2f2f37ffe0243d6cccbcaac8fab Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 10 Apr 2024 14:52:50 -0400 Subject: [PATCH 2/4] chore: bump version to 0.16.1 and update mako in lockfile --- poetry.lock | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 711999c..9cde784 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1000,13 +1000,13 @@ referencing = ">=0.31.0" [[package]] name = "mako" -version = "1.3.2" +version = "1.3.3" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." optional = false python-versions = ">=3.8" files = [ - {file = "Mako-1.3.2-py3-none-any.whl", hash = "sha256:32a99d70754dfce237019d17ffe4a282d2d3351b9c476e90d8a60e63f133b80c"}, - {file = "Mako-1.3.2.tar.gz", hash = "sha256:2a0c8ad7f6274271b3bb7467dd37cf9cc6dab4bc19cb69a4ef10669402de698e"}, + {file = "Mako-1.3.3-py3-none-any.whl", hash = "sha256:5324b88089a8978bf76d1629774fcc2f1c07b82acdf00f4c5dd8ceadfffc4b40"}, + {file = "Mako-1.3.3.tar.gz", hash = "sha256:e16c01d9ab9c11f7290eef1cfefc093fb5a45ee4a3da09e2fec2e4d1bae54e73"}, ] [package.dependencies] diff --git a/pyproject.toml b/pyproject.toml index 14e20ea..a9a4dc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "chord-drs" -version = "0.16.0" +version = "0.16.1" description = "An implementation of a data repository system (as per GA4GH's specs) for the Bento platform." authors = ["David Lougheed "] license = "LGPL-3.0" From 0fb274450eeac3588415abc89d99a82eff32643b Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 12 Apr 2024 11:39:01 -0400 Subject: [PATCH 3/4] lint: type hint --- chord_drs/request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chord_drs/request.py b/chord_drs/request.py index e6e7cc4..7a5e88c 100644 --- a/chord_drs/request.py +++ b/chord_drs/request.py @@ -1,6 +1,6 @@ from flask import Request, current_app from tempfile import SpooledTemporaryFile -from typing import Optional +from typing import IO, Optional __all__ = ["DrsRequest"] @@ -14,5 +14,5 @@ def _get_file_stream( content_type: Optional[str], filename: Optional[str] = None, content_length: Optional[int] = None, - ): + ) -> IO[bytes]: return SpooledTemporaryFile(max_size=MAX_IN_MEMORY_SIZE, dir=current_app.config["DRS_INGEST_TMP_DIR"]) From e362895381fb43502e8018779d038fe6fb12b421 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 12 Apr 2024 11:39:10 -0400 Subject: [PATCH 4/4] chore: add ingest debug log for temp path --- chord_drs/routes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/chord_drs/routes.py b/chord_drs/routes.py index ef7b97c..5bdfa03 100644 --- a/chord_drs/routes.py +++ b/chord_drs/routes.py @@ -449,6 +449,7 @@ def object_ingest(): filename: str | None = None # no override, use path filename if path is specified instead of a file upload if file is not None: logger.debug(f"ingest - received file object: {file}") + logger.debug(f"ingest - writing to temporary path: {t_obj_path}") file.save(t_obj_path) obj_path = t_obj_path filename = file.filename # still may be none, in which case the temporary filename will be used