From d699486bd05c593fb43e5eb3b100462cb0999bce Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 16 Sep 2023 19:20:29 +0200 Subject: [PATCH 1/2] List extra files only for terminal datasets Fixes: ``` WouldBlock: null File "anyio/streams/memory.py", line 98, in receive return self.receive_nowait() File "anyio/streams/memory.py", line 93, in receive_nowait raise WouldBlock EndOfStream: null File "starlette/middleware/base.py", line 78, in call_next message = await recv_stream.receive() File "anyio/streams/memory.py", line 118, in receive raise EndOfStream KeyError: None File "starlette/applications.py", line 122, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/middleware/errors.py", line 184, in __call__ raise exc File "starlette/middleware/errors.py", line 162, in __call__ await self.app(scope, receive, _send) File "starlette_context/middleware/raw_middleware.py", line 93, in __call__ await self.app(scope, receive, send_wrapper) File "starlette/middleware/base.py", line 108, in __call__ response = await self.dispatch_func(request, call_next) File "galaxy/webapps/galaxy/fast_app.py", line 103, in add_x_frame_options response = await call_next(request) File "starlette/middleware/base.py", line 84, in call_next raise app_exc File "starlette/middleware/base.py", line 70, in coro await self.app(scope, receive_or_disconnect, send_no_error) File "starlette/middleware/exceptions.py", line 79, in __call__ raise exc File "starlette/middleware/exceptions.py", line 68, in __call__ await self.app(scope, receive, sender) File "fastapi/middleware/asyncexitstack.py", line 20, in __call__ raise e File "fastapi/middleware/asyncexitstack.py", line 17, in __call__ await self.app(scope, receive, send) File "starlette/routing.py", line 718, in __call__ await route.handle(scope, receive, send) File "starlette/routing.py", line 276, in handle await self.app(scope, receive, send) File "starlette/routing.py", line 66, in app response = await func(request) File "fastapi/routing.py", line 241, in app raw_response = await run_endpoint_function( File "fastapi/routing.py", line 169, in run_endpoint_function return await run_in_threadpool(dependant.call, **values) File "starlette/concurrency.py", line 41, in run_in_threadpool return await anyio.to_thread.run_sync(func, *args) File "anyio/to_thread.py", line 33, in run_sync return await get_asynclib().run_sync_in_worker_thread( File "anyio/_backends/_asyncio.py", line 877, in run_sync_in_worker_thread return await future File "anyio/_backends/_asyncio.py", line 807, in run result = context.run(func, *args) File "galaxy/webapps/galaxy/api/datasets.py", line 253, in extra_files return self.service.extra_files(trans, history_content_id) File "galaxy/webapps/galaxy/services/datasets.py", line 548, in extra_files extra_files_path = hda.extra_files_path File "galaxy/model/__init__.py", line 4421, in extra_files_path return self.dataset.extra_files_path File "galaxy/model/__init__.py", line 3989, in get_extra_files_path return self.object_store.construct_path( File "galaxy/objectstore/__init__.py", line 419, in construct_path return self._invoke("construct_path", obj, **kwargs) File "galaxy/objectstore/__init__.py", line 413, in _invoke return self.__getattribute__(f"_{delegate}")(obj=obj, **kwargs) File "galaxy/objectstore/__init__.py", line 1157, in _construct_path return self.backends[obj.object_store_id].construct_path(obj, **kwargs) ``` --- lib/galaxy/webapps/galaxy/services/datasets.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index ff97caa142fb..b1ab34f5cf13 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -545,15 +545,16 @@ def extra_files( Generate list of extra files. """ hda = self.hda_manager.get_accessible(history_content_id, trans.user) - extra_files_path = hda.extra_files_path rval = [] - for root, directories, files in safe_walk(extra_files_path): - for directory in directories: - rval.append( - {"class": "Directory", "path": os.path.relpath(os.path.join(root, directory), extra_files_path)} - ) - for file in files: - rval.append({"class": "File", "path": os.path.relpath(os.path.join(root, file), extra_files_path)}) + if not hda.is_pending and hda.extra_files_path_exists(): + extra_files_path = hda.extra_files_path + for root, directories, files in safe_walk(extra_files_path): + for directory in directories: + rval.append( + {"class": "Directory", "path": os.path.relpath(os.path.join(root, directory), extra_files_path)} + ) + for file in files: + rval.append({"class": "File", "path": os.path.relpath(os.path.join(root, file), extra_files_path)}) return rval From 516a430b56883a01086143fef58cddcb834d277d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 16 Sep 2023 20:16:30 +0200 Subject: [PATCH 2/2] Improve various error conditions around downloading metadata files --- lib/galaxy/webapps/galaxy/services/datasets.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index b1ab34f5cf13..1d32b07a0d5b 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -643,12 +643,21 @@ def get_metadata_file( TODO: Remove the `open_file` parameter when removing the associated legacy endpoint. """ hda = self.hda_manager.get_accessible(history_content_id, trans.user) - file_ext = hda.metadata.spec.get(metadata_file).get("file_ext", metadata_file) + if hda.is_pending: + raise galaxy_exceptions.AcceptedRetryLater("dataset not in terminal state, retry later.", retry_after=60) + file_ext = hda.metadata.spec.get(metadata_file, {}).get("file_ext", metadata_file) fname = "".join(c in util.FILENAME_VALID_CHARS and c or "_" for c in hda.name)[0:150] headers = {} headers["Content-Type"] = "application/octet-stream" headers["Content-Disposition"] = f'attachment; filename="Galaxy{hda.hid}-[{fname}].{file_ext}"' - file_path = hda.metadata.get(metadata_file).file_name + metadata_file_instance = hda.metadata.get(metadata_file) + if not metadata_file_instance: + return galaxy_exceptions.RequestParameterInvalidException( + f"metadata file '{metadata_file}' not valid for dataset." + ) + file_path = metadata_file_instance.file_name + if not os.path.exists(file_path): + return galaxy_exceptions.ObjectNotFound(f"metadata file '{metadata_file}' not found.") if open_file: return open(file_path, "rb"), headers return file_path, headers