From 7d46e886f7905c6370e54bbc2aae708fb666fa77 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 30 Apr 2024 18:39:36 +0100 Subject: [PATCH] Fix ``InvocationClient.get_invocation_biocompute_object()`` method on upcoming Galaxy 24.1 xref: https://github.com/galaxyproject/galaxy/pull/16645 Also: - Set `enable_celery_tasks` for tests (needed for BioCompute objects export in Galaxy 24.1) - Add ``wait`` parameter to ``HistoryClient.delete_dataset()`` and ``HistoryDatasetAssociation.delete()`` methods (needed for testing purged datasets when Celery tasks are enabled). - Refactor methods that wait for something to make use of a new generic ``bioblend.wait_on()`` function. They now all raise ``TimeoutException``. --- CHANGELOG.md | 9 +++ bioblend/__init__.py | 38 +++++++++ bioblend/_tests/TestGalaxyDatasets.py | 11 +-- bioblend/_tests/TestGalaxyHistories.py | 2 +- bioblend/_tests/TestGalaxyJobs.py | 1 - bioblend/_tests/TestGalaxyObjects.py | 2 +- bioblend/_tests/TestGalaxyTools.py | 1 - bioblend/_tests/template_galaxy.yml | 1 + .../galaxy/dataset_collections/__init__.py | 36 +++------ bioblend/galaxy/datasets/__init__.py | 38 ++++----- bioblend/galaxy/histories/__init__.py | 22 ++++- bioblend/galaxy/invocations/__init__.py | 81 ++++++++++++++----- bioblend/galaxy/jobs/__init__.py | 24 +++--- bioblend/galaxy/libraries/__init__.py | 36 +++------ bioblend/galaxy/objects/wrappers.py | 6 +- 15 files changed, 182 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e60056a46..7036e4e76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ * Dropped support for Python 3.7. Added support for Python 3.12. Added support for Galaxy releases 23.2 and 24.0. +* Added ``wait`` parameter to ``HistoryClient.delete_dataset()`` and + ``HistoryDatasetAssociation.delete()`` methods. + * Dropped broken ``deleted`` parameter of ``DatasetClient.show_dataset()``. * Parameters after ``password`` in the ``__init__()`` method of the @@ -12,8 +15,14 @@ * Classes defined in ``bioblend.galaxy.objects.wrappers`` are no more re-exported by ``bioblend.galaxy.objects``. +* ``DatasetTimeoutException`` and ``DatasetCollectionTimeoutException`` are now + aliases for ``TimeoutException`` instead of subclasses. + * Added support for the new "cancelling" invocation state. +* Fixed ``InvocationClient.get_invocation_biocompute_object()`` method on + upcoming Galaxy 24.1 . + ### BioBlend v1.2.0 - 2023-06-30 * Dropped support for Galaxy releases 17.09-19.01. Added support for Galaxy diff --git a/bioblend/__init__.py b/bioblend/__init__.py index 2a526756b..d0062462a 100644 --- a/bioblend/__init__.py +++ b/bioblend/__init__.py @@ -2,8 +2,11 @@ import logging import logging.config import os +import time from typing import ( + Callable, Optional, + TypeVar, Union, ) @@ -116,3 +119,38 @@ def __str__(self) -> str: class TimeoutException(Exception): pass + + +class NotReady(Exception): + pass + + +T = TypeVar("T") + + +def wait_on(func: Callable[[], T], maxwait: float = 60, interval: float = 3) -> T: + """ + Wait until a function returns without raising a NotReady exception + + :param func: function to wait on. It should accept no parameters. + + :param maxwait: Total time (in seconds) to wait for the function to return + without raising a NotReady exception. After this time, a + ``TimeoutException`` will be raised. + + :param interval: Time (in seconds) to wait between 2 consecutive checks. + """ + assert maxwait >= 0 + assert interval > 0 + + time_left = maxwait + while True: + try: + return func() + except NotReady as e: + if time_left > 0: + log.info("%s. Will wait %s more s", e, time_left) + time.sleep(min(time_left, interval)) + time_left -= interval + else: + raise TimeoutException(f"{e} after {maxwait} s") diff --git a/bioblend/_tests/TestGalaxyDatasets.py b/bioblend/_tests/TestGalaxyDatasets.py index 8123bed8e..714ebbbc7 100644 --- a/bioblend/_tests/TestGalaxyDatasets.py +++ b/bioblend/_tests/TestGalaxyDatasets.py @@ -24,7 +24,6 @@ def setUp(self): def tearDown(self): self.gi.histories.delete_history(self.history_id, purge=True) - @test_util.skip_unless_galaxy("release_19.05") def test_show_nonexistent_dataset(self): with pytest.raises(ConnectionError): self.gi.datasets.show_dataset("nonexistent_id") @@ -65,25 +64,21 @@ def test_download_dataset(self): f.flush() assert f.read() == expected_contents - @test_util.skip_unless_galaxy("release_19.05") def test_get_datasets(self): datasets = self.gi.datasets.get_datasets() dataset_ids = [dataset["id"] for dataset in datasets] assert self.dataset_id in dataset_ids - @test_util.skip_unless_galaxy("release_19.05") def test_get_datasets_history(self): datasets = self.gi.datasets.get_datasets(history_id=self.history_id) assert len(datasets) == 1 - @test_util.skip_unless_galaxy("release_19.05") def test_get_datasets_limit_offset(self): datasets = self.gi.datasets.get_datasets(limit=1) assert len(datasets) == 1 datasets = self.gi.datasets.get_datasets(history_id=self.history_id, offset=1) assert datasets == [] - @test_util.skip_unless_galaxy("release_19.05") def test_get_datasets_name(self): datasets = self.gi.datasets.get_datasets(history_id=self.history_id, name="Pasted Entry") assert len(datasets) == 1 @@ -143,7 +138,6 @@ def test_get_datasets_visible(self): datasets = self.gi.datasets.get_datasets(history_id=self.history_id, visible=False) assert len(datasets) == 0 - @test_util.skip_unless_galaxy("release_19.05") def test_get_datasets_ordering(self): self.dataset_id2 = self._test_dataset(self.history_id, contents=self.dataset_contents) self.gi.datasets.wait_for_dataset(self.dataset_id2) @@ -156,7 +150,6 @@ def test_get_datasets_ordering(self): datasets = self.gi.datasets.get_datasets(history_id=self.history_id, order="hid-asc") assert datasets[0]["id"] == self.dataset_id - @test_util.skip_unless_galaxy("release_19.05") def test_get_datasets_deleted(self): deleted_datasets = self.gi.datasets.get_datasets(history_id=self.history_id, deleted=True) assert deleted_datasets == [] @@ -165,11 +158,10 @@ def test_get_datasets_deleted(self): assert len(deleted_datasets) == 1 purged_datasets = self.gi.datasets.get_datasets(history_id=self.history_id, purged=True) assert purged_datasets == [] - self.gi.histories.delete_dataset(self.history_id, self.dataset_id, purge=True) + self.gi.histories.delete_dataset(self.history_id, self.dataset_id, purge=True, wait=True) purged_datasets = self.gi.datasets.get_datasets(history_id=self.history_id, purged=True) assert len(purged_datasets) == 1 - @test_util.skip_unless_galaxy("release_19.05") def test_get_datasets_tool_id_and_tag(self): cat1_datasets = self.gi.datasets.get_datasets(history_id=self.history_id, tool_id="cat1") assert cat1_datasets == [] @@ -189,7 +181,6 @@ def test_wait_for_dataset(self): self.gi.histories.delete_history(history_id, purge=True) - @test_util.skip_unless_galaxy("release_19.05") def test_dataset_permissions(self): admin_user_id = self.gi.users.get_current_user()["id"] username = test_util.random_string() diff --git a/bioblend/_tests/TestGalaxyHistories.py b/bioblend/_tests/TestGalaxyHistories.py index c53292201..160b6eb5c 100644 --- a/bioblend/_tests/TestGalaxyHistories.py +++ b/bioblend/_tests/TestGalaxyHistories.py @@ -194,7 +194,7 @@ def test_delete_dataset(self): def test_purge_dataset(self): history_id = self.history["id"] dataset1_id = self._test_dataset(history_id) - self.gi.histories.delete_dataset(history_id, dataset1_id, purge=True) + self.gi.histories.delete_dataset(history_id, dataset1_id, purge=True, wait=True) dataset = self.gi.histories.show_dataset(history_id, dataset1_id) assert dataset["deleted"] assert dataset["purged"] diff --git a/bioblend/_tests/TestGalaxyJobs.py b/bioblend/_tests/TestGalaxyJobs.py index a43c6d7d3..4b52ac55f 100644 --- a/bioblend/_tests/TestGalaxyJobs.py +++ b/bioblend/_tests/TestGalaxyJobs.py @@ -144,7 +144,6 @@ def test_rerun_and_remap(self): assert last_dataset["id"] == history_contents[2]["id"] self._wait_and_verify_dataset(last_dataset["id"], b"line 1\tline 1\n") - @test_util.skip_unless_galaxy("release_19.05") @test_util.skip_unless_tool("random_lines1") def test_get_common_problems(self): job_id = self._run_tool()["jobs"][0]["id"] diff --git a/bioblend/_tests/TestGalaxyObjects.py b/bioblend/_tests/TestGalaxyObjects.py index efcfa00c1..69eece87a 100644 --- a/bioblend/_tests/TestGalaxyObjects.py +++ b/bioblend/_tests/TestGalaxyObjects.py @@ -966,7 +966,7 @@ def test_dataset_delete(self): assert not self.ds.purged def test_dataset_purge(self): - self.ds.delete(purge=True) + self.ds.delete(purge=True, wait=True) assert self.ds.deleted assert self.ds.purged diff --git a/bioblend/_tests/TestGalaxyTools.py b/bioblend/_tests/TestGalaxyTools.py index 722dcfa2d..2b3892a3c 100644 --- a/bioblend/_tests/TestGalaxyTools.py +++ b/bioblend/_tests/TestGalaxyTools.py @@ -125,7 +125,6 @@ def test_run_cat1(self): # TODO: Wait for results and verify it has 3 lines - 1 2 3, 4 5 6, # and 7 8 9. - @test_util.skip_unless_galaxy("release_19.05") @test_util.skip_unless_tool("CONVERTER_fasta_to_bowtie_color_index") def test_tool_dependency_install(self): installed_dependencies = self.gi.tools.install_dependencies("CONVERTER_fasta_to_bowtie_color_index") diff --git a/bioblend/_tests/template_galaxy.yml b/bioblend/_tests/template_galaxy.yml index 54f62d0d6..75bed0c8b 100644 --- a/bioblend/_tests/template_galaxy.yml +++ b/bioblend/_tests/template_galaxy.yml @@ -17,3 +17,4 @@ galaxy: master_api_key: $BIOBLEND_GALAXY_MASTER_API_KEY enable_quotas: true cleanup_job: onsuccess + enable_celery_tasks: true diff --git a/bioblend/galaxy/dataset_collections/__init__.py b/bioblend/galaxy/dataset_collections/__init__.py index 69c46fa80..addd84a32 100644 --- a/bioblend/galaxy/dataset_collections/__init__.py +++ b/bioblend/galaxy/dataset_collections/__init__.py @@ -1,5 +1,4 @@ import logging -import time from typing import ( Any, Dict, @@ -11,7 +10,9 @@ from bioblend import ( CHUNK_SIZE, + NotReady, TimeoutException, + wait_on, ) from bioblend.galaxy.client import Client from bioblend.galaxy.datasets import TERMINAL_STATES @@ -176,9 +177,8 @@ def wait_for_dataset_collection( :type maxwait: float :param maxwait: Total time (in seconds) to wait for the dataset - states in the dataset collection to become terminal. If not - all datasets are in a terminal state within this time, a - ``DatasetCollectionTimeoutException`` will be raised. + states in the dataset collection to become terminal. After this time, + a ``TimeoutException`` will be raised. :type interval: float :param interval: Time (in seconds) to wait between two consecutive checks. @@ -200,12 +200,9 @@ def wait_for_dataset_collection( :rtype: dict :return: Details of the given dataset collection. """ - assert maxwait >= 0 - assert interval > 0 assert 0 <= proportion_complete <= 1 - time_left = maxwait - while True: + def check_and_get_dataset_collection() -> Dict[str, Any]: dataset_collection = self.show_dataset_collection(dataset_collection_id) states = [elem["object"]["state"] for elem in dataset_collection["elements"]] terminal_states = [state for state in states if state in TERMINAL_STATES] @@ -217,24 +214,15 @@ def wait_for_dataset_collection( proportion = len(terminal_states) / len(states) if proportion >= proportion_complete: return dataset_collection - if time_left > 0: - log.info( - "The dataset collection %s has %s out of %s datasets in a terminal state. Will wait %s more s", - dataset_collection_id, - len(terminal_states), - len(states), - time_left, - ) - time.sleep(min(time_left, interval)) - time_left -= interval - else: - raise DatasetCollectionTimeoutException( - f"Less than {proportion_complete * 100}% of datasets in the dataset collection is in a terminal state after {maxwait} s" - ) + raise NotReady( + f"The dataset collection {dataset_collection_id} has only {proportion * 100}% of datasets in a terminal state" + ) + + return wait_on(check_and_get_dataset_collection, maxwait=maxwait, interval=interval) -class DatasetCollectionTimeoutException(TimeoutException): - pass +# Unused, for backward compatibility +DatasetCollectionTimeoutException = TimeoutException __all__ = ( diff --git a/bioblend/galaxy/datasets/__init__.py b/bioblend/galaxy/datasets/__init__.py index 6f889465c..f164fddeb 100644 --- a/bioblend/galaxy/datasets/__init__.py +++ b/bioblend/galaxy/datasets/__init__.py @@ -5,7 +5,6 @@ import logging import os import shlex -import time import warnings from typing import ( Any, @@ -21,8 +20,12 @@ from requests import Response -import bioblend -from bioblend import TimeoutException +from bioblend import ( + CHUNK_SIZE, + NotReady, + TimeoutException, + wait_on, +) from bioblend.galaxy.client import Client if TYPE_CHECKING: @@ -144,8 +147,8 @@ def download_dataset( :type maxwait: float :param maxwait: Total time (in seconds) to wait for the dataset state to - become terminal. If the dataset state is not terminal within this - time, a ``DatasetTimeoutException`` will be thrown. + become terminal. After this time, a ``TimeoutException`` will be + raised. :rtype: bytes or str :return: If a ``file_path`` argument is not provided, returns the file @@ -180,7 +183,7 @@ def download_dataset( file_local_path = file_path with open(file_local_path, "wb") as fp: - for chunk in r.iter_content(chunk_size=bioblend.CHUNK_SIZE): + for chunk in r.iter_content(chunk_size=CHUNK_SIZE): if chunk: fp.write(chunk) @@ -411,8 +414,7 @@ def wait_for_dataset( :type maxwait: float :param maxwait: Total time (in seconds) to wait for the dataset state to - become terminal. If the dataset state is not terminal within this - time, a ``DatasetTimeoutException`` will be raised. + become terminal. After this time, a ``TimeoutException`` will be raised. :type interval: float :param interval: Time (in seconds) to wait between 2 consecutive checks. @@ -423,25 +425,17 @@ def wait_for_dataset( :rtype: dict :return: Details of the given dataset. """ - assert maxwait >= 0 - assert interval > 0 - time_left = maxwait - while True: + def check_and_get_dataset() -> Dict[str, Any]: dataset = self.show_dataset(dataset_id) state = dataset["state"] if state in TERMINAL_STATES: if check and state != "ok": raise Exception(f"Dataset {dataset_id} is in terminal state {state}") return dataset - if time_left > 0: - log.info("Dataset %s is in non-terminal state %s. Will wait %s more s", dataset_id, state, time_left) - time.sleep(min(time_left, interval)) - time_left -= interval - else: - raise DatasetTimeoutException( - f"Dataset {dataset_id} is still in non-terminal state {state} after {maxwait} s" - ) + raise NotReady(f"Dataset {dataset_id} is in non-terminal state {state}") + + return wait_on(check_and_get_dataset, maxwait=maxwait, interval=interval) class DatasetStateException(Exception): @@ -452,5 +446,5 @@ class DatasetStateWarning(UserWarning): pass -class DatasetTimeoutException(TimeoutException): - pass +# Unused, just for backward compatibility +DatasetTimeoutException = TimeoutException diff --git a/bioblend/galaxy/histories/__init__.py b/bioblend/galaxy/histories/__init__.py index aa02b14ed..b1a2da726 100644 --- a/bioblend/galaxy/histories/__init__.py +++ b/bioblend/galaxy/histories/__init__.py @@ -21,7 +21,11 @@ ) import bioblend -from bioblend import ConnectionError +from bioblend import ( + ConnectionError, + NotReady, + wait_on, +) from bioblend.galaxy.client import Client from bioblend.galaxy.dataset_collections import CollectionDescription from bioblend.util import attach_file @@ -403,7 +407,7 @@ def show_history( params["keys"] = ",".join(keys) return self._get(id=history_id, contents=contents, params=params) - def delete_dataset(self, history_id: str, dataset_id: str, purge: bool = False) -> None: + def delete_dataset(self, history_id: str, dataset_id: str, purge: bool = False, wait: bool = False) -> None: """ Mark corresponding dataset as deleted. @@ -416,6 +420,8 @@ def delete_dataset(self, history_id: str, dataset_id: str, purge: bool = False) :type purge: bool :param purge: if ``True``, also purge (permanently delete) the dataset + :param wait: Whether to wait for the dataset to be purged. + :rtype: None :return: None @@ -426,9 +432,17 @@ def delete_dataset(self, history_id: str, dataset_id: str, purge: bool = False) """ url = "/".join((self._make_url(history_id, contents=True), dataset_id)) payload = {} - if purge is True: - payload["purge"] = purge + if purge: + payload["purge"] = True self._delete(payload=payload, url=url) + if purge and wait: + + def check_dataset_purged() -> None: + dataset = self.show_dataset(history_id, dataset_id) + if not dataset["purged"]: + raise NotReady(f"Dataset {dataset_id} in library {history_id} is not purged") + + wait_on(check_dataset_purged) def delete_dataset_collection(self, history_id: str, dataset_collection_id: str) -> None: """ diff --git a/bioblend/galaxy/invocations/__init__.py b/bioblend/galaxy/invocations/__init__.py index 7321c5c20..3d32113e1 100644 --- a/bioblend/galaxy/invocations/__init__.py +++ b/bioblend/galaxy/invocations/__init__.py @@ -3,7 +3,6 @@ """ import logging -import time from typing import ( Any, Dict, @@ -14,7 +13,9 @@ from bioblend import ( CHUNK_SIZE, - TimeoutException, + ConnectionError, + NotReady, + wait_on, ) from bioblend.galaxy.client import Client from bioblend.galaxy.workflows import InputsBy @@ -400,18 +401,66 @@ def get_invocation_report_pdf(self, invocation_id: str, file_path: str, chunk_si for chunk in r.iter_content(chunk_size): outf.write(chunk) - def get_invocation_biocompute_object(self, invocation_id: str) -> Dict[str, Any]: + # TODO: Move to a new ``bioblend.galaxy.short_term_storage`` module + def _wait_for_short_term_storage( + self, storage_request_id: str, maxwait: float = 60, interval: float = 3 + ) -> Dict[str, Any]: + """ + Wait until a short term storage request is ready + + :type storage_request_id: str + :param storage_request_id: Storage request ID to wait for. + + :type maxwait: float + :param maxwait: Total time (in seconds) to wait for the storage request + to become ready. After this time, a ``TimeoutException`` will be + raised. + + :type interval: float + :param interval: Time (in seconds) to wait between 2 consecutive checks. + + :rtype: dict + :return: The short term storage request. + """ + url = f"{self.gi.url}/short_term_storage/{storage_request_id}" + is_ready_url = f"{url}/ready" + + def check_and_get_short_term_storage() -> Dict[str, Any]: + if self._get(url=is_ready_url): + return self._get(url=url) + raise NotReady(f"Storage request {storage_request_id} is not ready") + + return wait_on(check_and_get_short_term_storage, maxwait=maxwait, interval=interval) + + def get_invocation_biocompute_object(self, invocation_id: str, maxwait: float = 1200) -> Dict[str, Any]: """ Get a BioCompute object for an invocation. :type invocation_id: str :param invocation_id: Encoded workflow invocation ID + :type maxwait: float + :param maxwait: Total time (in seconds) to wait for the BioCompute + object to become ready. After this time, a ``TimeoutException`` will + be raised. + :rtype: dict :return: The BioCompute object """ - url = self._make_url(invocation_id) + "/biocompute" - return self._get(url=url) + url = self._make_url(invocation_id) + "/prepare_store_download" + payload = {"model_store_format": "bco.json"} + try: + psd = self._post(url=url, payload=payload) + except ConnectionError as e: + if e.status_code not in (400, 404): + raise + # Galaxy release_22.05 and earlier + url = self._make_url(invocation_id) + "/biocompute" + return self._get(url=url) + else: + storage_request_id = psd["storage_request_id"] + url = f"{self.gi.url}/short_term_storage/{storage_request_id}/ready" + return self._wait_for_short_term_storage(storage_request_id, maxwait=maxwait) def wait_for_invocation( self, invocation_id: str, maxwait: float = 12000, interval: float = 3, check: bool = True @@ -424,8 +473,8 @@ def wait_for_invocation( :type maxwait: float :param maxwait: Total time (in seconds) to wait for the invocation state - to become terminal. If the invocation state is not terminal within - this time, a ``TimeoutException`` will be raised. + to become terminal. After this time, a ``TimeoutException`` will be + raised. :type interval: float :param interval: Time (in seconds) to wait between 2 consecutive checks. @@ -437,27 +486,17 @@ def wait_for_invocation( :rtype: dict :return: Details of the workflow invocation. """ - assert maxwait >= 0 - assert interval > 0 - time_left = maxwait - while True: + def check_and_get_invocation() -> Dict[str, Any]: invocation = self.gi.invocations.show_invocation(invocation_id) state = invocation["state"] if state in INVOCATION_TERMINAL_STATES: if check and state != "scheduled": raise Exception(f"Invocation {invocation_id} is in terminal state {state}") return invocation - if time_left > 0: - log.info( - "Invocation %s is in non-terminal state %s. Will wait %s more s", invocation_id, state, time_left - ) - time.sleep(min(time_left, interval)) - time_left -= interval - else: - raise TimeoutException( - f"Invocation {invocation_id} is still in non-terminal state {state} after {maxwait} s" - ) + raise NotReady(f"Invocation {invocation_id} is in non-terminal state {state}") + + return wait_on(check_and_get_invocation, maxwait=maxwait, interval=interval) def _invocation_step_url(self, invocation_id: str, step_id: str) -> str: return "/".join((self._make_url(invocation_id), "steps", step_id)) diff --git a/bioblend/galaxy/jobs/__init__.py b/bioblend/galaxy/jobs/__init__.py index 9b1c97243..4403d368b 100644 --- a/bioblend/galaxy/jobs/__init__.py +++ b/bioblend/galaxy/jobs/__init__.py @@ -3,7 +3,6 @@ """ import logging -import time from typing import ( Any, Dict, @@ -13,7 +12,10 @@ TYPE_CHECKING, ) -from bioblend import TimeoutException +from bioblend import ( + NotReady, + wait_on, +) from bioblend.galaxy.client import Client if TYPE_CHECKING: @@ -494,8 +496,8 @@ def wait_for_job( :type maxwait: float :param maxwait: Total time (in seconds) to wait for the job state to - become terminal. If the job state is not terminal within this time, a - ``TimeoutException`` will be raised. + become terminal. After this time, a ``TimeoutException`` will be + raised. :type interval: float :param interval: Time (in seconds) to wait between 2 consecutive checks. @@ -506,20 +508,14 @@ def wait_for_job( :rtype: dict :return: Details of the given job. """ - assert maxwait >= 0 - assert interval > 0 - time_left = maxwait - while True: + def check_and_get_job() -> Dict[str, Any]: job = self.show_job(job_id) state = job["state"] if state in JOB_TERMINAL_STATES: if check and state != "ok": raise Exception(f"Job {job_id} is in terminal state {state}") return job - if time_left > 0: - log.info("Job %s is in non-terminal state %s. Will wait %s more s", job_id, state, time_left) - time.sleep(min(time_left, interval)) - time_left -= interval - else: - raise TimeoutException(f"Job {job_id} is still in non-terminal state {state} after {maxwait} s") + raise NotReady(f"Job {job_id} is in non-terminal state {state}") + + return wait_on(check_and_get_job, maxwait=maxwait, interval=interval) diff --git a/bioblend/galaxy/libraries/__init__.py b/bioblend/galaxy/libraries/__init__.py index eb89ff157..e718b36c9 100644 --- a/bioblend/galaxy/libraries/__init__.py +++ b/bioblend/galaxy/libraries/__init__.py @@ -3,7 +3,6 @@ """ import logging -import time from typing import ( Any, Dict, @@ -13,11 +12,12 @@ TYPE_CHECKING, ) -from bioblend.galaxy.client import Client -from bioblend.galaxy.datasets import ( - DatasetTimeoutException, - TERMINAL_STATES, +from bioblend import ( + NotReady, + wait_on, ) +from bioblend.galaxy.client import Client +from bioblend.galaxy.datasets import TERMINAL_STATES from bioblend.util import attach_file if TYPE_CHECKING: @@ -174,8 +174,8 @@ def wait_for_dataset( :type maxwait: float :param maxwait: Total time (in seconds) to wait for the dataset state to - become terminal. If the dataset state is not terminal within this - time, a ``DatasetTimeoutException`` will be thrown. + become terminal. After this time, a ``TimeoutException`` will be + raised. :type interval: float :param interval: Time (in seconds) to wait between 2 consecutive checks. @@ -184,29 +184,15 @@ def wait_for_dataset( :return: A dictionary containing information about the dataset in the library """ - assert maxwait >= 0 - assert interval > 0 - time_left = maxwait - while True: + def check_and_get_library_dataset() -> Dict[str, Any]: dataset = self.show_dataset(library_id, dataset_id) state = dataset["state"] if state in TERMINAL_STATES: return dataset - if time_left > 0: - log.info( - "Dataset %s in library %s is in non-terminal state %s. Will wait %i more s", - dataset_id, - library_id, - state, - time_left, - ) - time.sleep(min(time_left, interval)) - time_left -= interval - else: - raise DatasetTimeoutException( - f"Dataset {dataset_id} in library {library_id} is still in non-terminal state {state} after {maxwait} s" - ) + raise NotReady(f"Dataset {dataset_id} in library {library_id} is in non-terminal state {state}") + + return wait_on(check_and_get_library_dataset, maxwait=maxwait, interval=interval) def show_folder(self, library_id: str, folder_id: str) -> Dict[str, Any]: """ diff --git a/bioblend/galaxy/objects/wrappers.py b/bioblend/galaxy/objects/wrappers.py index ff035458b..6ffaa058d 100644 --- a/bioblend/galaxy/objects/wrappers.py +++ b/bioblend/galaxy/objects/wrappers.py @@ -1025,19 +1025,21 @@ def update(self, **kwargs: Any) -> "HistoryDatasetAssociation": self.__init__(res, self.container, gi=self.gi) # type: ignore[misc] return self - def delete(self, purge: bool = False) -> None: + def delete(self, purge: bool = False, wait: bool = False) -> None: """ Delete this history dataset. :type purge: bool :param purge: if ``True``, also purge (permanently delete) the dataset + :param wait: Whether to wait for the dataset to be purged. + .. note:: The ``purge`` option works only if the Galaxy instance has the ``allow_user_dataset_purge`` option set to ``true`` in the ``config/galaxy.yml`` configuration file. """ - self.gi.gi.histories.delete_dataset(self.container.id, self.id, purge=purge) + self.gi.gi.histories.delete_dataset(self.container.id, self.id, purge=purge, wait=wait) self.container.refresh() self.refresh()