diff --git a/e2e/libs/keywords/k8s_keywords.py b/e2e/libs/keywords/k8s_keywords.py index ad5f66b65..64e54f8bf 100644 --- a/e2e/libs/keywords/k8s_keywords.py +++ b/e2e/libs/keywords/k8s_keywords.py @@ -35,6 +35,9 @@ async def restart_kubelet_tasks(): ) done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + for task in done: + if task.exception(): + assert False, task.exception() logging(f"All kubelets on nodes {node_list} are restarted after downtime {downtime_in_sec} seconds") await restart_kubelet_tasks() diff --git a/e2e/libs/keywords/network_keywords.py b/e2e/libs/keywords/network_keywords.py index 9566fb070..0400c3720 100644 --- a/e2e/libs/keywords/network_keywords.py +++ b/e2e/libs/keywords/network_keywords.py @@ -35,6 +35,9 @@ async def disconnect_network_tasks(): ) done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + for task in done: + if task.exception(): + assert False, task.exception() logging(f"All networks on nodes {node_list} are recovered after disconnection time {disconnection_time_in_sec} seconds") await disconnect_network_tasks() diff --git a/e2e/libs/keywords/persistentvolumeclaim_keywords.py b/e2e/libs/keywords/persistentvolumeclaim_keywords.py index 947bc2c0d..73ab14d35 100644 --- a/e2e/libs/keywords/persistentvolumeclaim_keywords.py +++ b/e2e/libs/keywords/persistentvolumeclaim_keywords.py @@ -1,4 +1,5 @@ from persistentvolumeclaim import PersistentVolumeClaim +from volume import Volume from utility.constant import ANNOT_EXPANDED_SIZE from utility.constant import LABEL_TEST @@ -12,6 +13,7 @@ class persistentvolumeclaim_keywords: def __init__(self): self.claim = PersistentVolumeClaim() + self.volume = Volume() def cleanup_persistentvolumeclaims(self): claims = self.claim.list(label_selector=f"{LABEL_TEST}={LABEL_TEST_VALUE}") @@ -19,6 +21,7 @@ def cleanup_persistentvolumeclaims(self): logging(f'Cleaning up {len(claims)} persistentvolumeclaims') for claim in claims: self.delete_persistentvolumeclaim(claim.metadata.name) + self.volume.wait_for_volume_deleted(claim.spec.volume_name) def create_persistentvolumeclaim(self, name, volume_type="RWO", sc_name="longhorn", storage_size="3GiB"): logging(f'Creating {volume_type} persistentvolumeclaim {name} with {sc_name} storageclass') diff --git a/e2e/libs/keywords/volume_keywords.py b/e2e/libs/keywords/volume_keywords.py index 4a7aa6aa5..02d891ffa 100644 --- a/e2e/libs/keywords/volume_keywords.py +++ b/e2e/libs/keywords/volume_keywords.py @@ -200,6 +200,9 @@ async def wait_for_both_replica_rebuildings(): ] done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + for task in done: + if task.exception(): + assert False, task.exception() logging(f"Observed {done.pop().get_name()} and {done.pop().get_name()} started replica rebuilding first") await wait_for_both_replica_rebuildings() @@ -215,6 +218,9 @@ async def wait_for_replica_rebuilding(): ] done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + for task in done: + if task.exception(): + assert False, task.exception() logging(f"Observed {done.pop().get_name()} started replica rebuilding") await wait_for_replica_rebuilding() diff --git a/e2e/libs/keywords/workload_keywords.py b/e2e/libs/keywords/workload_keywords.py index 1d7aae58f..bd5041c2b 100644 --- a/e2e/libs/keywords/workload_keywords.py +++ b/e2e/libs/keywords/workload_keywords.py @@ -138,6 +138,9 @@ async def wait_for_workloads_tasks(): ) done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + for task in done: + if task.exception(): + assert False, task.exception() logging(f"All workloads {workloads} pods are stably running now") await wait_for_workloads_tasks() diff --git a/e2e/libs/volume/crd.py b/e2e/libs/volume/crd.py index 22edf3289..bfe78b069 100644 --- a/e2e/libs/volume/crd.py +++ b/e2e/libs/volume/crd.py @@ -93,7 +93,7 @@ def delete(self, volume_name): plural="volumes", name=volume_name ) - self.wait_for_volume_delete(volume_name) + self.wait_for_volume_deleted(volume_name) except Exception as e: logging(f"Deleting volume error: {e}") @@ -208,8 +208,9 @@ def get_annotation_value(self, volume_name, annotation_key): volume = self.get(volume_name) return volume['metadata']['annotations'].get(annotation_key) - def wait_for_volume_delete(self, volume_name): + def wait_for_volume_deleted(self, volume_name): for i in range(self.retry_count): + logging(f"Waiting for volume {volume_name} deleted ... ({i})") try: self.obj_api.get_namespaced_custom_object( group="longhorn.io", diff --git a/e2e/libs/volume/volume.py b/e2e/libs/volume/volume.py index d664b908f..09e7f0d79 100644 --- a/e2e/libs/volume/volume.py +++ b/e2e/libs/volume/volume.py @@ -21,6 +21,9 @@ def create(self, volume_name, size, numberOfReplicas, frontend, migratable, data def delete(self, volume_name): return self.volume.delete(volume_name) + def wait_for_volume_deleted(self, volume_name): + return self.volume.wait_for_volume_deleted(volume_name) + def attach(self, volume_name, node_name, disable_frontend): return self.volume.attach(volume_name, node_name, disable_frontend)