test(robot): retrive asyncio task exception

Signed-off-by: Yang Chiu <[email protected]>

e2e/libs/keywords/k8s_keywords.py

@@ -35,6 +35,9 @@ async def restart_kubelet_tasks():
                 )
 
             done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
+            for task in done:
+                if task.exception():
+                    assert False, task.exception()
             logging(f"All kubelets on nodes {node_list} are restarted after downtime {downtime_in_sec} seconds")
 
         await restart_kubelet_tasks()

e2e/libs/keywords/network_keywords.py

@@ -35,6 +35,9 @@ async def disconnect_network_tasks():
                 )
 
             done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
+            for task in done:
+                if task.exception():
+                    assert False, task.exception()
             logging(f"All networks on nodes {node_list} are recovered after disconnection time {disconnection_time_in_sec} seconds")
 
         await disconnect_network_tasks()

e2e/libs/keywords/persistentvolumeclaim_keywords.py

@@ -1,4 +1,5 @@
 from persistentvolumeclaim import PersistentVolumeClaim
+from volume import Volume
 
 from utility.constant import ANNOT_EXPANDED_SIZE
 from utility.constant import LABEL_TEST
@@ -12,13 +13,15 @@ class persistentvolumeclaim_keywords:
 
     def __init__(self):
         self.claim = PersistentVolumeClaim()
+        self.volume = Volume()
 
     def cleanup_persistentvolumeclaims(self):
         claims = self.claim.list(label_selector=f"{LABEL_TEST}={LABEL_TEST_VALUE}")
 
         logging(f'Cleaning up {len(claims)} persistentvolumeclaims')
         for claim in claims:
             self.delete_persistentvolumeclaim(claim.metadata.name)
+            self.volume.wait_for_volume_deleted(claim.spec.volume_name)
 
     def create_persistentvolumeclaim(self, name, volume_type="RWO", sc_name="longhorn", storage_size="3GiB"):
         logging(f'Creating {volume_type} persistentvolumeclaim {name} with {sc_name} storageclass')

e2e/libs/keywords/volume_keywords.py

@@ -200,6 +200,9 @@ async def wait_for_both_replica_rebuildings():
             ]
 
             done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
+            for task in done:
+                if task.exception():
+                    assert False, task.exception()
             logging(f"Observed {done.pop().get_name()} and {done.pop().get_name()} started replica rebuilding first")
 
         await wait_for_both_replica_rebuildings()
@@ -215,6 +218,9 @@ async def wait_for_replica_rebuilding():
             ]
 
             done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
+            for task in done:
+                if task.exception():
+                    assert False, task.exception()
             logging(f"Observed {done.pop().get_name()} started replica rebuilding")
 
         await wait_for_replica_rebuilding()

e2e/libs/keywords/workload_keywords.py

@@ -138,6 +138,9 @@ async def wait_for_workloads_tasks():
                 )
 
             done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
+            for task in done:
+                if task.exception():
+                    assert False, task.exception()
             logging(f"All workloads {workloads} pods are stably running now")
 
         await wait_for_workloads_tasks()

e2e/libs/volume/crd.py

@@ -93,7 +93,7 @@ def delete(self, volume_name):
                 plural="volumes",
                 name=volume_name
             )
-            self.wait_for_volume_delete(volume_name)
+            self.wait_for_volume_deleted(volume_name)
         except Exception as e:
             logging(f"Deleting volume error: {e}")
 
@@ -208,8 +208,9 @@ def get_annotation_value(self, volume_name, annotation_key):
         volume = self.get(volume_name)
         return volume['metadata']['annotations'].get(annotation_key)
 
-    def wait_for_volume_delete(self, volume_name):
+    def wait_for_volume_deleted(self, volume_name):
         for i in range(self.retry_count):
+            logging(f"Waiting for volume {volume_name} deleted ... ({i})")
             try:
                 self.obj_api.get_namespaced_custom_object(
                     group="longhorn.io",

e2e/libs/volume/volume.py

@@ -21,6 +21,9 @@ def create(self, volume_name, size, numberOfReplicas, frontend, migratable, data
     def delete(self, volume_name):
         return self.volume.delete(volume_name)
 
+    def wait_for_volume_deleted(self, volume_name):
+        return self.volume.wait_for_volume_deleted(volume_name)
+
     def attach(self, volume_name, node_name, disable_frontend):
         return self.volume.attach(volume_name, node_name, disable_frontend)