test(robot): wait for pvc and volume deleted when cleaning up statefu…

…lset

Signed-off-by: Yang Chiu <[email protected]>

e2e/libs/keywords/persistentvolumeclaim_keywords.py

@@ -16,8 +16,8 @@ def __init__(self):
     def cleanup_persistentvolumeclaims(self):
         claims = self.claim.list(label_selector=f"{LABEL_TEST}={LABEL_TEST_VALUE}")
 
-        logging(f'Cleaning up {len(claims.items)} persistentvolumeclaims')
-        for claim in claims.items:
+        logging(f'Cleaning up {len(claims)} persistentvolumeclaims')
+        for claim in claims:
             self.delete_persistentvolumeclaim(claim.metadata.name)
 
     def create_persistentvolumeclaim(self, name, volume_type="RWO", sc_name="longhorn", storage_size="3GiB"):

e2e/libs/keywords/volume_keywords.py

@@ -26,8 +26,8 @@ def __init__(self):
     def cleanup_volumes(self):
         volumes = self.volume.list(label_selector=f"{LABEL_TEST}={LABEL_TEST_VALUE}")
 
-        logging(f'Cleaning up {len(volumes["items"])} volumes')
-        for volume in volumes['items']:
+        logging(f'Cleaning up {len(volumes)} volumes')
+        for volume in volumes:
             self.delete_volume(volume['metadata']['name'])
 
     def create_volume(self, volume_name, size="2Gi", numberOfReplicas=3, frontend="blockdev", migratable=False, dataLocality="disabled", accessMode="RWO", dataEngine="v1", backingImage="", Standby=False, fromBackup=""):

e2e/libs/persistentvolumeclaim/crd.py

@@ -22,7 +22,7 @@ def list(self, claim_namespace="default", label_selector=None):
         return self.core_v1_api.list_namespaced_persistent_volume_claim(
             namespace=claim_namespace,
             label_selector=label_selector
-        )
+        ).items
 
     def set_label(self, claim_name, label_key, label_value, claim_namespace="default"):
         for i in range(self.retry_count):

e2e/libs/volume/crd.py

@@ -178,7 +178,7 @@ def list(self, label_selector=None):
             namespace="longhorn-system",
             plural="volumes",
             label_selector=label_selector
-        )
+        )["items"]
 
     def set_annotation(self, volume_name, annotation_key, annotation_value):
         # retry conflict error

e2e/libs/volume/rest.py

@@ -238,6 +238,7 @@ def get_replica_name_on_node(self, volume_name, node_name):
                     return r.name
 
     def wait_for_replica_count(self, volume_name, node_name, replica_count):
+        condition_met = False
         for i in range(self.retry_count):
             running_replica_count = 0
             volume = get_longhorn_client().by_id_volume(volume_name)
@@ -248,10 +249,13 @@ def wait_for_replica_count(self, volume_name, node_name, replica_count):
                     running_replica_count += 1
             logging(f"Waiting for {replica_count if replica_count else ''} replicas for volume {volume_name} running on {node_name if node_name else 'nodes'}, currently it's {running_replica_count} ... ({i})")
             if replica_count and running_replica_count == int(replica_count):
+                condition_met = True
                 break
             elif not replica_count and running_replica_count:
+                condition_met = True
                 break
             time.sleep(self.retry_interval)
+        assert condition_met, f"Waiting for {replica_count if replica_count else ''} replicas for volume {volume_name} running on {node_name if node_name else 'nodes'} failed. There are only {running_replica_count} running replicas"
         return running_replica_count
 
     def wait_for_replica_rebuilding_complete(self, volume_name, node_name=None):

e2e/libs/volume/volume.py

@@ -34,7 +34,7 @@ def list(self, label_selector=None):
         return self.volume.list(label_selector=label_selector)
 
     def list_names(self, label_selector=None):
-        return [item['metadata']['name'] for item in self.list(label_selector)['items']]
+        return [item['metadata']['name'] for item in self.list(label_selector)]
 
     def set_annotation(self, volume_name, annotation_key, annotation_value):
         return self.volume.set_annotation(volume_name, annotation_key, annotation_value)

e2e/libs/workload/statefulset.py

@@ -4,6 +4,9 @@
 from kubernetes import client
 from kubernetes.client.rest import ApiException
 
+from persistentvolumeclaim import PersistentVolumeClaim
+from volume import Volume
+
 from utility.constant import LABEL_TEST
 from utility.constant import LABEL_TEST_VALUE
 from utility.utility import get_retry_count_and_interval
@@ -79,7 +82,9 @@ def delete_statefulset(name, namespace='default'):
         assert e.status == 404
 
     retry_count, retry_interval = get_retry_count_and_interval()
-    for _ in range(retry_count):
+    for i in range(retry_count):
+        time.sleep(retry_interval)
+        logging(f"Waiting for statefulset {name} deleted ... ({i})")
         resp = api.list_namespaced_stateful_set(namespace=namespace)
         deleted = True
         for item in resp.items:
@@ -88,9 +93,34 @@ def delete_statefulset(name, namespace='default'):
                 break
         if deleted:
             break
+    assert deleted
+
+    claim = PersistentVolumeClaim()
+    volume_name = None
+    for i in range(retry_count):
         time.sleep(retry_interval)
+        logging(f"Waiting for pvc of statefulset {name} deleted ... ({i})")
+        claims = claim.list(label_selector=f"app={name}")
+        deleted = False
+        if len(claims) == 0:
+            deleted = True
+            break
+        else:
+            volume_name = claims[0].spec.volume_name
     assert deleted
 
+    if volume_name:
+        volume = Volume()
+        for i in range(retry_count):
+            time.sleep(retry_interval)
+            logging(f"Waiting for volume {volume_name} deleted ... ({i})")
+            volumes = volume.list(label_selector=f"longhornvolume={volume_name}")
+            deleted = False
+            if len(volumes) == 0:
+                deleted = True
+                break
+        assert deleted
+
 
 def get_statefulset(name, namespace='default'):
     api = client.AppsV1Api()