From 70c989c5bc1953487f5d455fa2f01e6dd7defff5 Mon Sep 17 00:00:00 2001 From: Yang Chiu Date: Tue, 10 Dec 2024 16:16:28 +0800 Subject: [PATCH] test(robot): wait for pvc and volume deleted when cleaning up statefulset Signed-off-by: Yang Chiu --- .../persistentvolumeclaim_keywords.py | 4 +-- e2e/libs/keywords/volume_keywords.py | 4 +-- e2e/libs/persistentvolumeclaim/crd.py | 2 +- e2e/libs/volume/crd.py | 2 +- e2e/libs/volume/rest.py | 4 +++ e2e/libs/volume/volume.py | 2 +- e2e/libs/workload/statefulset.py | 32 ++++++++++++++++++- 7 files changed, 42 insertions(+), 8 deletions(-) diff --git a/e2e/libs/keywords/persistentvolumeclaim_keywords.py b/e2e/libs/keywords/persistentvolumeclaim_keywords.py index c58c212b6..947bc2c0d 100644 --- a/e2e/libs/keywords/persistentvolumeclaim_keywords.py +++ b/e2e/libs/keywords/persistentvolumeclaim_keywords.py @@ -16,8 +16,8 @@ def __init__(self): def cleanup_persistentvolumeclaims(self): claims = self.claim.list(label_selector=f"{LABEL_TEST}={LABEL_TEST_VALUE}") - logging(f'Cleaning up {len(claims.items)} persistentvolumeclaims') - for claim in claims.items: + logging(f'Cleaning up {len(claims)} persistentvolumeclaims') + for claim in claims: self.delete_persistentvolumeclaim(claim.metadata.name) def create_persistentvolumeclaim(self, name, volume_type="RWO", sc_name="longhorn", storage_size="3GiB"): diff --git a/e2e/libs/keywords/volume_keywords.py b/e2e/libs/keywords/volume_keywords.py index 0b7ec9353..4a7aa6aa5 100644 --- a/e2e/libs/keywords/volume_keywords.py +++ b/e2e/libs/keywords/volume_keywords.py @@ -26,8 +26,8 @@ def __init__(self): def cleanup_volumes(self): volumes = self.volume.list(label_selector=f"{LABEL_TEST}={LABEL_TEST_VALUE}") - logging(f'Cleaning up {len(volumes["items"])} volumes') - for volume in volumes['items']: + logging(f'Cleaning up {len(volumes)} volumes') + for volume in volumes: self.delete_volume(volume['metadata']['name']) def create_volume(self, volume_name, size="2Gi", numberOfReplicas=3, frontend="blockdev", migratable=False, dataLocality="disabled", accessMode="RWO", dataEngine="v1", backingImage="", Standby=False, fromBackup=""): diff --git a/e2e/libs/persistentvolumeclaim/crd.py b/e2e/libs/persistentvolumeclaim/crd.py index 688ca42f3..8a025b4ee 100644 --- a/e2e/libs/persistentvolumeclaim/crd.py +++ b/e2e/libs/persistentvolumeclaim/crd.py @@ -22,7 +22,7 @@ def list(self, claim_namespace="default", label_selector=None): return self.core_v1_api.list_namespaced_persistent_volume_claim( namespace=claim_namespace, label_selector=label_selector - ) + ).items def set_label(self, claim_name, label_key, label_value, claim_namespace="default"): for i in range(self.retry_count): diff --git a/e2e/libs/volume/crd.py b/e2e/libs/volume/crd.py index 5539cc57c..94e707b83 100644 --- a/e2e/libs/volume/crd.py +++ b/e2e/libs/volume/crd.py @@ -178,7 +178,7 @@ def list(self, label_selector=None): namespace="longhorn-system", plural="volumes", label_selector=label_selector - ) + )["items"] def set_annotation(self, volume_name, annotation_key, annotation_value): # retry conflict error diff --git a/e2e/libs/volume/rest.py b/e2e/libs/volume/rest.py index 65d25851b..45e119495 100644 --- a/e2e/libs/volume/rest.py +++ b/e2e/libs/volume/rest.py @@ -238,6 +238,7 @@ def get_replica_name_on_node(self, volume_name, node_name): return r.name def wait_for_replica_count(self, volume_name, node_name, replica_count): + condition_met = False for i in range(self.retry_count): running_replica_count = 0 volume = get_longhorn_client().by_id_volume(volume_name) @@ -248,10 +249,13 @@ def wait_for_replica_count(self, volume_name, node_name, replica_count): running_replica_count += 1 logging(f"Waiting for {replica_count if replica_count else ''} replicas for volume {volume_name} running on {node_name if node_name else 'nodes'}, currently it's {running_replica_count} ... ({i})") if replica_count and running_replica_count == int(replica_count): + condition_met = True break elif not replica_count and running_replica_count: + condition_met = True break time.sleep(self.retry_interval) + assert condition_met, f"Waiting for {replica_count if replica_count else ''} replicas for volume {volume_name} running on {node_name if node_name else 'nodes'} failed. There are only {running_replica_count} running replicas" return running_replica_count def wait_for_replica_rebuilding_complete(self, volume_name, node_name=None): diff --git a/e2e/libs/volume/volume.py b/e2e/libs/volume/volume.py index 8ba8eec4a..72159cd6c 100644 --- a/e2e/libs/volume/volume.py +++ b/e2e/libs/volume/volume.py @@ -34,7 +34,7 @@ def list(self, label_selector=None): return self.volume.list(label_selector=label_selector) def list_names(self, label_selector=None): - return [item['metadata']['name'] for item in self.list(label_selector)['items']] + return [item['metadata']['name'] for item in self.list(label_selector)] def set_annotation(self, volume_name, annotation_key, annotation_value): return self.volume.set_annotation(volume_name, annotation_key, annotation_value) diff --git a/e2e/libs/workload/statefulset.py b/e2e/libs/workload/statefulset.py index eebe8b501..0ecce07a7 100644 --- a/e2e/libs/workload/statefulset.py +++ b/e2e/libs/workload/statefulset.py @@ -4,6 +4,9 @@ from kubernetes import client from kubernetes.client.rest import ApiException +from persistentvolumeclaim import PersistentVolumeClaim +from volume import Volume + from utility.constant import LABEL_TEST from utility.constant import LABEL_TEST_VALUE from utility.utility import get_retry_count_and_interval @@ -79,7 +82,9 @@ def delete_statefulset(name, namespace='default'): assert e.status == 404 retry_count, retry_interval = get_retry_count_and_interval() - for _ in range(retry_count): + for i in range(retry_count): + time.sleep(retry_interval) + logging(f"Waiting for statefulset {name} deleted ... ({i})") resp = api.list_namespaced_stateful_set(namespace=namespace) deleted = True for item in resp.items: @@ -88,9 +93,34 @@ def delete_statefulset(name, namespace='default'): break if deleted: break + assert deleted + + claim = PersistentVolumeClaim() + volume_name = None + for i in range(retry_count): time.sleep(retry_interval) + logging(f"Waiting for pvc of statefulset {name} deleted ... ({i})") + claims = claim.list(label_selector=f"app={name}") + deleted = False + if len(claims) == 0: + deleted = True + break + else: + volume_name = claims[0].spec.volume_name assert deleted + if volume_name: + volume = Volume() + for i in range(retry_count): + time.sleep(retry_interval) + logging(f"Waiting for volume {volume_name} deleted ... ({i})") + volumes = volume.list(label_selector=f"longhornvolume={volume_name}") + deleted = False + if len(volumes) == 0: + deleted = True + break + assert deleted + def get_statefulset(name, namespace='default'): api = client.AppsV1Api()