test(robot): fix wait_for_workload_pods_stable assertion failed

Signed-off-by: Yang Chiu <[email protected]>
longhorn · Apr 11, 2024 · 7dc2631 · 7dc2631
1 parent 0bb5211
commit 7dc2631
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 27 deletions.
diff --git a/e2e/keywords/workload.resource b/e2e/keywords/workload.resource
@@ -4,6 +4,8 @@ Documentation    Workload Keywords
 Library    Collections
 Library    ../libs/keywords/common_keywords.py
 Library    ../libs/keywords/workload_keywords.py
+Library    ../libs/keywords/host_keywords.py
+Library    ../libs/keywords/kubelet_keywords.py
 
 *** Keywords ***
 Keep writing data to pod of ${workload_kind} ${workload_id}
@@ -12,15 +14,19 @@ Keep writing data to pod of ${workload_kind} ${workload_id}
 
 Power off volume node of ${workload_kind} ${workload_id} for ${duration} minutes
     ${workload_name} =   generate_name_with_suffix    ${workload_kind}    ${workload_id}
-    reboot_workload_volume_node    ${workload_name}    ${duration}
+    ${volume_name} =    get_workload_volume_name    ${workload_name}
+    reboot_volume_node    ${volume_name}    ${duration}
 
 Reboot volume node of ${workload_kind} ${workload_id}
     ${workload_name} =   generate_name_with_suffix    ${workload_kind}    ${workload_id}
-    reboot_workload_volume_node    ${workload_name}
+    ${volume_name} =    get_workload_volume_name    ${workload_name}
+    reboot_volume_node    ${volume_name}
 
 When Stop volume node kubelet of ${workload_kind} ${workload_id} for ${duration} seconds
     ${workload_name} =   generate_name_with_suffix    ${workload_kind}    ${workload_id}
-    restart_workload_kubelet    ${workload_name}    ${duration}
+    ${volume_name} =    get_workload_volume_name    ${workload_name}
+    ${node_name} =    get_volume_node    ${volume_name}
+    restart_kubelet    ${node_name}    ${duration}
 
 Wait for volume of ${workload_kind} ${workload_id} healthy
     ${workload_name} =   generate_name_with_suffix    ${workload_kind}    ${workload_id}

diff --git a/e2e/libs/workload/workload.py b/e2e/libs/workload/workload.py
@@ -151,34 +151,34 @@ def wait_for_workload_pods_running(workload_name, namespace="default"):
 def wait_for_workload_pods_stable(workload_name, namespace="default"):
     stable_pods = {}
     wait_for_stable_retry = {}
+    wait_for_stable_pod = []
 
     retry_count, retry_interval = get_retry_count_and_interval()
     for i in range(retry_count):
         pods = get_workload_pods(workload_name, namespace=namespace)
-        assert len(pods) > 0
-
-        for pod in pods:
-            pod_name = pod.metadata.name
-            if pod.status.phase == "Running":
-                if pod_name not in stable_pods or \
-                        stable_pods[pod_name].status.start_time != pod.status.start_time:
-                    stable_pods[pod_name] = pod
-                    wait_for_stable_retry[pod_name] = 0
-                else:
-                    wait_for_stable_retry[pod_name] += 1
-
-        wait_for_stable_pod = []
-        for pod in pods:
-            if pod.status.phase != "Running":
-                wait_for_stable_pod.append(pod.metadata.name)
-                continue
-
-            pod_name = pod.metadata.name
-            if wait_for_stable_retry[pod_name] < WAIT_FOR_POD_STABLE_MAX_RETRY:
-                wait_for_stable_pod.append(pod_name)
-
-        if len(wait_for_stable_pod) == 0:
-            return
+        if len(pods) > 0:
+            for pod in pods:
+                pod_name = pod.metadata.name
+                if pod.status.phase == "Running":
+                    if pod_name not in stable_pods or \
+                            stable_pods[pod_name].status.start_time != pod.status.start_time:
+                        stable_pods[pod_name] = pod
+                        wait_for_stable_retry[pod_name] = 0
+                    else:
+                        wait_for_stable_retry[pod_name] += 1
+
+            wait_for_stable_pod = []
+            for pod in pods:
+                if pod.status.phase != "Running":
+                    wait_for_stable_pod.append(pod.metadata.name)
+                    continue
+
+                pod_name = pod.metadata.name
+                if wait_for_stable_retry[pod_name] < WAIT_FOR_POD_STABLE_MAX_RETRY:
+                    wait_for_stable_pod.append(pod_name)
+
+            if len(wait_for_stable_pod) == 0:
+                return
 
         logging(f"Waiting for {workload_name} pods {wait_for_stable_pod} stable, retry ({i}) ...")
         time.sleep(retry_interval)