test: add volume node reboot test cases

Signed-off-by: Yang Chiu <[email protected]>
longhorn · Sep 22, 2023 · 318f1dd · 318f1dd
1 parent 7ae2289
commit 318f1dd
Show file tree

Hide file tree

Showing 9 changed files with 177 additions and 135 deletions.
diff --git a/e2e/keywords/common.resource b/e2e/keywords/common.resource
@@ -13,7 +13,7 @@ Library             ../libs/keywords/workload_keywords.py
 *** Keywords ***
 Set test environment
     init_k8s_api_client
-    init_node_exec    ${TEST NAME}
+    init_node_exec    ${SUITE NAME.rsplit('.')[1]}
     init_storageclasses
     @{volume_list} =    Create List
     Set Test Variable    ${volume_list}

diff --git a/e2e/keywords/node.resource b/e2e/keywords/node.resource
@@ -25,16 +25,26 @@ Reboot volume ${idx} replica node
     END
 
 Reboot node ${idx}
-    power_off_node    ${idx}
+    reboot_node_by_index    ${idx}
 
-Reboot all worker nodes
-    power_off_all_worker_nodes
+Restart all worker nodes
+    reboot_all_worker_nodes
+
+Reboot volume node of statefulset ${idx}
+    ${volume_name} =    get_workload_volume_name    ${statefulset_list}[${idx}]
+    ${node_name} =    get_volume_node    ${volume_name}
+    reboot_node_by_name    ${node_name}
 
 Power off node ${idx} for ${power_off_time_in_min} mins
-    power_off_node    ${idx}    ${power_off_time_in_min}
+    reboot_node_by_index    ${idx}    ${power_off_time_in_min}
 
 Power off all worker nodes for ${power_off_time_in_min} mins
-    power_off_all_worker_nodes    ${power_off_time_in_min}
+    reboot_all_worker_nodes    ${power_off_time_in_min}
+
+Power off volume node of statefulset ${idx} for ${power_off_time_in_min} mins
+    ${volume_name} =    get_workload_volume_name    ${statefulset_list}[${idx}]
+    ${node_name} =    get_volume_node    ${volume_name}
+    reboot_node_by_name    ${node_name}    ${power_off_time_in_min}
 
 Wait for longhorn ready
     wait_for_all_instance_manager_running

diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource
@@ -3,6 +3,7 @@ Documentation    Volume Keywords
 
 Library    Collections
 Library    ../libs/keywords/volume_keywords.py
+Library    ../libs/keywords/workload_keywords.py
 Library    ../libs/keywords/common_keywords.py
 
 *** Keywords ***
@@ -73,3 +74,7 @@ Wait until replica on replica node rebuilt
     ${node_name} =    get_replica_node    ${volume_name}
     wait_for_replica_rebuilding_start    ${volume_name}    ${node_name}
     wait_for_replica_rebuilding_complete    ${volume_name}    ${node_name}
+
+Wait for volume of statefulset ${idx} healthy
+    ${volume_name} =    get_workload_volume_name    ${statefulset_list}[${idx}]
+    wait_for_volume_healthy    ${volume_name}
diff --git a/e2e/keywords/workload.resource b/e2e/keywords/workload.resource
@@ -39,3 +39,5 @@ Check statefulset ${idx} works
     ${pod_data_checksum} =    write_pod_random_data    ${pod_name}    1024
     check_pod_data    ${pod_name}    ${pod_data_checksum}
 
+Wait for statefulset ${idx} stable
+    wait_for_workload_pod_stable   ${statefulset_list}[${idx}]
diff --git a/e2e/libs/keywords/node_keywords.py b/e2e/libs/keywords/node_keywords.py
@@ -21,15 +21,18 @@ def reboot_replica_node(self, volume_name):
         replica_node = volume_keywords.get_replica_node(volume_name)
         self.node.reboot_node(replica_node)
 
-    def power_off_node(self, idx, power_off_time_in_min=1):
+    def reboot_node_by_index(self, idx, power_off_time_in_min=1):
         node_name = get_node(idx)
         self.node.reboot_node(node_name, int(power_off_time_in_min) * 60)
 
-    def power_off_all_worker_nodes(self, power_off_time_in_min=1):
+    def reboot_all_worker_nodes(self, power_off_time_in_min=1):
         self.node.reboot_all_worker_nodes(int(power_off_time_in_min) * 60)
 
     def reboot_all_nodes(self):
         self.node.reboot_all_nodes()
 
+    def reboot_node_by_name(self, node_name, power_off_time_in_min=1):
+        self.node.reboot_node(node_name, int(power_off_time_in_min) * 60)
+
     def wait_for_all_instance_manager_running(self):
         wait_for_all_instance_manager_running()
diff --git a/e2e/libs/workload/workload.py b/e2e/libs/workload/workload.py
@@ -232,7 +232,8 @@ def get_workload_pods(workload_name):
     return resp.items
 
 def get_workload_volume_name(workload_name):
-    get_workload_pvc_name(workload_name)
+    api = client.CoreV1Api()
+    pvc_name = get_workload_pvc_name(workload_name)
     pvc = api.read_namespaced_persistent_volume_claim(
         name=pvc_name, namespace='default')
     return pvc.spec.volume_name

diff --git a/e2e/tests/node_reboot.robot b/e2e/tests/node_reboot.robot
@@ -1,5 +1,6 @@
 *** Settings ***
 Documentation    Negative Test Cases
+Resource    ../keywords/workload.resource
 Resource    ../keywords/volume.resource
 Resource    ../keywords/node.resource
 Resource    ../keywords/common.resource
@@ -11,22 +12,122 @@ Test Teardown    Cleanup test resources
 ${LOOP_COUNT}    1
 
 *** Test Cases ***
-Reboot Volume Node While Replica Rebuilding
-    Create a volume with 5 GB and 3 replicas
-    Write data to the volume
+Reboot Node One By One While Workload Heavy Writing
+    Create deployment 0 with rwo volume
+    Create deployment 1 with rwx volume
+    Create deployment 2 with rwo and strict-local volume
+    Create statefulset 0 with rwo volume
+    Create statefulset 1 with rwx volume
+    Create statefulset 2 with rwo and strict-local volume
     FOR    ${i}    IN RANGE    ${LOOP_COUNT}
-        Delete replica on volume node to trigger replica rebuilding
-        During replica rebuilding, reboot volume node
-        Wait until replica on volume node rebuilt
-        Check data is intact
+        Keep writing data to deployment 0
+        Keep writing data to deployment 1
+        Keep writing data to deployment 2
+        Keep writing data to statefulset 0
+        Keep writing data to statefulset 1
+        Keep writing data to statefulset 2
+        Reboot node 0
+        Reboot node 1
+        Reboot node 2
+        Wait for longhorn ready
+        Check deployment 0 works
+        Check deployment 1 works
+        Check deployment 2 works
+        Check statefulset 0 works
+        Check statefulset 1 works
+        Check statefulset 2 works
     END
 
-Reboot Replica Node While Replica Rebuilding
-    Create a volume with 5 GB and 3 replicas
-    Write data to the volume
+Power Off Node One By Once For More Than Pod Eviction Timeout While Workload Heavy Writing
+    Create deployment 0 with rwo volume
+    Create deployment 1 with rwx volume
+    Create deployment 2 with rwo and strict-local volume
+    Create statefulset 0 with rwo volume
+    Create statefulset 1 with rwx volume
+    Create statefulset 2 with rwo and strict-local volume
     FOR    ${i}    IN RANGE    ${LOOP_COUNT}
-        Delete replica on replica node to trigger replica rebuilding
-        During replica rebuilding, reboot replica node
-        Wait until replica on replica node rebuilt
-        Check data is intact
+        Keep writing data to deployment 0
+        Keep writing data to deployment 1
+        Keep writing data to deployment 2
+        Keep writing data to statefulset 0
+        Keep writing data to statefulset 1
+        Keep writing data to statefulset 2
+        Power off node 0 for 6 mins
+        Power off node 1 for 6 mins
+        Power off node 2 for 6 mins
+        Wait for longhorn ready
+        Check deployment 0 works
+        Check deployment 1 works
+        Check deployment 2 works
+        Check statefulset 0 works
+        Check statefulset 1 works
+        Check statefulset 2 works
+    END
+
+Reboot All Worker Nodes While Workload Heavy Writing
+    Create deployment 0 with rwo volume
+    Create deployment 1 with rwx volume
+    Create deployment 2 with rwo and strict-local volume
+    Create statefulset 0 with rwo volume
+    Create statefulset 1 with rwx volume
+    Create statefulset 2 with rwo and strict-local volume
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        Keep writing data to deployment 0
+        Keep writing data to deployment 1
+        Keep writing data to deployment 2
+        Keep writing data to statefulset 0
+        Keep writing data to statefulset 1
+        Keep writing data to statefulset 2
+        Restart all worker nodes
+        Wait for longhorn ready
+        Check deployment 0 works
+        Check deployment 1 works
+        Check deployment 2 works
+        Check statefulset 0 works
+        Check statefulset 1 works
+        Check statefulset 2 works
+    END
+
+Power Off All Worker Nodes For More Than Pod Eviction Timeout While Workload Heavy Writing
+    Create deployment 0 with rwo volume
+    Create deployment 1 with rwx volume
+    Create deployment 2 with rwo and strict-local volume
+    Create statefulset 0 with rwo volume
+    Create statefulset 1 with rwx volume
+    Create statefulset 2 with rwo and strict-local volume
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        Keep writing data to deployment 0
+        Keep writing data to deployment 1
+        Keep writing data to deployment 2
+        Keep writing data to statefulset 0
+        Keep writing data to statefulset 1
+        Keep writing data to statefulset 2
+        Power off all worker nodes for 6 mins
+        Wait for longhorn ready
+        Check deployment 0 works
+        Check deployment 1 works
+        Check deployment 2 works
+        Check statefulset 0 works
+        Check statefulset 1 works
+        Check statefulset 2 works
+    END
+
+Reboot Volume Node While Workload Heavy Writing
+    Create statefulset 0 with rwo volume
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        Keep writing data to statefulset 0
+        Reboot volume node of statefulset 0
+        Wait for volume of statefulset 0 healthy
+        Wait for statefulset 0 stable
+        Check statefulset 0 works
+    END
+
+Power Off Volume Node For More Than Pod Eviction Timeout While Workload Heavy Writing
+    Create statefulset 0 with rwo volume
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        Keep writing data to statefulset 0
+        Power off volume node of statefulset 0 for 6 mins
+        Wait for volume of statefulset 0 healthy
+        Wait for statefulset 0 stable
+        Check statefulset 0 works
     END
diff --git a/e2e/tests/nodes_reboot.robot b/e2e/tests/nodes_reboot.robot
diff --git a/e2e/tests/replica_rebuilding.robot b/e2e/tests/replica_rebuilding.robot
@@ -0,0 +1,32 @@
+*** Settings ***
+Documentation    Negative Test Cases
+Resource    ../keywords/volume.resource
+Resource    ../keywords/node.resource
+Resource    ../keywords/common.resource
+
+Test Setup    Set test environment
+Test Teardown    Cleanup test resources
+
+*** Variables ***
+${LOOP_COUNT}    1
+
+*** Test Cases ***
+Reboot Volume Node While Replica Rebuilding
+    Create a volume with 5 GB and 3 replicas
+    Write data to the volume
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        Delete replica on volume node to trigger replica rebuilding
+        During replica rebuilding, reboot volume node
+        Wait until replica on volume node rebuilt
+        Check data is intact
+    END
+
+Reboot Replica Node While Replica Rebuilding
+    Create a volume with 5 GB and 3 replicas
+    Write data to the volume
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        Delete replica on replica node to trigger replica rebuilding
+        During replica rebuilding, reboot replica node
+        Wait until replica on replica node rebuilt
+        Check data is intact
+    END