diff --git a/e2e/keywords/common.resource b/e2e/keywords/common.resource index bf3a47476e..078991263a 100644 --- a/e2e/keywords/common.resource +++ b/e2e/keywords/common.resource @@ -13,7 +13,7 @@ Library ../libs/keywords/workload_keywords.py *** Keywords *** Set test environment init_k8s_api_client - init_node_exec ${TEST NAME} + init_node_exec ${SUITE NAME.rsplit('.')[1]} init_storageclasses @{volume_list} = Create List Set Test Variable ${volume_list} diff --git a/e2e/keywords/node.resource b/e2e/keywords/node.resource index 9a288d7516..048b4459ef 100644 --- a/e2e/keywords/node.resource +++ b/e2e/keywords/node.resource @@ -25,16 +25,26 @@ Reboot volume ${idx} replica node END Reboot node ${idx} - power_off_node ${idx} + reboot_node_by_index ${idx} -Reboot all worker nodes - power_off_all_worker_nodes +Restart all worker nodes + reboot_all_worker_nodes + +Reboot volume node of statefulset ${idx} + ${volume_name} = get_workload_volume_name ${statefulset_list}[${idx}] + ${node_name} = get_volume_node ${volume_name} + reboot_node_by_name ${node_name} Power off node ${idx} for ${power_off_time_in_min} mins - power_off_node ${idx} ${power_off_time_in_min} + reboot_node_by_index ${idx} ${power_off_time_in_min} Power off all worker nodes for ${power_off_time_in_min} mins - power_off_all_worker_nodes ${power_off_time_in_min} + reboot_all_worker_nodes ${power_off_time_in_min} + +Power off volume node of statefulset ${idx} for ${power_off_time_in_min} mins + ${volume_name} = get_workload_volume_name ${statefulset_list}[${idx}] + ${node_name} = get_volume_node ${volume_name} + reboot_node_by_name ${node_name} ${power_off_time_in_min} Wait for longhorn ready wait_for_all_instance_manager_running diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource index 1fe66c8be4..e5d5eb1053 100644 --- a/e2e/keywords/volume.resource +++ b/e2e/keywords/volume.resource @@ -3,6 +3,7 @@ Documentation Volume Keywords Library Collections Library ../libs/keywords/volume_keywords.py +Library ../libs/keywords/workload_keywords.py Library ../libs/keywords/common_keywords.py *** Keywords *** @@ -73,3 +74,7 @@ Wait until replica on replica node rebuilt ${node_name} = get_replica_node ${volume_name} wait_for_replica_rebuilding_start ${volume_name} ${node_name} wait_for_replica_rebuilding_complete ${volume_name} ${node_name} + +Wait for volume of statefulset ${idx} healthy + ${volume_name} = get_workload_volume_name ${statefulset_list}[${idx}] + wait_for_volume_healthy ${volume_name} \ No newline at end of file diff --git a/e2e/keywords/workload.resource b/e2e/keywords/workload.resource index 00f5290d55..6d8b91d6d0 100644 --- a/e2e/keywords/workload.resource +++ b/e2e/keywords/workload.resource @@ -39,3 +39,5 @@ Check statefulset ${idx} works ${pod_data_checksum} = write_pod_random_data ${pod_name} 1024 check_pod_data ${pod_name} ${pod_data_checksum} +Wait for statefulset ${idx} stable + wait_for_workload_pod_stable ${statefulset_list}[${idx}] diff --git a/e2e/libs/keywords/node_keywords.py b/e2e/libs/keywords/node_keywords.py index d04fb6d13b..bf0ed11be7 100644 --- a/e2e/libs/keywords/node_keywords.py +++ b/e2e/libs/keywords/node_keywords.py @@ -21,15 +21,18 @@ def reboot_replica_node(self, volume_name): replica_node = volume_keywords.get_replica_node(volume_name) self.node.reboot_node(replica_node) - def power_off_node(self, idx, power_off_time_in_min=1): + def reboot_node_by_index(self, idx, power_off_time_in_min=1): node_name = get_node(idx) self.node.reboot_node(node_name, int(power_off_time_in_min) * 60) - def power_off_all_worker_nodes(self, power_off_time_in_min=1): + def reboot_all_worker_nodes(self, power_off_time_in_min=1): self.node.reboot_all_worker_nodes(int(power_off_time_in_min) * 60) def reboot_all_nodes(self): self.node.reboot_all_nodes() + def reboot_node_by_name(self, node_name, power_off_time_in_min=1): + self.node.reboot_node(node_name, int(power_off_time_in_min) * 60) + def wait_for_all_instance_manager_running(self): wait_for_all_instance_manager_running() diff --git a/e2e/libs/workload/workload.py b/e2e/libs/workload/workload.py index b55fa493d8..9229b28076 100644 --- a/e2e/libs/workload/workload.py +++ b/e2e/libs/workload/workload.py @@ -232,7 +232,8 @@ def get_workload_pods(workload_name): return resp.items def get_workload_volume_name(workload_name): - get_workload_pvc_name(workload_name) + api = client.CoreV1Api() + pvc_name = get_workload_pvc_name(workload_name) pvc = api.read_namespaced_persistent_volume_claim( name=pvc_name, namespace='default') return pvc.spec.volume_name diff --git a/e2e/tests/node_reboot.robot b/e2e/tests/node_reboot.robot index e97fc10c21..3acc4015e8 100644 --- a/e2e/tests/node_reboot.robot +++ b/e2e/tests/node_reboot.robot @@ -1,5 +1,6 @@ *** Settings *** Documentation Negative Test Cases +Resource ../keywords/workload.resource Resource ../keywords/volume.resource Resource ../keywords/node.resource Resource ../keywords/common.resource @@ -11,22 +12,122 @@ Test Teardown Cleanup test resources ${LOOP_COUNT} 1 *** Test Cases *** -Reboot Volume Node While Replica Rebuilding - Create a volume with 5 GB and 3 replicas - Write data to the volume +Reboot Node One By One While Workload Heavy Writing + Create deployment 0 with rwo volume + Create deployment 1 with rwx volume + Create deployment 2 with rwo and strict-local volume + Create statefulset 0 with rwo volume + Create statefulset 1 with rwx volume + Create statefulset 2 with rwo and strict-local volume FOR ${i} IN RANGE ${LOOP_COUNT} - Delete replica on volume node to trigger replica rebuilding - During replica rebuilding, reboot volume node - Wait until replica on volume node rebuilt - Check data is intact + Keep writing data to deployment 0 + Keep writing data to deployment 1 + Keep writing data to deployment 2 + Keep writing data to statefulset 0 + Keep writing data to statefulset 1 + Keep writing data to statefulset 2 + Reboot node 0 + Reboot node 1 + Reboot node 2 + Wait for longhorn ready + Check deployment 0 works + Check deployment 1 works + Check deployment 2 works + Check statefulset 0 works + Check statefulset 1 works + Check statefulset 2 works END -Reboot Replica Node While Replica Rebuilding - Create a volume with 5 GB and 3 replicas - Write data to the volume +Power Off Node One By Once For More Than Pod Eviction Timeout While Workload Heavy Writing + Create deployment 0 with rwo volume + Create deployment 1 with rwx volume + Create deployment 2 with rwo and strict-local volume + Create statefulset 0 with rwo volume + Create statefulset 1 with rwx volume + Create statefulset 2 with rwo and strict-local volume FOR ${i} IN RANGE ${LOOP_COUNT} - Delete replica on replica node to trigger replica rebuilding - During replica rebuilding, reboot replica node - Wait until replica on replica node rebuilt - Check data is intact + Keep writing data to deployment 0 + Keep writing data to deployment 1 + Keep writing data to deployment 2 + Keep writing data to statefulset 0 + Keep writing data to statefulset 1 + Keep writing data to statefulset 2 + Power off node 0 for 6 mins + Power off node 1 for 6 mins + Power off node 2 for 6 mins + Wait for longhorn ready + Check deployment 0 works + Check deployment 1 works + Check deployment 2 works + Check statefulset 0 works + Check statefulset 1 works + Check statefulset 2 works + END + +Reboot All Worker Nodes While Workload Heavy Writing + Create deployment 0 with rwo volume + Create deployment 1 with rwx volume + Create deployment 2 with rwo and strict-local volume + Create statefulset 0 with rwo volume + Create statefulset 1 with rwx volume + Create statefulset 2 with rwo and strict-local volume + FOR ${i} IN RANGE ${LOOP_COUNT} + Keep writing data to deployment 0 + Keep writing data to deployment 1 + Keep writing data to deployment 2 + Keep writing data to statefulset 0 + Keep writing data to statefulset 1 + Keep writing data to statefulset 2 + Restart all worker nodes + Wait for longhorn ready + Check deployment 0 works + Check deployment 1 works + Check deployment 2 works + Check statefulset 0 works + Check statefulset 1 works + Check statefulset 2 works + END + +Power Off All Worker Nodes For More Than Pod Eviction Timeout While Workload Heavy Writing + Create deployment 0 with rwo volume + Create deployment 1 with rwx volume + Create deployment 2 with rwo and strict-local volume + Create statefulset 0 with rwo volume + Create statefulset 1 with rwx volume + Create statefulset 2 with rwo and strict-local volume + FOR ${i} IN RANGE ${LOOP_COUNT} + Keep writing data to deployment 0 + Keep writing data to deployment 1 + Keep writing data to deployment 2 + Keep writing data to statefulset 0 + Keep writing data to statefulset 1 + Keep writing data to statefulset 2 + Power off all worker nodes for 6 mins + Wait for longhorn ready + Check deployment 0 works + Check deployment 1 works + Check deployment 2 works + Check statefulset 0 works + Check statefulset 1 works + Check statefulset 2 works + END + +Reboot Volume Node While Workload Heavy Writing + Create statefulset 0 with rwo volume + FOR ${i} IN RANGE ${LOOP_COUNT} + Keep writing data to statefulset 0 + Reboot volume node of statefulset 0 + Wait for volume of statefulset 0 healthy + Wait for statefulset 0 stable + Check statefulset 0 works + END + +Power Off Volume Node For More Than Pod Eviction Timeout While Workload Heavy Writing + Create statefulset 0 with rwo volume + FOR ${i} IN RANGE ${LOOP_COUNT} + Keep writing data to statefulset 0 + Power off volume node of statefulset 0 for 6 mins + Wait for volume of statefulset 0 healthy + Wait for statefulset 0 stable + Check statefulset 0 works END \ No newline at end of file diff --git a/e2e/tests/nodes_reboot.robot b/e2e/tests/nodes_reboot.robot deleted file mode 100644 index aa9e5dbbed..0000000000 --- a/e2e/tests/nodes_reboot.robot +++ /dev/null @@ -1,112 +0,0 @@ -*** Settings *** -Documentation Negative Test Cases -Resource ../keywords/workload.resource -Resource ../keywords/node.resource -Resource ../keywords/common.resource - -Test Setup Set test environment -Test Teardown Cleanup test resources - -*** Variables *** -${LOOP_COUNT} 1 - -*** Test Cases *** -Reboot Node One By One While Workload Heavy Writing - Create deployment 0 with rwo volume - Create deployment 1 with rwx volume - Create deployment 2 with rwo and strict-local volume - Create statefulset 0 with rwo volume - Create statefulset 1 with rwx volume - Create statefulset 2 with rwo and strict-local volume - FOR ${i} IN RANGE ${LOOP_COUNT} - Keep writing data to deployment 0 - Keep writing data to deployment 1 - Keep writing data to deployment 2 - Keep writing data to statefulset 0 - Keep writing data to statefulset 1 - Keep writing data to statefulset 2 - Reboot node 0 - Reboot node 1 - Reboot node 2 - Wait for longhorn ready - Check deployment 0 works - Check deployment 1 works - Check deployment 2 works - Check statefulset 0 works - Check statefulset 1 works - Check statefulset 2 works - END - -Power Off Node One By Once For More Than Pod Eviction Timeout While Workload Heavy Writing - Create deployment 0 with rwo volume - Create deployment 1 with rwx volume - Create deployment 2 with rwo and strict-local volume - Create statefulset 0 with rwo volume - Create statefulset 1 with rwx volume - Create statefulset 2 with rwo and strict-local volume - FOR ${i} IN RANGE ${LOOP_COUNT} - Keep writing data to deployment 0 - Keep writing data to deployment 1 - Keep writing data to deployment 2 - Keep writing data to statefulset 0 - Keep writing data to statefulset 1 - Keep writing data to statefulset 2 - Power off node 0 for 6 mins - Power off node 1 for 6 mins - Power off node 2 for 6 mins - Wait for longhorn ready - Check deployment 0 works - Check deployment 1 works - Check deployment 2 works - Check statefulset 0 works - Check statefulset 1 works - Check statefulset 2 works - END - -Reboot All Worker Nodes While Workload Heavy Writing - Create deployment 0 with rwo volume - Create deployment 1 with rwx volume - Create deployment 2 with rwo and strict-local volume - Create statefulset 0 with rwo volume - Create statefulset 1 with rwx volume - Create statefulset 2 with rwo and strict-local volume - FOR ${i} IN RANGE ${LOOP_COUNT} - Keep writing data to deployment 0 - Keep writing data to deployment 1 - Keep writing data to deployment 2 - Keep writing data to statefulset 0 - Keep writing data to statefulset 1 - Keep writing data to statefulset 2 - Reboot all worker nodes - Wait for longhorn ready - Check deployment 0 works - Check deployment 1 works - Check deployment 2 works - Check statefulset 0 works - Check statefulset 1 works - Check statefulset 2 works - END - -Power Off All Worker Nodes For More Than Pod Eviction Timeout While Workload Heavy Writing - Create deployment 0 with rwo volume - Create deployment 1 with rwx volume - Create deployment 2 with rwo and strict-local volume - Create statefulset 0 with rwo volume - Create statefulset 1 with rwx volume - Create statefulset 2 with rwo and strict-local volume - FOR ${i} IN RANGE ${LOOP_COUNT} - Keep writing data to deployment 0 - Keep writing data to deployment 1 - Keep writing data to deployment 2 - Keep writing data to statefulset 0 - Keep writing data to statefulset 1 - Keep writing data to statefulset 2 - Power off all worker nodes for 6 mins - Wait for longhorn ready - Check deployment 0 works - Check deployment 1 works - Check deployment 2 works - Check statefulset 0 works - Check statefulset 1 works - Check statefulset 2 works - END \ No newline at end of file diff --git a/e2e/tests/replica_rebuilding.robot b/e2e/tests/replica_rebuilding.robot new file mode 100644 index 0000000000..e97fc10c21 --- /dev/null +++ b/e2e/tests/replica_rebuilding.robot @@ -0,0 +1,32 @@ +*** Settings *** +Documentation Negative Test Cases +Resource ../keywords/volume.resource +Resource ../keywords/node.resource +Resource ../keywords/common.resource + +Test Setup Set test environment +Test Teardown Cleanup test resources + +*** Variables *** +${LOOP_COUNT} 1 + +*** Test Cases *** +Reboot Volume Node While Replica Rebuilding + Create a volume with 5 GB and 3 replicas + Write data to the volume + FOR ${i} IN RANGE ${LOOP_COUNT} + Delete replica on volume node to trigger replica rebuilding + During replica rebuilding, reboot volume node + Wait until replica on volume node rebuilt + Check data is intact + END + +Reboot Replica Node While Replica Rebuilding + Create a volume with 5 GB and 3 replicas + Write data to the volume + FOR ${i} IN RANGE ${LOOP_COUNT} + Delete replica on replica node to trigger replica rebuilding + During replica rebuilding, reboot replica node + Wait until replica on replica node rebuilt + Check data is intact + END \ No newline at end of file