diff --git a/e2e/tests/cluster_restart.robot b/e2e/tests/cluster_restart.robot index 73630f7ba4..f1644fd4e2 100644 --- a/e2e/tests/cluster_restart.robot +++ b/e2e/tests/cluster_restart.robot @@ -1,8 +1,11 @@ *** Settings *** Documentation Negative Test Cases + Resource ../keywords/common.resource Resource ../keywords/deployment.resource -Resource ../keywords/node.resource +Resource ../keywords/longhorn.resource +Resource ../keywords/host.resource +Resource ../keywords/persistentvolumeclaim.resource Resource ../keywords/statefulset.resource Resource ../keywords/workload.resource @@ -17,22 +20,32 @@ ${CONTROL_PLANE_NODE_NETWORK_LATENCY_IN_MS} 0 *** Test Cases *** Restart Cluster While Workload Heavy Writing - Given Create deployment 0 with rwo volume - And Create deployment 1 with rwx volume - And Create deployment 2 with rwo and strict-local volume - And Create statefulset 0 with rwo volume - And Create statefulset 1 with rwx volume - And Create statefulset 2 with rwo and strict-local volume + Given Create persistentvolumeclaim 0 using RWO volume + And Create persistentvolumeclaim 1 using RWX volume + And Create persistentvolumeclaim 2 using RWO volume with strict-local storageclass + And Create deployment 0 with persistentvolumeclaim 0 + And Create deployment 1 with persistentvolumeclaim 1 + And Create deployment 2 with persistentvolumeclaim 2 + And Create statefulset 0 using RWO volume + And Create statefulset 1 using RWX volume + And Create statefulset 2 using RWO volume with strict-local storageclass FOR ${i} IN RANGE ${LOOP_COUNT} - And Keep writing data to deployment 0 - And Keep writing data to deployment 1 - And Keep writing data to deployment 2 - And Keep writing data to statefulset 0 - And Keep writing data to statefulset 1 - And Keep writing data to statefulset 2 + And Keep writing data to pod of deployment 0 + And Keep writing data to pod of deployment 1 + And Keep writing data to pod of deployment 2 + And Keep writing data to pod of statefulset 0 + And Keep writing data to pod of statefulset 1 + And Keep writing data to pod of statefulset 2 When Restart cluster + And Wait for longhorn ready + And Wait for deployment 0 pods stable + And Wait for deployment 1 pods stable + And Wait for deployment 2 pods stable + And Wait for statefulset 0 pods stable + And Wait for statefulset 1 pods stable + And Wait for statefulset 2 pods stable Then Check deployment 0 works And Check deployment 1 works diff --git a/e2e/tests/kubelet_restart.robot b/e2e/tests/kubelet_restart.robot index a270b08058..67ad85c7c8 100644 --- a/e2e/tests/kubelet_restart.robot +++ b/e2e/tests/kubelet_restart.robot @@ -1,6 +1,8 @@ *** Settings *** Documentation Negative Test Cases + Resource ../keywords/common.resource +Resource ../keywords/persistentvolumeclaim.resource Resource ../keywords/statefulset.resource Resource ../keywords/workload.resource @@ -14,21 +16,27 @@ ${RETRY_INTERVAL} 1 *** Test Cases *** Restart Volume Node Kubelet While Workload Heavy Writing - Given Create statefulset 0 with rwo volume + Given Create statefulset 0 using RWO volume + FOR ${i} IN RANGE ${LOOP_COUNT} - And Keep writing data to statefulset 0 + And Keep writing data to pod of statefulset 0 + When Stop volume node kubelet of statefulset 0 for 10 seconds And Wait for volume of statefulset 0 healthy - And Wait for statefulset 0 stable + And Wait for statefulset 0 pods stable + Then Check statefulset 0 works END Stop Volume Node Kubelet For More Than Pod Eviction Timeout While Workload Heavy Writing - Given Create statefulset 0 with rwo volume + Given Create statefulset 0 using RWO volume + FOR ${i} IN RANGE ${LOOP_COUNT} - And Keep writing data to statefulset 0 + And Keep writing data to pod of statefulset 0 + When Stop volume node kubelet of statefulset 0 for 360 seconds And Wait for volume of statefulset 0 healthy - And Wait for statefulset 0 stable + And Wait for statefulset 0 pods stable + Then Check statefulset 0 works END diff --git a/e2e/tests/node_reboot.robot b/e2e/tests/node_reboot.robot index e4928d11ca..7a873c106c 100644 --- a/e2e/tests/node_reboot.robot +++ b/e2e/tests/node_reboot.robot @@ -1,8 +1,11 @@ *** Settings *** Documentation Negative Test Cases + Resource ../keywords/common.resource Resource ../keywords/deployment.resource -Resource ../keywords/node.resource +Resource ../keywords/longhorn.resource +Resource ../keywords/host.resource +Resource ../keywords/persistentvolumeclaim.resource Resource ../keywords/recurringjob.resource Resource ../keywords/statefulset.resource Resource ../keywords/volume.resource @@ -15,30 +18,41 @@ Test Teardown Cleanup test resources ${LOOP_COUNT} 1 ${RETRY_COUNT} 300 ${RETRY_INTERVAL} 1 -${VOLUME_TYPE} rwo +${VOLUME_TYPE} RWO ${CONTROL_PLANE_NODE_NETWORK_LATENCY_IN_MS} 0 *** Test Cases *** Reboot Node One By One While Workload Heavy Writing - Given Create deployment 0 with rwo volume - And Create deployment 1 with rwx volume - And Create deployment 2 with rwo and strict-local volume - And Create statefulset 0 with rwo volume - And Create statefulset 1 with rwx volume - And Create statefulset 2 with rwo and strict-local volume + Given Create persistentvolumeclaim 0 using RWO volume + And Create persistentvolumeclaim 1 using RWX volume + And Create persistentvolumeclaim 2 using RWO volume with strict-local storageclass + + And Create deployment 0 with persistentvolumeclaim 0 + And Create deployment 1 with persistentvolumeclaim 1 + And Create deployment 2 with persistentvolumeclaim 2 + + And Create statefulset 0 using RWO volume + And Create statefulset 1 using RWX volume + And Create statefulset 2 using RWO volume with strict-local storageclass FOR ${i} IN RANGE ${LOOP_COUNT} - And Keep writing data to deployment 0 - And Keep writing data to deployment 1 - And Keep writing data to deployment 2 - And Keep writing data to statefulset 0 - And Keep writing data to statefulset 1 - And Keep writing data to statefulset 2 + And Keep writing data to pod of deployment 0 + And Keep writing data to pod of deployment 1 + And Keep writing data to pod of deployment 2 + And Keep writing data to pod of statefulset 0 + And Keep writing data to pod of statefulset 1 + And Keep writing data to pod of statefulset 2 When Reboot node 0 And Reboot node 1 And Reboot node 2 And Wait for longhorn ready + And Wait for deployment 0 pods stable + And Wait for deployment 1 pods stable + And Wait for deployment 2 pods stable + And Wait for statefulset 0 pods stable + And Wait for statefulset 1 pods stable + And Wait for statefulset 2 pods stable Then Check deployment 0 works And Check deployment 1 works @@ -49,25 +63,36 @@ Reboot Node One By One While Workload Heavy Writing END Power Off Node One By Once For More Than Pod Eviction Timeout While Workload Heavy Writing - Given Create deployment 0 with rwo volume - And Create deployment 1 with rwx volume - And Create deployment 2 with rwo and strict-local volume - And Create statefulset 0 with rwo volume - And Create statefulset 1 with rwx volume - And Create statefulset 2 with rwo and strict-local volume + Given Create persistentvolumeclaim 0 using RWO volume + And Create persistentvolumeclaim 1 using RWX volume + And Create persistentvolumeclaim 2 using RWO volume with strict-local storageclass + + And Create deployment 0 with persistentvolumeclaim 0 + And Create deployment 1 with persistentvolumeclaim 1 + And Create deployment 2 with persistentvolumeclaim 2 + + And Create statefulset 0 using RWO volume + And Create statefulset 1 using RWX volume + And Create statefulset 2 using RWO volume with strict-local storageclass FOR ${i} IN RANGE ${LOOP_COUNT} - And Keep writing data to deployment 0 - And Keep writing data to deployment 1 - And Keep writing data to deployment 2 - And Keep writing data to statefulset 0 - And Keep writing data to statefulset 1 - And Keep writing data to statefulset 2 + And Keep writing data to pod of deployment 0 + And Keep writing data to pod of deployment 1 + And Keep writing data to pod of deployment 2 + And Keep writing data to pod of statefulset 0 + And Keep writing data to pod of statefulset 1 + And Keep writing data to pod of statefulset 2 When Power off node 0 for 6 mins And Power off node 1 for 6 mins And Power off node 2 for 6 mins And Wait for longhorn ready + And Wait for deployment 0 pods stable + And Wait for deployment 1 pods stable + And Wait for deployment 2 pods stable + And Wait for statefulset 0 pods stable + And Wait for statefulset 1 pods stable + And Wait for statefulset 2 pods stable Then Check deployment 0 works And Check deployment 1 works @@ -78,23 +103,34 @@ Power Off Node One By Once For More Than Pod Eviction Timeout While Workload Hea END Reboot All Worker Nodes While Workload Heavy Writing - Given Create deployment 0 with rwo volume - And Create deployment 1 with rwx volume - And Create deployment 2 with rwo and strict-local volume - And Create statefulset 0 with rwo volume - And Create statefulset 1 with rwx volume - And Create statefulset 2 with rwo and strict-local volume + Given Create persistentvolumeclaim 0 using RWO volume + And Create persistentvolumeclaim 1 using RWX volume + And Create persistentvolumeclaim 2 using RWO volume with strict-local storageclass + + And Create deployment 0 with persistentvolumeclaim 0 + And Create deployment 1 with persistentvolumeclaim 1 + And Create deployment 2 with persistentvolumeclaim 2 + + And Create statefulset 0 using RWO volume + And Create statefulset 1 using RWX volume + And Create statefulset 2 using RWO volume with strict-local storageclass FOR ${i} IN RANGE ${LOOP_COUNT} - And Keep writing data to deployment 0 - And Keep writing data to deployment 1 - And Keep writing data to deployment 2 - And Keep writing data to statefulset 0 - And Keep writing data to statefulset 1 - And Keep writing data to statefulset 2 + And Keep writing data to pod of deployment 0 + And Keep writing data to pod of deployment 1 + And Keep writing data to pod of deployment 2 + And Keep writing data to pod of statefulset 0 + And Keep writing data to pod of statefulset 1 + And Keep writing data to pod of statefulset 2 When Restart all worker nodes And Wait for longhorn ready + And Wait for deployment 0 pods stable + And Wait for deployment 1 pods stable + And Wait for deployment 2 pods stable + And Wait for statefulset 0 pods stable + And Wait for statefulset 1 pods stable + And Wait for statefulset 2 pods stable Then Check deployment 0 works And Check deployment 1 works @@ -105,23 +141,34 @@ Reboot All Worker Nodes While Workload Heavy Writing END Power Off All Worker Nodes For More Than Pod Eviction Timeout While Workload Heavy Writing - Given Create deployment 0 with rwo volume - And Create deployment 1 with rwx volume - And Create deployment 2 with rwo and strict-local volume - And Create statefulset 0 with rwo volume - And Create statefulset 1 with rwx volume - And Create statefulset 2 with rwo and strict-local volume + Given Create persistentvolumeclaim 0 using RWO volume + And Create persistentvolumeclaim 1 using RWX volume + And Create persistentvolumeclaim 2 using RWO volume with strict-local storageclass + + And Create deployment 0 with persistentvolumeclaim 0 + And Create deployment 1 with persistentvolumeclaim 1 + And Create deployment 2 with persistentvolumeclaim 2 + + And Create statefulset 0 using RWO volume + And Create statefulset 1 using RWX volume + And Create statefulset 2 using RWO volume with strict-local storageclass FOR ${i} IN RANGE ${LOOP_COUNT} - And Keep writing data to deployment 0 - And Keep writing data to deployment 1 - And Keep writing data to deployment 2 - And Keep writing data to statefulset 0 - And Keep writing data to statefulset 1 - And Keep writing data to statefulset 2 + And Keep writing data to pod of deployment 0 + And Keep writing data to pod of deployment 1 + And Keep writing data to pod of deployment 2 + And Keep writing data to pod of statefulset 0 + And Keep writing data to pod of statefulset 1 + And Keep writing data to pod of statefulset 2 When Power off all worker nodes for 6 mins And Wait for longhorn ready + And Wait for deployment 0 pods stable + And Wait for deployment 1 pods stable + And Wait for deployment 2 pods stable + And Wait for statefulset 0 pods stable + And Wait for statefulset 1 pods stable + And Wait for statefulset 2 pods stable Then Check deployment 0 works And Check deployment 1 works @@ -132,26 +179,26 @@ Power Off All Worker Nodes For More Than Pod Eviction Timeout While Workload Hea END Reboot Volume Node While Workload Heavy Writing - Given Create statefulset 0 with ${VOLUME_TYPE} volume + Given Create statefulset 0 using ${VOLUME_TYPE} volume FOR ${i} IN RANGE ${LOOP_COUNT} - And Keep writing data to statefulset 0 + And Keep writing data to pod of statefulset 0 When Reboot volume node of statefulset 0 And Wait for volume of statefulset 0 healthy - And Wait for statefulset 0 stable + And Wait for statefulset 0 pods stable Then Check statefulset 0 works END Power Off Volume Node For More Than Pod Eviction Timeout While Workload Heavy Writing - Given Create statefulset 0 with rwo volume + Given Create statefulset 0 using RWO volume FOR ${i} IN RANGE ${LOOP_COUNT} - And Keep writing data to statefulset 0 + And Keep writing data to pod of statefulset 0 - When Power off volume node of statefulset 0 for 6 mins + When Power off volume node of statefulset 0 for 6 minutes And Wait for volume of statefulset 0 healthy - And Wait for statefulset 0 stable + And Wait for statefulset 0 pods stable Then Check statefulset 0 works END @@ -159,6 +206,8 @@ Power Off Volume Node For More Than Pod Eviction Timeout While Workload Heavy Wr Reboot Volume Node While Heavy Writing And Recurring Jobs Exist Given Create volume 0 with 2 GB and 1 replicas And Create volume 1 with 2 GB and 3 replicas + And Attach volume 0 + And Attach volume 1 And Keep writing data to volume 0 And Keep Writing data to volume 1 And Create snapshot and backup recurringjob for volume 0 @@ -176,6 +225,8 @@ Reboot Volume Node While Heavy Writing And Recurring Jobs Exist Reboot Replica Node While Heavy Writing And Recurring Jobs Exist Given Create volume 0 with 2 GB and 1 replicas And Create volume 1 with 2 GB and 3 replicas + And Attach volume 0 + And Attach volume 1 And Keep Writing data to volume 0 And Keep Writing data to volume 1 And Create snapshot and backup recurringjob for volume 0 diff --git a/e2e/tests/replica_rebuilding.robot b/e2e/tests/replica_rebuilding.robot index 86375088b9..56d764f46c 100644 --- a/e2e/tests/replica_rebuilding.robot +++ b/e2e/tests/replica_rebuilding.robot @@ -1,7 +1,9 @@ *** Settings *** Documentation Negative Test Cases + Resource ../keywords/common.resource -Resource ../keywords/node.resource +Resource ../keywords/host.resource +Resource ../keywords/persistentvolumeclaim.resource Resource ../keywords/volume.resource Test Setup Set test environment @@ -14,38 +16,45 @@ ${RETRY_INTERVAL} 1 *** Test Cases *** Delete Replica While Replica Rebuilding - Given Create a volume with 2 GB and 3 replicas - And Write data to the volume + Given Create volume 0 with 2 GB and 3 replicas + And Attach volume 0 + And Write data to volume 0 FOR ${i} IN RANGE ${LOOP_COUNT} - When Delete replica 0 to trigger replica 0 rebuilding - And During replica 0 rebuilding, delete replica 1 - And Wait until replica 0 rebuilt, delete replica 2 - - Then Check data is intact - And Wait until all replicas rebuilt + When Delete volume 0 replica on volume node + And Wait until volume 0 replica rebuilding started on volume node + And Delete volume 0 replica on replica node + And Wait until volume 0 replica rebuilding completed on volume node + And Delete volume 0 replica on test pod node + + Then Check volume 0 data is intact + And Wait until volume 0 replicas rebuilding completed END Reboot Volume Node While Replica Rebuilding - Given Create a volume with 5 GB and 3 replicas - And Write data to the volume + Given Create volume 0 with 5 GB and 3 replicas + And Attach volume 0 + And Write data to volume 0 FOR ${i} IN RANGE ${LOOP_COUNT} - When Delete replica on volume node to trigger replica rebuilding - And During replica rebuilding, reboot volume node + When Delete volume 0 replica on volume node + And Wait until volume 0 replica rebuilding started on volume node + And Reboot volume 0 volume node - Then Wait until replica on volume node rebuilt - And Check data is intact + Then Wait until volume 0 replica rebuilding completed on volume node + And Check volume 0 data is intact END Reboot Replica Node While Replica Rebuilding - Given Create a volume with 5 GB and 3 replicas - And Write data to the volume + Given Create volume 0 with 5 GB and 3 replicas + And Attach volume 0 + And Write data to volume 0 FOR ${i} IN RANGE ${LOOP_COUNT} - When Delete replica on replica node to trigger replica rebuilding - And During replica rebuilding, reboot replica node + When Delete volume 0 replica on replica node + And Wait until volume 0 replica rebuilding started on replica node + And Reboot volume 0 replica node - Then Wait until replica on replica node rebuilt - And Check data is intact + Then Wait until volume 0 replica rebuilding completed on replica node + And Check volume 0 data is intact END diff --git a/e2e/tests/stress_cpu.robot b/e2e/tests/stress_cpu.robot index c5a3859c98..293d459c90 100644 --- a/e2e/tests/stress_cpu.robot +++ b/e2e/tests/stress_cpu.robot @@ -1,7 +1,8 @@ *** Settings *** Documentation Negative Test Cases + Resource ../keywords/common.resource -Resource ../keywords/node.resource +Resource ../keywords/persistentvolumeclaim.resource Resource ../keywords/statefulset.resource Resource ../keywords/stress.resource Resource ../keywords/volume.resource @@ -14,65 +15,60 @@ Test Teardown Cleanup test resources ${LOOP_COUNT} 1 ${RETRY_COUNT} 300 ${RETRY_INTERVAL} 1 - *** Test Cases *** Stress Volume Node CPU When Replica Is Rebuilding - Given Create a volume with 5 GB and 3 replicas - And Write data to the volume + Given Create volume 0 with 5 GB and 3 replicas + And Attach volume 0 + And Write data to volume 0 FOR ${i} IN RANGE ${LOOP_COUNT} - When Delete replica on volume node to trigger replica rebuilding - And Stress the CPU of all volume nodes + When Delete volume 0 replica on volume node + And Wait until volume 0 replica rebuilding started on volume node + And Stress CPU of node with volume 0 - Then Wait until replica on volume node rebuilt - And Check data is intact + Then Wait until volume 0 replica rebuilding completed on volume node + And Check volume 0 data is intact END Stress Volume Node CPU When Volume Is Detaching and Attaching - Given Create a volume with 5 GB and 3 replicas - And Write data to the volume + Given Create volume 0 with 5 GB and 3 replicas + And Attach volume 0 + And Write data to volume 0 FOR ${i} IN RANGE ${LOOP_COUNT} - When Stress the CPU of all volume nodes + When Stress CPU of node with volume 0 + And Detach volume 0 + And Attach volume 0 - And Detach volume from node - And Attach volume to node - - And Check data is intact + Then Check volume 0 data is intact END Stress Volume Node CPU When Volume Is Online Expanding - @{data_checksum_list} = Create List - Set Test Variable ${data_checksum_list} - - Given Create statefulset 0 with rwo volume - And Write 1024 MB data to statefulset 0 + Given Create statefulset 0 using RWO volume + And Write 1024 MB data to file 0.txt in statefulset 0 FOR ${i} IN RANGE ${LOOP_COUNT} - And Stress the CPU of all volume nodes + And Stress CPU of volume nodes + When Expand statefulset 0 volume by 100 MiB Then Wait for statefulset 0 volume size expanded - And Check statefulset 0 data is intact + And Check statefulset 0 data in file 0.txt is intact END Stress Volume Node CPU When Volume Is Offline Expanding - @{data_checksum_list} = Create List - Set Test Variable ${data_checksum_list} - - Given Create statefulset 0 with rwo volume - And Write 1024 MB data to statefulset 0 + Given Create statefulset 0 using RWO volume + And Write 1024 MB data to file 0.txt in statefulset 0 FOR ${i} IN RANGE ${LOOP_COUNT} And Scale down statefulset 0 to detach volume - And Stress the CPU of all worker nodes + And Stress CPU of all worker nodes When Expand statefulset 0 volume by 100 MiB Then Wait for statefulset 0 volume size expanded And Wait for statefulset 0 volume detached - And Scale up statefulset 0 to attach volume - And Check statefulset 0 data is intact + And Check statefulset 0 data in file 0.txt is intact END diff --git a/e2e/tests/stress_memory.robot b/e2e/tests/stress_memory.robot index 34a46f703e..d69329c393 100644 --- a/e2e/tests/stress_memory.robot +++ b/e2e/tests/stress_memory.robot @@ -1,7 +1,8 @@ *** Settings *** Documentation Negative Test Cases + Resource ../keywords/common.resource -Resource ../keywords/node.resource +Resource ../keywords/persistentvolumeclaim.resource Resource ../keywords/statefulset.resource Resource ../keywords/stress.resource Resource ../keywords/volume.resource @@ -18,61 +19,58 @@ ${RETRY_INTERVAL} 1 *** Test Cases *** Stress Volume Node Memory When Replica Is Rebuilding - Given Create a volume with 5 GB and 3 replicas - And Write data to the volume + Given Create volume 0 with 5 GB and 3 replicas + And Attach volume 0 + And Write data to volume 0 FOR ${i} IN RANGE ${LOOP_COUNT} - When Delete replica on volume node to trigger replica rebuilding - And Stress the memory of all volume nodes + When Delete volume 0 replica on volume node + And Wait until volume 0 replica rebuilding started on volume node + And Stress memory of node with volume 0 - Then Wait until replica on volume node rebuilt - And Check data is intact + Then Wait until volume 0 replica rebuilding completed on volume node + And Check volume 0 data is intact END Stress Volume Node Memory When Volume Is Detaching and Attaching - Given Create a volume with 5 GB and 3 replicas - And Write data to the volume + Given Create volume 0 with 5 GB and 3 replicas + And Attach volume 0 + And Write data to volume 0 FOR ${i} IN RANGE ${LOOP_COUNT} - When Stress the memory of all volume nodes + When Stress memory of node with volume 0 - And Detach volume from node - And Attach volume to node + And Detach volume 0 + And Attach volume 0 - And Check data is intact + Then Check volume 0 data is intact END Stress Volume Node Memory When Volume Is Online Expanding - @{data_checksum_list} = Create List - Set Test Variable ${data_checksum_list} - - Given Create statefulset 0 with rwo volume - And Write 1024 MB data to statefulset 0 + Given Create statefulset 0 using RWO volume + And Write 1024 MB data to file 0.txt in statefulset 0 FOR ${i} IN RANGE ${LOOP_COUNT} - And Stress the memory of all volume nodes + And Stress memory of volume nodes + When Expand statefulset 0 volume by 100 MiB Then Wait for statefulset 0 volume size expanded - And Check statefulset 0 data is intact + And Check statefulset 0 data in file 0.txt is intact END Stress Volume Node Memory When Volume Is Offline Expanding - @{data_checksum_list} = Create List - Set Test Variable ${data_checksum_list} - - Given Create statefulset 0 with rwo volume - And Write 1024 MB data to statefulset 0 + Given Create statefulset 0 using RWO volume + And Write 1024 MB data to file 0.txt in statefulset 0 FOR ${i} IN RANGE ${LOOP_COUNT} And Scale down statefulset 0 to detach volume - And Stress the memory of all worker nodes + And Stress memory of all worker nodes When Expand statefulset 0 volume by 100 MiB Then Wait for statefulset 0 volume size expanded And Wait for statefulset 0 volume detached - And Scale up statefulset 0 to attach volume - And Check statefulset 0 data is intact + And Check statefulset 0 data in file 0.txt is intact END