test(robot): add more rebuilding test cases

ref: longhorn/longhorn#9240

Signed-off-by: Chris <[email protected]>

e2e/keywords/deployment.resource

@@ -5,6 +5,7 @@ Library    Collections
 Library    ../libs/keywords/common_keywords.py
 Library    ../libs/keywords/deployment_keywords.py
 Library    ../libs/keywords/node_keywords.py
+Library    ../libs/keywords/workload_keywords.py
 
 *** Keywords ***
 Create deployment ${deployment_id} with persistentvolumeclaim ${claim_id}
@@ -39,3 +40,13 @@ Check deployment ${deployment_id} works
 Wait for deployment ${deployment_id} pods stable
     ${deployment_name} =    generate_name_with_suffix    deployment    ${deployment_id}
     wait_for_workload_pods_stable   ${deployment_name}
+
+Get deployment ${deployment_id} pod name
+    ${deployment_name} =   generate_name_with_suffix    deployment    ${deployment_id}
+    ${pod_name} =    get_workload_pod_name    ${deployment_name}
+    Set Test Variable    ${pod_name}
+
+Check deployment ${deployment_id} pod not restarted
+    ${deployment_name} =   generate_name_with_suffix    deployment    ${deployment_id}
+    ${current_pod_name} =    get_workload_pod_name    ${deployment_name}
+    Should Be Equal    ${pod_name}    ${current_pod_name}

e2e/tests/replica_rebuilding.robot

@@ -6,6 +6,10 @@ Test Tags    negative
 Resource    ../keywords/common.resource
 Resource    ../keywords/host.resource
 Resource    ../keywords/volume.resource
+Resource    ../keywords/storageclass.resource
+Resource    ../keywords/persistentvolumeclaim.resource
+Resource    ../keywords/deployment.resource
+Resource    ../keywords/workload.resource
 
 Test Setup    Set test environment
 Test Teardown    Cleanup test resources
@@ -66,3 +70,53 @@ Reboot Replica Node While Replica Rebuilding
         And Wait for volume 0 healthy
         And Check volume 0 data is intact
     END
+
+Delete replicas one by one after the volume is healthy
+    Given Create storageclass longhorn-test with    dataEngine=${DATA_ENGINE}
+    And Create persistentvolumeclaim 0 using RWO volume with longhorn-test storageclass
+    And Create deployment 0 with persistentvolumeclaim 0
+    And Wait for volume of deployment 0 attached
+    And Write 2048 MB data to file data.txt in deployment 0
+
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        When Delete replica of deployment 0 volume on node 0
+        And Wait for volume of deployment 0 attached and degraded
+        And Wait for volume of deployment 0 healthy
+
+        Then Delete replica of deployment 0 volume on node 1
+        And Wait for volume of deployment 0 attached and degraded
+        And Wait for volume of deployment 0 healthy
+
+        Then Delete replica of deployment 0 volume on node 2
+        And Wait for volume of deployment 0 attached and degraded
+        And Wait for volume of deployment 0 healthy
+
+        And Wait for deployment 0 pods stable
+        Then Check deployment 0 data in file data.txt is intact
+    END
+
+Delete replicas one by one regardless of the volume health
+    [Documentation]    Currently v2 data engine have a chance to hit
+    ...                https://github.com/longhorn/longhorn/issues/9216 and will be fixed
+    ...                in v1.9.0
+    Given Create storageclass longhorn-test with    dataEngine=${DATA_ENGINE}
+    And Create persistentvolumeclaim 0 using RWO volume with longhorn-test storageclass
+    And Create deployment 0 with persistentvolumeclaim 0
+    And Wait for volume of deployment 0 attached
+    And Get deployment 0 pod name
+    And Write 2048 MB data to file data.txt in deployment 0
+
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        When Delete replica of deployment 0 volume on node 0
+        And And Wait until volume of deployment 0 replica rebuilding started on node 0
+
+        Then Delete replica of deployment 0 volume on node 1
+        And And Wait until volume of deployment 0 replica rebuilding started on node 1
+
+        Then Delete replica of deployment 0 volume on node 2
+        And And Wait until volume of deployment 0 replica rebuilding started on node 2
+    END
+
+    And Wait for deployment 0 pods stable
+    And Check deployment 0 pod not restarted
+    Then Check deployment 0 data in file data.txt is intact