test(robot): migrate test case test_migration_rollback

Signed-off-by: Yang Chiu <[email protected]>
longhorn · Dec 6, 2024 · 8af8d7b · 8af8d7b
1 parent e269a61
commit 8af8d7b
Show file tree

Hide file tree

Showing 8 changed files with 96 additions and 39 deletions.
diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource
@@ -167,14 +167,19 @@ Check all replicas of volume ${volume_id} kept in error
         Sleep    ${RETRY_INTERVAL}
     END
 
-Wait for volume ${volume_id} migration ready
+Wait for volume ${volume_id} migration to be ready
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
-    wait_for_volume_migration_ready    ${volume_name}
+    wait_for_volume_migration_to_be_ready    ${volume_name}
 
-Wait for volume ${volume_id} migrated to node ${node_id}
+Wait for volume ${volume_id} to migrate to node ${node_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     ${node_name} =    get_node_by_index    ${node_id}
-    wait_for_volume_migration_completed    ${volume_name}    ${node_name}
+    wait_for_volume_migration_complete    ${volume_name}    ${node_name}
+
+Wait for volume ${volume_id} to stay on node ${node_id}
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${node_name} =    get_node_by_index    ${node_id}
+    wait_for_volume_migration_to_rollback    ${volume_name}    ${node_name}
 
 Wait for volume ${volume_id} restoration from backup ${backup_id} completed
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}

diff --git a/e2e/libs/engine/crd.py b/e2e/libs/engine/crd.py
@@ -3,21 +3,21 @@
 from kubernetes import client
 
 from engine.base import Base
+from utility.utility import logging
 
 
 class CRD(Base):
     def __init__(self):
         self.obj_api = client.CustomObjectsApi()
 
     def get_engines(self, volume_name, node_name=None):
-        if volume_name == "" or node_name == "":
-            logging.info("getting all engines")
+        if not node_name:
+            logging(f"Getting all engines of {volume_name}")
         else:
-            logging.info(
-                f"getting the volume {volume_name} on node {node_name} engine")
+            logging(f"Getting engine of volume {volume_name} on node {node_name}")
 
         label_selector = []
-        if volume_name != "":
+        if volume_name:
             label_selector.append(f"longhornvolume={volume_name}")
         if node_name:
             label_selector.append(f"longhornnode={node_name}")
@@ -31,20 +31,19 @@ def get_engines(self, volume_name, node_name=None):
         )
 
         if api_response == "" or api_response is None:
-            raise Exception(f"failed to get the volume {volume_name} engine")
+            raise Exception(f"failed to get volume {volume_name} engine")
 
         engines = api_response["items"]
         if len(engines) == 0:
-            logging.warning(f"cannot get the volume {volume_name} engines")
+            logging(f"Cannot get volume {volume_name} engines")
 
         return engines
 
     def delete_engine(self, volume_name, node_name):
         if volume_name == "" or node_name == "":
-            logging.info("deleting all engines")
+            logging("deleting all engines")
         else:
-            logging.info(
-                f"delete the volume {volume_name} on node {node_name} engine")
+            logging(f"delete the volume {volume_name} on node {node_name} engine")
 
         for engine in self.get_engine(volume_name, node_name):
             engine_name = engine['metadata']['name']
@@ -55,7 +54,7 @@ def delete_engine(self, volume_name, node_name):
                 plural="engines",
                 name=engine_name
             )
-        logging.info("finished delete engines")
+        logging("finished delete engines")
 
     def validate_engine_setting(self, volume_name, setting_name, value):
         engines = self.get_engines(volume_name)

diff --git a/e2e/libs/keywords/volume_keywords.py b/e2e/libs/keywords/volume_keywords.py
@@ -271,13 +271,17 @@ def wait_for_volume_faulted(self, volume_name):
         logging(f'Waiting for volume {volume_name} to be in faulted')
         self.volume.wait_for_volume_faulted(volume_name)
 
-    def wait_for_volume_migration_ready(self, volume_name):
+    def wait_for_volume_migration_to_be_ready(self, volume_name):
         logging(f'Waiting for volume {volume_name} migration to be ready')
-        self.volume.wait_for_volume_migration_ready(volume_name)
+        self.volume.wait_for_volume_migration_to_be_ready(volume_name)
 
-    def wait_for_volume_migration_completed(self, volume_name, node_name):
-        logging(f'Waiting for volume {volume_name} migration to node {node_name} completed')
-        self.volume.wait_for_volume_migration_completed(volume_name, node_name)
+    def wait_for_volume_migration_complete(self, volume_name, node_name):
+        logging(f'Waiting for volume {volume_name} migration to node {node_name} complete')
+        self.volume.wait_for_volume_migration_complete(volume_name, node_name)
+
+    def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
+        logging(f'Waiting for volume {volume_name} migration to rollback to node {node_name}')
+        self.volume.wait_for_volume_migration_to_rollback(volume_name, node_name)
 
     def wait_for_volume_restoration_completed(self, volume_name, backup_name):
         logging(f'Waiting for volume {volume_name} restoration from {backup_name} completed')

diff --git a/e2e/libs/volume/base.py b/e2e/libs/volume/base.py
@@ -81,11 +81,15 @@ def wait_for_volume_state(self, volume_name, desired_state):
         return NotImplemented
 
     @abstractmethod
-    def wait_for_volume_migration_ready(self, volume_name):
+    def wait_for_volume_migration_to_be_ready(self, volume_name):
         return NotImplemented
 
     @abstractmethod
-    def wait_for_volume_migration_completed(self, volume_name, node_name):
+    def wait_for_volume_migration_complete(self, volume_name, node_name):
+        return NotImplemented
+
+    @abstractmethod
+    def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
         return NotImplemented
 
     @abstractmethod

diff --git a/e2e/libs/volume/crd.py b/e2e/libs/volume/crd.py
@@ -301,10 +301,10 @@ def wait_for_volume_robustness_not(self, volume_name, not_desired_state):
             time.sleep(self.retry_interval)
         assert volume["status"]["robustness"] != not_desired_state
 
-    def wait_for_volume_migration_ready(self, volume_name):
+    def wait_for_volume_migration_to_be_ready(self, volume_name):
         ready = False
         for i in range(self.retry_count):
-            logging(f"Waiting for volume {volume_name} migration ready ({i}) ...")
+            logging(f"Waiting for volume {volume_name} migration to be ready ({i}) ...")
             try:
                 engines = self.engine.get_engines(volume_name)
                 ready = len(engines) == 2
@@ -317,19 +317,33 @@ def wait_for_volume_migration_ready(self, volume_name):
             time.sleep(self.retry_interval)
         assert ready
 
-    def wait_for_volume_migration_completed(self, volume_name, node_name):
-        completed = False
+    def wait_for_volume_migration_complete(self, volume_name, node_name):
+        complete = False
         for i in range(self.retry_count):
-            logging(f"Waiting for volume {volume_name} migration to node {node_name} completed ({i}) ...")
+            logging(f"Waiting for volume {volume_name} migration to node {node_name} complete ({i}) ...")
             try:
-                engines = self.engine.get_engines(volume_name, node_name)
-                completed = len(engines) == 1 and engines[0]['status']['endpoint']
-                if completed:
+                engines = self.engine.get_engines(volume_name)
+                complete = len(engines) == 1 and engines[0]['status']['endpoint'] and engines[0]['status']['ownerID'] == node_name
+                if complete:
                     break
             except Exception as e:
                 logging(f"Getting volume {volume_name} engines error: {e}")
             time.sleep(self.retry_interval)
-        assert completed
+        assert complete
+
+    def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
+        rollback = False
+        for i in range(self.retry_count):
+            logging(f"Waiting for volume {volume_name} migration to rollback to node {node_name} ({i}) ...")
+            try:
+                engines = self.engine.get_engines(volume_name)
+                rollback = len(engines) == 1 and engines[0]['status']['endpoint'] and engines[0]['status']['ownerID'] == node_name
+                if rollback:
+                    break
+            except Exception as e:
+                logging(f"Getting volume {volume_name} engines error: {e}")
+            time.sleep(self.retry_interval)
+        assert rollback
 
     def wait_for_volume_restoration_completed(self, volume_name, backup_name):
         completed = False

diff --git a/e2e/libs/volume/rest.py b/e2e/libs/volume/rest.py
@@ -71,10 +71,13 @@ def wait_for_volume_state(self, volume_name, desired_state):
     def wait_for_restore_required_status(self, volume_name, restore_required_state):
         return NotImplemented
 
-    def wait_for_volume_migration_ready(self, volume_name):
+    def wait_for_volume_migration_to_be_ready(self, volume_name):
         return NotImplemented
 
-    def wait_for_volume_migration_completed(self, volume_name, node_name):
+    def wait_for_volume_migration_complete(self, volume_name, node_name):
+        return NotImplemented
+
+    def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
         return NotImplemented
 
     def wait_for_volume_restoration_completed(self, volume_name):

diff --git a/e2e/libs/volume/volume.py b/e2e/libs/volume/volume.py
@@ -69,11 +69,14 @@ def wait_for_volume_healthy(self, volume_name):
         self.volume.wait_for_volume_state(volume_name, "attached")
         self.volume.wait_for_volume_robustness(volume_name, "healthy")
 
-    def wait_for_volume_migration_ready(self, volume_name):
-        self.volume.wait_for_volume_migration_ready(volume_name)
+    def wait_for_volume_migration_to_be_ready(self, volume_name):
+        self.volume.wait_for_volume_migration_to_be_ready(volume_name)
 
-    def wait_for_volume_migration_completed(self, volume_name, node_name):
-        self.volume.wait_for_volume_migration_completed(volume_name, node_name)
+    def wait_for_volume_migration_complete(self, volume_name, node_name):
+        self.volume.wait_for_volume_migration_complete(volume_name, node_name)
+
+    def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
+        self.volume.wait_for_volume_migration_to_rollback(volume_name, node_name)
 
     def wait_for_volume_restoration_completed(self, volume_name, backup_name):
         self.volume.wait_for_volume_restoration_completed(volume_name, backup_name)

diff --git a/e2e/tests/regression/test_migration.robot b/e2e/tests/regression/test_migration.robot
@@ -39,8 +39,33 @@ Test Migration Confirm
     And Wait for volume 0 healthy
     And Write data to volume 0
     And Attach volume 0 to node 1
-    Then Wait for volume 0 migration ready
+    Then Wait for volume 0 migration to be ready
     And Detach volume 0 from node 0
-    And Wait for volume 0 migrated to node 1
+    And Wait for volume 0 to migrate to node 1
+    And Wait for volume 0 healthy
+    And Check volume 0 data is intact
+
+Test Migration Rollback
+    [Tags]    coretest    migration
+    [Documentation]    Test that a migratable RWX volume can be rolled back to initial node.
+    ...
+    ...                1. Create a new RWX migratable volume.
+    ...                2. Attach to test node to write some test data on it.
+    ...                3. Detach from test node.
+    ...                4. Get set of nodes excluding the test node
+    ...                5. Attach volume to node 1 (initial node)
+    ...                6. Attach volume to node 2 (migration target)
+    ...                7. Wait for migration ready (engine running on node 2)
+    ...                8. Detach volume from node 2
+    ...                9. Observe volume stayed on node 1 (single active engine)
+    ...                10. Validate initially written test data
+    Given Create volume 0 with    migratable=True    accessMode=RWX    dataEngine=${DATA_ENGINE}
+    When Attach volume 0 to node 0
+    And Wait for volume 0 healthy
+    And Write data to volume 0
+    And Attach volume 0 to node 1
+    Then Wait for volume 0 migration to be ready
+    And Detach volume 0 from node 1
+    And Wait for volume 0 to stay on node 0
     And Wait for volume 0 healthy
     And Check volume 0 data is intact