From 8af8d7b22b3eb6de3b008c4b54162938cc7f5393 Mon Sep 17 00:00:00 2001 From: Yang Chiu Date: Thu, 5 Dec 2024 11:34:24 +0800 Subject: [PATCH] test(robot): migrate test case test_migration_rollback Signed-off-by: Yang Chiu --- e2e/keywords/volume.resource | 13 ++++++--- e2e/libs/engine/crd.py | 21 +++++++-------- e2e/libs/keywords/volume_keywords.py | 14 ++++++---- e2e/libs/volume/base.py | 8 ++++-- e2e/libs/volume/crd.py | 32 ++++++++++++++++------- e2e/libs/volume/rest.py | 7 +++-- e2e/libs/volume/volume.py | 11 +++++--- e2e/tests/regression/test_migration.robot | 29 ++++++++++++++++++-- 8 files changed, 96 insertions(+), 39 deletions(-) diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource index 6dfe25423a..921de837c3 100644 --- a/e2e/keywords/volume.resource +++ b/e2e/keywords/volume.resource @@ -167,14 +167,19 @@ Check all replicas of volume ${volume_id} kept in error Sleep ${RETRY_INTERVAL} END -Wait for volume ${volume_id} migration ready +Wait for volume ${volume_id} migration to be ready ${volume_name} = generate_name_with_suffix volume ${volume_id} - wait_for_volume_migration_ready ${volume_name} + wait_for_volume_migration_to_be_ready ${volume_name} -Wait for volume ${volume_id} migrated to node ${node_id} +Wait for volume ${volume_id} to migrate to node ${node_id} ${volume_name} = generate_name_with_suffix volume ${volume_id} ${node_name} = get_node_by_index ${node_id} - wait_for_volume_migration_completed ${volume_name} ${node_name} + wait_for_volume_migration_complete ${volume_name} ${node_name} + +Wait for volume ${volume_id} to stay on node ${node_id} + ${volume_name} = generate_name_with_suffix volume ${volume_id} + ${node_name} = get_node_by_index ${node_id} + wait_for_volume_migration_to_rollback ${volume_name} ${node_name} Wait for volume ${volume_id} restoration from backup ${backup_id} completed ${volume_name} = generate_name_with_suffix volume ${volume_id} diff --git a/e2e/libs/engine/crd.py b/e2e/libs/engine/crd.py index a38f5f0b45..fdaa94f627 100644 --- a/e2e/libs/engine/crd.py +++ b/e2e/libs/engine/crd.py @@ -3,6 +3,7 @@ from kubernetes import client from engine.base import Base +from utility.utility import logging class CRD(Base): @@ -10,14 +11,13 @@ def __init__(self): self.obj_api = client.CustomObjectsApi() def get_engines(self, volume_name, node_name=None): - if volume_name == "" or node_name == "": - logging.info("getting all engines") + if not node_name: + logging(f"Getting all engines of {volume_name}") else: - logging.info( - f"getting the volume {volume_name} on node {node_name} engine") + logging(f"Getting engine of volume {volume_name} on node {node_name}") label_selector = [] - if volume_name != "": + if volume_name: label_selector.append(f"longhornvolume={volume_name}") if node_name: label_selector.append(f"longhornnode={node_name}") @@ -31,20 +31,19 @@ def get_engines(self, volume_name, node_name=None): ) if api_response == "" or api_response is None: - raise Exception(f"failed to get the volume {volume_name} engine") + raise Exception(f"failed to get volume {volume_name} engine") engines = api_response["items"] if len(engines) == 0: - logging.warning(f"cannot get the volume {volume_name} engines") + logging(f"Cannot get volume {volume_name} engines") return engines def delete_engine(self, volume_name, node_name): if volume_name == "" or node_name == "": - logging.info("deleting all engines") + logging("deleting all engines") else: - logging.info( - f"delete the volume {volume_name} on node {node_name} engine") + logging(f"delete the volume {volume_name} on node {node_name} engine") for engine in self.get_engine(volume_name, node_name): engine_name = engine['metadata']['name'] @@ -55,7 +54,7 @@ def delete_engine(self, volume_name, node_name): plural="engines", name=engine_name ) - logging.info("finished delete engines") + logging("finished delete engines") def validate_engine_setting(self, volume_name, setting_name, value): engines = self.get_engines(volume_name) diff --git a/e2e/libs/keywords/volume_keywords.py b/e2e/libs/keywords/volume_keywords.py index 829c73e6c1..369be72668 100644 --- a/e2e/libs/keywords/volume_keywords.py +++ b/e2e/libs/keywords/volume_keywords.py @@ -271,13 +271,17 @@ def wait_for_volume_faulted(self, volume_name): logging(f'Waiting for volume {volume_name} to be in faulted') self.volume.wait_for_volume_faulted(volume_name) - def wait_for_volume_migration_ready(self, volume_name): + def wait_for_volume_migration_to_be_ready(self, volume_name): logging(f'Waiting for volume {volume_name} migration to be ready') - self.volume.wait_for_volume_migration_ready(volume_name) + self.volume.wait_for_volume_migration_to_be_ready(volume_name) - def wait_for_volume_migration_completed(self, volume_name, node_name): - logging(f'Waiting for volume {volume_name} migration to node {node_name} completed') - self.volume.wait_for_volume_migration_completed(volume_name, node_name) + def wait_for_volume_migration_complete(self, volume_name, node_name): + logging(f'Waiting for volume {volume_name} migration to node {node_name} complete') + self.volume.wait_for_volume_migration_complete(volume_name, node_name) + + def wait_for_volume_migration_to_rollback(self, volume_name, node_name): + logging(f'Waiting for volume {volume_name} migration to rollback to node {node_name}') + self.volume.wait_for_volume_migration_to_rollback(volume_name, node_name) def wait_for_volume_restoration_completed(self, volume_name, backup_name): logging(f'Waiting for volume {volume_name} restoration from {backup_name} completed') diff --git a/e2e/libs/volume/base.py b/e2e/libs/volume/base.py index c58489e866..57eb51f462 100644 --- a/e2e/libs/volume/base.py +++ b/e2e/libs/volume/base.py @@ -81,11 +81,15 @@ def wait_for_volume_state(self, volume_name, desired_state): return NotImplemented @abstractmethod - def wait_for_volume_migration_ready(self, volume_name): + def wait_for_volume_migration_to_be_ready(self, volume_name): return NotImplemented @abstractmethod - def wait_for_volume_migration_completed(self, volume_name, node_name): + def wait_for_volume_migration_complete(self, volume_name, node_name): + return NotImplemented + + @abstractmethod + def wait_for_volume_migration_to_rollback(self, volume_name, node_name): return NotImplemented @abstractmethod diff --git a/e2e/libs/volume/crd.py b/e2e/libs/volume/crd.py index 84acc0fab9..80078c081b 100644 --- a/e2e/libs/volume/crd.py +++ b/e2e/libs/volume/crd.py @@ -301,10 +301,10 @@ def wait_for_volume_robustness_not(self, volume_name, not_desired_state): time.sleep(self.retry_interval) assert volume["status"]["robustness"] != not_desired_state - def wait_for_volume_migration_ready(self, volume_name): + def wait_for_volume_migration_to_be_ready(self, volume_name): ready = False for i in range(self.retry_count): - logging(f"Waiting for volume {volume_name} migration ready ({i}) ...") + logging(f"Waiting for volume {volume_name} migration to be ready ({i}) ...") try: engines = self.engine.get_engines(volume_name) ready = len(engines) == 2 @@ -317,19 +317,33 @@ def wait_for_volume_migration_ready(self, volume_name): time.sleep(self.retry_interval) assert ready - def wait_for_volume_migration_completed(self, volume_name, node_name): - completed = False + def wait_for_volume_migration_complete(self, volume_name, node_name): + complete = False for i in range(self.retry_count): - logging(f"Waiting for volume {volume_name} migration to node {node_name} completed ({i}) ...") + logging(f"Waiting for volume {volume_name} migration to node {node_name} complete ({i}) ...") try: - engines = self.engine.get_engines(volume_name, node_name) - completed = len(engines) == 1 and engines[0]['status']['endpoint'] - if completed: + engines = self.engine.get_engines(volume_name) + complete = len(engines) == 1 and engines[0]['status']['endpoint'] and engines[0]['status']['ownerID'] == node_name + if complete: break except Exception as e: logging(f"Getting volume {volume_name} engines error: {e}") time.sleep(self.retry_interval) - assert completed + assert complete + + def wait_for_volume_migration_to_rollback(self, volume_name, node_name): + rollback = False + for i in range(self.retry_count): + logging(f"Waiting for volume {volume_name} migration to rollback to node {node_name} ({i}) ...") + try: + engines = self.engine.get_engines(volume_name) + rollback = len(engines) == 1 and engines[0]['status']['endpoint'] and engines[0]['status']['ownerID'] == node_name + if rollback: + break + except Exception as e: + logging(f"Getting volume {volume_name} engines error: {e}") + time.sleep(self.retry_interval) + assert rollback def wait_for_volume_restoration_completed(self, volume_name, backup_name): completed = False diff --git a/e2e/libs/volume/rest.py b/e2e/libs/volume/rest.py index b048f34db2..65d25851ba 100644 --- a/e2e/libs/volume/rest.py +++ b/e2e/libs/volume/rest.py @@ -71,10 +71,13 @@ def wait_for_volume_state(self, volume_name, desired_state): def wait_for_restore_required_status(self, volume_name, restore_required_state): return NotImplemented - def wait_for_volume_migration_ready(self, volume_name): + def wait_for_volume_migration_to_be_ready(self, volume_name): return NotImplemented - def wait_for_volume_migration_completed(self, volume_name, node_name): + def wait_for_volume_migration_complete(self, volume_name, node_name): + return NotImplemented + + def wait_for_volume_migration_to_rollback(self, volume_name, node_name): return NotImplemented def wait_for_volume_restoration_completed(self, volume_name): diff --git a/e2e/libs/volume/volume.py b/e2e/libs/volume/volume.py index 578480c22c..8ba8eec4a8 100644 --- a/e2e/libs/volume/volume.py +++ b/e2e/libs/volume/volume.py @@ -69,11 +69,14 @@ def wait_for_volume_healthy(self, volume_name): self.volume.wait_for_volume_state(volume_name, "attached") self.volume.wait_for_volume_robustness(volume_name, "healthy") - def wait_for_volume_migration_ready(self, volume_name): - self.volume.wait_for_volume_migration_ready(volume_name) + def wait_for_volume_migration_to_be_ready(self, volume_name): + self.volume.wait_for_volume_migration_to_be_ready(volume_name) - def wait_for_volume_migration_completed(self, volume_name, node_name): - self.volume.wait_for_volume_migration_completed(volume_name, node_name) + def wait_for_volume_migration_complete(self, volume_name, node_name): + self.volume.wait_for_volume_migration_complete(volume_name, node_name) + + def wait_for_volume_migration_to_rollback(self, volume_name, node_name): + self.volume.wait_for_volume_migration_to_rollback(volume_name, node_name) def wait_for_volume_restoration_completed(self, volume_name, backup_name): self.volume.wait_for_volume_restoration_completed(volume_name, backup_name) diff --git a/e2e/tests/regression/test_migration.robot b/e2e/tests/regression/test_migration.robot index c61cc892fd..c9ce064ebf 100644 --- a/e2e/tests/regression/test_migration.robot +++ b/e2e/tests/regression/test_migration.robot @@ -39,8 +39,33 @@ Test Migration Confirm And Wait for volume 0 healthy And Write data to volume 0 And Attach volume 0 to node 1 - Then Wait for volume 0 migration ready + Then Wait for volume 0 migration to be ready And Detach volume 0 from node 0 - And Wait for volume 0 migrated to node 1 + And Wait for volume 0 to migrate to node 1 + And Wait for volume 0 healthy + And Check volume 0 data is intact + +Test Migration Rollback + [Tags] coretest migration + [Documentation] Test that a migratable RWX volume can be rolled back to initial node. + ... + ... 1. Create a new RWX migratable volume. + ... 2. Attach to test node to write some test data on it. + ... 3. Detach from test node. + ... 4. Get set of nodes excluding the test node + ... 5. Attach volume to node 1 (initial node) + ... 6. Attach volume to node 2 (migration target) + ... 7. Wait for migration ready (engine running on node 2) + ... 8. Detach volume from node 2 + ... 9. Observe volume stayed on node 1 (single active engine) + ... 10. Validate initially written test data + Given Create volume 0 with migratable=True accessMode=RWX dataEngine=${DATA_ENGINE} + When Attach volume 0 to node 0 + And Wait for volume 0 healthy + And Write data to volume 0 + And Attach volume 0 to node 1 + Then Wait for volume 0 migration to be ready + And Detach volume 0 from node 1 + And Wait for volume 0 to stay on node 0 And Wait for volume 0 healthy And Check volume 0 data is intact