From cf7e52dd4ea8c1d5bfc7644a70e71ae35c36e3ad Mon Sep 17 00:00:00 2001 From: Yang Chiu Date: Thu, 7 Nov 2024 15:35:25 +0800 Subject: [PATCH] test(robot): migrate test_replica_rebuild_per_volume_limit Signed-off-by: Yang Chiu --- e2e/keywords/volume.resource | 12 +++++ e2e/libs/keywords/volume_keywords.py | 28 ++++++++-- e2e/libs/volume/crd.py | 19 ++++++- e2e/libs/volume/rest.py | 69 ++++++++++++++++++------- e2e/libs/volume/volume.py | 9 ++-- e2e/tests/regression/test_replica.robot | 51 ++++++++++++++++++ 6 files changed, 160 insertions(+), 28 deletions(-) create mode 100644 e2e/tests/regression/test_replica.robot diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource index ce595a3228..b322b369e0 100644 --- a/e2e/keywords/volume.resource +++ b/e2e/keywords/volume.resource @@ -99,6 +99,10 @@ Delete volume ${volume_id} replica on ${replica_locality} ${volume_name} = generate_name_with_suffix volume ${volume_id} delete_replica_on_node ${volume_name} ${replica_locality} +Delete ${count} replicas of volume ${volume_id} + ${volume_name} = generate_name_with_suffix volume ${volume_id} + delete_replicas ${volume_name} ${count} + Wait for volume ${volume_id} healthy ${volume_name} = generate_name_with_suffix volume ${volume_id} wait_for_volume_healthy ${volume_name} @@ -178,6 +182,10 @@ Wait until volume ${volume_id} replicas rebuilding completed ${volume_name} = generate_name_with_suffix volume ${volume_id} wait_for_replica_rebuilding_to_complete ${volume_name} +Monitor only one replica rebuilding will start at a time for volume ${volume_id} + ${volume_name} = generate_name_with_suffix volume ${volume_id} + wait_for_replica_rebuilding_to_complete ${volume_name} + Wait until volume ${volume_id} replica rebuilding stopped on ${replica_locality} ${volume_name} = generate_name_with_suffix volume ${volume_id} wait_for_replica_rebuilding_to_stop_on_node ${volume_name} ${replica_locality} @@ -192,6 +200,10 @@ Both volume ${volume_id_0} and volume ${volume_id_1} replica rebuilding on ${rep ${volume_name_1} = generate_name_with_suffix volume ${volume_id_1} both_replica_rebuildings_will_start_at_the_same_time_on_node ${volume_name_0} ${volume_name_1} ${replica_locality} +Only one replica rebuilding will start at a time for volume ${volume_id} + ${volume_name} = generate_name_with_suffix volume ${volume_id} + only_one_replica_rebuilding_will_start_at_a_time ${volume_name} + Crash volume ${volume_id} replica processes ${volume_name} = generate_name_with_suffix volume ${volume_id} crash_replica_processes ${volume_name} diff --git a/e2e/libs/keywords/volume_keywords.py b/e2e/libs/keywords/volume_keywords.py index 69f096b0fb..2ef9c77ac0 100644 --- a/e2e/libs/keywords/volume_keywords.py +++ b/e2e/libs/keywords/volume_keywords.py @@ -128,6 +128,13 @@ def delete_replica_on_nodes(self, volume_name, replica_locality): logging(f"Deleting volume {volume_name}'s replica on node {node_id}") self.volume.delete_replica(volume_name, node_id) + def delete_replicas(self, volume_name, count): + replica_list = self.replica.get(volume_name, node_name="") + replica_names = [replica['metadata']['name'] for replica in replica_list] + for i in range(int(count)): + logging(f"Deleting volume {volume_name} replica volume {replica_names[i]}") + self.volume.delete_replica_by_name(volume_name, replica_names[i]) + def set_annotation(self, volume_name, annotation_key, annotation_value): self.volume.set_annotation(volume_name, annotation_key, annotation_value) @@ -152,11 +159,7 @@ def wait_for_replica_rebuilding_to_complete_on_node(self, volume_name, replica_l self.volume.wait_for_replica_rebuilding_complete(volume_name, node_name) def wait_for_replica_rebuilding_to_complete(self, volume_name): - for node_name in self.node.list_node_names_by_role("worker"): - if self.node.is_node_schedulable(node_name) == "False": - continue - logging(f"Waiting for volume {volume_name}'s replica on node {node_name} rebuilding completed") - self.volume.wait_for_replica_rebuilding_complete(volume_name, node_name) + self.volume.wait_for_replica_rebuilding_complete(volume_name) async def only_one_replica_rebuilding_will_start_at_a_time_on_node(self, volume_name_0, volume_name_1, replica_locality): @@ -203,6 +206,21 @@ async def wait_for_both_replica_rebuildings(): assert self.volume.is_replica_rebuilding_in_progress(volume_name_0, node_id) and self.volume.is_replica_rebuilding_in_progress(volume_name_1, node_id), \ f"Expect {volume_name_0} and {volume_name_1} replica rebuilding at the same time" + async def only_one_replica_rebuilding_will_start_at_a_time(self, volume_name): + + async def wait_for_replica_rebuilding(): + tasks = [ + asyncio.create_task(self.volume.wait_for_replica_rebuilding_start(volume_name), name=volume_name), + ] + + done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + logging(f"Observed {done.pop().get_name()} started replica rebuilding") + + await wait_for_replica_rebuilding() + + assert self.volume.is_replica_rebuilding_in_progress(volume_name), \ + f"Expect {volume_name} replica rebuilding in progress" + def crash_replica_processes(self, volume_name): self.volume.crash_replica_processes(volume_name) diff --git a/e2e/libs/volume/crd.py b/e2e/libs/volume/crd.py index eea996d79b..72166495ae 100644 --- a/e2e/libs/volume/crd.py +++ b/e2e/libs/volume/crd.py @@ -420,6 +420,23 @@ def delete_replica(self, volume_name, node_name): name=replica_list['items'][0]['metadata']['name'] ) + def delete_replica_by_name(self, volume_name, replica_name): + replica = self.obj_api.get_namespaced_custom_object( + group="longhorn.io", + version="v1beta2", + namespace="longhorn-system", + plural="replicas", + name=replica_name + ) + logging(f"Deleting replica {replica['metadata']['name']}") + self.obj_api.delete_namespaced_custom_object( + group="longhorn.io", + version="v1beta2", + namespace="longhorn-system", + plural="replicas", + name=replica['metadata']['name'] + ) + def wait_for_replica_rebuilding_start(self, volume_name, node_name): return Rest().wait_for_replica_rebuilding_start(volume_name, node_name) @@ -432,7 +449,7 @@ def crash_replica_processes(self, volume_name): def crash_node_replica_process(self, volume_name, node_name): return Rest().crash_node_replica_process(volume_name, node_name) - def wait_for_replica_rebuilding_complete(self, volume_name, node_name): + def wait_for_replica_rebuilding_complete(self, volume_name, node_name=None): return Rest().wait_for_replica_rebuilding_complete(volume_name, node_name) def check_data_checksum(self, volume_name, data_id): diff --git a/e2e/libs/volume/rest.py b/e2e/libs/volume/rest.py index 502d8f64e7..93c83efdfb 100644 --- a/e2e/libs/volume/rest.py +++ b/e2e/libs/volume/rest.py @@ -118,22 +118,26 @@ def keep_writing_data(self, volume_name, size): def delete_replica(self, volume_name, node_name): return NotImplemented - async def wait_for_replica_rebuilding_start(self, volume_name, node_name): + async def wait_for_replica_rebuilding_start(self, volume_name, node_name=None): rebuilding_replica_name = None for i in range(self.retry_count): try: v = get_longhorn_client().by_id_volume(volume_name) logging(f"Trying to get volume {volume_name} rebuilding replicas ... ({i})") for replica in v.replicas: - if replica.hostId == node_name: + if node_name and replica.hostId == node_name and replica.mode == "WO": rebuilding_replica_name = replica.name break + elif replica.mode == "WO": + rebuilding_replica_name = replica.name + node_name = replica.hostId + break if rebuilding_replica_name: break except Exception as e: logging(f"Failed to get volume {volume_name} with error: {e}") await asyncio.sleep(self.retry_interval) - assert rebuilding_replica_name != None + assert rebuilding_replica_name != None, f"Waiting for replica rebuilding start for volume {volume_name} on node {node_name} failed: replicas = {v.replicas}" logging(f"Got volume {volume_name} rebuilding replica = {rebuilding_replica_name} on node {node_name}") started = False @@ -141,6 +145,15 @@ async def wait_for_replica_rebuilding_start(self, volume_name, node_name): try: v = get_longhorn_client().by_id_volume(volume_name) logging(f"Got volume {volume_name} rebuild status = {v.rebuildStatus}") + + # During monitoring replica rebuilding + # at the same time monitoring if there are unexpected concurrent replica rebuilding + rebuilding_count = 0 + for replica in v.replicas: + if replica.mode == "WO": + rebuilding_count +=1 + assert rebuilding_count <= 1, f"Unexpected concurrent replica rebuilding = {rebuilding_count}, replicas = {v.replicas}" + for status in v.rebuildStatus: for replica in v.replicas: if status.replica == replica.name and \ @@ -156,7 +169,7 @@ async def wait_for_replica_rebuilding_start(self, volume_name, node_name): await asyncio.sleep(self.retry_interval) assert started, f"wait for replica on node {node_name} rebuilding timeout: {v}" - def is_replica_rebuilding_in_progress(self, volume_name, node_name): + def is_replica_rebuilding_in_progress(self, volume_name, node_name=None): in_progress = False for i in range(self.retry_count): try: @@ -165,8 +178,9 @@ def is_replica_rebuilding_in_progress(self, volume_name, node_name): for status in v.rebuildStatus: for replica in v.replicas: if status.replica == replica.name and \ - replica.hostId == node_name and \ + (node_name is None or replica.hostId == node_name) and \ status.state == "in_progress": + node_name = replica.hostId if not node_name else node_name logging(f"Volume {volume_name} replica rebuilding {replica.name} in progress on {node_name}") in_progress = True break @@ -217,31 +231,48 @@ def get_replica_name_on_node(self, volume_name, node_name): if r.hostId == node_name: return r.name - def wait_for_replica_rebuilding_complete(self, volume_name, node_name): + def wait_for_replica_rebuilding_complete(self, volume_name, node_name=None): completed = False for i in range(self.retry_count): - logging(f"wait for {volume_name} replica rebuilding completed on {node_name} ... ({i})") + logging(f"wait for {volume_name} replica rebuilding completed on {'all nodes' if not node_name else node_name} ... ({i})") try: v = get_longhorn_client().by_id_volume(volume_name) + + # During monitoring replica rebuilding + # at the same time monitoring if there are unexpected concurrent replica rebuilding + rebuilding_count = 0 for replica in v.replicas: - # use replica.mode is RW or RO to check if this replica - # has been rebuilt or not - # because rebuildStatus is not reliable - # when the rebuild progress reaches 100% - # it will be removed from rebuildStatus immediately - # and you will just get an empty rebuildStatus [] - # so it's no way to distinguish "rebuilding not started yet" - # or "rebuilding already completed" using rebuildStatus - if replica.hostId == node_name and replica.mode == "RW": + if replica.mode == "WO": + rebuilding_count +=1 + assert rebuilding_count <= 1, f"Unexpected concurrent replica rebuilding = {rebuilding_count}, replicas = {v.replicas}" + + if node_name: + for replica in v.replicas: + # use replica.mode is RW or RO to check if this replica + # has been rebuilt or not + # because rebuildStatus is not reliable + # when the rebuild progress reaches 100% + # it will be removed from rebuildStatus immediately + # and you will just get an empty rebuildStatus [] + # so it's no way to distinguish "rebuilding not started yet" + # or "rebuilding already completed" using rebuildStatus + if replica.hostId == node_name and replica.mode == "RW": + completed = True + break + else: + rw_replica_count = 0 + for replica in v.replicas: + if replica.mode == "RW": + rw_replica_count += 1 + if rw_replica_count == v.numberOfReplicas: completed = True - break if completed: break except Exception as e: logging(f"Failed to get volume {volume_name} with error: {e}") time.sleep(self.retry_interval) - logging(f"Completed volume {volume_name} replica rebuilding on {node_name}") - assert completed, f"Expect volume {volume_name} replica rebuilding completed on {node_name}" + logging(f"Completed volume {volume_name} replica rebuilding on {'all nodes' if not node_name else node_name}") + assert completed, f"Expect volume {volume_name} replica rebuilding completed on {'all nodes' if not node_name else node_name}" def check_data_checksum(self, volume_name, data_id): return NotImplemented diff --git a/e2e/libs/volume/volume.py b/e2e/libs/volume/volume.py index bbfb2832bf..2617a30c2a 100644 --- a/e2e/libs/volume/volume.py +++ b/e2e/libs/volume/volume.py @@ -101,10 +101,13 @@ def keep_writing_data(self, volume_name): def delete_replica(self, volume_name, node_name): return self.volume.delete_replica(volume_name, node_name) - def wait_for_replica_rebuilding_start(self, volume_name, node_name): + def delete_replica_by_name(self, volume_name, replica_name): + return self.volume.delete_replica_by_name(volume_name, replica_name) + + def wait_for_replica_rebuilding_start(self, volume_name, node_name=None): return self.volume.wait_for_replica_rebuilding_start(volume_name, node_name) - def is_replica_rebuilding_in_progress(self, volume_name, node_name): + def is_replica_rebuilding_in_progress(self, volume_name, node_name=None): return self.volume.is_replica_rebuilding_in_progress(volume_name, node_name) def crash_replica_processes(self, volume_name): @@ -122,7 +125,7 @@ def wait_for_replica_running(self, volume_name, node_name): def get_replica_name_on_node(self, volume_name, node_name): return self.volume.get_replica_name_on_node(volume_name, node_name) - def wait_for_replica_rebuilding_complete(self, volume_name, node_name): + def wait_for_replica_rebuilding_complete(self, volume_name, node_name=None): return self.volume.wait_for_replica_rebuilding_complete(volume_name, node_name) def check_data_checksum(self, volume_name, data_id): diff --git a/e2e/tests/regression/test_replica.robot b/e2e/tests/regression/test_replica.robot new file mode 100644 index 0000000000..1555d3e5bd --- /dev/null +++ b/e2e/tests/regression/test_replica.robot @@ -0,0 +1,51 @@ +*** Settings *** +Documentation Replica Test Cases + +Test Tags regression + +Resource ../keywords/common.resource +Resource ../keywords/volume.resource +Resource ../keywords/setting.resource +Resource ../keywords/deployment.resource +Resource ../keywords/persistentvolumeclaim.resource +Resource ../keywords/workload.resource + +Test Setup Set test environment +Test Teardown Cleanup test resources + +*** Variables *** +${LOOP_COUNT} 1 +${RETRY_COUNT} 300 +${RETRY_INTERVAL} 1 +${DATA_ENGINE} v1 + +*** Test Cases *** +Test Replica Rebuilding Per Volume Limit + [Tags] coretest + [Documentation] Test the volume always only have one replica scheduled for rebuild + ... + ... 1. Set soft anti-affinity to `true`. + ... 2. Create a volume with 1 replica. + ... 3. Attach the volume and write a few hundreds MB data to it. + ... 4. Scale the volume replica to 5. + ... 5. Monitor the volume replica list to make sure there should be only 1 replica in WO state. + ... 6. Wait for the volume to complete rebuilding. Then remove 4 of the 5 replicas. + ... 7. Monitoring the volume replica list again. + ... 8. Once the rebuild was completed again, verify the data checksum. + Given Set setting replica-soft-anti-affinity to true + And Create volume 0 with numberOfReplicas=3 dataEngine=${DATA_ENGINE} + And Attach volume 0 + And Wait for volume 0 healthy + And Write data to volume 0 + + When Update volume 0 replica count to 5 + Then Only one replica rebuilding will start at a time for volume 0 + And Monitor only one replica rebuilding will start at a time for volume 0 + And Wait until volume 0 replicas rebuilding completed + + When Delete 4 replicas of volume 0 + Then Only one replica rebuilding will start at a time for volume 0 + And Monitor only one replica rebuilding will start at a time for volume 0 + And Wait until volume 0 replicas rebuilding completed + And Wait for volume 0 healthy + And Check volume 0 data is intact