diff --git a/e2e/keywords/k8s.resource b/e2e/keywords/k8s.resource index 0b7c18a7a..a68157591 100644 --- a/e2e/keywords/k8s.resource +++ b/e2e/keywords/k8s.resource @@ -31,6 +31,9 @@ Delete volume of ${workload_kind} ${workload_id} replica node Add deleted node back reboot_node_by_name ${deleted_node} + ${host_provider}= Get Environment Variable HOST_PROVIDER + ${disk_path}= Set Variable If "${host_provider}" == "harvester" /dev/vdc /dev/xvdh + add_disk block-disk ${deleted_node} block ${disk_path} Force drain volume of ${workload_kind} ${workload_id} volume node ${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id} diff --git a/e2e/libs/node/node.py b/e2e/libs/node/node.py index a45f15554..fd2238a5e 100644 --- a/e2e/libs/node/node.py +++ b/e2e/libs/node/node.py @@ -62,10 +62,19 @@ def wait_for_disk_update(self, node_name, disk_num): assert len(node.disks) == disk_num, f"Waiting for node {node_name} disk updated to {disk_num} failed: {disks}" def add_disk(self, node_name, disk): - node = get_longhorn_client().by_id_node(node_name) - disks = node.disks - disks.update(disk) - self.update_disks(node_name, disks) + added = False + for i in range(self.retry_count): + try: + node = get_longhorn_client().by_id_node(node_name) + disks = node.disks + disks.update(disk) + self.update_disks(node_name, disks) + added = True + break + except Exception as e: + logging(f"Adding disk {disk} to node {node_name} error: {e}") + time.sleep(self.retry_interval) + assert added, f"Adding disk {disk} to node {node_name} failed" def reset_disks(self, node_name): node = get_longhorn_client().by_id_node(node_name)