Skip to content

Commit

Permalink
test(robot): migrate test_migration_confirm
Browse files Browse the repository at this point in the history 
Signed-off-by: Yang Chiu <[email protected]>
  • Loading branch information
@yangchiu @khushboo-rancher
yangchiu authored and khushboo-rancher committed May 1, 2024
1 parent a11e2e1 commit da29ef3
Show file tree
Hide file tree
Showing 13 changed files with 203 additions and 76 deletions.
24 changes: 22 additions & 2 deletions e2e/keywords/volume.resource
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ Create volume ${volume_id}
${volume_name} = generate_name_with_suffix volume ${volume_id}
create_volume ${volume_name} 2 3

Create volume ${volume_id} with frontend ${frontend_name}
Create volume ${volume_id} with
[Arguments] &{config}
${volume_name} = generate_name_with_suffix volume ${volume_id}
create_volume ${volume_name} 2 3 ${frontend_name}
create_volume ${volume_name} 2 3 &{config}

No volume created
${volumes} = list_volumes
Expand All @@ -29,6 +30,11 @@ Create volume ${volume_id} with ${size} GB and ${replica_count} replicas
Update volume ${volume_name} replica count to ${replica_count}
update_volume_spec ${volume_name} numberOfReplicas 3

Attach volume ${volume_id} to node ${node_id}
${volume_name} = generate_name_with_suffix volume ${volume_id}
${node_name} = get_node_by_index ${node_id}
attach_volume ${volume_name} ${node_name}

Attach volume ${volume_id}
${volume_name} = generate_name_with_suffix volume ${volume_id}
attach_volume ${volume_name}
Expand All @@ -37,6 +43,11 @@ Detach volume ${volume_id}
${volume_name} = generate_name_with_suffix volume ${volume_id}
detach_volume ${volume_name}

Detach volume ${volume_id} from node ${node_id}
${volume_name} = generate_name_with_suffix volume ${volume_id}
${node_name} = get_node_by_index ${node_id}
detach_volume ${volume_name} ${node_name}

Delete volume ${volume_id}
${volume_name} = generate_name_with_suffix volume ${volume_id}
delete_volume ${volume_name}
Expand All @@ -61,6 +72,15 @@ Wait for volume ${volume_id} healthy
${volume_name} = generate_name_with_suffix volume ${volume_id}
wait_for_volume_healthy ${volume_name}

Wait for volume ${volume_id} migration ready
${volume_name} = generate_name_with_suffix volume ${volume_id}
wait_for_volume_migration_ready ${volume_name}

Wait for volume ${volume_id} migrated to node ${node_id}
${volume_name} = generate_name_with_suffix volume ${volume_id}
${node_name} = get_node_by_index ${node_id}
wait_for_volume_migration_completed ${volume_name} ${node_name}

Wait until volume ${volume_id} replica rebuilding started on ${replica_locality}
${volume_name} = generate_name_with_suffix volume ${volume_id}
wait_for_replica_rebuilding_to_start_on_node ${volume_name} ${replica_locality}
Expand Down
2 changes: 1 addition & 1 deletion e2e/libs/engine/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
from engine.engine import Engine
from engine.engine import Engine
2 changes: 1 addition & 1 deletion e2e/libs/engine/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
class Base(ABC):

@abstractmethod
def get_engine(self, volume_name, node_name):
def get_engines(self, volume_name, node_name):
return NotImplemented

@abstractmethod
Expand Down
4 changes: 2 additions & 2 deletions e2e/libs/engine/crd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ class CRD(Base):
def __init__(self):
self.obj_api = client.CustomObjectsApi()

def get_engine(self, volume_name, node_name):
def get_engines(self, volume_name, node_name):
if volume_name == "" or node_name == "":
logging.info("getting all engines")
else:
Expand All @@ -19,7 +19,7 @@ def get_engine(self, volume_name, node_name):
label_selector = []
if volume_name != "":
label_selector.append(f"longhornvolume={volume_name}")
if node_name != "":
if node_name:
label_selector.append(f"longhornnode={node_name}")

api_response = self.obj_api.list_namespaced_custom_object(
Expand Down
6 changes: 3 additions & 3 deletions e2e/libs/engine/engine.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ def __init__(self):
if self._strategy == LonghornOperationStrategy.CRD:
self.engine = CRD()

def get_engine(self, volume_name, node_name):
return self.engine.get_engine(volume_name, node_name)
def get_engines(self, volume_name, node_name=None):
return self.engine.get_engines(volume_name, node_name)

def get_engine_by_volume(self, volume):
engines = self.engine.get_engine(volume["metadata"]["name"], "")
engines = self.engine.get_engines(volume["metadata"]["name"])
assert len(engines) == 1, \
f"Expected exactly one engine but found {len(engines)}"

Expand Down
3 changes: 3 additions & 0 deletions e2e/libs/keywords/common_keywords.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,6 @@ def generate_name_with_suffix(self, kind, suffix):

def get_worker_nodes(self):
return Node().list_node_names_by_role("worker")

def get_node_by_index(self, node_id):
return Node().get_node_by_index(node_id)
29 changes: 20 additions & 9 deletions e2e/libs/keywords/volume_keywords.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,25 @@ def cleanup_volumes(self):
for volume in volumes['items']:
self.delete_volume(volume['metadata']['name'])

def create_volume(self, volume_name, size, replica_count, frontend="blockdev"):
def create_volume(self, volume_name, size, replica_count, frontend="blockdev", migratable=False, access_mode="RWO"):
logging(f'Creating volume {volume_name}')
self.volume.create(volume_name, size, replica_count, frontend)
self.volume.create(volume_name, size, replica_count, frontend, migratable, access_mode)

def delete_volume(self, volume_name):
logging(f'Deleting volume {volume_name}')
self.volume.delete(volume_name)

def attach_volume(self, volume_name):
attach_node = self.node.get_test_pod_not_running_node()
logging(f'Attaching volume {volume_name} to {attach_node}')
self.volume.attach(volume_name, attach_node)
def attach_volume(self, volume_name, node_name=None):
if not node_name:
node_name = self.node.get_node_by_index(0)
logging(f'Attaching volume {volume_name} to node {node_name}')
self.volume.attach(volume_name, node_name)

def detach_volume(self, volume_name):
logging(f'Detaching volume {volume_name}')
self.volume.detach(volume_name)
def detach_volume(self, volume_name, node_name=None):
if not node_name:
node_name = self.node.get_node_by_index(0)
logging(f'Detaching volume {volume_name} from node {node_name}')
self.volume.detach(volume_name, node_name)

def list_volumes(self):
logging(f'Listing volumes')
Expand Down Expand Up @@ -232,6 +235,14 @@ def wait_for_volume_healthy(self, volume_name):
logging(f'Waiting for volume {volume_name} to be healthy')
self.volume.wait_for_volume_healthy(volume_name)

def wait_for_volume_migration_ready(self, volume_name):
logging(f'Waiting for volume {volume_name} migration to be ready')
self.volume.wait_for_volume_migration_ready(volume_name)

def wait_for_volume_migration_completed(self, volume_name, node_name):
logging(f'Waiting for volume {volume_name} migration to node {node_name} completed')
self.volume.wait_for_volume_migration_completed(volume_name, node_name)

def validate_volume_replicas_anti_affinity(self, volume_name):
self.volume.validate_volume_replicas_anti_affinity(volume_name)

Expand Down
12 changes: 12 additions & 0 deletions e2e/libs/volume/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ def create(self, volume_name, size, replica_count):
def attach(self, volume_name, node_name):
return NotImplemented

@abstractmethod
def detach(self, volume_name, node_name):
return NotImplemented

@abstractmethod
def delete(self, volume_name):
return NotImplemented
Expand All @@ -23,6 +27,14 @@ def delete(self, volume_name):
def wait_for_volume_state(self, volume_name, desired_state):
return NotImplemented

@abstractmethod
def wait_for_volume_migration_ready(self, volume_name):
return NotImplemented

@abstractmethod
def wait_for_volume_migration_completed(self, volume_name, node_name):
return NotImplemented

@abstractmethod
def get_endpoint(self, volume_name):
return NotImplemented
Expand Down
127 changes: 74 additions & 53 deletions e2e/libs/volume/crd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from utility.constant import LABEL_TEST_VALUE
from utility.utility import get_retry_count_and_interval
from utility.utility import logging

from utility.utility import get_cr
from volume.base import Base
from volume.constant import GIBIBYTE
from volume.rest import Rest
Expand All @@ -19,9 +19,11 @@ def __init__(self, node_exec):
self.obj_api = client.CustomObjectsApi()
self.node_exec = node_exec
self.retry_count, self.retry_interval = get_retry_count_and_interval()
self.engine = Engine()

def create(self, volume_name, size, replica_count, frontend="blockdev"):
def create(self, volume_name, size, replica_count, frontend, migratable, access_mode):
size = str(int(size) * GIBIBYTE)
access_mode = access_mode.lower()
body = {
"apiVersion": "longhorn.io/v1beta2",
"kind": "Volume",
Expand All @@ -35,7 +37,9 @@ def create(self, volume_name, size, replica_count, frontend="blockdev"):
"frontend": frontend,
"replicaAutoBalance": "ignored",
"size": size,
"numberOfReplicas": int(replica_count)
"numberOfReplicas": int(replica_count),
"migratable": migratable,
"accessMode": access_mode
}
}
try:
Expand All @@ -52,6 +56,8 @@ def create(self, volume_name, size, replica_count, frontend="blockdev"):
assert volume['spec']['size'] == size, f"expect volume size is {size}, but it's {volume['spec']['size']}"
assert volume['spec']['numberOfReplicas'] == int(replica_count), f"expect volume numberOfReplicas is {replica_count}, but it's {volume['spec']['numberOfReplicas']}"
assert volume['spec']['frontend'] == frontend, f"expect volume frontend is {frontend}, but it's {volume['spec']['frontend']}"
assert volume['spec']['migratable'] == migratable, f"expect volume migratable is {migratable}, but it's {volume['spec']['migratable']}"
assert volume['spec']['accessMode'] == access_mode, f"expect volume accessMode is {access_mode}, but it's {volume['spec']['accessMode']}"
assert volume['status']['restoreRequired'] is False, f"expect volume restoreRequired is False, but it's {volume['status']['restoreRequired']}"
#assert volume['backingImage'] == backing_image, f"expect volume backingImage is {backing_image}, but it's {volume['backingImage']}"
except ApiException as e:
Expand All @@ -71,42 +77,35 @@ def delete(self, volume_name):
logging(f"Deleting volume error: {e}")

def attach(self, volume_name, node_name):
self.obj_api.patch_namespaced_custom_object(
group="longhorn.io",
version="v1beta2",
namespace="longhorn-system",
plural="volumes",
name=volume_name,
body={
"spec": {
"nodeID": node_name
}
}
)

migratable = self.get(volume_name)['spec']['migratable']
type = "longhorn-api" if not migratable else "csi-attacher"

try:
body = get_cr(
group="longhorn.io",
version="v1beta2",
namespace="longhorn-system",
plural="volumeattachments",
name=volume_name
)
body['spec']['attachmentTickets'][node_name] = {
"id": node_name,
"nodeID": node_name,
"parameters": {
"disableFrontend": "false",
"lastAttachedBy": ""
},
"type": type
}

self.obj_api.patch_namespaced_custom_object(
group="longhorn.io",
version="v1beta2",
namespace="longhorn-system",
plural="volumeattachments",
name=volume_name,
body={
"spec": {
"attachmentTickets": {
"": {
"generation": 0,
"id": "",
"nodeID": node_name,
"parameters": {
"disableFrontend": "false",
"lastAttachedBy": ""
},
"type": "longhorn-api"
}
}
}
}
body=body
)
except Exception as e:
# new CRD: volumeattachments was added since from 1.5.0
Expand All @@ -115,40 +114,32 @@ def attach(self, volume_name, node_name):
Exception(f'exception for creating volumeattachments:', e)
self.wait_for_volume_state(volume_name, "attached")

def detach(self, volume_name):
def detach(self, volume_name, node_name):
try:
self.obj_api.patch_namespaced_custom_object(
body = get_cr(
group="longhorn.io",
version="v1beta2",
namespace="longhorn-system",
plural="volumeattachments",
name=volume_name,
body={
"spec": {
"attachmentTickets": None,
}
}
name=volume_name
)
except Exception as e:
# new CRD: volumeattachments was added since from 1.5.0
# https://github.com/longhorn/longhorn/issues/3715
if e.reason != "Not Found":
Exception(f'exception for patching volumeattachments:', e)
del body['spec']['attachmentTickets'][node_name]

self.obj_api.patch_namespaced_custom_object(
self.obj_api.replace_namespaced_custom_object(
group="longhorn.io",
version="v1beta2",
namespace="longhorn-system",
plural="volumes",
plural="volumeattachments",
name=volume_name,
body={
"spec": {
"nodeID": ""
}
}
body=body
)

self.wait_for_volume_state(volume_name, "detached")
except Exception as e:
# new CRD: volumeattachments was added since from 1.5.0
# https://github.com/longhorn/longhorn/issues/3715
if e.reason != "Not Found":
Exception(f'exception for patching volumeattachments:', e)
if len(body['spec']['attachmentTickets']) == 0:
self.wait_for_volume_state(volume_name, "detached")

def get(self, volume_name):
return self.obj_api.get_namespaced_custom_object(
Expand Down Expand Up @@ -254,6 +245,36 @@ def wait_for_volume_robustness_not(self, volume_name, not_desired_state):
time.sleep(self.retry_interval)
assert volume["status"]["robustness"] != not_desired_state

def wait_for_volume_migration_ready(self, volume_name):
ready = False
for i in range(self.retry_count):
logging(f"Waiting for volume {volume_name} migration ready ({i}) ...")
try:
engines = self.engine.get_engines(volume_name)
ready = len(engines) == 2
for engine in engines:
ready = ready and engine['status']['endpoint']
if ready:
break
except Exception as e:
logging(f"Getting volume {volume_name} engines error: {e}")
time.sleep(self.retry_interval)
assert ready

def wait_for_volume_migration_completed(self, volume_name, node_name):
completed = False
for i in range(self.retry_count):
logging(f"Waiting for volume {volume_name} migration to node {node_name} completed ({i}) ...")
try:
engines = self.engine.get_engines(volume_name, node_name)
completed = len(engines) == 1 and engines[0]['status']['endpoint']
if completed:
break
except Exception as e:
logging(f"Getting volume {volume_name} engines error: {e}")
time.sleep(self.retry_interval)
assert completed

def wait_for_volume_expand_to_size(self, volume_name, expected_size):
engine = None
engine_current_size = 0
Expand Down
Loading

0 comments on commit da29ef3

Please sign in to comment.