Skip to content

Commit

Permalink
Merge branch 'master' into shut-down
Browse files Browse the repository at this point in the history
  • Loading branch information
@yangchiu
yangchiu authored Dec 6, 2024
2 parents 8d3517c + 8af8d7b commit 10f6e22
Show file tree
Hide file tree
Showing 8 changed files with 96 additions and 39 deletions.
13 changes: 9 additions & 4 deletions e2e/keywords/volume.resource
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,14 +167,19 @@ Check all replicas of volume ${volume_id} kept in error
Sleep ${RETRY_INTERVAL}
END

Wait for volume ${volume_id} migration ready
Wait for volume ${volume_id} migration to be ready
${volume_name} = generate_name_with_suffix volume ${volume_id}
wait_for_volume_migration_ready ${volume_name}
wait_for_volume_migration_to_be_ready ${volume_name}

Wait for volume ${volume_id} migrated to node ${node_id}
Wait for volume ${volume_id} to migrate to node ${node_id}
${volume_name} = generate_name_with_suffix volume ${volume_id}
${node_name} = get_node_by_index ${node_id}
wait_for_volume_migration_completed ${volume_name} ${node_name}
wait_for_volume_migration_complete ${volume_name} ${node_name}

Wait for volume ${volume_id} to stay on node ${node_id}
${volume_name} = generate_name_with_suffix volume ${volume_id}
${node_name} = get_node_by_index ${node_id}
wait_for_volume_migration_to_rollback ${volume_name} ${node_name}

Wait for volume ${volume_id} restoration from backup ${backup_id} completed
${volume_name} = generate_name_with_suffix volume ${volume_id}
Expand Down
21 changes: 10 additions & 11 deletions e2e/libs/engine/crd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,21 @@
from kubernetes import client

from engine.base import Base
from utility.utility import logging


class CRD(Base):
def __init__(self):
self.obj_api = client.CustomObjectsApi()

def get_engines(self, volume_name, node_name=None):
if volume_name == "" or node_name == "":
logging.info("getting all engines")
if not node_name:
logging(f"Getting all engines of {volume_name}")
else:
logging.info(
f"getting the volume {volume_name} on node {node_name} engine")
logging(f"Getting engine of volume {volume_name} on node {node_name}")

label_selector = []
if volume_name != "":
if volume_name:
label_selector.append(f"longhornvolume={volume_name}")
if node_name:
label_selector.append(f"longhornnode={node_name}")
Expand All @@ -31,20 +31,19 @@ def get_engines(self, volume_name, node_name=None):
)

if api_response == "" or api_response is None:
raise Exception(f"failed to get the volume {volume_name} engine")
raise Exception(f"failed to get volume {volume_name} engine")

engines = api_response["items"]
if len(engines) == 0:
logging.warning(f"cannot get the volume {volume_name} engines")
logging(f"Cannot get volume {volume_name} engines")

return engines

def delete_engine(self, volume_name, node_name):
if volume_name == "" or node_name == "":
logging.info("deleting all engines")
logging("deleting all engines")
else:
logging.info(
f"delete the volume {volume_name} on node {node_name} engine")
logging(f"delete the volume {volume_name} on node {node_name} engine")

for engine in self.get_engine(volume_name, node_name):
engine_name = engine['metadata']['name']
Expand All @@ -55,7 +54,7 @@ def delete_engine(self, volume_name, node_name):
plural="engines",
name=engine_name
)
logging.info("finished delete engines")
logging("finished delete engines")

def validate_engine_setting(self, volume_name, setting_name, value):
engines = self.get_engines(volume_name)
Expand Down
14 changes: 9 additions & 5 deletions e2e/libs/keywords/volume_keywords.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,13 +271,17 @@ def wait_for_volume_faulted(self, volume_name):
logging(f'Waiting for volume {volume_name} to be in faulted')
self.volume.wait_for_volume_faulted(volume_name)

def wait_for_volume_migration_ready(self, volume_name):
def wait_for_volume_migration_to_be_ready(self, volume_name):
logging(f'Waiting for volume {volume_name} migration to be ready')
self.volume.wait_for_volume_migration_ready(volume_name)
self.volume.wait_for_volume_migration_to_be_ready(volume_name)

def wait_for_volume_migration_completed(self, volume_name, node_name):
logging(f'Waiting for volume {volume_name} migration to node {node_name} completed')
self.volume.wait_for_volume_migration_completed(volume_name, node_name)
def wait_for_volume_migration_complete(self, volume_name, node_name):
logging(f'Waiting for volume {volume_name} migration to node {node_name} complete')
self.volume.wait_for_volume_migration_complete(volume_name, node_name)

def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
logging(f'Waiting for volume {volume_name} migration to rollback to node {node_name}')
self.volume.wait_for_volume_migration_to_rollback(volume_name, node_name)

def wait_for_volume_restoration_completed(self, volume_name, backup_name):
logging(f'Waiting for volume {volume_name} restoration from {backup_name} completed')
Expand Down
8 changes: 6 additions & 2 deletions e2e/libs/volume/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,15 @@ def wait_for_volume_state(self, volume_name, desired_state):
return NotImplemented

@abstractmethod
def wait_for_volume_migration_ready(self, volume_name):
def wait_for_volume_migration_to_be_ready(self, volume_name):
return NotImplemented

@abstractmethod
def wait_for_volume_migration_completed(self, volume_name, node_name):
def wait_for_volume_migration_complete(self, volume_name, node_name):
return NotImplemented

@abstractmethod
def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
return NotImplemented

@abstractmethod
Expand Down
32 changes: 23 additions & 9 deletions e2e/libs/volume/crd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,10 +301,10 @@ def wait_for_volume_robustness_not(self, volume_name, not_desired_state):
time.sleep(self.retry_interval)
assert volume["status"]["robustness"] != not_desired_state

def wait_for_volume_migration_ready(self, volume_name):
def wait_for_volume_migration_to_be_ready(self, volume_name):
ready = False
for i in range(self.retry_count):
logging(f"Waiting for volume {volume_name} migration ready ({i}) ...")
logging(f"Waiting for volume {volume_name} migration to be ready ({i}) ...")
try:
engines = self.engine.get_engines(volume_name)
ready = len(engines) == 2
Expand All @@ -317,19 +317,33 @@ def wait_for_volume_migration_ready(self, volume_name):
time.sleep(self.retry_interval)
assert ready

def wait_for_volume_migration_completed(self, volume_name, node_name):
completed = False
def wait_for_volume_migration_complete(self, volume_name, node_name):
complete = False
for i in range(self.retry_count):
logging(f"Waiting for volume {volume_name} migration to node {node_name} completed ({i}) ...")
logging(f"Waiting for volume {volume_name} migration to node {node_name} complete ({i}) ...")
try:
engines = self.engine.get_engines(volume_name, node_name)
completed = len(engines) == 1 and engines[0]['status']['endpoint']
if completed:
engines = self.engine.get_engines(volume_name)
complete = len(engines) == 1 and engines[0]['status']['endpoint'] and engines[0]['status']['ownerID'] == node_name
if complete:
break
except Exception as e:
logging(f"Getting volume {volume_name} engines error: {e}")
time.sleep(self.retry_interval)
assert completed
assert complete

def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
rollback = False
for i in range(self.retry_count):
logging(f"Waiting for volume {volume_name} migration to rollback to node {node_name} ({i}) ...")
try:
engines = self.engine.get_engines(volume_name)
rollback = len(engines) == 1 and engines[0]['status']['endpoint'] and engines[0]['status']['ownerID'] == node_name
if rollback:
break
except Exception as e:
logging(f"Getting volume {volume_name} engines error: {e}")
time.sleep(self.retry_interval)
assert rollback

def wait_for_volume_restoration_completed(self, volume_name, backup_name):
completed = False
Expand Down
7 changes: 5 additions & 2 deletions e2e/libs/volume/rest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,13 @@ def wait_for_volume_state(self, volume_name, desired_state):
def wait_for_restore_required_status(self, volume_name, restore_required_state):
return NotImplemented

def wait_for_volume_migration_ready(self, volume_name):
def wait_for_volume_migration_to_be_ready(self, volume_name):
return NotImplemented

def wait_for_volume_migration_completed(self, volume_name, node_name):
def wait_for_volume_migration_complete(self, volume_name, node_name):
return NotImplemented

def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
return NotImplemented

def wait_for_volume_restoration_completed(self, volume_name):
Expand Down
11 changes: 7 additions & 4 deletions e2e/libs/volume/volume.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,14 @@ def wait_for_volume_healthy(self, volume_name):
self.volume.wait_for_volume_state(volume_name, "attached")
self.volume.wait_for_volume_robustness(volume_name, "healthy")

def wait_for_volume_migration_ready(self, volume_name):
self.volume.wait_for_volume_migration_ready(volume_name)
def wait_for_volume_migration_to_be_ready(self, volume_name):
self.volume.wait_for_volume_migration_to_be_ready(volume_name)

def wait_for_volume_migration_completed(self, volume_name, node_name):
self.volume.wait_for_volume_migration_completed(volume_name, node_name)
def wait_for_volume_migration_complete(self, volume_name, node_name):
self.volume.wait_for_volume_migration_complete(volume_name, node_name)

def wait_for_volume_migration_to_rollback(self, volume_name, node_name):
self.volume.wait_for_volume_migration_to_rollback(volume_name, node_name)

def wait_for_volume_restoration_completed(self, volume_name, backup_name):
self.volume.wait_for_volume_restoration_completed(volume_name, backup_name)
Expand Down
29 changes: 27 additions & 2 deletions e2e/tests/regression/test_migration.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,33 @@ Test Migration Confirm
And Wait for volume 0 healthy
And Write data to volume 0
And Attach volume 0 to node 1
Then Wait for volume 0 migration ready
Then Wait for volume 0 migration to be ready
And Detach volume 0 from node 0
And Wait for volume 0 migrated to node 1
And Wait for volume 0 to migrate to node 1
And Wait for volume 0 healthy
And Check volume 0 data is intact

Test Migration Rollback
[Tags] coretest migration
[Documentation] Test that a migratable RWX volume can be rolled back to initial node.
...
... 1. Create a new RWX migratable volume.
... 2. Attach to test node to write some test data on it.
... 3. Detach from test node.
... 4. Get set of nodes excluding the test node
... 5. Attach volume to node 1 (initial node)
... 6. Attach volume to node 2 (migration target)
... 7. Wait for migration ready (engine running on node 2)
... 8. Detach volume from node 2
... 9. Observe volume stayed on node 1 (single active engine)
... 10. Validate initially written test data
Given Create volume 0 with migratable=True accessMode=RWX dataEngine=${DATA_ENGINE}
When Attach volume 0 to node 0
And Wait for volume 0 healthy
And Write data to volume 0
And Attach volume 0 to node 1
Then Wait for volume 0 migration to be ready
And Detach volume 0 from node 1
And Wait for volume 0 to stay on node 0
And Wait for volume 0 healthy
And Check volume 0 data is intact

0 comments on commit 10f6e22

Please sign in to comment.