test(robot): add replica rebuilding after engine image upgrade test

Signed-off-by: Yang Chiu <[email protected]>

e2e/keywords/common.resource

@@ -48,6 +48,7 @@ Cleanup test resources
     cleanup_backups
     cleanup_disks
     cleanup_backing_images
+    cleanup_engine_images
     reset_backupstore
 
 Cleanup test resources include off nodes

e2e/keywords/engine_image.resource

@@ -0,0 +1,9 @@
+*** Settings ***
+Documentation       Longhorn engine image related keywords
+
+Library             ../libs/keywords/engine_image_keywords.py
+
+*** Keywords ***
+Create compatible engine image
+    ${compatible_engine_image_name} =    deploy_compatible_engine_image
+    Set Test Variable    ${compatible_engine_image_name}

e2e/keywords/volume.resource

@@ -255,3 +255,8 @@ Record volume ${volume_id} replica names
 Check volume ${volume_id} replica names are as recorded
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     check_volume_replica_names_recorded    ${volume_name}
+
+Upgrade volume ${volume_id} engine to compatible engine image
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    upgrade_engine_image    ${volume_name}    ${compatible_engine_image_name}
+    wait_for_engine_image_upgrade_completed    ${volume_name}    ${compatible_engine_image_name}

e2e/libs/engine_image/__init__.py

@@ -0,0 +1 @@
+from engine_image.engine_image import EngineImage

e2e/libs/engine_image/engine_image.py

@@ -0,0 +1,105 @@
+import time
+from utility.utility import logging
+from utility.utility import get_longhorn_client
+from utility.utility import get_retry_count_and_interval
+
+
+class EngineImage():
+
+    UPGRADE_TEST_IMAGE_PREFIX = "longhornio/longhorn-test:upgrade-test"
+
+    def __init__(self):
+        self.retry_count, self.retry_interval = get_retry_count_and_interval()
+
+    def get_default_engine_image(self):
+        images = get_longhorn_client().list_engine_image()
+        for img in images:
+            if img.default:
+                return img
+        assert False, f"Failed to get default engine image: {images}"
+
+    def get_default_engine_image_versions(self):
+        default_img = self.get_default_engine_image()
+        cli_v = default_img.cliAPIVersion
+        cli_minv = default_img.cliAPIMinVersion
+        ctl_v = default_img.controllerAPIVersion
+        ctl_minv = default_img.controllerAPIMinVersion
+        data_v = default_img.dataFormatVersion
+        data_minv = default_img.dataFormatMinVersion
+        return cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv
+
+    def deploy_compatible_engine_image(self):
+        cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv = self.get_default_engine_image_versions()
+        compatible_engine_image_name = \
+            f"{self.UPGRADE_TEST_IMAGE_PREFIX}.{cli_v}-{cli_minv}.{ctl_v}-{ctl_minv}.{data_v}-{data_minv}"
+        image = self.create_engine_image(compatible_engine_image_name)
+        return image.image
+
+    def create_engine_image(self, image_name):
+        image = get_longhorn_client().create_engine_image(image=image_name)
+        image = self.wait_for_engine_image_deployed(image.name)
+        assert image.refCount == 0, f"Expected new engine image {image_name} refCount == 0, but it's {image.refCount}"
+        assert image.noRefSince != "", f"Expected new engine image {image_name} noRefSince not empty, but it's {image.noRefSince}"
+        return image
+
+    def wait_for_engine_image_deployed(self, image_name):
+        self.wait_for_engine_image_created(image_name)
+        for i in range(self.retry_count):
+            image = get_longhorn_client().by_id_engine_image(image_name)
+            if image.state == "deployed":
+                break
+            time.sleep(self.retry_interval)
+        assert image.state == "deployed", f"Failed to deploy engine image {image_name}: {image}"
+        return image
+
+    def wait_for_engine_image_ref_count(self, image_name, count):
+        self.wait_for_engine_image_created(image_name)
+        for i in range(self.retry_count):
+            image = get_longhorn_client().by_id_engine_image(image_name)
+            if image.refCount == count:
+                break
+            time.sleep(self.retry_interval)
+        assert image.refCount == count, f"Failed to wait engine image {image_name} reference count {count}: {image}"
+        if count == 0:
+            assert image.noRefSince != "", f"Expected engine image {image_name} noRefSince non-empty: {image}"
+        return image
+
+    def wait_for_engine_image_created(self, image_name):
+        for i in range(self.retry_count):
+            images = get_longhorn_client().list_engine_image()
+            found = False
+            for img in images:
+                if img.name == image_name:
+                    found = True
+                    break
+            if found:
+                break
+            time.sleep(self.retry_interval)
+        assert found, f"Failed to create engine image {image_name}: {images}"
+
+    def cleanup_engine_images(self):
+        images = get_longhorn_client().list_engine_image().data
+        for image in images:
+            if not image.default:
+                self.wait_for_engine_image_ref_count(image.name, 0)
+                get_longhorn_client().delete(image)
+                self.wait_for_engine_image_deleted(image.name)
+
+    def wait_for_engine_image_deleted(self, image_name):
+
+        deleted = False
+
+        for i in range(self.retry_count):
+
+            time.sleep(self.retry_interval)
+            deleted = True
+
+            images = get_longhorn_client().list_engine_image().data
+            for image in images:
+                if image.name == image_name:
+                    deleted = False
+                    break
+            if deleted:
+                break
+
+        assert deleted, f"Failed to delete engine image {image_name}: {images}"

e2e/libs/keywords/engine_image_keywords.py

@@ -0,0 +1,13 @@
+from engine_image import EngineImage
+from utility.utility import logging
+
+class engine_image_keywords:
+
+    def __init__(self):
+        self.engine_image = EngineImage()
+
+    def deploy_compatible_engine_image(self):
+        return self.engine_image.deploy_compatible_engine_image()
+
+    def cleanup_engine_images(self):
+        return self.engine_image.cleanup_engine_images()

e2e/libs/keywords/volume_keywords.py

@@ -305,3 +305,9 @@ def check_volume_replica_names_recorded(self, volume_name):
             f"Volume {volume_name} replica names mismatched:\n" \
             f"Want: {expected_replica_names}\n" \
             f"Got: {actual_replica_names}"
+
+    def upgrade_engine_image(self, volume_name, engine_image_name):
+        self.volume.upgrade_engine_image(volume_name, engine_image_name)
+
+    def wait_for_engine_image_upgrade_completed(self, volume_name, engine_image_name):
+        self.volume.wait_for_engine_image_upgrade_completed(volume_name, engine_image_name)

e2e/libs/volume/crd.py

@@ -53,7 +53,9 @@ def create(self, volume_name, size, numberOfReplicas, frontend, migratable, acce
                 "dataEngine": dataEngine,
                 "backingImage": backingImage,
                 "Standby": Standby,
-                "fromBackup": fromBackup
+                "fromBackup": fromBackup,
+                # disable revision counter by default from v1.7.0
+                "revisionCounterDisabled": True
             }
         }
         try:
@@ -487,3 +489,9 @@ def create_persistentvolume(self, volume_name, retry):
 
     def create_persistentvolumeclaim(self, volume_name, retry):
         return Rest(self.node_exec).create_persistentvolumeclaim(volume_name, retry)
+
+    def upgrade_engine_image(self, volume_name, engine_image_name):
+        return Rest(self.node_exec).upgrade_engine_image(volume_name, engine_image_name)
+
+    def wait_for_engine_image_upgrade_completed(self, volume_name, engine_image_name):
+        return Rest(self.node_exec).wait_for_engine_image_upgrade_completed(volume_name, engine_image_name)

e2e/libs/volume/rest.py

@@ -315,3 +315,36 @@ def create_persistentvolumeclaim(self, volume_name, retry):
                 break
             time.sleep(self.retry_interval)
         assert created
+
+    def upgrade_engine_image(self, volume_name, engine_image_name):
+        volume = self.get(volume_name)
+        volume.engineUpgrade(image=engine_image_name)
+
+    def wait_for_engine_image_upgrade_completed(self, volume_name, engine_image_name):
+        for i in range(self.retry_count):
+            volume = self.get(volume_name)
+            if volume.currentImage == engine_image_name:
+                break
+            time.sleep(self.retry_interval)
+        assert volume.currentImage == engine_image_name, f"Failed to upgrade engine image to {engine_image_name}: {volume}"
+        self.wait_for_replica_ready_to_rw(volume_name)
+
+    def wait_for_replica_ready_to_rw(self, volume_name):
+        for _ in range(self.retry_count):
+            ready = True
+            volume = self.get(volume_name)
+            replicas = volume.replicas
+            if len(replicas) != volume.numberOfReplicas:
+                logging(f"Waiting for volume {volume_name} replica count = {volume.numberOfReplicas}, current count = {len(replicas)}")
+                ready = False
+                time.sleep(self.retry_interval)
+                continue
+            else:
+                for replica in replicas:
+                    if replica.mode != "RW":
+                        ready = False
+                        break
+            if ready:
+                break
+            time.sleep(self.retry_interval)
+        assert ready, f"Failed to get volume {volume_name} replicas ready: {replicas}"

e2e/libs/volume/volume.py

@@ -145,3 +145,9 @@ def create_persistentvolume(self, volume_name, retry):
 
     def create_persistentvolumeclaim(self, volume_name, retry):
         return self.volume.create_persistentvolumeclaim(volume_name, retry)
+
+    def upgrade_engine_image(self, volume_name, engine_image_name):
+        return self.volume.upgrade_engine_image(volume_name, engine_image_name)
+
+    def wait_for_engine_image_upgrade_completed(self, volume_name, engine_image_name):
+        return self.volume.wait_for_engine_image_upgrade_completed(volume_name, engine_image_name)

e2e/tests/regression/test_engine_image.robot

@@ -0,0 +1,32 @@
+*** Settings ***
+Documentation    Engine Image Test Cases
+
+Test Tags    regression    engine_image
+
+Resource    ../keywords/common.resource
+Resource    ../keywords/volume.resource
+Resource    ../keywords/engine_image.resource
+
+Test Setup    Set test environment
+Test Teardown    Cleanup test resources
+
+*** Variables ***
+${LOOP_COUNT}    1
+${RETRY_COUNT}    300
+${RETRY_INTERVAL}    1
+${DATA_ENGINE}    v1
+
+*** Test Cases ***
+Test Replica Rebuilding After Engine Upgrade
+    [Tags]    coretest
+    Given Create compatible engine image
+    And Create volume 0
+    And Attach volume 0
+    And Wait for volume 0 healthy
+    And Write data to volume 0
+    When Upgrade volume 0 engine to compatible engine image
+    Then Delete volume 0 replica on node 1
+    And Wait until volume 0 replica rebuilding started on node 1
+    And Wait until volume 0 replica rebuilding completed on node 1
+    And Wait for volume 0 healthy
+    And Check volume 0 data is intact