Merge branch 'master' into force-drain

e2e/keywords/snapshot.resource

@@ -0,0 +1,42 @@
+*** Settings ***
+Documentation    Snapshot Keywords
+
+Library    ../libs/keywords/common_keywords.py
+Library    ../libs/keywords/snapshot_keywords.py
+
+*** Keywords ***
+Create snapshot ${snapshot_id} of volume ${volume_id}
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    create_snapshot    ${volume_name}    ${snapshot_id}
+
+Delete snapshot ${snapshot_id} of volume ${volume_id}
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    delete_snapshot    ${volume_name}    ${snapshot_id}
+
+Revert volume ${volume_id} to snapshot ${snapshot_id}
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    revert_snapshot    ${volume_name}    ${snapshot_id}
+
+Purge volume ${volume_id} snapshot
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    purge_snapshot    ${volume_name}
+
+Validate snapshot ${parent_id} is parent of snapshot ${child_id} in volume ${volume_id} snapshot list
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    is_parent_of   ${volume_name}    ${parent_id}    ${child_id}
+
+Validate snapshot ${parent_id} is parent of volume-head in volume ${volume_id} snapshot list
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    is_parent_of_volume_head   ${volume_name}    ${parent_id}
+
+Validate snapshot ${snapshot_id} is marked as removed in volume ${volume_id} snapshot list
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    is_marked_as_removed    ${volume_name}    ${snapshot_id}
+
+Validate snapshot ${snapshot_id} is not in volume ${volume_id} snapshot list
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    is_not_existing    ${volume_name}    ${snapshot_id}
+
+Validate snapshot ${snapshot_id} is in volume ${volume_id} snapshot list
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    is_existing    ${volume_name}    ${snapshot_id}

e2e/keywords/volume.resource

@@ -8,7 +8,7 @@ Library    ../libs/keywords/volume_keywords.py
 *** Keywords ***
 Create volume ${volume_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
-    create_volume   ${volume_name}    2    3
+    create_volume   ${volume_name}
 
 Create volume ${volume_id} with
     [Arguments]    &{config}
@@ -39,6 +39,10 @@ Attach volume ${volume_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     attach_volume    ${volume_name}
 
+Attach volume ${volume_id} in maintenance mode
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    attach_volume_in_maintenance_mode    ${volume_name}
+
 Detach volume ${volume_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     detach_volume    ${volume_name}
@@ -60,6 +64,10 @@ Write ${size} GB data to volume ${volume_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     write_volume_random_data   ${volume_name}    ${${size} * 1024}
 
+Write data ${data_id} to volume ${volume_id}
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    write_volume_random_data   ${volume_name}    2048    ${data_id}
+
 Keep writing data to volume ${volume_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     keep_writing_data    ${volume_name}
@@ -115,6 +123,10 @@ Check volume ${volume_id} data is intact
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     check_data_checksum    ${volume_name}
 
+Check volume ${volume_id} data is data ${data_id}
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    check_data_checksum    ${volume_name}    ${data_id}
+
 Check volume ${volume_id} works
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     ${volume_data_checksum} =     write_volume_random_data   ${volume_name}    1024

e2e/libs/keywords/snapshot_keywords.py

@@ -0,0 +1,35 @@
+from snapshot import Snapshot
+from utility.utility import logging
+
+
+class snapshot_keywords:
+
+    def __init__(self):
+        self.snapshot = Snapshot()
+
+    def create_snapshot(self, volume_name, snapshot_id):
+        self.snapshot.create(volume_name, snapshot_id)
+
+    def delete_snapshot(self, volume_name, snapshot_id):
+        self.snapshot.delete(volume_name, snapshot_id)
+
+    def revert_snapshot(self, volume_name, snapshot_id):
+        self.snapshot.revert(volume_name, snapshot_id)
+
+    def purge_snapshot(self, volume_name):
+        self.snapshot.purge(volume_name)
+
+    def is_parent_of(self, volume_name, parent_id, child_id):
+        self.snapshot.is_parent_of(volume_name, parent_id, child_id)
+
+    def is_parent_of_volume_head(self, volume_name, parent_id):
+        self.snapshot.is_parent_of_volume_head(volume_name, parent_id)
+
+    def is_marked_as_removed(self, volume_name, snapshot_id):
+        self.snapshot.is_marked_as_removed(volume_name, snapshot_id)
+
+    def is_not_existing(self, volume_name, snapshot_id):
+        assert not self.snapshot.is_existing(volume_name, snapshot_id)
+
+    def is_existing(self, volume_name, snapshot_id):
+        assert self.snapshot.is_existing(volume_name, snapshot_id)

e2e/libs/keywords/volume_keywords.py

@@ -24,7 +24,7 @@ def cleanup_volumes(self):
         for volume in volumes['items']:
             self.delete_volume(volume['metadata']['name'])
 
-    def create_volume(self, volume_name, size, replica_count, frontend="blockdev", migratable=False, access_mode="RWO", data_engine="v1"):
+    def create_volume(self, volume_name, size=2, replica_count=3, frontend="blockdev", migratable=False, access_mode="RWO", data_engine="v1"):
         logging(f'Creating volume {volume_name}')
         self.volume.create(volume_name, size, replica_count, frontend, migratable, access_mode, data_engine)
 
@@ -36,7 +36,13 @@ def attach_volume(self, volume_name, node_name=None):
         if not node_name:
             node_name = self.node.get_node_by_index(0)
         logging(f'Attaching volume {volume_name} to node {node_name}')
-        self.volume.attach(volume_name, node_name)
+        self.volume.attach(volume_name, node_name, disable_frontend=False)
+
+    def attach_volume_in_maintenance_mode(self, volume_name, node_name=None):
+        if not node_name:
+            node_name = self.node.get_node_by_index(0)
+        logging(f'Attaching volume {volume_name} to node {node_name} in maintenance mode')
+        self.volume.attach(volume_name, node_name, disable_frontend=True)
 
     def detach_volume(self, volume_name, node_name=None):
         if not node_name:
@@ -80,21 +86,17 @@ def get_node_ids_by_replica_locality(self, volume_name, replica_locality):
 
         raise Exception(f"Failed to get node ID of the replica on {replica_locality}")
 
-    def write_volume_random_data(self, volume_name, size_in_mb):
+    def write_volume_random_data(self, volume_name, size_in_mb, data_id=0):
         logging(f'Writing {size_in_mb} MB random data to volume {volume_name}')
-        checksum = self.volume.write_random_data(volume_name, size_in_mb)
-
-        self.volume.set_annotation(volume_name, ANNOT_CHECKSUM, checksum)
+        checksum = self.volume.write_random_data(volume_name, size_in_mb, data_id)
 
     def keep_writing_data(self, volume_name):
         logging(f'Keep writing data to volume {volume_name}')
         self.volume.keep_writing_data(volume_name)
 
-    def check_data_checksum(self, volume_name):
-        checksum = self.volume.get_annotation_value(volume_name, ANNOT_CHECKSUM)
-
-        logging(f"Checking volume {volume_name} data checksum is {checksum}")
-        self.volume.check_data_checksum(volume_name, checksum)
+    def check_data_checksum(self, volume_name, data_id=0):
+        logging(f"Checking volume {volume_name} data {data_id} checksum")
+        self.volume.check_data_checksum(volume_name, data_id)
 
     def delete_replica_on_node(self, volume_name, replica_locality):
         node_name = None

e2e/libs/snapshot/__init__.py

@@ -0,0 +1 @@
+from snapshot.snapshot import Snapshot

e2e/libs/snapshot/base.py

@@ -0,0 +1,72 @@
+from abc import ABC, abstractmethod
+from utility.utility import set_annotation
+from utility.utility import get_annotation_value
+
+class Base(ABC):
+
+    ANNOT_ID = "test.longhorn.io/snapshot-id"
+
+    @abstractmethod
+    def create(self, volume_name, snapshot_id):
+        return NotImplemented
+
+    def set_snapshot_id(self, snapshot_name, snapshot_id):
+        set_annotation(
+            group="longhorn.io",
+            version="v1beta2",
+            namespace="longhorn-system",
+            plural="snapshots",
+            name=snapshot_name,
+            annotation_key=self.ANNOT_ID,
+            annotation_value=snapshot_id
+        )
+
+    def get_snapshot_id(self, snapshot_name):
+        return get_annotation_value(
+            group="longhorn.io",
+            version="v1beta2",
+            namespace="longhorn-system",
+            plural="snapshots",
+            name=snapshot_name,
+            annotation_key=self.ANNOT_ID
+        )
+
+    @abstractmethod
+    def get(self, volume_name, snapshot_id):
+        return NotImplemented
+
+    @abstractmethod
+    def get_volume_head(self, volume_name):
+        return NotImplemented
+
+    @abstractmethod
+    def list(self, volume_name):
+        return NotImplemented
+
+    @abstractmethod
+    def delete(self, volume_name, snapshot_id):
+        return NotImplemented
+
+    @abstractmethod
+    def revert(self, volume_name, snapshot_id):
+        return NotImplemented
+
+    @abstractmethod
+    def purge(self, volume_name):
+        return NotImplemented
+
+    @abstractmethod
+    def is_parent_of(self, volume_name, parent_id, child_id):
+        return NotImplemented
+
+    @abstractmethod
+    def is_parent_of_volume_head(self, volume_name, parent_id):
+        return NotImplemented
+
+    @abstractmethod
+    def is_existing(self, volume_name, snapshot_id):
+        return NotImplemented
+
+    @abstractmethod
+    def is_marked_as_removed(self, volume_name, snapshot_id):
+        return NotImplemented

e2e/libs/snapshot/crd.py

@@ -0,0 +1,7 @@
+from snapshot.base import Base
+
+
+class CRD(Base):
+
+    def __init__(self):
+        pass

e2e/libs/snapshot/rest.py

@@ -0,0 +1,117 @@
+from snapshot.base import Base
+from utility.utility import logging
+from utility.utility import get_longhorn_client
+from utility.utility import get_retry_count_and_interval
+from node_exec import NodeExec
+from volume import Rest as RestVolume
+import time
+
+
+class Rest(Base):
+
+    def __init__(self):
+        self.volume = RestVolume(NodeExec.get_instance())
+        self.retry_count, self.retry_interval = get_retry_count_and_interval()
+
+    def create(self, volume_name, snapshot_id):
+        logging(f"Creating volume {volume_name} snapshot {snapshot_id}")
+        volume = self.volume.get(volume_name)
+        snapshot = volume.snapshotCreate()
+        snap_name = snapshot.name
+
+        snapshot_created = False
+        for i in range(self.retry_count):
+            snapshots = volume.snapshotList().data
+            for vs in snapshots:
+                if vs.name == snap_name:
+                    snapshot_created = True
+                    break
+            if snapshot_created is True:
+                break
+            time.sleep(self.retry_interval)
+
+        assert snapshot_created
+
+        self.set_snapshot_id(snap_name, snapshot_id)
+
+        return snapshot
+
+    def get(self, volume_name, snapshot_id):
+        snapshots = self.list(volume_name)
+        for snapshot in snapshots:
+            if snapshot.name != "volume-head" and self.get_snapshot_id(snapshot.name) == snapshot_id:
+                return snapshot
+        return None
+
+    def get_volume_head(self, volume_name):
+        snapshots = self.list(volume_name)
+        for snapshot in snapshots:
+            if snapshot.name == "volume-head":
+                return snapshot
+        assert False
+
+    def list(self, volume_name):
+        return self.volume.get(volume_name).snapshotList().data
+
+    def delete(self, volume_name, snapshot_id):
+        logging(f"Deleting volume {volume_name} snapshot {snapshot_id}")
+        snapshot = self.get(volume_name, snapshot_id)
+        self.volume.get(volume_name).snapshotDelete(name=snapshot.name)
+
+    def revert(self, volume_name, snapshot_id):
+        logging(f"Reverting volume {volume_name} to snapshot {snapshot_id}")
+        snapshot = self.get(volume_name, snapshot_id)
+        self.volume.get(volume_name).snapshotRevert(name=snapshot.name)
+
+    def purge(self, volume_name):
+        logging(f"Purging volume {volume_name} snapshot")
+
+        volume = self.volume.get(volume_name)
+        volume.snapshotPurge()
+
+        completed = 0
+        last_purge_progress = {}
+        purge_status = {}
+        for i in range(self.retry_count):
+            completed = 0
+            volume = self.volume.get(volume_name)
+            purge_status = volume.purgeStatus
+            for status in purge_status:
+                assert status.error == "", f"Expect purge without error, but its' {status.error}"
+
+                progress = status.progress
+                assert progress <= 100, f"Expect purge progress <= 100, but it's {status}"
+                replica = status.replica
+                last = last_purge_progress.get(replica)
+                assert last is None or last <= status.progress, f"Expect purge progress increasing, but it didn't: current status = {status}, last progress = {last_purge_progress}"
+                last_purge_progress["replica"] = progress
+
+                if status.state == "complete":
+                    assert progress == 100
+                    completed += 1
+            if completed == len(purge_status):
+                break
+            time.sleep(self.retry_interval)
+        assert completed == len(purge_status)
+
+    def is_parent_of(self, volume_name, parent_id, child_id):
+        logging(f"Checking volume {volume_name} snapshot {parent_id} is parent of snapshot {child_id}")
+        parent = self.get(volume_name, parent_id)
+        child = self.get(volume_name, child_id)
+        if child.name not in parent.children.keys() or child.parent != parent.name:
+            logging(f"Expect snapshot {parent_id} is parent of snapshot {child_id}, but it's not")
+            time.sleep(self.retry_count)
+
+    def is_parent_of_volume_head(self, volume_name, parent_id):
+        parent = self.get(volume_name, parent_id)
+        volume_head = self.get_volume_head(volume_name)
+        if volume_head.name not in parent.children.keys() or volume_head.parent != parent.name:
+            logging(f"Expect snapshot {parent_id} is parent of volume-head, but it's not")
+            time.sleep(self.retry_count)
+
+    def is_existing(self, volume_name, snapshot_id):
+        return self.get(volume_name, snapshot_id)
+
+    def is_marked_as_removed(self, volume_name, snapshot_id):
+        snapshot = self.get(volume_name, snapshot_id)
+        assert snapshot.removed