Merge branch 'master' into backup_in_other_cluster

e2e/keywords/node.resource

@@ -11,6 +11,11 @@ Add ${disk_type} type disk ${disk_path} for all worker nodes
         add_disk    ${worker_node}    ${disk_type}    ${disk_path}
     END
 
+Set node ${node_id} with
+    [Arguments]    &{config}
+    ${node_name} =    get_node_by_index    ${node_id}
+    set_node   ${node_name}    &{config}
+
 Disable node ${node_id} scheduling
     ${node_name} =    get_node_by_index    ${node_id}
     disable_node_scheduling    ${node_name}

e2e/keywords/sharemanager.resource

@@ -21,12 +21,12 @@ Check sharemanager ${condition} using headless service
 Wait for all sharemanager to be deleted
     wait_for_sharemanagers_deleted
 
-Delete sharemanager of deployment ${deployment_id} and wait for recreation
+Delete sharemanager pod of deployment ${deployment_id} and wait for recreation
     ${deployment_name} =   generate_name_with_suffix    deployment    ${deployment_id}
     ${volume_name} =    get_workload_volume_name    ${deployment_name}
-    delete_sharemanager_and_wait_for_recreation    ${volume_name}
+    delete_sharemanager_pod_and_wait_for_recreation    ${volume_name}
 
-Wait for sharemanager of deployment ${deployment_id} running
+Wait for sharemanager pod of deployment ${deployment_id} running
     ${deployment_name} =   generate_name_with_suffix    deployment    ${deployment_id}
     ${volume_name} =    get_workload_volume_name    ${deployment_name}
-    wait_for_share_manager_running    ${volume_name}
+    wait_for_share_manager_pod_running    ${volume_name}

e2e/keywords/volume.resource

@@ -123,6 +123,13 @@ Wait for volume ${volume_id} detached
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     wait_for_volume_detached    ${volume_name}
 
+Assert volume ${volume_id} remains detached for at least ${period} seconds
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    FOR    ${i}    IN RANGE    ${period}
+        wait_for_volume_detached    ${volume_name}
+        Sleep    1
+    END
+
 Wait for volume ${volume_id} faulted
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     wait_for_volume_faulted    ${volume_name}

e2e/libs/keywords/backing_image_keywords.py

@@ -23,7 +23,7 @@ def cleanup_backing_images(self):
 
     def delete_backing_image_manager(self, name):
         self.backing_image.delete_backing_image_manager(name)
-    
+
     def wait_all_backing_image_managers_running(self):
         self.backing_image.wait_all_backing_image_managers_running()
 

e2e/libs/keywords/backup_keywords.py

@@ -1,4 +1,5 @@
 from backup import Backup
+
 from utility.utility import logging
 from utility.utility import get_backupstore
 

e2e/libs/keywords/backupstore_keywords.py

@@ -1,8 +1,9 @@
 from backupstore import Nfs, S3
-from utility.utility import get_backupstore
-from utility.utility import logging
+
 import os
 
+from utility.utility import get_backupstore
+
 
 class backupstore_keywords:
 

e2e/libs/keywords/common_keywords.py

@@ -1,5 +1,6 @@
 from node import Node
 from node_exec import NodeExec
+
 from utility.utility import init_k8s_api_client
 from utility.utility import generate_name_with_suffix
 

e2e/libs/keywords/engine_image_keywords.py

@@ -1,5 +1,5 @@
 from engine_image import EngineImage
-from utility.utility import logging
+
 
 class engine_image_keywords:
 

e2e/libs/keywords/engine_keywords.py

@@ -1,5 +1,5 @@
 from engine import Engine
-from utility.utility import logging
+
 
 class engine_keywords:
 

e2e/libs/keywords/host_keywords.py

@@ -1,6 +1,7 @@
-import os
 from robot.libraries.BuiltIn import BuiltIn
 
+import os
+
 from host import Harvester, Aws
 from host.constant import NODE_REBOOT_DOWN_TIME_SECOND
 

e2e/libs/keywords/k8s_keywords.py

@@ -1,5 +1,7 @@
-import asyncio
 from robot.libraries.BuiltIn import BuiltIn
+
+import asyncio
+
 from k8s.k8s import restart_kubelet
 from k8s.k8s import delete_node
 from k8s.k8s import drain_node, force_drain_node
@@ -10,9 +12,12 @@
 from k8s.k8s import get_instance_manager_on_node
 from k8s.k8s import check_instance_manager_pdb_not_exist
 from k8s.k8s import wait_for_namespace_pods_running
-from utility.utility import logging
+
 from node import Node
 
+from utility.utility import logging
+
+
 class k8s_keywords:
 
     async def restart_kubelet(self, node_name, downtime_in_sec):

e2e/libs/keywords/longhorn_deploy_keywords.py

@@ -1,4 +1,6 @@
 from longhorn_deploy import LonghornDeploy
+
+
 class longhorn_deploy_keywords:
 
     def __init__(self):

e2e/libs/keywords/network_keywords.py

@@ -44,6 +44,6 @@ def disconnect_node_network_without_waiting_completion(self, node_name, disconne
 
     def drop_pod_egress_traffic(self, pod_name, drop_time_in_sec):
         drop_pod_egress_traffic(pod_name, drop_time_in_sec)
-    
+
     def wait_for_block_network_pod_completed(self, pod_name, status, namespace='default'):
         wait_for_pod_status(pod_name, status, namespace)

e2e/libs/keywords/node_keywords.py

@@ -1,6 +1,8 @@
 from node import Node
+
 from utility.utility import logging
 
+
 class node_keywords:
 
     def __init__(self):
@@ -32,6 +34,10 @@ def disable_default_disk(self, node_name):
     def enable_default_disk(self, node_name):
         self.node.set_default_disk_scheduling(node_name, allowScheduling=True)
 
+    def set_node(self, node_name, allowScheduling=True, evictionRequested=False):
+        logging(f"Setting node {node_name}; scheduling={allowScheduling}; evictionRequested={evictionRequested}")
+        self.node.set_node(node_name, allowScheduling, evictionRequested)
+
     def disable_node_scheduling(self, node_name):
         self.node.set_node_scheduling(node_name, allowScheduling=False)
 

e2e/libs/keywords/persistentvolume_keywords.py

@@ -1,4 +1,5 @@
 from persistentvolume import PersistentVolume
+
 from utility.utility import logging
 
 

e2e/libs/keywords/replica_keywords.py

@@ -1,5 +1,6 @@
 from replica import Replica
 
+
 class replica_keywords:
 
     def __init__(self):

e2e/libs/keywords/setting_keywords.py

@@ -1,5 +1,6 @@
 from setting import Setting
 
+
 class setting_keywords:
 
     def __init__(self):

e2e/libs/keywords/sharemanager_keywords.py

@@ -7,7 +7,7 @@
 
 from utility.utility import get_retry_count_and_interval
 from utility.utility import logging
-
+from utility.utility import get_pod, delete_pod
 
 class sharemanager_keywords:
 
@@ -48,14 +48,32 @@ def wait_for_sharemanagers_deleted(self, name=[]):
 
         assert AssertionError, f"Failed to wait for all sharemanagers to be deleted"
 
-    def delete_sharemanager(self, name):
-        return self.sharemanager.delete(name)
 
-    def delete_sharemanager_and_wait_for_recreation(self, name):        
-        sharemanager = self.sharemanager.get(name)
-        last_creation_time = sharemanager["metadata"]["creationTimestamp"]        
-        self.sharemanager.delete(name)
-        self.sharemanager.wait_for_restart(name, last_creation_time)
+    def delete_sharemanager_pod_and_wait_for_recreation(self, name):
+        sharemanager_pod_name = "share-manager-" + name
+        sharemanager_pod = get_pod(sharemanager_pod_name, "longhorn-system")
+        last_creation_time = sharemanager_pod.metadata.creation_timestamp
+        delete_pod(sharemanager_pod_name, "longhorn-system")
+
+        retry_count, retry_interval = get_retry_count_and_interval()
+        for i in range(retry_count):
+            time.sleep(retry_interval)
+            sharemanager_pod = get_pod(sharemanager_pod_name, "longhorn-system")
+            if sharemanager_pod == None:
+                continue
+            creation_time = sharemanager_pod.metadata.creation_timestamp
+            if creation_time > last_creation_time:
+                return
+
+        assert False, f"sharemanager pod {sharemanager_pod_name} not recreated"
+
+
+    def wait_for_share_manager_pod_running(self, name):
+        sharemanager_pod_name = "share-manager-" + name
+        retry_count, retry_interval = get_retry_count_and_interval()
+        for i in range(retry_count):
+            sharemanager_pod = get_pod(sharemanager_pod_name, "longhorn-system")
+            if sharemanager_pod.status.phase == "Running":
+                return
 
-    def wait_for_share_manager_running(self, name):
-        return self.sharemanager.wait_for_running(name)
+        assert False, f"sharemanager pod {sharemanager_pod_name} not running"

e2e/libs/keywords/statefulset_keywords.py

@@ -15,7 +15,6 @@
 from workload.workload import get_workload_volume_name
 
 
-
 class statefulset_keywords:
 
     def __init__(self):

e2e/libs/keywords/storageclass_keywords.py

@@ -1,6 +1,7 @@
 from utility.utility import logging
 from storageclass import StorageClass
 
+
 class storageclass_keywords:
 
     def __init__(self):

e2e/libs/keywords/workload_keywords.py

@@ -29,7 +29,6 @@
 from utility.constant import ANNOT_EXPANDED_SIZE
 from utility.constant import LABEL_LONGHORN_COMPONENT
 from utility.utility import logging
-from node.node import Node
 
 from volume import Volume
 from volume.constant import MEBIBYTE

e2e/libs/node/node.py

@@ -10,7 +10,6 @@
 from utility.utility import get_retry_count_and_interval
 from utility.utility import logging
 
-from k8s.k8s import uncordon_node
 
 class Node:
 
@@ -129,6 +128,28 @@ def filter_nodes(nodes, condition):
         elif role == "worker":
             return worker_nodes
 
+    def set_node(self, node_name: str, allowScheduling: bool, evictionRequested: bool) -> object:
+        for _ in range(self.retry_count):
+            try:
+                node = get_longhorn_client().by_id_node(node_name)
+
+                get_longhorn_client().update(
+                    node,
+                    allowScheduling=allowScheduling,
+                    evictionRequested=evictionRequested
+                )
+
+                node = get_longhorn_client().by_id_node(node_name)
+                assert node.allowScheduling == allowScheduling
+                assert node.evictionRequested == evictionRequested
+                return node
+            except Exception as e:
+                logging(f"Updating node {node_name} error: {e}")
+
+            time.sleep(self.retry_interval)
+
+        raise AssertionError(f"Updating node {node_name} failed")
+
     def set_node_scheduling(self, node_name, allowScheduling=True, retry=False):
         node = get_longhorn_client().by_id_node(node_name)
 

e2e/libs/volume/crd.py

@@ -162,8 +162,6 @@ def detach(self, volume_name, node_name):
             # https://github.com/longhorn/longhorn/issues/3715
             if e.reason != "Not Found":
                 Exception(f'exception for patching volumeattachments:', e)
-        if len(body['spec']['attachmentTickets']) == 0:
-            self.wait_for_volume_state(volume_name, "detached")
 
     def get(self, volume_name):
         return self.obj_api.get_namespaced_custom_object(

e2e/tests/negative/component_resilience.robot

@@ -174,8 +174,8 @@ Test Longhorn dynamic provisioned RWX volume recovery
         And Wait until volume of deployment 0 replica rebuilding started on replica node
         Then Delete instance-manager of deployment 0 volume and wait for recover
 
-        When Delete sharemanager of deployment 0 and wait for recreation
-        And Wait for sharemanager of deployment 0 running
+        When Delete sharemanager pod of deployment 0 and wait for recreation
+        And Wait for sharemanager pod of deployment 0 running
         And Wait for deployment 0 pods stable
         And Check deployment 0 data in file data.txt is intact
     END

e2e/tests/negative/network_disconnect.robot

@@ -164,4 +164,5 @@ Node Disconnect With Statefulset
     And Wait for volume 0 healthy
 
     And Detach volume 0 from attached node
+    And Wait for volume 0 detached
     Then Wait for volume 0 detached

e2e/tests/negative/stress_cpu.robot

@@ -41,6 +41,7 @@ Stress Volume Node CPU When Volume Is Detaching and Attaching
     FOR    ${i}    IN RANGE    ${LOOP_COUNT}
         When Stress CPU of node with volume 0
         And Detach volume 0
+        And Wait for volume 0 detached
         And Attach volume 0
         And Wait for volume 0 healthy
         Then Check volume 0 data is intact

e2e/tests/negative/stress_filesystem.robot

@@ -42,6 +42,7 @@ Stress Volume Node Filesystem When Volume Is Detaching and Attaching
 
     FOR    ${i}    IN RANGE    ${LOOP_COUNT}
         And Detach volume 0
+        And Wait for volume 0 detached
         And Attach volume 0
         And Wait for volume 0 healthy
         And Check volume 0 data is intact

e2e/tests/negative/stress_memory.robot

@@ -43,6 +43,7 @@ Stress Volume Node Memory When Volume Is Detaching and Attaching
         When Stress memory of node with volume 0
 
         And Detach volume 0
+        And Wait for volume 0 detached
         And Attach volume 0
         And Wait for volume 0 healthy
         Then Check volume 0 data is intact

e2e/tests/regression/test_backing_image.robot

@@ -30,6 +30,7 @@ Test Backing Image Basic Operation
     And Verify clean up backing image bi from a disk will fail
     And Verify delete backing image bi will fail
     And Detach volume 0
+    And Wait for volume 0 detached
     And Delete volume 0
     And Clean up backing image bi from a disk
     And Delete backing image bi

e2e/tests/regression/test_backup.robot

@@ -102,6 +102,7 @@ Test Incremental Restore
     And Wait for volume 3 healthy
     And Check volume 3 data is backup 2
     And Detach volume 3
+    And Wait for volume 3 detached
 
     When Create persistentvolume for volume 3
     And Create persistentvolumeclaim for volume 3

e2e/tests/regression/test_basic.robot

@@ -53,6 +53,7 @@ Test Volume Basic
     Then Check volume 0 data is intact
 
     And Detach volume 0
+    And Wait for volume 0 detached
     And Delete volume 0
 
 Test Snapshot
@@ -80,11 +81,13 @@ Test Snapshot
     And Check volume 0 data is data 2
 
     When Detach volume 0
+    And Wait for volume 0 detached
     And Attach volume 0 in maintenance mode
     And Wait for volume 0 healthy
 
     And Revert volume 0 to snapshot 1
     And Detach volume 0
+    And Wait for volume 0 detached
     And Attach volume 0
     And Wait for volume 0 healthy
     Then Check volume 0 data is data 1

e2e/tests/regression/test_settings.robot

@@ -64,6 +64,7 @@ Test Setting Concurrent Rebuild Limit
     # Test the setting won't intervene normal attachment.
     Given Set setting concurrent-replica-rebuild-per-node-limit to 1
     When Detach volume 1
+    And Wait for volume 1 detached
     And Delete volume 0 replica on replica node
     And Wait until volume 0 replica rebuilding started on replica node
     And Attach volume 1

e2e/tests/regression/test_v2.robot

@@ -32,6 +32,7 @@ Test V2 Volume Basic
     And Write data to volume 0
     Then Check volume 0 data is intact
     And Detach volume 0
+    And Wait for volume 0 detached
     And Delete volume 0
 
 Degraded Volume Replica Rebuilding