longhorn · c3y1huang · Nov 15, 2024 · Nov 14, 2024 · Nov 14, 2024 · Nov 14, 2024
@@ -11,6 +11,11 @@ Add ${disk_type} type disk ${disk_path} for all worker nodes
         add_disk    ${worker_node}    ${disk_type}    ${disk_path}
     END
 
+Set node ${node_id} with
+    [Arguments]    &{config}
+    ${node_name} =    get_node_by_index    ${node_id}
+    set_node   ${node_name}    &{config}
+
 Disable node ${node_id} scheduling
     ${node_name} =    get_node_by_index    ${node_id}
     disable_node_scheduling    ${node_name}

@@ -119,6 +119,13 @@ Wait for volume ${volume_id} detached
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     wait_for_volume_detached    ${volume_name}
 
+Assert volume ${volume_id} remains detached for at least ${period} seconds
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    FOR    ${i}    IN RANGE    ${period}
+        wait_for_volume_detached    ${volume_name}
+        Sleep    1
+    END
+
 Wait for volume ${volume_id} faulted
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     wait_for_volume_faulted    ${volume_name}

@@ -23,7 +23,7 @@ def cleanup_backing_images(self):
 
     def delete_backing_image_manager(self, name):
         self.backing_image.delete_backing_image_manager(name)
-    
+
     def wait_all_backing_image_managers_running(self):
         self.backing_image.wait_all_backing_image_managers_running()
 

@@ -1,4 +1,5 @@
 from backup import Backup
+
 from utility.utility import logging
 from utility.utility import get_backupstore
 

@@ -1,8 +1,9 @@
 from backupstore import Nfs, S3
-from utility.utility import get_backupstore
-from utility.utility import logging
+
 import os
 
+from utility.utility import get_backupstore
+
 
 class backupstore_keywords:
 

@@ -1,5 +1,6 @@
 from node import Node
 from node_exec import NodeExec
+
 from utility.utility import init_k8s_api_client
 from utility.utility import generate_name_with_suffix
 

@@ -1,5 +1,5 @@
 from engine_image import EngineImage
-from utility.utility import logging
+
 
 class engine_image_keywords:
 

@@ -1,5 +1,5 @@
 from engine import Engine
-from utility.utility import logging
+
 
 class engine_keywords:
 

@@ -1,6 +1,7 @@
-import os
 from robot.libraries.BuiltIn import BuiltIn
 
+import os
+
 from host import Harvester, Aws
 from host.constant import NODE_REBOOT_DOWN_TIME_SECOND
 

@@ -1,5 +1,7 @@
-import asyncio
 from robot.libraries.BuiltIn import BuiltIn
+
+import asyncio
+
 from k8s.k8s import restart_kubelet
 from k8s.k8s import delete_node
 from k8s.k8s import drain_node, force_drain_node
@@ -10,9 +12,12 @@
 from k8s.k8s import get_instance_manager_on_node
 from k8s.k8s import check_instance_manager_pdb_not_exist
 from k8s.k8s import wait_for_namespace_pods_running
-from utility.utility import logging
+
 from node import Node
 
+from utility.utility import logging
+
+
 class k8s_keywords:
 
     async def restart_kubelet(self, node_name, downtime_in_sec):

@@ -1,4 +1,6 @@
 from longhorn_deploy import LonghornDeploy
+
+
 class longhorn_deploy_keywords:
 
     def __init__(self):

@@ -44,6 +44,6 @@ def disconnect_node_network_without_waiting_completion(self, node_name, disconne
 
     def drop_pod_egress_traffic(self, pod_name, drop_time_in_sec):
         drop_pod_egress_traffic(pod_name, drop_time_in_sec)
-    
+
     def wait_for_block_network_pod_completed(self, pod_name, status, namespace='default'):
         wait_for_pod_status(pod_name, status, namespace)
@@ -1,6 +1,8 @@
 from node import Node
+
 from utility.utility import logging
 
+
 class node_keywords:
 
     def __init__(self):
@@ -32,6 +34,10 @@ def disable_default_disk(self, node_name):
     def enable_default_disk(self, node_name):
         self.node.set_default_disk_scheduling(node_name, allowScheduling=True)
 
+    def set_node(self, node_name, allowScheduling=True, evictionRequested=False):
+        logging(f"Setting node {node_name}; scheduling={allowScheduling}; evictionRequested={evictionRequested}")
+        self.node.set_node(node_name, allowScheduling, evictionRequested)
+
     def disable_node_scheduling(self, node_name):
         self.node.set_node_scheduling(node_name, allowScheduling=False)
 

@@ -1,4 +1,5 @@
 from persistentvolume import PersistentVolume
+
 from utility.utility import logging
 
 

@@ -1,5 +1,6 @@
 from replica import Replica
 
+
 class replica_keywords:
 
     def __init__(self):

@@ -1,5 +1,6 @@
 from setting import Setting
 
+
 class setting_keywords:
 
     def __init__(self):

@@ -51,9 +51,9 @@ def wait_for_sharemanagers_deleted(self, name=[]):
     def delete_sharemanager(self, name):
         return self.sharemanager.delete(name)
 
-    def delete_sharemanager_and_wait_for_recreation(self, name):        
+    def delete_sharemanager_and_wait_for_recreation(self, name):
         sharemanager = self.sharemanager.get(name)
-        last_creation_time = sharemanager["metadata"]["creationTimestamp"]        
+        last_creation_time = sharemanager["metadata"]["creationTimestamp"]
         self.sharemanager.delete(name)
         self.sharemanager.wait_for_restart(name, last_creation_time)
 

@@ -15,7 +15,6 @@
 from workload.workload import get_workload_volume_name
 
 
-
 class statefulset_keywords:
 
     def __init__(self):

@@ -1,6 +1,7 @@
 from utility.utility import logging
 from storageclass import StorageClass
 
+
 class storageclass_keywords:
 
     def __init__(self):

@@ -29,7 +29,6 @@
 from utility.constant import ANNOT_EXPANDED_SIZE
 from utility.constant import LABEL_LONGHORN_COMPONENT
 from utility.utility import logging
-from node.node import Node
 
 from volume import Volume
 from volume.constant import MEBIBYTE

@@ -10,7 +10,6 @@
 from utility.utility import get_retry_count_and_interval
 from utility.utility import logging
 
-from k8s.k8s import uncordon_node
 
 class Node:
 
@@ -129,6 +128,28 @@ def filter_nodes(nodes, condition):
         elif role == "worker":
             return worker_nodes
 
+    def set_node(self, node_name: str, allowScheduling: bool, evictionRequested: bool) -> object:
+        for _ in range(self.retry_count):
+            try:
+                node = get_longhorn_client().by_id_node(node_name)
+
+                get_longhorn_client().update(
+                    node,
+                    allowScheduling=allowScheduling,
+                    evictionRequested=evictionRequested
+                )
+
+                node = get_longhorn_client().by_id_node(node_name)
+                assert node.allowScheduling == allowScheduling
+                assert node.evictionRequested == evictionRequested
+                return node
+            except Exception as e:
+                logging(f"Updating node {node_name} error: {e}")
+
+            time.sleep(self.retry_interval)
+
+        raise AssertionError(f"Updating node {node_name} failed")
+
     def set_node_scheduling(self, node_name, allowScheduling=True, retry=False):
         node = get_longhorn_client().by_id_node(node_name)
 

@@ -162,8 +162,6 @@ def detach(self, volume_name, node_name):
             # https://github.com/longhorn/longhorn/issues/3715
             if e.reason != "Not Found":
                 Exception(f'exception for patching volumeattachments:', e)
-        if len(body['spec']['attachmentTickets']) == 0:
-            self.wait_for_volume_state(volume_name, "detached")
 
     def get(self, volume_name):
         return self.obj_api.get_namespaced_custom_object(

@@ -164,4 +164,5 @@ Node Disconnect With Statefulset
     And Wait for volume 0 healthy
 
     And Detach volume 0 from attached node
+    And Wait for volume 0 detached
     Then Wait for volume 0 detached
@@ -41,6 +41,7 @@ Stress Volume Node CPU When Volume Is Detaching and Attaching
     FOR    ${i}    IN RANGE    ${LOOP_COUNT}
         When Stress CPU of node with volume 0
         And Detach volume 0
+        And Wait for volume 0 detached
         And Attach volume 0
         And Wait for volume 0 healthy
         Then Check volume 0 data is intact

@@ -42,6 +42,7 @@ Stress Volume Node Filesystem When Volume Is Detaching and Attaching
 
     FOR    ${i}    IN RANGE    ${LOOP_COUNT}
         And Detach volume 0
+        And Wait for volume 0 detached
         And Attach volume 0
         And Wait for volume 0 healthy
         And Check volume 0 data is intact

@@ -43,6 +43,7 @@ Stress Volume Node Memory When Volume Is Detaching and Attaching
         When Stress memory of node with volume 0
 
         And Detach volume 0
+        And Wait for volume 0 detached
         And Attach volume 0
         And Wait for volume 0 healthy
         Then Check volume 0 data is intact

@@ -30,6 +30,7 @@ Test Backing Image Basic Operation
     And Verify clean up backing image bi from a disk will fail
     And Verify delete backing image bi will fail
     And Detach volume 0
+    And Wait for volume 0 detached
     And Delete volume 0
     And Clean up backing image bi from a disk
     And Delete backing image bi
@@ -102,6 +102,7 @@ Test Incremental Restore
     And Wait for volume 3 healthy
     And Check volume 3 data is backup 2
     And Detach volume 3
+    And Wait for volume 3 detached
 
     When Create persistentvolume for volume 3
     And Create persistentvolumeclaim for volume 3

@@ -53,6 +53,7 @@ Test Volume Basic
     Then Check volume 0 data is intact
 
     And Detach volume 0
+    And Wait for volume 0 detached
     And Delete volume 0
 
 Test Snapshot
@@ -80,11 +81,13 @@ Test Snapshot
     And Check volume 0 data is data 2
 
     When Detach volume 0
+    And Wait for volume 0 detached
     And Attach volume 0 in maintenance mode
     And Wait for volume 0 healthy
 
     And Revert volume 0 to snapshot 1
     And Detach volume 0
+    And Wait for volume 0 detached
     And Attach volume 0
     And Wait for volume 0 healthy
     Then Check volume 0 data is data 1

@@ -64,6 +64,7 @@ Test Setting Concurrent Rebuild Limit
     # Test the setting won't intervene normal attachment.
     Given Set setting concurrent-replica-rebuild-per-node-limit to 1
     When Detach volume 1
+    And Wait for volume 1 detached
     And Delete volume 0 replica on replica node
     And Wait until volume 0 replica rebuilding started on replica node
     And Attach volume 1

@@ -32,6 +32,7 @@ Test V2 Volume Basic
     And Write data to volume 0
     Then Check volume 0 data is intact
     And Detach volume 0
+    And Wait for volume 0 detached
     And Delete volume 0
 
 Degraded Volume Replica Rebuilding

@@ -9,6 +9,8 @@ Resource    ../keywords/longhorn.resource
 Resource    ../keywords/persistentvolumeclaim.resource
 Resource    ../keywords/setting.resource
 Resource    ../keywords/workload.resource
+Resource    ../keywords/node.resource
+Resource    ../keywords/volume.resource
 
 Test Setup    Set test environment
 Test Teardown    Cleanup test resources
@@ -42,3 +44,19 @@ Test RWX volume data integrity after CSI plugin pod restart
         ...    longhorn-csi-plugin
 
     Then Check deployment 0 data in file data.txt is intact
+
+Test detached volume should not reattach after node eviction
+    [Tags]    volume    node-eviction
+    [Documentation]    Test detached volume should not reattach after node eviction.
+    ...
+    ...                Issue: https://github.com/longhorn/longhorn/issues/9781
+
+    Given Create volume 0 with    dataEngine=${DATA_ENGINE}
+    And Attach volume 0
+    And Wait for volume 0 healthy
+
+    When Detach volume 0
+    And Set node 1 with    allowScheduling=false    evictionRequested=true
+
+    Then Wait for volume 0 detached
+    And Assert volume 0 remains detached for at least 60 seconds
Original file line number	Diff line number	Diff line change
		@@ -1,4 +1,5 @@
		from persistentvolume import PersistentVolume

		from utility.utility import logging


Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -15,7 +15,6 @@
		from workload.workload import get_workload_volume_name



		class statefulset_keywords:

		def __init__(self):
Expand Down