test: fix flaky test case test_offline_node

Signed-off-by: Yang Chiu <[email protected]>
longhorn · Dec 29, 2023 · 0e52aa5 · 0e52aa5
1 parent a3ea9f1
commit 0e52aa5
Show file tree

Hide file tree

Showing 11 changed files with 72 additions and 38 deletions.
diff --git a/manager/integration/deploy/test.yaml b/manager/integration/deploy/test.yaml
@@ -55,6 +55,8 @@ spec:
           fieldPath: spec.nodeName
     - name: MANAGED_K8S_CLUSTER
       value: "false"
+    - name: RESOURCE_SUFFIX
+      value: ""
     volumeMounts:
     - name: dev
       mountPath: /dev

diff --git a/manager/integration/tests/aws.py b/manager/integration/tests/aws.py
@@ -32,12 +32,17 @@ def __init__(self):
             region_name=default_region)
 
     def instance_id_by_ip(self, instance_ip):
+        resource_suffix = os.getenv("RESOURCE_SUFFIX")
         response = aws.ec2_client.describe_instances(
             Filters=[
                 {
                     'Name': 'private-ip-address',
                     'Values': [instance_ip]
                 },
+                {
+                    'Name': 'tag:Name',
+                    'Values': [f"*{resource_suffix}*"]
+                }
             ],
         )
 

diff --git a/manager/integration/tests/test_infra.py b/manager/integration/tests/test_infra.py
@@ -141,35 +141,23 @@ def is_infra_k3s():
 @pytest.fixture
 def reset_cluster_ready_status(request):
     yield
-    node_worker_label = 'node-role.kubernetes.io/worker'
     node_controlplane_label = 'node-role.kubernetes.io/control-plane'
     node_ip_annotation = "flannel.alpha.coreos.com/public-ip"
 
     k8s_api_client = get_core_api_client()
     longhorn_api_client = get_longhorn_api_client()
     cloudprovider = detect_cloudprovider()
 
-    k3s = is_infra_k3s()
-
     print('==> test completed! reset cluster ready status ...')
 
     for node_item in k8s_api_client.list_node().items:
 
-        if k3s is True:
-            if node_controlplane_label not in node_item.metadata.labels:
-                node_name = node_item.metadata.name
-                node_ip = node_item.metadata.annotations[node_ip_annotation]
-                node = cloudprovider.instance_id_by_ip(node_ip)
-            else:
-                continue
-
+        if node_controlplane_label not in node_item.metadata.labels:
+            node_name = node_item.metadata.name
+            node_ip = node_item.metadata.annotations[node_ip_annotation]
+            node = cloudprovider.instance_id_by_ip(node_ip)
         else:
-            if node_worker_label in node_item.metadata.labels and \
-                    node_item.metadata.labels[node_worker_label] == 'true':
-                node_name = node_item.metadata.name
-                node = cloudprovider.instance_id(node_name)
-            else:
-                continue
+            continue
 
         if is_node_ready_k8s(node_name, k8s_api_client) is False:
 
@@ -199,7 +187,6 @@ def test_offline_node(reset_cluster_ready_status):
     1. Bring down one of the nodes in Kuberntes cluster (avoid current node)
     2. Make sure the Longhorn node state become `down`
     """
-    node_worker_label = 'node-role.kubernetes.io/worker'
     pod_lable_selector = "longhorn-test=test-job"
     node_controlplane_label = 'node-role.kubernetes.io/control-plane'
     node_ip_annotation = "flannel.alpha.coreos.com/public-ip"
@@ -214,27 +201,15 @@ def test_offline_node(reset_cluster_ready_status):
         if pod.metadata.name == "longhorn-test":
             longhorn_test_node_name = pod.spec.node_name
 
-    k3s = is_infra_k3s()
-
     for node_item in k8s_api_client.list_node().items:
-        if k3s is True:
-            if node_controlplane_label not in node_item.metadata.labels:
-                node_name = node_item.metadata.name
-                node_ip = node_item.metadata.annotations[node_ip_annotation]
-                if node_name == longhorn_test_node_name:
-                    continue
-                else:
-                    node = cloudprovider.instance_id_by_ip(node_ip)
-                    break
-        else:
-            if node_worker_label in node_item.metadata.labels and \
-                    node_item.metadata.labels[node_worker_label] == 'true':
-                node_name = node_item.metadata.name
-                if node_name == longhorn_test_node_name:
-                    continue
-                else:
-                    node = cloudprovider.instance_id(node_name)
-                    break
+        if node_controlplane_label not in node_item.metadata.labels:
+            node_name = node_item.metadata.name
+            node_ip = node_item.metadata.annotations[node_ip_annotation]
+            if node_name == longhorn_test_node_name:
+                continue
+            else:
+                node = cloudprovider.instance_id_by_ip(node_ip)
+                break
 
     print(f'==> stop node: {node_name}')
 

diff --git a/test_framework/scripts/longhorn-setup.sh b/test_framework/scripts/longhorn-setup.sh
@@ -323,6 +323,8 @@ run_longhorn_upgrade_test(){
 
   yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env[4].value="'${LONGHORN_UPGRADE_TYPE}'"' ${LONGHORN_UPGRADE_TESTS_MANIFEST_FILE_PATH}
 
+  yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env[7].value="'${RESOURCE_SUFFIX}'"' ${LONGHORN_UPGRADE_TESTS_MANIFEST_FILE_PATH}
+
   kubectl apply -f ${LONGHORN_UPGRADE_TESTS_MANIFEST_FILE_PATH}
 
   # wait upgrade test pod to start running

diff --git a/test_framework/scripts/terraform-setup.sh b/test_framework/scripts/terraform-setup.sh
@@ -32,6 +32,8 @@ terraform_setup(){
   if [[ "${TF_VAR_create_load_balancer}" == true ]]; then
     terraform -chdir=${TF_VAR_tf_workspace}/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw load_balancer_url > ${TF_VAR_tf_workspace}/load_balancer_url
   fi
+
+  export RESOURCE_SUFFIX=$(terraform -chdir=${TF_VAR_tf_workspace}/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw resource_suffix)
 }
 
 

diff --git a/test_framework/terraform/aws/oracle/output.tf b/test_framework/terraform/aws/oracle/output.tf
@@ -46,3 +46,11 @@ output "load_balancer_url" {
 
   value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null
 }
+
+output "resource_suffix" {
+  depends_on = [
+    random_string.random_suffix
+  ]
+
+  value = random_string.random_suffix.id
+}
diff --git a/test_framework/terraform/aws/rhel/output.tf b/test_framework/terraform/aws/rhel/output.tf
@@ -46,3 +46,11 @@ output "load_balancer_url" {
 
   value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null
 }
+
+output "resource_suffix" {
+  depends_on = [
+    random_string.random_suffix
+  ]
+
+  value = random_string.random_suffix.id
+}
diff --git a/test_framework/terraform/aws/rockylinux/output.tf b/test_framework/terraform/aws/rockylinux/output.tf
@@ -46,3 +46,11 @@ output "load_balancer_url" {
 
   value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null
 }
+
+output "resource_suffix" {
+  depends_on = [
+    random_string.random_suffix
+  ]
+
+  value = random_string.random_suffix.id
+}
diff --git a/test_framework/terraform/aws/sle-micro/output.tf b/test_framework/terraform/aws/sle-micro/output.tf
@@ -45,3 +45,11 @@ output "load_balancer_url" {
 
   value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null
 }
+
+output "resource_suffix" {
+  depends_on = [
+    random_string.random_suffix
+  ]
+
+  value = random_string.random_suffix.id
+}
diff --git a/test_framework/terraform/aws/sles/output.tf b/test_framework/terraform/aws/sles/output.tf
@@ -78,3 +78,11 @@ output "controlplane_public_ip" {
   ]
   value = aws_eip.lh_aws_eip_controlplane[0].public_ip
 }
+
+output "resource_suffix" {
+  depends_on = [
+    random_string.random_suffix
+  ]
+
+  value = random_string.random_suffix.id
+}
diff --git a/test_framework/terraform/aws/ubuntu/output.tf b/test_framework/terraform/aws/ubuntu/output.tf
@@ -46,3 +46,11 @@ output "load_balancer_url" {
 
   value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null
 }
+
+output "resource_suffix" {
+  depends_on = [
+    random_string.random_suffix
+  ]
+
+  value = random_string.random_suffix.id
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -32,6 +32,8 @@ terraform_setup(){ @@
       if [[ "${TF_VAR_create_load_balancer}" == true ]]; then
         terraform -chdir=${TF_VAR_tf_workspace}/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw load_balancer_url > ${TF_VAR_tf_workspace}/load_balancer_url
       fi
+      export RESOURCE_SUFFIX=$(terraform -chdir=${TF_VAR_tf_workspace}/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw resource_suffix)
     }
@@ Expand Down @@