diff --git a/manager/integration/tests/common.py b/manager/integration/tests/common.py index 47dd4daee9..2c5a79ba8c 100644 --- a/manager/integration/tests/common.py +++ b/manager/integration/tests/common.py @@ -273,6 +273,9 @@ K8S_ZONE_LABEL = "topology.kubernetes.io/zone" +K8S_GKE_OS_DISTRO_LABEL = "cloud.google.com/gke-os-distribution" +K8S_GKE_OS_DISTRO_COS = "cos" + K8S_CLUSTER_AUTOSCALER_EVICT_KEY = \ "cluster-autoscaler.kubernetes.io/safe-to-evict" K8S_CLUSTER_AUTOSCALER_SCALE_DOWN_DISABLED_KEY = \ @@ -3346,12 +3349,32 @@ def set_k8s_node_label(core_api, node_name, key, value): core_api.patch_node(node_name, body=payload) +def is_k8s_node_label(core_api, label_key, label_value, node_name): + node = core_api.read_node(node_name) + + if label_key in node.metadata.labels: + if node.metadata.labels[label_key] == label_value: + return True + return False + + def set_k8s_node_zone_label(core_api, node_name, zone_name): + if is_k8s_node_label(core_api, K8S_ZONE_LABEL, zone_name, node_name): + return + k8s_zone_label = get_k8s_zone_label() set_k8s_node_label(core_api, node_name, k8s_zone_label, zone_name) +def is_k8s_node_gke_cos(core_api): + node = core_api.read_node(get_self_host_id()) + return is_k8s_node_label(core_api, + K8S_GKE_OS_DISTRO_LABEL, + K8S_GKE_OS_DISTRO_COS, + node.metadata.name) + + def get_k8s_zone_label(): ver_api = get_version_api_client() k8s_ver_data = ver_api.get_code() diff --git a/manager/integration/tests/test_zone.py b/manager/integration/tests/test_zone.py index 5965a7bf39..14f1d05c6e 100644 --- a/manager/integration/tests/test_zone.py +++ b/manager/integration/tests/test_zone.py @@ -30,6 +30,7 @@ from common import get_host_replica_count from common import get_k8s_zone_label +from common import is_k8s_node_gke_cos from common import set_k8s_node_zone_label from common import set_node_cordon from common import set_node_tags @@ -337,10 +338,16 @@ def test_replica_auto_balance_zone_least_effort(client, core_api, volume_name): n1, n2, n3 = client.list_node() - set_k8s_node_zone_label(core_api, n1.name, ZONE1) - set_k8s_node_zone_label(core_api, n2.name, ZONE2) - set_k8s_node_zone_label(core_api, n3.name, ZONE3) - wait_longhorn_node_zone_updated(client) + # The GKE zone label is periodically updated with the actual zone. + # Invoke _set_k8s_node_zone_label to refresh the zone label with each + # retry iteration to maintain the expected zone label. + def _set_k8s_node_zone_label(): + set_k8s_node_zone_label(core_api, n1.name, ZONE1) + set_k8s_node_zone_label(core_api, n2.name, ZONE2) + set_k8s_node_zone_label(core_api, n3.name, ZONE3) + wait_longhorn_node_zone_updated(client) + + _set_k8s_node_zone_label() client.update(n2, allowScheduling=False) client.update(n3, allowScheduling=False) @@ -361,6 +368,9 @@ def test_replica_auto_balance_zone_least_effort(client, core_api, volume_name): if z1_r_count == 6 and z2_r_count == z3_r_count == 0: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert z1_r_count == 6 assert z2_r_count == 0 @@ -380,6 +390,9 @@ def test_replica_auto_balance_zone_least_effort(client, core_api, volume_name): if z2_r_count != 0 and all_r_count == n_replicas: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert z1_r_count != z2_r_count assert z2_r_count != 0 @@ -399,6 +412,9 @@ def test_replica_auto_balance_zone_least_effort(client, core_api, volume_name): if z3_r_count != 0 and all_r_count == n_replicas: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert z1_r_count != z3_r_count assert z2_r_count != 0 @@ -442,10 +458,16 @@ def test_replica_auto_balance_zone_best_effort(client, core_api, volume_name): n1, n2, n3 = client.list_node() - set_k8s_node_zone_label(core_api, n1.name, ZONE1) - set_k8s_node_zone_label(core_api, n2.name, ZONE2) - set_k8s_node_zone_label(core_api, n3.name, ZONE3) - wait_longhorn_node_zone_updated(client) + # The GKE zone label is periodically updated with the actual zone. + # Invoke _set_k8s_node_zone_label to refresh the zone label with each + # retry iteration to maintain the expected zone label. + def _set_k8s_node_zone_label(): + set_k8s_node_zone_label(core_api, n1.name, ZONE1) + set_k8s_node_zone_label(core_api, n2.name, ZONE2) + set_k8s_node_zone_label(core_api, n3.name, ZONE3) + wait_longhorn_node_zone_updated(client) + + _set_k8s_node_zone_label() client.update(n2, allowScheduling=False) client.update(n3, allowScheduling=False) @@ -466,6 +488,9 @@ def test_replica_auto_balance_zone_best_effort(client, core_api, volume_name): if z1_r_count == 6 and z2_r_count == z3_r_count == 0: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert z1_r_count == 6 assert z2_r_count == 0 @@ -484,6 +509,9 @@ def test_replica_auto_balance_zone_best_effort(client, core_api, volume_name): if z1_r_count == z2_r_count == 3 and z3_r_count == 0: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL_LONG) assert z1_r_count == 3 assert z2_r_count == 3 @@ -502,6 +530,9 @@ def test_replica_auto_balance_zone_best_effort(client, core_api, volume_name): if z1_r_count == z2_r_count == z3_r_count == 2: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL_LONG) assert z1_r_count == 2 assert z2_r_count == 2 @@ -536,10 +567,17 @@ def test_replica_auto_balance_when_disabled_disk_scheduling_in_zone(client, core # Assign nodes to respective zones node1, node2, node3 = client.list_node() - set_k8s_node_zone_label(core_api, node1.name, ZONE1) - set_k8s_node_zone_label(core_api, node2.name, ZONE2) - set_k8s_node_zone_label(core_api, node3.name, ZONE3) - wait_longhorn_node_zone_updated(client) + + # The GKE zone label is periodically updated with the actual zone. + # Invoke _set_k8s_node_zone_label to refresh the zone label with each + # retry iteration to maintain the expected zone label. + def _set_k8s_node_zone_label(): + set_k8s_node_zone_label(core_api, node1.name, ZONE1) + set_k8s_node_zone_label(core_api, node2.name, ZONE2) + set_k8s_node_zone_label(core_api, node3.name, ZONE3) + wait_longhorn_node_zone_updated(client) + + _set_k8s_node_zone_label() # Disable disk scheduling on node 3 cleanup_node_disks(client, node3.name) @@ -556,6 +594,9 @@ def test_replica_auto_balance_when_disabled_disk_scheduling_in_zone(client, core # Define a function to assert replica count def assert_replica_count(is_stable=False): for _ in range(RETRY_COUNTS): + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) zone3_replica_count = get_zone_replica_count( @@ -614,10 +655,17 @@ def test_replica_auto_balance_when_no_storage_available_in_zone(client, core_api # Assign nodes to respective zones node1, node2, node3 = client.list_node() - set_k8s_node_zone_label(core_api, node1.name, ZONE1) - set_k8s_node_zone_label(core_api, node2.name, ZONE2) - set_k8s_node_zone_label(core_api, node3.name, ZONE3) - wait_longhorn_node_zone_updated(client) + + # The GKE zone label is periodically updated with the actual zone. + # Invoke _set_k8s_node_zone_label to refresh the zone label with each + # retry iteration to maintain the expected zone label. + def _set_k8s_node_zone_label(): + set_k8s_node_zone_label(core_api, node1.name, ZONE1) + set_k8s_node_zone_label(core_api, node2.name, ZONE2) + set_k8s_node_zone_label(core_api, node3.name, ZONE3) + wait_longhorn_node_zone_updated(client) + + _set_k8s_node_zone_label # Fill up the storage on node 3 for _, disk in node3.disks.items(): @@ -638,6 +686,9 @@ def test_replica_auto_balance_when_no_storage_available_in_zone(client, core_api # Define a function to assert replica count def assert_replica_count(is_stable=False): for _ in range(RETRY_COUNTS): + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) zone3_replica_count = get_zone_replica_count( @@ -651,7 +702,11 @@ def assert_replica_count(is_stable=False): client, volume_name, ZONE2, chk_running=True) if is_stable: - assert total_replica_count == num_of_replicas + try: + assert total_replica_count == num_of_replicas + except AssertionError as e: + if not is_k8s_node_gke_cos(core_api): + raise AssertionError(e) elif total_replica_count == num_of_replicas: break @@ -701,10 +756,16 @@ def test_replica_auto_balance_when_replica_on_unschedulable_node(client, core_ap n1, n2, n3 = client.list_node() - set_k8s_node_zone_label(core_api, n1.name, ZONE1) - set_k8s_node_zone_label(core_api, n2.name, ZONE2) - set_k8s_node_zone_label(core_api, n3.name, ZONE3) - wait_longhorn_node_zone_updated(client) + # The GKE zone label is periodically updated with the actual zone. + # Invoke _set_k8s_node_zone_label to refresh the zone label with each + # retry iteration to maintain the expected zone label. + def _set_k8s_node_zone_label(): + set_k8s_node_zone_label(core_api, n1.name, ZONE1) + set_k8s_node_zone_label(core_api, n2.name, ZONE2) + set_k8s_node_zone_label(core_api, n3.name, ZONE3) + wait_longhorn_node_zone_updated(client) + + _set_k8s_node_zone_label() client.update(n2, allowScheduling=True, tags=["AVAIL"]) client.update(n3, allowScheduling=True, tags=["AVAIL"]) @@ -728,6 +789,10 @@ def test_replica_auto_balance_when_replica_on_unschedulable_node(client, core_ap if z1_r_count == 0 and (z2_r_count and z3_r_count == 1): break + + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert z1_r_count == 0 and (z2_r_count and z3_r_count == 1) @@ -751,6 +816,9 @@ def finalizer(): for status in volume.rebuildStatus: assert not status.isRebuilding + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) @@ -809,10 +877,16 @@ def test_replica_auto_balance_zone_best_effort_with_data_locality(client, core_a n1, n2, n3 = client.list_node() - set_k8s_node_zone_label(core_api, n1.name, ZONE1) - set_k8s_node_zone_label(core_api, n2.name, ZONE1) - set_k8s_node_zone_label(core_api, n3.name, ZONE2) - wait_longhorn_node_zone_updated(client) + # The GKE zone label is periodically updated with the actual zone. + # Invoke _set_k8s_node_zone_label to refresh the zone label with each + # retry iteration to maintain the expected zone label. + def _set_k8s_node_zone_label(): + set_k8s_node_zone_label(core_api, n1.name, ZONE1) + set_k8s_node_zone_label(core_api, n2.name, ZONE1) + set_k8s_node_zone_label(core_api, n3.name, ZONE2) + wait_longhorn_node_zone_updated(client) + + _set_k8s_node_zone_label() n_replicas = 2 volume = create_and_check_volume(client, volume_name, @@ -838,6 +912,9 @@ def test_replica_auto_balance_zone_best_effort_with_data_locality(client, core_a duplicate_node = [n1.name, n2.name] duplicate_node.remove(pod_node_name) for _ in range(RETRY_COUNTS): + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + pod_node_r_count = get_host_replica_count( client, volume_name, pod_node_name, chk_running=True) duplicate_node_r_count = get_host_replica_count( @@ -857,6 +934,9 @@ def test_replica_auto_balance_zone_best_effort_with_data_locality(client, core_a client.update(n3, allowScheduling=True) for _ in range(RETRY_COUNTS): + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + pod_node_r_count = get_host_replica_count( client, volume_name, pod_node_name, chk_running=True) duplicate_node_r_count = get_host_replica_count( @@ -882,6 +962,9 @@ def test_replica_auto_balance_zone_best_effort_with_data_locality(client, core_a # loop 3 times and each to wait 5 seconds to ensure there is no # re-scheduling happening. for _ in range(3): + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(5) assert pod_node_r_count == get_host_replica_count( client, volume_name, pod_node_name, chk_running=True) @@ -924,10 +1007,16 @@ def test_replica_auto_balance_node_duplicates_in_multiple_zones(client, core_api n1, n2, n3 = client.list_node() - set_k8s_node_zone_label(core_api, n1.name, ZONE1) - set_k8s_node_zone_label(core_api, n2.name, ZONE2) - set_k8s_node_zone_label(core_api, n3.name, "temp") - wait_longhorn_node_zone_updated(client) + # The GKE zone label is periodically updated with the actual zone. + # Invoke _set_k8s_node_zone_label to refresh the zone label with each + # retry iteration to maintain the expected zone label. + def _set_k8s_node_zone_label(): + set_k8s_node_zone_label(core_api, n1.name, ZONE1) + set_k8s_node_zone_label(core_api, n2.name, ZONE2) + set_k8s_node_zone_label(core_api, n3.name, "temp") + wait_longhorn_node_zone_updated(client) + + _set_k8s_node_zone_label() client.update(n3, allowScheduling=False) @@ -956,6 +1045,10 @@ def test_replica_auto_balance_node_duplicates_in_multiple_zones(client, core_api if n1_r_count == n2_r_count == n3_r_count == 1: break + + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert n1_r_count == 1 assert n2_r_count == 1 @@ -1022,12 +1115,18 @@ def test_replica_auto_balance_zone_best_effort_with_uneven_node_in_zones(client, n1, n2, n3, n4, n5 = client.list_node() - set_k8s_node_zone_label(core_api, n1.name, ZONE1) - set_k8s_node_zone_label(core_api, n2.name, ZONE1) - set_k8s_node_zone_label(core_api, n3.name, ZONE1) - set_k8s_node_zone_label(core_api, n4.name, ZONE2) - set_k8s_node_zone_label(core_api, n5.name, ZONE2) - wait_longhorn_node_zone_updated(client) + # The GKE zone label is periodically updated with the actual zone. + # Invoke _set_k8s_node_zone_label to refresh the zone label with each + # retry iteration to maintain the expected zone label. + def _set_k8s_node_zone_label(): + set_k8s_node_zone_label(core_api, n1.name, ZONE1) + set_k8s_node_zone_label(core_api, n2.name, ZONE1) + set_k8s_node_zone_label(core_api, n3.name, ZONE1) + set_k8s_node_zone_label(core_api, n4.name, ZONE2) + set_k8s_node_zone_label(core_api, n5.name, ZONE2) + wait_longhorn_node_zone_updated(client) + + _set_k8s_node_zone_label() client.update(n2, allowScheduling=False) client.update(n3, allowScheduling=False) @@ -1055,6 +1154,9 @@ def test_replica_auto_balance_zone_best_effort_with_uneven_node_in_zones(client, n2_r_count == n3_r_count == n4_r_count == n5_r_count == 0: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert n1_r_count == 4 assert n2_r_count == 0 @@ -1073,6 +1175,9 @@ def test_replica_auto_balance_zone_best_effort_with_uneven_node_in_zones(client, if z1_r_count == z2_r_count == 2: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert z1_r_count == 2 @@ -1097,6 +1202,9 @@ def test_replica_auto_balance_zone_best_effort_with_uneven_node_in_zones(client, n5_r_count == 0: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert n1_r_count == 1 assert n2_r_count == 1 @@ -1115,6 +1223,9 @@ def test_replica_auto_balance_zone_best_effort_with_uneven_node_in_zones(client, if z1_r_count == z2_r_count == 2: break + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(RETRY_INTERVAL) assert z1_r_count == 2 @@ -1158,10 +1269,16 @@ def test_replica_auto_balance_should_respect_node_selector(client, core_api, vol set_node_tags(client, node, tags=[node_tag]) wait_for_node_tag_update(client, node.name, [node_tag]) - set_k8s_node_zone_label(core_api, n1.name, ZONE1) - set_k8s_node_zone_label(core_api, n2.name, ZONE2) - set_k8s_node_zone_label(core_api, n3.name, "should-not-schedule") - wait_longhorn_node_zone_updated(client) + # The GKE zone label is periodically updated with the actual zone. + # Invoke _set_k8s_node_zone_label to refresh the zone label with each + # retry iteration to maintain the expected zone label. + def _set_k8s_node_zone_label(): + set_k8s_node_zone_label(core_api, n1.name, ZONE1) + set_k8s_node_zone_label(core_api, n2.name, ZONE2) + set_k8s_node_zone_label(core_api, n3.name, "should-not-schedule") + wait_longhorn_node_zone_updated(client) + + _set_k8s_node_zone_label() n_replicas = 3 client.create_volume(name=volume_name, @@ -1177,6 +1294,9 @@ def test_replica_auto_balance_should_respect_node_selector(client, core_api, vol # Check over 10 seconds to check for unexpected re-scheduling. for _ in range(10): + if is_k8s_node_gke_cos(core_api): + _set_k8s_node_zone_label() + time.sleep(1) check_z1_r_count = get_zone_replica_count(client, volume_name, ZONE1)