diff --git a/integration/test_metric.html b/integration/test_metric.html index 80b38b6e3f..afa831812b 100644 --- a/integration/test_metric.html +++ b/integration/test_metric.html @@ -50,11 +50,13 @@

Module tests.test_metric

from common import wait_for_volume_healthy from common import write_volume_data from common import write_volume_random_data - +from common import set_node_scheduling +from common import set_node_cordon from common import Mi from common import LONGHORN_NAMESPACE from common import RETRY_COUNTS from common import RETRY_INTERVAL +from common import DEFAULT_DISK_PATH # The dictionaries use float type of value because the value obtained from # prometheus_client is in float type. @@ -106,6 +108,33 @@

Module tests.test_metric

return metrics +def check_metric_with_condition(core_api, metric_name, metric_labels, expected_value=None, metric_node_id=get_self_host_id()): # NOQA) + """ + Some metric have multiple conditions, for exameple metric + longhorn_node_status have condition + - allowScheduling + - mountpropagation + - ready + - schedulable + metric longhorn_disk_status have conditions + - ready + - schedulable + Use this function to get specific condition of a mertic + """ + metric_data = get_metrics(core_api, metric_node_id) + + found_metric = next( + (sample for family in metric_data for sample in family.samples + if sample.name == metric_name and + sample.labels.get("condition") == metric_labels.get("condition")), + None + ) + + assert found_metric is not None + + examine_metric_value(found_metric, metric_labels, expected_value) + + def check_metric(core_api, metric_name, metric_labels, expected_value=None, metric_node_id=get_self_host_id()): # NOQA metric_data = get_metrics(core_api, metric_node_id) @@ -117,6 +146,10 @@

Module tests.test_metric

assert found_metric is not None + examine_metric_value(found_metric, metric_labels, expected_value) + + +def examine_metric_value(found_metric, metric_labels, expected_value=None): for key, value in metric_labels.items(): assert found_metric.labels[key] == value @@ -436,7 +469,86 @@

Module tests.test_metric

user_snapshot_metric_labels, 4) wait_for_metric_count_all_nodes(client, core_api, "longhorn_snapshot_actual_size_bytes", - system_snapshot_metric_labels, 1) + system_snapshot_metric_labels, 1) + + +def test_node_metrics(client, core_api): # NOQA + lht_hostId = get_self_host_id() + node = client.by_id_node(lht_hostId) + disks = node.disks + for _, disk in iter(disks.items()): + if disk.path == DEFAULT_DISK_PATH: + default_disk = disk + break + assert default_disk is not None + + metric_labels = {} + check_metric(core_api, "longhorn_node_count_total", + metric_labels, expected_value=3.0) + + metric_labels = { + "node": lht_hostId, + } + check_metric(core_api, "longhorn_node_cpu_capacity_millicpu", + metric_labels) + check_metric(core_api, "longhorn_node_cpu_usage_millicpu", + metric_labels) + check_metric(core_api, "longhorn_node_memory_capacity_bytes", + metric_labels) + check_metric(core_api, "longhorn_node_memory_usage_bytes", + metric_labels) + check_metric(core_api, "longhorn_node_storage_capacity_bytes", + metric_labels, default_disk.storageMaximum) + check_metric(core_api, "longhorn_node_storage_usage_bytes", + metric_labels) + check_metric(core_api, "longhorn_node_storage_reservation_bytes", + metric_labels, default_disk.storageReserved) + + # check longhorn_node_status by 4 different conditions + metric_labels = { + "condition": "mountpropagation", + "condition_reason": "", + "node": lht_hostId + } + check_metric_with_condition(core_api, "longhorn_node_status", + metric_labels, 1.0) + + metric_labels = { + "condition": "ready", + "condition_reason": "", + "node": lht_hostId + } + check_metric_with_condition(core_api, "longhorn_node_status", + metric_labels, 1.0) + + metric_labels = { + "condition": "allowScheduling", + "condition_reason": "", + "node": lht_hostId, + } + check_metric_with_condition(core_api, "longhorn_node_status", + metric_labels, 1.0) + node = client.by_id_node(lht_hostId) + set_node_scheduling(client, node, allowScheduling=False, retry=True) + check_metric_with_condition(core_api, "longhorn_node_status", + metric_labels, 0.0) + + metric_labels = { + "condition": "schedulable", + "condition_reason": "", + "node": lht_hostId + } + check_metric_with_condition(core_api, "longhorn_node_status", + metric_labels, 1.0) + + metric_labels = { + "condition": "schedulable", + "condition_reason": "KubernetesNodeCordoned", + "node": lht_hostId + } + set_node_cordon(core_api, lht_hostId, True) + check_metric_with_condition(core_api, "longhorn_node_status", + metric_labels, 0.0)
@@ -466,15 +578,7 @@

Functions

assert found_metric is not None - for key, value in metric_labels.items(): - assert found_metric.labels[key] == value - - assert isinstance(found_metric.value, float) - - if expected_value is not None: - assert found_metric.value == expected_value - else: - assert found_metric.value >= 0.0 + examine_metric_value(found_metric, metric_labels, expected_value)
@@ -591,6 +695,72 @@

Functions

assert total_metrics["value"] >= 0.0 +
+def check_metric_with_condition(core_api, metric_name, metric_labels, expected_value=None, metric_node_id=None) +
+
+

Some metric have multiple conditions, for exameple metric +longhorn_node_status have condition +- allowScheduling +- mountpropagation +- ready +- schedulable +metric longhorn_disk_status have conditions +- ready +- schedulable +Use this function to get specific condition of a mertic

+
+ +Expand source code + +
def check_metric_with_condition(core_api, metric_name, metric_labels, expected_value=None, metric_node_id=get_self_host_id()): # NOQA)
+    """
+    Some metric have multiple conditions, for exameple metric
+    longhorn_node_status have condition
+    - allowScheduling
+    - mountpropagation
+    - ready
+    - schedulable
+    metric longhorn_disk_status have conditions
+    - ready
+    - schedulable
+    Use this function to get specific condition of a mertic
+    """
+    metric_data = get_metrics(core_api, metric_node_id)
+
+    found_metric = next(
+        (sample for family in metric_data for sample in family.samples
+            if sample.name == metric_name and
+            sample.labels.get("condition") == metric_labels.get("condition")),
+        None
+        )
+
+    assert found_metric is not None
+
+    examine_metric_value(found_metric, metric_labels, expected_value)
+
+
+
+def examine_metric_value(found_metric, metric_labels, expected_value=None) +
+
+
+
+ +Expand source code + +
def examine_metric_value(found_metric, metric_labels, expected_value=None):
+    for key, value in metric_labels.items():
+        assert found_metric.labels[key] == value
+
+    assert isinstance(found_metric.value, float)
+
+    if expected_value is not None:
+        assert found_metric.value == expected_value
+    else:
+        assert found_metric.value >= 0.0
+
+
def find_metric(metric_data, metric_name)
@@ -783,6 +953,94 @@

Functions

system_snapshot_metric_labels, 1) +
+def test_node_metrics(client, core_api) +
+
+
+
+ +Expand source code + +
def test_node_metrics(client, core_api): # NOQA
+    lht_hostId = get_self_host_id()
+    node = client.by_id_node(lht_hostId)
+    disks = node.disks
+    for _, disk in iter(disks.items()):
+        if disk.path == DEFAULT_DISK_PATH:
+            default_disk = disk
+            break
+    assert default_disk is not None
+
+    metric_labels = {}
+    check_metric(core_api, "longhorn_node_count_total",
+                 metric_labels, expected_value=3.0)
+
+    metric_labels = {
+        "node": lht_hostId,
+    }
+    check_metric(core_api, "longhorn_node_cpu_capacity_millicpu",
+                 metric_labels)
+    check_metric(core_api, "longhorn_node_cpu_usage_millicpu",
+                 metric_labels)
+    check_metric(core_api, "longhorn_node_memory_capacity_bytes",
+                 metric_labels)
+    check_metric(core_api, "longhorn_node_memory_usage_bytes",
+                 metric_labels)
+    check_metric(core_api, "longhorn_node_storage_capacity_bytes",
+                 metric_labels, default_disk.storageMaximum)
+    check_metric(core_api, "longhorn_node_storage_usage_bytes",
+                 metric_labels)
+    check_metric(core_api, "longhorn_node_storage_reservation_bytes",
+                 metric_labels, default_disk.storageReserved)
+
+    # check longhorn_node_status by 4 different conditions
+    metric_labels = {
+        "condition": "mountpropagation",
+        "condition_reason": "",
+        "node": lht_hostId
+    }
+    check_metric_with_condition(core_api, "longhorn_node_status",
+                                metric_labels, 1.0)
+
+    metric_labels = {
+        "condition": "ready",
+        "condition_reason": "",
+        "node": lht_hostId
+    }
+    check_metric_with_condition(core_api, "longhorn_node_status",
+                                metric_labels, 1.0)
+
+    metric_labels = {
+        "condition": "allowScheduling",
+        "condition_reason": "",
+        "node": lht_hostId,
+    }
+    check_metric_with_condition(core_api, "longhorn_node_status",
+                                metric_labels, 1.0)
+    node = client.by_id_node(lht_hostId)
+    set_node_scheduling(client, node, allowScheduling=False, retry=True)
+    check_metric_with_condition(core_api, "longhorn_node_status",
+                                metric_labels, 0.0)
+
+    metric_labels = {
+        "condition": "schedulable",
+        "condition_reason": "",
+        "node": lht_hostId
+    }
+    check_metric_with_condition(core_api, "longhorn_node_status",
+                                metric_labels, 1.0)
+
+    metric_labels = {
+        "condition": "schedulable",
+        "condition_reason": "KubernetesNodeCordoned",
+        "node": lht_hostId
+    }
+    set_node_cordon(core_api, lht_hostId, True)
+    check_metric_with_condition(core_api, "longhorn_node_status",
+                                metric_labels, 0.0)
+
+
def test_volume_metrics(client, core_api, volume_name, pvc_namespace)
@@ -933,10 +1191,13 @@

Index

  • check_metric
  • check_metric_count_all_nodes
  • check_metric_sum_on_all_nodes
    • +
  • check_metric_with_condition
    • +
  • examine_metric_value
  • find_metric
  • find_metrics
  • get_metrics
  • test_metric_longhorn_snapshot_actual_size_bytes
    • +
  • test_node_metrics
  • test_volume_metrics
  • wait_for_metric_count_all_nodes