Add geo load-balancing tests

Signed-off-by: averevki <[email protected]>

config/settings.local.yaml.tpl

@@ -52,6 +52,13 @@
 #    issuer:                                       # Issuer object for testing TLSPolicy
 #      name: "selfsigned-cluster-issuer"           # Name of Issuer CR
 #      kind: "ClusterIssuer"                       # Kind of Issuer, can be "Issuer" or "ClusterIssuer"
+#  dns:
+#    dns_server:
+#      geo_code: "DE"                       # dns provider geo code of the dns server
+#      address: "ns1.seolizer.de"           # dns nameserver hostname or ip
+#    dns_server2:
+#      geo_code: "AU"                       # dns provider geo code of the second dns server
+#      address: "ns2.seolizer.de"           # second dns nameserver hostname or ip
 #  letsencrypt:
 #    issuer:                                       # Issuer object for testing TLSPolicy
 #      name: "letsencrypt-staging-issuer"          # Name of Issuer CR

testsuite/config/__init__.py

@@ -2,6 +2,7 @@
 
 from dynaconf import Dynaconf, Validator
 
+from testsuite.utils import hostname_to_ip
 from testsuite.config.tools import fetch_route, fetch_service, fetch_secret, fetch_service_ip
 
 
@@ -59,6 +60,12 @@ def __init__(self, name, default, **kwargs) -> None:
             Validator("letsencrypt.issuer.name", must_exist=True, ne=None)
             & Validator("letsencrypt.issuer.kind", must_exist=True, is_in={"Issuer", "ClusterIssuer"})
         ),
+        (
+            Validator("dns.dns_server.address", must_exist=True, ne=None, cast=hostname_to_ip)
+            & Validator("dns.dns_server.geo_code", must_exist=True, ne=None)
+            & Validator("dns.dns_server2.address", must_exist=True, ne=None, cast=hostname_to_ip)
+            & Validator("dns.dns_server2.geo_code", must_exist=True, ne=None)
+        ),
         DefaultValueValidator("keycloak.url", default=fetch_service_ip("keycloak", force_http=True, port=8080)),
         DefaultValueValidator("keycloak.password", default=fetch_secret("credential-sso", "ADMIN_PASSWORD")),
         DefaultValueValidator("mockserver.url", default=fetch_service_ip("mockserver", force_http=True, port=1080)),

testsuite/kuadrant/policy/dns.py

@@ -1,8 +1,26 @@
 """Module for DNSPolicy related classes"""
 
+from dataclasses import dataclass
+
 from testsuite.gateway import Referencable
 from testsuite.kubernetes.client import KubernetesClient
 from testsuite.kuadrant.policy import Policy
+from testsuite.utils import asdict
+
+
+@dataclass
+class LoadBalancing:
+    """Dataclass for DNSPolicy load-balancing spec"""
+
+    default_geo: str
+    default_weight: int
+
+    def asdict(self):
+        """Custom asdict due to nested structure."""
+        return {
+            "geo": {"defaultGeo": self.default_geo},
+            "weighted": {"defaultWeight": self.default_weight},
+        }
 
 
 class DNSPolicy(Policy):
@@ -14,6 +32,7 @@ def create_instance(
         cluster: KubernetesClient,
         name: str,
         parent: Referencable,
+        load_balancing: LoadBalancing = None,
         labels: dict[str, str] = None,
     ):
         """Creates new instance of DNSPolicy"""
@@ -22,7 +41,14 @@ def create_instance(
             "apiVersion": "kuadrant.io/v1alpha1",
             "kind": "DNSPolicy",
             "metadata": {"name": name, "labels": labels},
-            "spec": {"targetRef": parent.reference, "routingStrategy": "simple"},
+            "spec": {
+                "routingStrategy": "simple",
+                "targetRef": parent.reference,
+            },
         }
 
+        if load_balancing:
+            model["spec"]["routingStrategy"] = "loadbalanced"
+            model["spec"]["loadBalancing"] = asdict(load_balancing)
+
         return cls(model, context=cluster.context)

testsuite/tests/multicluster/load_balanced/conftest.py

@@ -0,0 +1,38 @@
+"""Conftest for load-balanced multicluster tests"""
+
+import pytest
+
+from testsuite.kuadrant.policy.dns import DNSPolicy, LoadBalancing
+
+
+@pytest.fixture(scope="package")
+def dns_config(testconfig):
+    """Configuration for DNS tests"""
+    testconfig.validators.validate(only="dns")
+    return testconfig["dns"]
+
+
+@pytest.fixture(scope="package")
+def dns_server(dns_config):
+    """DNS server in the first geo region"""
+    return dns_config["dns_server"]
+
+
+@pytest.fixture(scope="package")
+def dns_server2(dns_config):
+    """DNS server in the second geo region"""
+    return dns_config["dns_server2"]
+
+
+@pytest.fixture(scope="module")
+def dns_policy(blame, cluster, gateway, dns_server, module_label):
+    """DNSPolicy with load-balancing for the first cluster"""
+    load_balancing = LoadBalancing(default_geo=dns_server["geo_code"], default_weight=10)
+    return DNSPolicy.create_instance(cluster, blame("dns"), gateway, load_balancing, labels={"app": module_label})
+
+
+@pytest.fixture(scope="module")
+def dns_policy2(blame, cluster2, gateway2, dns_server, module_label):
+    """DNSPolicy with load-balancing for the second cluster"""
+    load_balancing = LoadBalancing(default_geo=dns_server["geo_code"], default_weight=10)
+    return DNSPolicy.create_instance(cluster2, blame("dns"), gateway2, load_balancing, labels={"app": module_label})

testsuite/tests/multicluster/load_balanced/test_load_balanced_geo.py

@@ -0,0 +1,32 @@
+"""Test load-balancing based on geolocation"""
+
+import pytest
+import dns.name
+import dns.resolver
+
+pytestmark = [pytest.mark.multicluster]
+
+
+@pytest.fixture(scope="module")
+def gateway2(gateway2, dns_server2):
+    """Overwrite second gateway to have a different geocode"""
+    gateway2.label({"kuadrant.io/lb-attribute-geo-code": dns_server2["geo_code"]})
+    return gateway2
+
+
+def test_load_balanced_geo(client, hostname, gateway, gateway2, dns_server, dns_server2):
+    """
+    - Verify that request to the hostname is successful
+    - Verify that DNS resolution through nameservers from different regions returns according IPs
+    """
+    result = client.get("/get")
+    assert not result.has_dns_error(), result.error
+    assert not result.has_cert_verify_error(), result.error
+    assert result.status_code == 200
+
+    resolver = dns.resolver.Resolver(configure=False)
+    resolver.nameservers = [dns_server["address"]]
+    assert resolver.resolve(hostname.hostname)[0].address == gateway.external_ip().split(":")[0]
+
+    resolver.nameservers = [dns_server2["address"]]
+    assert resolver.resolve(hostname.hostname)[0].address == gateway2.external_ip().split(":")[0]

testsuite/tests/multicluster/load_balanced/test_unsupported_geocode.py

@@ -0,0 +1,15 @@
+"""Test not supported geocode in geo load-balancing"""
+
+import pytest
+
+from testsuite.kuadrant.policy import has_condition
+
+pytestmark = [pytest.mark.multicluster]
+
+
+def test_unsupported_geocode(dns_policy):
+    """Change default geocode to not existent one and verify that policy became not enforced"""
+    dns_policy.model.spec.loadBalancing.geo.defaultGeo = "XX"
+    dns_policy.apply()
+
+    assert dns_policy.wait_until(has_condition("Enforced", "False"))

testsuite/utils.py

@@ -14,6 +14,7 @@
 from typing import Dict, Union
 from urllib.parse import urlparse, ParseResult
 
+import dns.resolver
 from weakget import weakget
 
 from testsuite.certificates import Certificate, CFSSLClient, CertInfo
@@ -176,3 +177,13 @@ def check_condition(condition, condition_type, status, reason=None, message=None
     ):
         return True
     return False
+
+
+def hostname_to_ip(address: str) -> str:
+    """Resolves hostname to IP if necessary"""
+    if any(c.isalpha() for c in address):
+        try:
+            return dns.resolver.resolve(address)[0].address
+        except dns.resolver.NXDOMAIN as e:
+            raise ValueError(f"Hostname {address} can't be resolved to an IP address") from e
+    return address