From 4b03286607b1999ce8cd7129fbedf99d202f8cf2 Mon Sep 17 00:00:00 2001 From: averevki Date: Tue, 20 Aug 2024 16:04:04 +0200 Subject: [PATCH] Add geo load-balancing tests Signed-off-by: averevki --- config/settings.local.yaml.tpl | 7 ++++ testsuite/config/__init__.py | 7 ++++ testsuite/kuadrant/policy/dns.py | 23 +++++++++++ .../multicluster/load_balanced/__init__.py | 0 .../multicluster/load_balanced/conftest.py | 38 +++++++++++++++++++ .../load_balanced/test_load_balanced_geo.py | 32 ++++++++++++++++ .../load_balanced/test_unsupported_geocode.py | 15 ++++++++ testsuite/utils.py | 10 +++++ 8 files changed, 132 insertions(+) create mode 100644 testsuite/tests/multicluster/load_balanced/__init__.py create mode 100644 testsuite/tests/multicluster/load_balanced/conftest.py create mode 100644 testsuite/tests/multicluster/load_balanced/test_load_balanced_geo.py create mode 100644 testsuite/tests/multicluster/load_balanced/test_unsupported_geocode.py diff --git a/config/settings.local.yaml.tpl b/config/settings.local.yaml.tpl index dca4fad7..764f92b2 100644 --- a/config/settings.local.yaml.tpl +++ b/config/settings.local.yaml.tpl @@ -52,6 +52,13 @@ # issuer: # Issuer object for testing TLSPolicy # name: "selfsigned-cluster-issuer" # Name of Issuer CR # kind: "ClusterIssuer" # Kind of Issuer, can be "Issuer" or "ClusterIssuer" +# dns: +# dns_server: +# geo_code: "DE" # dns provider geo code of the dns server +# address: "ns1.seolizer.de" # dns nameserver hostname or ip +# dns_server2: +# geo_code: "AU" # dns provider geo code of the second dns server +# address: "ns2.seolizer.de" # second dns nameserver hostname or ip # letsencrypt: # issuer: # Issuer object for testing TLSPolicy # name: "letsencrypt-staging-issuer" # Name of Issuer CR diff --git a/testsuite/config/__init__.py b/testsuite/config/__init__.py index 6c351c38..9af6945f 100644 --- a/testsuite/config/__init__.py +++ b/testsuite/config/__init__.py @@ -2,6 +2,7 @@ from dynaconf import Dynaconf, Validator +from testsuite.utils import hostname_to_ip from testsuite.config.tools import fetch_route, fetch_service, fetch_secret, fetch_service_ip @@ -59,6 +60,12 @@ def __init__(self, name, default, **kwargs) -> None: Validator("letsencrypt.issuer.name", must_exist=True, ne=None) & Validator("letsencrypt.issuer.kind", must_exist=True, is_in={"Issuer", "ClusterIssuer"}) ), + ( + Validator("dns.dns_server.address", must_exist=True, ne=None, cast=hostname_to_ip) + & Validator("dns.dns_server.geo_code", must_exist=True, ne=None) + & Validator("dns.dns_server2.address", must_exist=True, ne=None, cast=hostname_to_ip) + & Validator("dns.dns_server2.geo_code", must_exist=True, ne=None) + ), DefaultValueValidator("keycloak.url", default=fetch_service_ip("keycloak", force_http=True, port=8080)), DefaultValueValidator("keycloak.password", default=fetch_secret("credential-sso", "ADMIN_PASSWORD")), DefaultValueValidator("mockserver.url", default=fetch_service_ip("mockserver", force_http=True, port=1080)), diff --git a/testsuite/kuadrant/policy/dns.py b/testsuite/kuadrant/policy/dns.py index 481d7d67..eddeff38 100644 --- a/testsuite/kuadrant/policy/dns.py +++ b/testsuite/kuadrant/policy/dns.py @@ -1,8 +1,26 @@ """Module for DNSPolicy related classes""" +from dataclasses import dataclass + from testsuite.gateway import Referencable from testsuite.kubernetes.client import KubernetesClient from testsuite.kuadrant.policy import Policy +from testsuite.utils import asdict + + +@dataclass +class LoadBalancing: + """Dataclass for DNSPolicy load-balancing spec""" + + default_geo: str + default_weight: int + + def asdict(self): + """Custom asdict due to nested structure.""" + return { + "geo": {"defaultGeo": self.default_geo}, + "weighted": {"defaultWeight": self.default_weight}, + } class DNSPolicy(Policy): @@ -15,6 +33,7 @@ def create_instance( name: str, parent: Referencable, provider_secret_name: str, + load_balancing: LoadBalancing = None, labels: dict[str, str] = None, ): """Creates new instance of DNSPolicy""" @@ -30,4 +49,8 @@ def create_instance( }, } + if load_balancing: + model["spec"]["routingStrategy"] = "loadbalanced" + model["spec"]["loadBalancing"] = asdict(load_balancing) + return cls(model, context=cluster.context) diff --git a/testsuite/tests/multicluster/load_balanced/__init__.py b/testsuite/tests/multicluster/load_balanced/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testsuite/tests/multicluster/load_balanced/conftest.py b/testsuite/tests/multicluster/load_balanced/conftest.py new file mode 100644 index 00000000..576cb0d0 --- /dev/null +++ b/testsuite/tests/multicluster/load_balanced/conftest.py @@ -0,0 +1,38 @@ +"""Conftest for load-balanced multicluster tests""" + +import pytest + +from testsuite.kuadrant.policy.dns import DNSPolicy, LoadBalancing + + +@pytest.fixture(scope="package") +def dns_config(testconfig): + """Configuration for DNS tests""" + testconfig.validators.validate(only="dns") + return testconfig["dns"] + + +@pytest.fixture(scope="package") +def dns_server(dns_config): + """DNS server in the first geo region""" + return dns_config["dns_server"] + + +@pytest.fixture(scope="package") +def dns_server2(dns_config): + """DNS server in the second geo region""" + return dns_config["dns_server2"] + + +@pytest.fixture(scope="module") +def dns_policy(blame, cluster, gateway, dns_server, module_label): + """DNSPolicy with load-balancing for the first cluster""" + load_balancing = LoadBalancing(default_geo=dns_server["geo_code"], default_weight=10) + return DNSPolicy.create_instance(cluster, blame("dns"), gateway, load_balancing, labels={"app": module_label}) + + +@pytest.fixture(scope="module") +def dns_policy2(blame, cluster2, gateway2, dns_server, module_label): + """DNSPolicy with load-balancing for the second cluster""" + load_balancing = LoadBalancing(default_geo=dns_server["geo_code"], default_weight=10) + return DNSPolicy.create_instance(cluster2, blame("dns"), gateway2, load_balancing, labels={"app": module_label}) diff --git a/testsuite/tests/multicluster/load_balanced/test_load_balanced_geo.py b/testsuite/tests/multicluster/load_balanced/test_load_balanced_geo.py new file mode 100644 index 00000000..b02615b0 --- /dev/null +++ b/testsuite/tests/multicluster/load_balanced/test_load_balanced_geo.py @@ -0,0 +1,32 @@ +"""Test load-balancing based on geolocation""" + +import pytest +import dns.name +import dns.resolver + +pytestmark = [pytest.mark.multicluster] + + +@pytest.fixture(scope="module") +def gateway2(gateway2, dns_server2): + """Overwrite second gateway to have a different geocode""" + gateway2.label({"kuadrant.io/lb-attribute-geo-code": dns_server2["geo_code"]}) + return gateway2 + + +def test_load_balanced_geo(client, hostname, gateway, gateway2, dns_server, dns_server2): + """ + - Verify that request to the hostname is successful + - Verify that DNS resolution through nameservers from different regions returns according IPs + """ + result = client.get("/get") + assert not result.has_dns_error(), result.error + assert not result.has_cert_verify_error(), result.error + assert result.status_code == 200 + + resolver = dns.resolver.Resolver(configure=False) + resolver.nameservers = [dns_server["address"]] + assert resolver.resolve(hostname.hostname)[0].address == gateway.external_ip().split(":")[0] + + resolver.nameservers = [dns_server2["address"]] + assert resolver.resolve(hostname.hostname)[0].address == gateway2.external_ip().split(":")[0] diff --git a/testsuite/tests/multicluster/load_balanced/test_unsupported_geocode.py b/testsuite/tests/multicluster/load_balanced/test_unsupported_geocode.py new file mode 100644 index 00000000..c06e16b6 --- /dev/null +++ b/testsuite/tests/multicluster/load_balanced/test_unsupported_geocode.py @@ -0,0 +1,15 @@ +"""Test not supported geocode in geo load-balancing""" + +import pytest + +from testsuite.kuadrant.policy import has_condition + +pytestmark = [pytest.mark.multicluster] + + +def test_unsupported_geocode(dns_policy): + """Change default geocode to not existent one and verify that policy became not enforced""" + dns_policy.model.spec.loadBalancing.geo.defaultGeo = "XX" + dns_policy.apply() + + assert dns_policy.wait_until(has_condition("Enforced", "False")) diff --git a/testsuite/utils.py b/testsuite/utils.py index 2652a6e9..c5fbf388 100644 --- a/testsuite/utils.py +++ b/testsuite/utils.py @@ -180,6 +180,16 @@ def check_condition(condition, condition_type, status, reason=None, message=None return False +def hostname_to_ip(address: str) -> str: + """Resolves hostname to IP if necessary""" + if any(c.isalpha() for c in address): + try: + return dns.resolver.resolve(address)[0].address + except dns.resolver.NXDOMAIN as e: + raise ValueError(f"Hostname {address} can't be resolved to an IP address") from e + return address + + def is_nxdomain(hostname: str): """ Returns True if hostname has no `A` record in DNS. False otherwise.