From bb48dd9c8dc864972b1c567681cdda1299d3b2f6 Mon Sep 17 00:00:00 2001 From: averevki Date: Tue, 20 Aug 2024 16:01:59 +0200 Subject: [PATCH] Refactor multicluster conftest & settings Signed-off-by: averevki --- config/settings.local.yaml.tpl | 13 +- testsuite/capabilities.py | 9 +- testsuite/config/exposer.py | 2 +- testsuite/config/openshift_loader.py | 14 +- testsuite/tests/conftest.py | 2 +- testsuite/tests/multicluster/conftest.py | 176 +++++++++--------- ...cluster_dns.py => test_simple_strategy.py} | 10 +- .../operator/clusterwide/conftest.py | 4 +- .../clusterwide/test_all_namespace_api_key.py | 8 +- .../clusterwide/test_wildcard_collision.py | 4 +- testsuite/tests/singlecluster/conftest.py | 7 +- 11 files changed, 128 insertions(+), 121 deletions(-) rename testsuite/tests/multicluster/{test_multicluster_dns.py => test_simple_strategy.py} (67%) diff --git a/config/settings.local.yaml.tpl b/config/settings.local.yaml.tpl index f420b8de..dca4fad7 100644 --- a/config/settings.local.yaml.tpl +++ b/config/settings.local.yaml.tpl @@ -1,9 +1,5 @@ #default: # tester: "someuser" # Optional: name of the user, who is running the tests, defaults to whoami/uid -# cluster: # Primary cluster where tests should run -# api_url: "https://api.kubernetes.com" # Optional: Kubernetes API URL, if None it will use Kubernetes that you are logged in -# token: "KUADRANT_RULEZ" # Optional: Kubernetes Token, if None it will Kubernetes that you are logged in -# kubeconfig_path: "~/.kube/config" # Optional: Kubeconfig to use, if None the default one is used # kuadrantctl: kuadrantctl # tools: # project: "tools" # Optional: Kubernetes project, where external tools are located @@ -44,7 +40,14 @@ # metrics_service_name: "" # controller metrics service name for already deployed Authorino # default_exposer: "kubernetes" # Force Exposer typem options: 'openshift', 'kind', 'kubernetes' # control_plane: -# additional_clusters: [] # List of additional clusters for Multicluster testing, see 'cluster' option for more details +# cluster: # Primary cluster where tests should run +# api_url: "https://api.kubernetes.com" # Optional: Kubernetes API URL, if None it will use Kubernetes that you are logged in +# token: "KUADRANT_RULEZ" # Optional: Kubernetes Token, if None it will Kubernetes that you are logged in +# kubeconfig_path: "~/.kube/config" # Optional: Kubeconfig to use, if None the default one is used +# cluster2: # Second cluster for the multicluster tests +# api_url: "https://api.kubernetes2.com" +# token: "KUADRANT_RULEZ" +# kubeconfig_path: "~/.kube/config2" # provider_secret: "aws-credentials" # Name of the Secret resource that contains DNS provider credentials # issuer: # Issuer object for testing TLSPolicy # name: "selfsigned-cluster-issuer" # Name of Issuer CR diff --git a/testsuite/capabilities.py b/testsuite/capabilities.py index d0146ec7..1e5332ac 100644 --- a/testsuite/capabilities.py +++ b/testsuite/capabilities.py @@ -3,7 +3,6 @@ import functools from openshift_client import selector -from weakget import weakget from testsuite.config import settings @@ -12,14 +11,14 @@ def has_kuadrant(): """Returns True, if Kuadrant deployment is present and should be used""" project = settings["service_protection"]["system_project"] - clusters = weakget(settings)["control_plane"]["additional_clusters"] % [] - clusters.append(settings["cluster"]) + clusters = [settings["control_plane"]["cluster"]] + if cluster2 := settings["control_plane"]["cluster2"]: + clusters.append(cluster2) for cluster in clusters: system_project = cluster.change_project(project) - # Try if Kuadrant is deployed if not system_project.connected: return False, f"Cluster {cluster.api_url} is not connected, or namespace {project} does not exist" - system_project = cluster.change_project(project) + with system_project.context: if selector("kuadrant").count_existing() == 0: return False, f"Cluster {cluster.api_url} does not have Kuadrant resource in project {project}" diff --git a/testsuite/config/exposer.py b/testsuite/config/exposer.py index 7f9d63a5..1328cb91 100644 --- a/testsuite/config/exposer.py +++ b/testsuite/config/exposer.py @@ -9,7 +9,7 @@ def load(obj, env=None, silent=True, key=None, filename=None): """Selects proper Exposes class""" if "default_exposer" not in obj or not obj["default_exposer"]: - client = obj["cluster"] + client = obj["control_plane"]["cluster"] if "route.openshift.io/v1" in client.do_action("api-versions").out(): obj["default_exposer"] = EXPOSERS["openshift"] else: diff --git a/testsuite/config/openshift_loader.py b/testsuite/config/openshift_loader.py index 09d039ba..c5eb8498 100644 --- a/testsuite/config/openshift_loader.py +++ b/testsuite/config/openshift_loader.py @@ -15,18 +15,19 @@ def inject_client(obj, base_client, path): # pylint: disable=unused-argument, too-many-locals def load(obj, env=None, silent=True, key=None, filename=None): """Creates all KubernetesClients""" - section = obj.setdefault("cluster", {}) + control_plane = obj.setdefault("control_plane", {}) + + cluster = control_plane.setdefault("cluster", {}) client = KubernetesClient( - section.get("project"), section.get("api_url"), section.get("token"), section.get("kubeconfig_path") + cluster.get("project"), cluster.get("api_url"), cluster.get("token"), cluster.get("kubeconfig_path") ) - obj["cluster"] = client + obj["control_plane"]["cluster"] = client tools = None if "tools" in obj and "project" in obj["tools"]: tools = client.change_project(obj["tools"]["project"]) obj["tools"] = tools - control_plane = obj.setdefault("control_plane", {}) clients = [] clusters = control_plane.setdefault("additional_clusters", []) for value in clusters: @@ -37,3 +38,8 @@ def load(obj, env=None, silent=True, key=None, filename=None): ) if len(clients) > 0: control_plane["additional_clusters"] = clients + + if cluster2 := control_plane.setdefault("cluster2", {}): + obj["control_plane"]["cluster2"] = KubernetesClient( + cluster2.get("project"), cluster2.get("api_url"), cluster2.get("token"), cluster2.get("kubeconfig_path") + ) diff --git a/testsuite/tests/conftest.py b/testsuite/tests/conftest.py index 0ff0a589..3f318f1a 100644 --- a/testsuite/tests/conftest.py +++ b/testsuite/tests/conftest.py @@ -239,7 +239,7 @@ def module_label(label): def cluster(testconfig): """Kubernetes client for the primary namespace""" project = testconfig["service_protection"]["project"] - client = testconfig["cluster"].change_project(testconfig["service_protection"]["project"]) + client = testconfig["control_plane"]["cluster"].change_project(testconfig["service_protection"]["project"]) if not client.connected: pytest.fail(f"You are not logged into Kubernetes or the {project} namespace doesn't exist") return client diff --git a/testsuite/tests/multicluster/conftest.py b/testsuite/tests/multicluster/conftest.py index 1c9f273f..0df5fe24 100644 --- a/testsuite/tests/multicluster/conftest.py +++ b/testsuite/tests/multicluster/conftest.py @@ -1,29 +1,31 @@ """Conftest for Multicluster tests""" from importlib import resources -from typing import TypeVar import pytest from openshift_client import selector, OpenShiftPythonException from testsuite.backend.httpbin import Httpbin from testsuite.certificates import Certificate -from testsuite.gateway import Exposer, Gateway, CustomReference, Hostname +from testsuite.gateway import Exposer, CustomReference, Hostname from testsuite.gateway.gateway_api.gateway import KuadrantGateway from testsuite.gateway.gateway_api.hostname import DNSPolicyExposer from testsuite.gateway.gateway_api.route import HTTPRoute -from testsuite.kubernetes.client import KubernetesClient -from testsuite.kuadrant.policy import Policy from testsuite.kuadrant.policy.dns import DNSPolicy from testsuite.kuadrant.policy.tls import TLSPolicy -AnyPolicy = TypeVar("AnyPolicy", bound=Policy) - +@pytest.fixture(scope="session") +def cluster2(testconfig): + """Kubernetes client for the primary namespace""" + if not testconfig["control_plane"]["cluster2"]: + pytest.skip("Second cluster is not configured properly") -def generate_policies(clusters: list[KubernetesClient], policy: AnyPolicy) -> dict[KubernetesClient, AnyPolicy]: - """Copy policies for each cluster""" - return {cluster: policy.__class__(policy.as_dict(), context=cluster.context) for cluster in clusters} + project = testconfig["service_protection"]["project"] + client = testconfig["control_plane"]["cluster2"].change_project(project) + if not client.connected: + pytest.fail(f"You are not logged into the second cluster or the {project} namespace doesn't exist") + return client @pytest.fixture(scope="module") @@ -43,128 +45,124 @@ def cluster_issuer(testconfig, cluster, skip_or_fail): ) -@pytest.fixture(scope="session") -def clusters(testconfig, cluster, skip_or_fail) -> list[KubernetesClient]: - """Returns list of all clusters on which to run Multicluster tests""" - additional_clusters = testconfig["control_plane"]["additional_clusters"] - if len(additional_clusters) == 0: - skip_or_fail("Only one cluster was provided for multi-cluster tests") - return [ - cluster, - *(cluster.change_project(testconfig["service_protection"]["project"]) for cluster in additional_clusters), - ] +@pytest.fixture(scope="module") +def hostname(gateway, exposer, blame) -> Hostname: + """Exposed Hostname object""" + return exposer.expose_hostname(blame("hostname"), gateway) + + +@pytest.fixture(scope="module") +def exposer(request, cluster) -> Exposer: + """Expose using DNSPolicy""" + exposer = DNSPolicyExposer(cluster) + request.addfinalizer(exposer.delete) + exposer.commit() + return exposer + + +@pytest.fixture(scope="module") +def base_domain(exposer): + """Returns preconfigured base domain""" + return exposer.base_domain + + +@pytest.fixture(scope="module") +def wildcard_domain(base_domain): + """ + Wildcard domain for the exposer + """ + return f"*.{base_domain}" @pytest.fixture(scope="session") -def backends(request, clusters, blame, label, testconfig) -> dict[KubernetesClient, Httpbin]: +def backends(request, cluster, cluster2, blame, label, testconfig) -> list[Httpbin]: """Deploys Backend to each Kubernetes cluster""" - backends = {} + backends = [] name = blame("httpbin") image = testconfig["httpbin"]["image"] - for cluster in clusters: - httpbin = Httpbin(cluster, name, label, image) + for client in [cluster, cluster2]: + httpbin = Httpbin(client, name, label, image) request.addfinalizer(httpbin.delete) httpbin.commit() - backends[cluster] = httpbin + backends.append(httpbin) return backends @pytest.fixture(scope="module") -def gateways(request, clusters, blame, label, wildcard_domain) -> dict[KubernetesClient, Gateway]: - """Deploys Gateway to each Kubernetes cluster""" - gateways = {} - name = blame("gw") - for cluster in clusters: - gw = KuadrantGateway.create_instance(cluster, name, wildcard_domain, {"app": label}, tls=True) - request.addfinalizer(gw.delete) - gw.commit() - gateways[cluster] = gw - for gateway in gateways.values(): - gateway.wait_for_ready() - return gateways - - -@pytest.fixture(scope="module") -def routes(request, gateways, blame, hostname, backends, module_label) -> dict[KubernetesClient, HTTPRoute]: - """Deploys HttpRoute to each Kubernetes cluster""" - routes = {} +def routes(request, gateway, gateway2, blame, hostname, backends, module_label) -> list[HTTPRoute]: + """Deploys HttpRoute for each gateway""" + routes = [] name = blame("route") - for client, gateway in gateways.items(): - route = HTTPRoute.create_instance(gateway.cluster, name, gateway, {"app": module_label}) + for i, gateway_ in enumerate([gateway, gateway2]): + route = HTTPRoute.create_instance(gateway_.cluster, name, gateway_, {"app": module_label}) route.add_hostname(hostname.hostname) - route.add_backend(backends[client]) + route.add_backend(backends[i]) request.addfinalizer(route.delete) route.commit() - routes[client] = route + routes.append(route) return routes @pytest.fixture(scope="module") -def hostname(gateways, cluster, exposer, blame) -> Hostname: - """Exposed Hostname object""" - hostname = exposer.expose_hostname(blame("hostname"), gateways[cluster]) - return hostname - - -@pytest.fixture(scope="module") -def exposer(request, cluster) -> Exposer: - """Expose using DNSPolicy""" - exposer = DNSPolicyExposer(cluster) - request.addfinalizer(exposer.delete) - exposer.commit() - return exposer +def gateway(request, cluster, blame, label, wildcard_domain): + """Deploys Gateway to first Kubernetes cluster""" + gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": label}, tls=True) + request.addfinalizer(gw.delete) + gw.commit() + gw.wait_for_ready() + return gw @pytest.fixture(scope="module") -def base_domain(exposer): - """Returns preconfigured base domain""" - return exposer.base_domain +def gateway2(request, cluster2, blame, label, wildcard_domain): + """Deploys Gateway to second Kubernetes cluster""" + gw = KuadrantGateway.create_instance(cluster2, blame("gw"), wildcard_domain, {"app": label}, tls=True) + request.addfinalizer(gw.delete) + gw.commit() + gw.wait_for_ready() + return gw @pytest.fixture(scope="module") -def wildcard_domain(base_domain): - """ - Wildcard domain for the exposer - """ - return f"*.{base_domain}" +def dns_policy(blame, cluster, gateway, dns_provider_secret, module_label): + """DNSPolicy for the first cluster""" + return DNSPolicy.create_instance(cluster, blame("dns"), gateway, dns_provider_secret, labels={"app": module_label}) @pytest.fixture(scope="module") -def dns_policy(blame, cluster, gateways, module_label, dns_provider_secret): - """DNSPolicy fixture""" - policy = DNSPolicy.create_instance( - cluster, blame("dns"), gateways[cluster], dns_provider_secret, labels={"app": module_label} +def dns_policy2(blame, cluster2, gateway2, dns_provider_secret, module_label): + """DNSPolicy for the second cluster""" + return DNSPolicy.create_instance( + cluster2, blame("dns"), gateway2, dns_provider_secret, labels={"app": module_label} ) - return policy @pytest.fixture(scope="module") -def tls_policy(blame, cluster, gateways, module_label, cluster_issuer): - """TLSPolicy fixture""" - policy = TLSPolicy.create_instance( +def tls_policy(blame, cluster, gateway, module_label, cluster_issuer): + """TLSPolicy for the first cluster""" + return TLSPolicy.create_instance( cluster, blame("tls"), - parent=gateways[cluster], + parent=gateway, issuer=cluster_issuer, labels={"app": module_label}, ) - return policy @pytest.fixture(scope="module") -def dns_policies(clusters, dns_policy) -> dict[KubernetesClient, DNSPolicy]: - """Creates DNSPolicy for each Kubernetes cluster""" - return generate_policies(clusters, dns_policy) - - -@pytest.fixture(scope="module") -def tls_policies(clusters, tls_policy) -> dict[KubernetesClient, TLSPolicy]: - """Creates TLSPolicy for each Kubernetes cluster""" - return generate_policies(clusters, tls_policy) +def tls_policy2(blame, cluster2, gateway2, module_label, cluster_issuer): + """TLSPolicy for the second cluster""" + return TLSPolicy.create_instance( + cluster2, + blame("tls"), + parent=gateway2, + issuer=cluster_issuer, + labels={"app": module_label}, + ) @pytest.fixture(scope="module") -def client(hostname, gateways): # pylint: disable=unused-argument +def client(hostname, gateway, gateway2): # pylint: disable=unused-argument """Returns httpx client to be used for requests""" root_cert = resources.files("testsuite.resources").joinpath("letsencrypt-stg-root-x1.pem").read_text() client = hostname.client(verify=Certificate(certificate=root_cert, chain=root_cert, key="")) @@ -173,9 +171,9 @@ def client(hostname, gateways): # pylint: disable=unused-argument @pytest.fixture(scope="module", autouse=True) -def commit(request, routes, dns_policies, tls_policies): # pylint: disable=unused-argument +def commit(request, routes, dns_policy, dns_policy2, tls_policy, tls_policy2): # pylint: disable=unused-argument """Commits all policies before tests""" - components = [*dns_policies.values(), *tls_policies.values()] + components = [dns_policy, dns_policy2, tls_policy, tls_policy2] for component in components: request.addfinalizer(component.delete) component.commit() diff --git a/testsuite/tests/multicluster/test_multicluster_dns.py b/testsuite/tests/multicluster/test_simple_strategy.py similarity index 67% rename from testsuite/tests/multicluster/test_multicluster_dns.py rename to testsuite/tests/multicluster/test_simple_strategy.py index ac59b685..0055b495 100644 --- a/testsuite/tests/multicluster/test_multicluster_dns.py +++ b/testsuite/tests/multicluster/test_simple_strategy.py @@ -6,13 +6,13 @@ pytestmark = [pytest.mark.multicluster] -def test_gateway_readiness(gateways): +def test_gateway_readiness(gateway, gateway2): """Tests whether the Gateway was successfully placed by having its IP address assigned""" - for client, gateway in gateways.items(): - assert gateway.is_ready(), f"Gateway {gateway.name()} on a server {client.api_url} did not get ready" + assert gateway.is_ready(), "Gateway on the first cluster did not get ready in time" + assert gateway2.is_ready(), "Gateway on the second cluster did not get ready in time" -def test_multicluster_dns(client, hostname, gateways): +def test_simple_strategy(client, hostname, gateway, gateway2): """ Tests DNS/TLS across multiple clusters - Checks that all Gateways will get ready @@ -24,6 +24,6 @@ def test_multicluster_dns(client, hostname, gateways): assert not result.has_cert_verify_error(), result.error assert result.status_code == 200 - ips = {gateway.external_ip().split(":")[0] for gateway in gateways.values()} + ips = {gateway.external_ip().split(":")[0], gateway2.external_ip().split(":")[0]} dns_ips = {ip.address for ip in dns.resolver.resolve(hostname.hostname)} assert ips == dns_ips, f"Expected IPs and actual IP mismatch, got {dns_ips}, expected {ips}" diff --git a/testsuite/tests/singlecluster/authorino/operator/clusterwide/conftest.py b/testsuite/tests/singlecluster/authorino/operator/clusterwide/conftest.py index 24d6fc57..ea5de720 100644 --- a/testsuite/tests/singlecluster/authorino/operator/clusterwide/conftest.py +++ b/testsuite/tests/singlecluster/authorino/operator/clusterwide/conftest.py @@ -29,9 +29,9 @@ def route2(request, gateway, blame, hostname2): @pytest.fixture(scope="module") -def authorization2(route2, blame, cluster2, label, oidc_provider): +def authorization2(route2, blame, second_namespace, label, oidc_provider): """Second valid hostname""" - auth = AuthConfig.create_instance(cluster2, blame("ac"), route2, labels={"testRun": label}) + auth = AuthConfig.create_instance(second_namespace, blame("ac"), route2, labels={"testRun": label}) auth.identity.add_oidc("default", oidc_provider.well_known["issuer"]) return auth diff --git a/testsuite/tests/singlecluster/authorino/operator/clusterwide/test_all_namespace_api_key.py b/testsuite/tests/singlecluster/authorino/operator/clusterwide/test_all_namespace_api_key.py index f34f1d66..07ba36d4 100644 --- a/testsuite/tests/singlecluster/authorino/operator/clusterwide/test_all_namespace_api_key.py +++ b/testsuite/tests/singlecluster/authorino/operator/clusterwide/test_all_namespace_api_key.py @@ -11,10 +11,10 @@ @pytest.fixture(scope="module") -def api_key(create_api_key, module_label, cluster2): +def api_key(create_api_key, module_label, second_namespace): """Creates API key Secret""" api_key = "cluster_wide_api_key" - return create_api_key("wide-api-key", module_label, api_key, cluster2) + return create_api_key("wide-api-key", module_label, api_key, second_namespace) @pytest.fixture(scope="module") @@ -30,9 +30,9 @@ def invalid_label_selector(): @pytest.fixture(scope="module") -def invalid_api_key(create_api_key, invalid_label_selector, cluster2): +def invalid_api_key(create_api_key, invalid_label_selector, second_namespace): """Creates API key Secret with label that does not match any of the labelSelectors defined by AuthConfig""" - return create_api_key("invalid-api-key", invalid_label_selector, "invalid_api_key", cluster2) + return create_api_key("invalid-api-key", invalid_label_selector, "invalid_api_key", second_namespace) @pytest.fixture(scope="module") diff --git a/testsuite/tests/singlecluster/authorino/operator/clusterwide/test_wildcard_collision.py b/testsuite/tests/singlecluster/authorino/operator/clusterwide/test_wildcard_collision.py index 301241ea..8f230d7f 100644 --- a/testsuite/tests/singlecluster/authorino/operator/clusterwide/test_wildcard_collision.py +++ b/testsuite/tests/singlecluster/authorino/operator/clusterwide/test_wildcard_collision.py @@ -29,9 +29,9 @@ def authorization(authorino, blame, route, cluster, label, gateway): # pylint: disable = unused-argument @pytest.fixture(scope="module") -def authorization2(authorino, blame, route, cluster2, label, gateway): +def authorization2(authorino, blame, route, second_namespace, label, gateway): """Create AuthConfig with host set to wildcard_domain in another project""" - auth = AuthConfig.create_instance(cluster2, blame("ac"), route, labels={"testRun": label}) + auth = AuthConfig.create_instance(second_namespace, blame("ac"), route, labels={"testRun": label}) auth.responses.add_success_header("header", JsonResponse({"anything": Value("two")})) return auth diff --git a/testsuite/tests/singlecluster/conftest.py b/testsuite/tests/singlecluster/conftest.py index 06730f5d..1490257e 100644 --- a/testsuite/tests/singlecluster/conftest.py +++ b/testsuite/tests/singlecluster/conftest.py @@ -13,13 +13,14 @@ from testsuite.kuadrant import KuadrantCR from testsuite.kuadrant.policy.authorization.auth_policy import AuthPolicy from testsuite.kuadrant.policy.rate_limit import RateLimitPolicy +from testsuite.kubernetes.client import KubernetesClient @pytest.fixture(scope="session") -def cluster2(testconfig, skip_or_fail): +def second_namespace(testconfig, skip_or_fail) -> KubernetesClient: """Kubernetes client for the secondary namespace located on the same cluster as primary cluster""" project = testconfig["service_protection"]["project2"] - client = testconfig["cluster"].change_project(testconfig["service_protection"]["project2"]) + client = testconfig["control_plane"]["cluster"].change_project(testconfig["service_protection"]["project2"]) if client is None: skip_or_fail("Tests requires second_project but service_protection.project2 is not set") if not client.connected: @@ -74,7 +75,7 @@ def kuadrant(request, testconfig): if request.config.getoption("--standalone"): return None - ocp = testconfig["cluster"] + ocp = testconfig["control_plane"]["cluster"] project = testconfig["service_protection"]["system_project"] kuadrant_openshift = ocp.change_project(project)