From 5c171e2c269115c43604ece942a59a34b5df4405 Mon Sep 17 00:00:00 2001 From: phala Date: Thu, 14 Sep 2023 17:33:51 +0200 Subject: [PATCH] MGC rewritten for new API and DNSPolicy usage --- config/settings.local.yaml.tpl | 9 +- config/settings.yaml | 2 +- pyproject.toml | 4 +- testsuite/config/openshift_loader.py | 24 ++-- testsuite/openshift/client.py | 7 +- testsuite/openshift/objects/dnspolicy.py | 27 ++++ .../openshift/objects/gateway_api/gateway.py | 79 +++++++++--- .../openshift/objects/gateway_api/route.py | 6 +- testsuite/tests/conftest.py | 23 ++-- testsuite/tests/mgc/conftest.py | 117 ++++++++++++++++++ testsuite/tests/mgc/test_basic.py | 38 ------ 11 files changed, 255 insertions(+), 81 deletions(-) create mode 100644 testsuite/openshift/objects/dnspolicy.py create mode 100644 testsuite/tests/mgc/conftest.py diff --git a/config/settings.local.yaml.tpl b/config/settings.local.yaml.tpl index 828aecd7a..1811ae6f1 100644 --- a/config/settings.local.yaml.tpl +++ b/config/settings.local.yaml.tpl @@ -40,4 +40,11 @@ # namespace: "kuadrant" # Namespaces where Kuadrant resides # gateway: # Reference to Gateway that should be used # namespace: "istio-system" -# name: "istio-ingressgateway" \ No newline at end of file +# name: "istio-ingressgateway" +# mgc: +# spokes: +# local-cluster: +# project: "kuadrant" # Optional: namespace for tests to run, if None uses current project +# api_url: "https://api.openshift.com" # Optional: OpenShift API URL, if None it will OpenShift that you are logged in +# token: "KUADRANT_RULEZ" # Optional: OpenShift Token, if None it will OpenShift that you are logged in +# kubeconfig_path: "~/.kube/config" # Optional: Kubeconfig to use, if None the default one is used \ No newline at end of file diff --git a/config/settings.yaml b/config/settings.yaml index 493c4dd7b..c3daa1cbf 100644 --- a/config/settings.yaml +++ b/config/settings.yaml @@ -22,4 +22,4 @@ default: name: "istio-ingressgateway" hyperfoil: generate_reports: True - reports_dir: "reports" \ No newline at end of file + reports_dir: "reports" diff --git a/pyproject.toml b/pyproject.toml index 0de4208d2..96095c2b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,14 +69,14 @@ disable = [ good-names=["i","j","k", "pytestmark", "logger", - "ca"] + "ca", "gw"] # Mypy: [tool.mypy] implicit_optional = true [[tool.mypy.overrides]] -module = ["dynaconf.*", "keycloak.*", "weakget.*", "openshift.*", "apyproxy.*", "click.*"] +module = ["dynaconf.*", "keycloak.*", "weakget.*", "openshift.*", "apyproxy.*", "click.*", "py.*"] ignore_missing_imports = true [build-system] diff --git a/testsuite/config/openshift_loader.py b/testsuite/config/openshift_loader.py index 990733dab..6f640f59b 100644 --- a/testsuite/config/openshift_loader.py +++ b/testsuite/config/openshift_loader.py @@ -24,11 +24,19 @@ def load(obj, env=None, silent=True, key=None, filename=None): openshift2 = client.change_project(obj["openshift2"]["project"]) obj["openshift2"] = openshift2 - kcp = None - if "kcp" in obj and "project" in obj["kcp"]: - kcp_section = config["kcp"] - kcp = client.change_project(kcp_section["project"] % None) - # when advanced scheduling is enabled on kcp/syncer, status field is not synced back from workload cluster - # deployment, is_ready method depends on status field that is not available yet hence we have to mock it - kcp.is_ready = lambda _: True - obj["kcp"] = kcp + clients = {} + spokes = weakget(obj)["mgc"]["spokes"] % {} + for name, value in spokes.items(): + value = weakget(value) + clients[name] = OpenShiftClient( + value["project"] % None, value["api_url"] % None, value["token"] % None, value["kubeconfig_path"] % None + ) + if len(clients) > 0: + obj["mgc"]["spokes"] = clients + # if "kcp" in obj and "project" in obj["kcp"]: + # kcp_section = config["kcp"] + # kcp = client.change_project(kcp_section["project"] % None) + # # when advanced scheduling is enabled on kcp/syncer, status field is not synced back from workload cluster + # # deployment, is_ready method depends on status field that is not available yet hence we have to mock it + # kcp.is_ready = lambda _: True + # obj["kcp"] = kcp diff --git a/testsuite/openshift/client.py b/testsuite/openshift/client.py index a0e54584c..dfebaca57 100644 --- a/testsuite/openshift/client.py +++ b/testsuite/openshift/client.py @@ -34,6 +34,11 @@ def __init__(self, project: str, api_url: str = None, token: str = None, kubecon self._token = token self._kubeconfig_path = kubeconfig_path + @classmethod + def from_context(cls, context: Context) -> "OpenShiftClient": + """Creates OpenShiftClient from the context""" + return cls(context.get_project(), context.get_api_url(), context.get_token(), context.get_kubeconfig_path()) + def change_project(self, project) -> "OpenShiftClient": """Return new OpenShiftClient with a different project""" return OpenShiftClient(project, self._api_url, self._token, self._kubeconfig_path) @@ -44,7 +49,7 @@ def context(self): context = Context() context.project_name = self._project - context.api_url = self._api_url + context.api_server = self._api_url context.token = self._token context.kubeconfig_path = self._kubeconfig_path diff --git a/testsuite/openshift/objects/dnspolicy.py b/testsuite/openshift/objects/dnspolicy.py new file mode 100644 index 000000000..8d228b162 --- /dev/null +++ b/testsuite/openshift/objects/dnspolicy.py @@ -0,0 +1,27 @@ +"""Module for DNSPolicy related classes""" +from testsuite.openshift.client import OpenShiftClient +from testsuite.openshift.objects import OpenShiftObject +from testsuite.openshift.objects.gateway_api import Referencable + + +class DNSPolicy(OpenShiftObject): + """DNSPolicy object""" + + @classmethod + def create_instance( + cls, + openshift: OpenShiftClient, + name: str, + parent: Referencable, + labels: dict[str, str] = None, + ): + """Creates new instance of DNSPolicy""" + + model = { + "apiVersion": "kuadrant.io/v1alpha1", + "kind": "DNSPolicy", + "metadata": {"name": name, "labels": labels}, + "spec": {"targetRef": parent.reference}, + } + + return cls(model, context=openshift.context) diff --git a/testsuite/openshift/objects/gateway_api/gateway.py b/testsuite/openshift/objects/gateway_api/gateway.py index 22ba4f437..da49ee053 100644 --- a/testsuite/openshift/objects/gateway_api/gateway.py +++ b/testsuite/openshift/objects/gateway_api/gateway.py @@ -1,7 +1,8 @@ """Module containing all gateway classes""" +import json import typing -from openshift import Selector, ModelError, timeout +from openshift import Selector, timeout, selector from testsuite.openshift.client import OpenShiftClient from testsuite.openshift.objects import OpenShiftObject @@ -37,7 +38,7 @@ def create_instance( "listeners": [ { "name": "api", - "port": 8080, + "port": 80, "protocol": "HTTP", "hostname": hostname, "allowedRoutes": {"namespaces": {"from": "All"}}, @@ -52,6 +53,11 @@ def wait_for_ready(self) -> bool: """Waits for the gateway to be ready""" return True + @property + def openshift(self): + """Hostname of the first listener""" + return OpenShiftClient.from_context(self.context) + @property def hostname(self): """Hostname of the first listener""" @@ -87,33 +93,70 @@ def create_instance( if placement is not None: labels["cluster.open-cluster-management.io/placement"] = placement - return Gateway.create_instance(openshift, name, gateway_class, hostname, labels) + return super(MGCGateway, cls).create_instance(openshift, name, gateway_class, hostname, labels) + + def get_spoke_gateway(self, spokes: dict[str, OpenShiftClient]) -> "MGCGateway": + """Returns spoke gateway on one of the spoke clusters""" + # time.sleep(20) + self.refresh() + cluster_name = json.loads(self.model.metadata.annotations["kuadrant.io/gateway-clusters"])[0] + spoke_client = spokes[cluster_name] + # prefix = json.loads(self.model.metadata.annotations["kuadrant.io/namespace"]) + prefix = "kuadrant" + spoke_client = spoke_client.change_project(f"{prefix}-{self.namespace()}") + with spoke_client.context: + return selector(f"gateway/{self.name()}").object(cls=self.__class__) def is_ready(self): - """Checks whether the gateway got its IP address assigned thus is ready""" - try: - addresses = self.model["status"]["addresses"] - multi_cluster_addresses = [ - address for address in addresses if address["type"] == "kuadrant.io/MultiClusterIPAddress" - ] - return len(multi_cluster_addresses) > 0 - except (KeyError, ModelError): - return False + """Check the programmed status""" + for condition in self.model.status.conditions: + if condition.type == "Programmed" and condition.status == "True": + return True + return False + + # def is_ready(self): + # """Checks whether the gateway got its IP address assigned thus is ready""" + # try: + # addresses = self.model["status"]["addresses"] + # multi_cluster_addresses = [ + # address for address in addresses if address["type"] == "kuadrant.io/MultiClusterIPAddress" + # ] + # return len(multi_cluster_addresses) > 0 + # except (KeyError, ModelError): + # return False def wait_for_ready(self): """Waits for the gateway to be ready in the sense of is_ready(self)""" - with timeout(90): - success, _, _ = self.self_selector().until_all(success_func=lambda obj: MGCGateway(obj.model).is_ready()) + with timeout(600): + success, _, _ = self.self_selector().until_all( + success_func=lambda obj: self.__class__(obj.model).is_ready() + ) assert success, "Gateway didn't get ready in time" self.refresh() + return success + + def delete(self, ignore_not_found=True, cmd_args=None): + with timeout(90): + super().delete(ignore_not_found, cmd_args) + + +# class DownstreamMGCGateway(MGCGateway): +# def is_ready(self): +# """Checks whether the gateway got its IP address assigned thus is ready""" +# try: +# addresses = self.model.status.addresses +# multi_cluster_addresses = [address for address in addresses if address["type"] == "IPAddress"] +# return len(multi_cluster_addresses) > 0 +# except (KeyError, ModelError): +# return False class GatewayProxy(Proxy): """Wrapper for Gateway object to make it a Proxy implementation e.g. exposing hostnames outside of the cluster""" - def __init__(self, openshift: OpenShiftClient, gateway: Gateway, label, backend: "Httpbin") -> None: + def __init__(self, gateway: Gateway, label, backend: "Httpbin") -> None: super().__init__() - self.openshift = openshift + self.openshift = gateway.openshift self.gateway = gateway self.name = gateway.name() self.label = label @@ -145,4 +188,6 @@ def commit(self): pass def delete(self): - self.selector.delete() + if self.selector: + self.selector.delete() + self.selector = None diff --git a/testsuite/openshift/objects/gateway_api/route.py b/testsuite/openshift/objects/gateway_api/route.py index 60cd91874..f50789af1 100644 --- a/testsuite/openshift/objects/gateway_api/route.py +++ b/testsuite/openshift/objects/gateway_api/route.py @@ -21,6 +21,10 @@ class HTTPRoute(OpenShiftObject, Referencable): """HTTPRoute object, serves as replacement for Routes and Ingresses""" + def client(self, **kwargs) -> Client: + """Returns HTTPX client""" + return HttpxBackoffClient(base_url=f"http://{self.hostnames[0]}", **kwargs) + @classmethod def create_instance( cls, @@ -33,7 +37,7 @@ def create_instance( ): """Creates new instance of HTTPRoute""" model = { - "apiVersion": "gateway.networking.k8s.io/v1alpha2", + "apiVersion": "gateway.networking.k8s.io/v1beta1", "kind": "HTTPRoute", "metadata": {"name": name, "namespace": openshift.project, "labels": labels}, "spec": { diff --git a/testsuite/tests/conftest.py b/testsuite/tests/conftest.py index e9181e56e..ca3d37dc8 100644 --- a/testsuite/tests/conftest.py +++ b/testsuite/tests/conftest.py @@ -5,17 +5,16 @@ import pytest from dynaconf import ValidationError from keycloak import KeycloakAuthenticationError -from openshift import OpenShiftPythonException from weakget import weakget +from testsuite.certificates import CFSSLClient +from testsuite.config import settings from testsuite.mockserver import Mockserver from testsuite.oidc import OIDCProvider -from testsuite.config import settings -from testsuite.certificates import CFSSLClient from testsuite.oidc.auth0 import Auth0Provider -from testsuite.openshift.httpbin import Httpbin -from testsuite.openshift.envoy import Envoy from testsuite.oidc.rhsso import RHSSO +from testsuite.openshift.envoy import Envoy +from testsuite.openshift.httpbin import Httpbin from testsuite.openshift.objects.gateway_api.gateway import GatewayProxy, Gateway from testsuite.openshift.objects.proxy import Proxy from testsuite.openshift.objects.route import Route @@ -230,12 +229,12 @@ def kuadrant(testconfig, openshift): pytest.fail("Running Kuadrant tests, but Kuadrant resource was not found") # Try if the configured Gateway is deployed - gateway_openshift = openshift.change_project(settings["kuadrant"]["gateway"]["project"] % None) - name = testconfig["kuadrant"]["gateway"]["name"] - try: - gateway_openshift.do_action("get", f"Gateway/{name}") - except OpenShiftPythonException: - pytest.fail(f"Running Kuadrant tests, but Gateway/{name} was not found") + # gateway_openshift = openshift.change_project(settings["kuadrant"]["gateway"]["project"] % None) + # name = testconfig["kuadrant"]["gateway"]["name"] + # try: + # gateway_openshift.do_action("get", f"Gateway/{name}") + # except OpenShiftPythonException: + # pytest.fail(f"Running Kuadrant tests, but Gateway/{name} was not found") # TODO: Return actual Kuadrant object return True @@ -265,7 +264,7 @@ def proxy(request, kuadrant, authorino, openshift, blame, backend, module_label, """Deploys Envoy that wire up the Backend behind the reverse-proxy and Authorino instance""" if kuadrant: gateway_object = request.getfixturevalue("gateway") - envoy: Proxy = GatewayProxy(openshift, gateway_object, module_label, backend) + envoy: Proxy = GatewayProxy(gateway_object, module_label, backend) else: envoy = Envoy(openshift, authorino, blame("envoy"), module_label, backend, testconfig["envoy"]["image"]) request.addfinalizer(envoy.delete) diff --git a/testsuite/tests/mgc/conftest.py b/testsuite/tests/mgc/conftest.py new file mode 100644 index 000000000..2c6a0fcf2 --- /dev/null +++ b/testsuite/tests/mgc/conftest.py @@ -0,0 +1,117 @@ +"""Conftest for MGC tests""" +import pytest +from openshift import selector +from weakget import weakget + +from testsuite.openshift.httpbin import Httpbin +from testsuite.openshift.objects.dnspolicy import DNSPolicy +from testsuite.openshift.objects.gateway_api.gateway import MGCGateway, GatewayProxy +from testsuite.openshift.objects.gateway_api.route import HTTPRoute +from testsuite.openshift.objects.proxy import Proxy +from testsuite.openshift.objects.route import Route + + +@pytest.fixture(scope="module") +def backend(request, gateway, blame, label): + """Deploys Httpbin backend""" + httpbin = Httpbin(gateway.openshift, blame("httpbin"), label) + request.addfinalizer(httpbin.delete) + httpbin.commit() + return httpbin + + +@pytest.fixture(scope="session") +def spokes(testconfig): + """Returns Map of spokes names and their respective clients""" + spokes = weakget(testconfig)["mgc"]["spokes"] % {} + assert len(spokes) > 0, "No spokes configured" + return spokes + + +@pytest.fixture(scope="module") +def upstream_gateway(request, openshift, blame, hostname, module_label): + """Creates and returns configured and ready upstream Gateway""" + upstream_gateway = MGCGateway.create_instance( + openshift=openshift, + name=blame("mgc-gateway"), + gateway_class="kuadrant-multi-cluster-gateway-instance-per-cluster", + hostname=f"*.{hostname}", + placement="http-gateway", + labels={"app": module_label}, + ) + request.addfinalizer(upstream_gateway.delete) + upstream_gateway.commit() + upstream_gateway.wait_for_ready() + + return upstream_gateway + + +@pytest.fixture(scope="module") +def proxy(request, gateway, backend, module_label) -> Proxy: + """Deploys Envoy that wire up the Backend behind the reverse-proxy and Authorino instance""" + envoy: Proxy = GatewayProxy(gateway, module_label, backend) + request.addfinalizer(envoy.delete) + envoy.commit() + return envoy + + +@pytest.fixture(scope="module") +def initial_host(hostname): + """Hostname that will be added to HTTPRoute""" + return f"route.{hostname}" + + +@pytest.fixture(scope="module") +def route(request, proxy, blame, gateway, initial_host, backend) -> Route: + """Exposed Route object""" + route = HTTPRoute.create_instance( + gateway.openshift, + blame("route"), + gateway, + initial_host, + backend, + labels={"app": proxy.label}, + ) + request.addfinalizer(route.delete) + route.commit() + return route + + +@pytest.fixture(scope="module") +def gateway(upstream_gateway, spokes): + """Downstream gateway, e.g. gateway on a spoke cluster""" + gw = upstream_gateway.get_spoke_gateway(spokes) + gw.wait_for_ready() + return gw + + +@pytest.fixture(scope="module") +def base_domain(openshift): + """Returns preconfigured base domain""" + with openshift.context: + zone = selector("managedzone/mgc-dev-mz").object() + return zone.model["spec"]["domainName"] + + +@pytest.fixture(scope="module") +def hostname(blame, base_domain): + """Returns domain used for testing""" + return f"{blame('mgc')}.{base_domain}" + + +@pytest.fixture(scope="module") +def dns_policy(blame, upstream_gateway, module_label): + """DNSPolicy fixture""" + policy = DNSPolicy.create_instance( + upstream_gateway.openshift, blame("dns"), upstream_gateway, labels={"app": module_label} + ) + return policy + + +@pytest.fixture(scope="module", autouse=True) +def commit(request, dns_policy): + """Commits all important stuff before tests""" + for component in [dns_policy]: + if component is not None: + request.addfinalizer(component.delete) + component.commit() diff --git a/testsuite/tests/mgc/test_basic.py b/testsuite/tests/mgc/test_basic.py index da4972f6d..235d9a50c 100644 --- a/testsuite/tests/mgc/test_basic.py +++ b/testsuite/tests/mgc/test_basic.py @@ -17,47 +17,9 @@ import pytest -from testsuite.openshift.objects.gateway_api.gateway import MGCGateway - pytestmark = [pytest.mark.mgc] -@pytest.fixture(scope="module") -def base_domain(openshift): - """Returns preconfigured base domain""" - managed_zone = openshift.do_action("get", ["managedzone", "mgc-dev-mz", "-o", "yaml"], parse_output=True) - return managed_zone.model["spec"]["domainName"] - - -@pytest.fixture(scope="module") -def hostname(blame, base_domain): - """Returns domain used for testing""" - return f"{blame('mgc')}.{base_domain}" - - -@pytest.fixture(scope="module") -def gateway(request, openshift, blame, hostname, module_label): - """Creates and returns configured and ready upstream Gateway""" - upstream_gateway = MGCGateway.create_instance( - openshift=openshift, - name=blame("mgc-gateway"), - gateway_class="kuadrant-multi-cluster-gateway-instance-per-cluster", - hostname=hostname, - placement="local-gateway", - labels={"app": module_label}, - ) - request.addfinalizer(upstream_gateway.delete) - upstream_gateway.commit() - upstream_gateway.wait_for_ready() - - openshift = openshift.change_project(f"kuadrant-{upstream_gateway.namespace()}") - downstream_gateway = openshift.do_action( - "get", ["gateway", upstream_gateway.name(), "-o", "yaml"], parse_output=False - ) - downstream_gateway = MGCGateway(string_to_model=downstream_gateway.out(), context=openshift.context) - return downstream_gateway - - def test_gateway_readiness(gateway): """Tests whether the Gateway was successfully placed by having its IP address assigned""" assert gateway.is_ready()