Skip to content

Commit

Permalink
Merge pull request #237 from pehala/fix_mgc_multiple_clusters
Browse files Browse the repository at this point in the history
MGC rewritten for new API and DNSPolicy usage
  • Loading branch information
pehala authored Sep 18, 2023
2 parents 61b56e6 + 0351aa2 commit 1cac07a
Show file tree
Hide file tree
Showing 11 changed files with 221 additions and 83 deletions.
9 changes: 8 additions & 1 deletion config/settings.local.yaml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,11 @@
# namespace: "kuadrant" # Namespaces where Kuadrant resides
# gateway: # Reference to Gateway that should be used
# namespace: "istio-system"
# name: "istio-ingressgateway"
# name: "istio-ingressgateway"
# mgc:
# spokes:
# local-cluster:
# project: "kuadrant" # Optional: namespace for tests to run, if None uses current project
# api_url: "https://api.openshift.com" # Optional: OpenShift API URL, if None it will OpenShift that you are logged in
# token: "KUADRANT_RULEZ" # Optional: OpenShift Token, if None it will OpenShift that you are logged in
# kubeconfig_path: "~/.kube/config" # Optional: Kubeconfig to use, if None the default one is used
2 changes: 1 addition & 1 deletion config/settings.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@ default:
name: "istio-ingressgateway"
hyperfoil:
generate_reports: True
reports_dir: "reports"
reports_dir: "reports"
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,14 @@ disable = [
good-names=["i","j","k",
"pytestmark",
"logger",
"ca"]
"ca", "gw"]

# Mypy:
[tool.mypy]
implicit_optional = true

[[tool.mypy.overrides]]
module = ["dynaconf.*", "keycloak.*", "weakget.*", "openshift.*", "apyproxy.*", "click.*"]
module = ["dynaconf.*", "keycloak.*", "weakget.*", "openshift.*", "apyproxy.*", "click.*", "py.*"]
ignore_missing_imports = true

[build-system]
Expand Down
17 changes: 9 additions & 8 deletions testsuite/config/openshift_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@ def load(obj, env=None, silent=True, key=None, filename=None):
openshift2 = client.change_project(obj["openshift2"]["project"])
obj["openshift2"] = openshift2

kcp = None
if "kcp" in obj and "project" in obj["kcp"]:
kcp_section = config["kcp"]
kcp = client.change_project(kcp_section["project"] % None)
# when advanced scheduling is enabled on kcp/syncer, status field is not synced back from workload cluster
# deployment, is_ready method depends on status field that is not available yet hence we have to mock it
kcp.is_ready = lambda _: True
obj["kcp"] = kcp
clients = {}
spokes = weakget(obj)["mgc"]["spokes"] % {}
for name, value in spokes.items():
value = weakget(value)
clients[name] = OpenShiftClient(
value["project"] % None, value["api_url"] % None, value["token"] % None, value["kubeconfig_path"] % None
)
if len(clients) > 0:
obj["mgc"]["spokes"] = clients
7 changes: 6 additions & 1 deletion testsuite/openshift/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ def __init__(self, project: str, api_url: str = None, token: str = None, kubecon
self._token = token
self._kubeconfig_path = kubeconfig_path

@classmethod
def from_context(cls, context: Context) -> "OpenShiftClient":
"""Creates OpenShiftClient from the context"""
return cls(context.get_project(), context.get_api_url(), context.get_token(), context.get_kubeconfig_path())

def change_project(self, project) -> "OpenShiftClient":
"""Return new OpenShiftClient with a different project"""
return OpenShiftClient(project, self._api_url, self._token, self._kubeconfig_path)
Expand All @@ -44,7 +49,7 @@ def context(self):
context = Context()

context.project_name = self._project
context.api_url = self._api_url
context.api_server = self._api_url
context.token = self._token
context.kubeconfig_path = self._kubeconfig_path

Expand Down
27 changes: 27 additions & 0 deletions testsuite/openshift/objects/dnspolicy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
"""Module for DNSPolicy related classes"""
from testsuite.openshift.client import OpenShiftClient
from testsuite.openshift.objects import OpenShiftObject
from testsuite.openshift.objects.gateway_api import Referencable


class DNSPolicy(OpenShiftObject):
"""DNSPolicy object"""

@classmethod
def create_instance(
cls,
openshift: OpenShiftClient,
name: str,
parent: Referencable,
labels: dict[str, str] = None,
):
"""Creates new instance of DNSPolicy"""

model = {
"apiVersion": "kuadrant.io/v1alpha1",
"kind": "DNSPolicy",
"metadata": {"name": name, "labels": labels},
"spec": {"targetRef": parent.reference},
}

return cls(model, context=openshift.context)
58 changes: 41 additions & 17 deletions testsuite/openshift/objects/gateway_api/gateway.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""Module containing all gateway classes"""
import json
import typing

from openshift import Selector, ModelError, timeout
from openshift import Selector, timeout, selector

from testsuite.openshift.client import OpenShiftClient
from testsuite.openshift.objects import OpenShiftObject
Expand Down Expand Up @@ -37,7 +38,7 @@ def create_instance(
"listeners": [
{
"name": "api",
"port": 8080,
"port": 80,
"protocol": "HTTP",
"hostname": hostname,
"allowedRoutes": {"namespaces": {"from": "All"}},
Expand All @@ -52,6 +53,11 @@ def wait_for_ready(self) -> bool:
"""Waits for the gateway to be ready"""
return True

@property
def openshift(self):
"""Hostname of the first listener"""
return OpenShiftClient.from_context(self.context)

@property
def hostname(self):
"""Hostname of the first listener"""
Expand Down Expand Up @@ -87,33 +93,49 @@ def create_instance(
if placement is not None:
labels["cluster.open-cluster-management.io/placement"] = placement

return Gateway.create_instance(openshift, name, gateway_class, hostname, labels)
return super(MGCGateway, cls).create_instance(openshift, name, gateway_class, hostname, labels)

def get_spoke_gateway(self, spokes: dict[str, OpenShiftClient]) -> "MGCGateway":
"""
Returns spoke gateway on an arbitrary, and sometimes, random spoke cluster.
Works only for GW deployed on Hub
"""
self.refresh()
cluster_name = json.loads(self.model.metadata.annotations["kuadrant.io/gateway-clusters"])[0]
spoke_client = spokes[cluster_name]
prefix = "kuadrant"
spoke_client = spoke_client.change_project(f"{prefix}-{self.namespace()}")
with spoke_client.context:
return selector(f"gateway/{self.name()}").object(cls=self.__class__)

def is_ready(self):
"""Checks whether the gateway got its IP address assigned thus is ready"""
try:
addresses = self.model["status"]["addresses"]
multi_cluster_addresses = [
address for address in addresses if address["type"] == "kuadrant.io/MultiClusterIPAddress"
]
return len(multi_cluster_addresses) > 0
except (KeyError, ModelError):
return False
"""Check the programmed status"""
for condition in self.model.status.conditions:
if condition.type == "Programmed" and condition.status == "True":
return True
return False

def wait_for_ready(self):
"""Waits for the gateway to be ready in the sense of is_ready(self)"""
with timeout(90):
success, _, _ = self.self_selector().until_all(success_func=lambda obj: MGCGateway(obj.model).is_ready())
with timeout(600):
success, _, _ = self.self_selector().until_all(
success_func=lambda obj: self.__class__(obj.model).is_ready()
)
assert success, "Gateway didn't get ready in time"
self.refresh()
return success

def delete(self, ignore_not_found=True, cmd_args=None):
with timeout(90):
super().delete(ignore_not_found, cmd_args)


class GatewayProxy(Proxy):
"""Wrapper for Gateway object to make it a Proxy implementation e.g. exposing hostnames outside of the cluster"""

def __init__(self, openshift: OpenShiftClient, gateway: Gateway, label, backend: "Httpbin") -> None:
def __init__(self, gateway: Gateway, label, backend: "Httpbin") -> None:
super().__init__()
self.openshift = openshift
self.openshift = gateway.openshift
self.gateway = gateway
self.name = gateway.name()
self.label = label
Expand Down Expand Up @@ -145,4 +167,6 @@ def commit(self):
pass

def delete(self):
self.selector.delete()
if self.selector:
self.selector.delete()
self.selector = None
6 changes: 5 additions & 1 deletion testsuite/openshift/objects/gateway_api/route.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@
class HTTPRoute(OpenShiftObject, Referencable):
"""HTTPRoute object, serves as replacement for Routes and Ingresses"""

def client(self, **kwargs) -> Client:
"""Returns HTTPX client"""
return HttpxBackoffClient(base_url=f"http://{self.hostnames[0]}", **kwargs)

@classmethod
def create_instance(
cls,
Expand All @@ -33,7 +37,7 @@ def create_instance(
):
"""Creates new instance of HTTPRoute"""
model = {
"apiVersion": "gateway.networking.k8s.io/v1alpha2",
"apiVersion": "gateway.networking.k8s.io/v1beta1",
"kind": "HTTPRoute",
"metadata": {"name": name, "namespace": openshift.project, "labels": labels},
"spec": {
Expand Down
19 changes: 5 additions & 14 deletions testsuite/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,16 @@
import pytest
from dynaconf import ValidationError
from keycloak import KeycloakAuthenticationError
from openshift import OpenShiftPythonException
from weakget import weakget

from testsuite.certificates import CFSSLClient
from testsuite.config import settings
from testsuite.mockserver import Mockserver
from testsuite.oidc import OIDCProvider
from testsuite.config import settings
from testsuite.certificates import CFSSLClient
from testsuite.oidc.auth0 import Auth0Provider
from testsuite.openshift.httpbin import Httpbin
from testsuite.openshift.envoy import Envoy
from testsuite.oidc.rhsso import RHSSO
from testsuite.openshift.envoy import Envoy
from testsuite.openshift.httpbin import Httpbin
from testsuite.openshift.objects.gateway_api.gateway import GatewayProxy, Gateway
from testsuite.openshift.objects.proxy import Proxy
from testsuite.openshift.objects.route import Route
Expand Down Expand Up @@ -229,14 +228,6 @@ def kuadrant(testconfig, openshift):
if len(kuadrants.model["items"]) == 0:
pytest.fail("Running Kuadrant tests, but Kuadrant resource was not found")

# Try if the configured Gateway is deployed
gateway_openshift = openshift.change_project(settings["kuadrant"]["gateway"]["project"] % None)
name = testconfig["kuadrant"]["gateway"]["name"]
try:
gateway_openshift.do_action("get", f"Gateway/{name}")
except OpenShiftPythonException:
pytest.fail(f"Running Kuadrant tests, but Gateway/{name} was not found")

# TODO: Return actual Kuadrant object
return True

Expand Down Expand Up @@ -265,7 +256,7 @@ def proxy(request, kuadrant, authorino, openshift, blame, backend, module_label,
"""Deploys Envoy that wire up the Backend behind the reverse-proxy and Authorino instance"""
if kuadrant:
gateway_object = request.getfixturevalue("gateway")
envoy: Proxy = GatewayProxy(openshift, gateway_object, module_label, backend)
envoy: Proxy = GatewayProxy(gateway_object, module_label, backend)
else:
envoy = Envoy(openshift, authorino, blame("envoy"), module_label, backend, testconfig["envoy"]["image"])
request.addfinalizer(envoy.delete)
Expand Down
117 changes: 117 additions & 0 deletions testsuite/tests/mgc/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
"""Conftest for MGC tests"""
import pytest
from openshift import selector
from weakget import weakget

from testsuite.openshift.httpbin import Httpbin
from testsuite.openshift.objects.dnspolicy import DNSPolicy
from testsuite.openshift.objects.gateway_api.gateway import MGCGateway, GatewayProxy
from testsuite.openshift.objects.gateway_api.route import HTTPRoute
from testsuite.openshift.objects.proxy import Proxy
from testsuite.openshift.objects.route import Route


@pytest.fixture(scope="module")
def backend(request, gateway, blame, label):
"""Deploys Httpbin backend"""
httpbin = Httpbin(gateway.openshift, blame("httpbin"), label)
request.addfinalizer(httpbin.delete)
httpbin.commit()
return httpbin


@pytest.fixture(scope="session")
def spokes(testconfig):
"""Returns Map of spokes names and their respective clients"""
spokes = weakget(testconfig)["mgc"]["spokes"] % {}
assert len(spokes) > 0, "No spokes configured"
return spokes


@pytest.fixture(scope="module")
def upstream_gateway(request, openshift, blame, hostname, module_label):
"""Creates and returns configured and ready upstream Gateway"""
upstream_gateway = MGCGateway.create_instance(
openshift=openshift,
name=blame("mgc-gateway"),
gateway_class="kuadrant-multi-cluster-gateway-instance-per-cluster",
hostname=f"*.{hostname}",
placement="http-gateway",
labels={"app": module_label},
)
request.addfinalizer(upstream_gateway.delete)
upstream_gateway.commit()
upstream_gateway.wait_for_ready()

return upstream_gateway


@pytest.fixture(scope="module")
def proxy(request, gateway, backend, module_label) -> Proxy:
"""Deploys Envoy that wire up the Backend behind the reverse-proxy and Authorino instance"""
envoy: Proxy = GatewayProxy(gateway, module_label, backend)
request.addfinalizer(envoy.delete)
envoy.commit()
return envoy


@pytest.fixture(scope="module")
def initial_host(hostname):
"""Hostname that will be added to HTTPRoute"""
return f"route.{hostname}"


@pytest.fixture(scope="module")
def route(request, proxy, blame, gateway, initial_host, backend) -> Route:
"""Exposed Route object"""
route = HTTPRoute.create_instance(
gateway.openshift,
blame("route"),
gateway,
initial_host,
backend,
labels={"app": proxy.label},
)
request.addfinalizer(route.delete)
route.commit()
return route


@pytest.fixture(scope="module")
def gateway(upstream_gateway, spokes):
"""Downstream gateway, e.g. gateway on a spoke cluster"""
gw = upstream_gateway.get_spoke_gateway(spokes)
gw.wait_for_ready()
return gw


@pytest.fixture(scope="module")
def base_domain(openshift):
"""Returns preconfigured base domain"""
with openshift.context:
zone = selector("managedzone/mgc-dev-mz").object()
return zone.model["spec"]["domainName"]


@pytest.fixture(scope="module")
def hostname(blame, base_domain):
"""Returns domain used for testing"""
return f"{blame('mgc')}.{base_domain}"


@pytest.fixture(scope="module")
def dns_policy(blame, upstream_gateway, module_label):
"""DNSPolicy fixture"""
policy = DNSPolicy.create_instance(
upstream_gateway.openshift, blame("dns"), upstream_gateway, labels={"app": module_label}
)
return policy


@pytest.fixture(scope="module", autouse=True)
def commit(request, dns_policy):
"""Commits all important stuff before tests"""
for component in [dns_policy]:
if component is not None:
request.addfinalizer(component.delete)
component.commit()
Loading

0 comments on commit 1cac07a

Please sign in to comment.