Merge pull request Kuadrant#288 from pehala/capabilities

Add capabilities

Makefile

@@ -1,4 +1,4 @@
-.PHONY: commit-acceptance pylint mypy black reformat test performance authorino poetry poetry-no-dev mgc container-image polish-junit reportportal
+.PHONY: commit-acceptance pylint mypy black reformat test performance authorino poetry poetry-no-dev mgc container-image polish-junit reportportal authorino-standalone limitador kuadrant kuadrant-only
 
 TB ?= short
 LOGLEVEL ?= INFO
@@ -38,23 +38,39 @@ all-is-package:
 
 # pattern to run individual testfile or all testfiles in directory
 testsuite/%: FORCE poetry-no-dev
-	$(PYTEST) --performance --mgc -v $(flags) $@
+	$(PYTEST) --performance -v $(flags) $@
 
-test: ## Run tests
+test: ## Run all non mgc tests
 test pytest tests: poetry-no-dev
-	$(PYTEST) -n4 -m 'not flaky' --dist loadfile $(flags) testsuite
+	$(PYTEST) -n4 -m 'not mgc' --dist loadfile $(flags) testsuite
 
-authorino: ## Run test
+authorino: ## Run only authorino related tests
 authorino: poetry-no-dev
-	$(PYTEST) -n4 -m 'not flaky' --dist loadfile $(flags) testsuite/tests/kuadrant/authorino
+	$(PYTEST) -n4 -m 'authorino' --dist loadfile --enforce $(flags) testsuite
 
-performance: ## Run performance tests
-performance: poetry-no-dev
-	$(PYTEST) --performance $(flags) testsuite/tests/kuadrant/authorino/performance
+authorino-standalone: ## Run only test capable of running with standalone Authorino
+authorino-standalone: poetry-no-dev
+	$(PYTEST) -n4 -m 'authorino and not kuadrant_only' --dist loadfile --enforce $(flags) testsuite/tests/kuadrant/authorino
+
+limitador: ## Run only Limitador related tests
+limitador: poetry-no-dev
+	$(PYTEST) -n4 -m 'limitador' --dist loadfile --enforce $(flags) testsuite
+
+kuadrant: ## Run all tests available on Kuadrant
+kuadrant: poetry-no-dev
+	$(PYTEST) -n4 -m 'not standalone_only' --dist loadfile --enforce $(flags) testsuite
+
+kuadrant-only: ## Run Kuadrant-only tests
+kuadrant-only: poetry-no-dev
+	$(PYTEST) -n4 -m 'kuadrant_only' --dist loadfile --enforce $(flags) testsuite
 
 mgc: ## Run mgc tests
 mgc: poetry-no-dev
-	$(PYTEST) --mgc $(flags) testsuite/tests/mgc
+	$(PYTEST) -m "mgc" --enforce $(flags) testsuite
+
+performance: ## Run performance tests
+performance: poetry-no-dev
+	$(PYTEST) --performance $(flags) testsuite/tests/kuadrant/authorino/performance
 
 poetry.lock: pyproject.toml
 	poetry lock

config/settings.local.yaml.tpl

@@ -1,7 +1,7 @@
 #default:
 #  skip_cleanup: false
 #  tester: "someuser"                          # Optional: name of the user, who is running the tests, defaults to whoami/uid
-#  gateway_api: true                           # True, if Testsuite should test with Gateway API enabled (e.g. Full Kuadrant) or individual components (e.g. Authorino)
+#  standalone: false                           # True, if Testsuite should test only individual components (e.g. Authorino/limitador operators)
 #  cluster:                                    # Workload cluster where tests should run, will get overriden if run on Multicluster
 #    project: "kuadrant"                       # Optional: Default namespace for this cluster
 #    api_url: "https://api.openshift.com"      # Optional: OpenShift API URL, if None it will OpenShift that you are logged in

config/settings.yaml

@@ -1,7 +1,7 @@
 default:
   skip_cleanup: false
   dynaconf_merge: true
-  gateway_api: true
+  standalone: false
   cluster: {}
   tools:
     project: "tools"

pyproject.toml

@@ -43,7 +43,11 @@ line-length = 120
 markers = [
     "issue: Reference to covered issue",
     "performance: Performance tests have unique needs",
-    "mgc: MGC tests have specific needs"
+    "mgc: Test is using MGC specific features",
+    "authorino: Test is using Authorino features",
+    "standalone_only: Test is using features available only in standalone mode, without Kuadrant",
+    "kuadrant_only: Test is using features available only in Kuadrant mode",
+    "limitador: Test is using Limitador features",
 ]
 filterwarnings = [
     "ignore: WARNING the new order is not taken into account:UserWarning",

testsuite/capabilities.py

@@ -0,0 +1,56 @@
+"""Contains capability related classes"""
+
+import functools
+
+from weakget import weakget
+
+from testsuite.config import settings
+
+
+@functools.cache
+def has_kuadrant():
+    """Returns True, if Kuadrant deployment is present and should be used"""
+    spokes = weakget(settings)["control_plane"]["spokes"] % {}
+
+    if settings.get("standalone", False):
+        return False, "Standalone mode is enabled"
+
+    for name, openshift in spokes.items():
+        # Try if Kuadrant is deployed
+        if not openshift.connected:
+            return False, f"Spoke {name} is not connected"
+        project = settings["service_protection"]["system_project"]
+        kuadrant_openshift = openshift.change_project(project)
+        kuadrants = kuadrant_openshift.do_action("get", "kuadrant", "-o", "json", parse_output=True)
+        if len(kuadrants.model["items"]) == 0:
+            return False, f"Spoke {name} does not have Kuadrant resource in project {project}"
+
+    return True, None
+
+
+@functools.cache
+def is_standalone():
+    """Return True, if the testsuite is configured to run with envoy in standalone mode, without Gateway API"""
+    if not settings.get("standalone", False):
+        return False, "Standalone mode is disabled"
+    return True, None
+
+
+@functools.cache
+def has_mgc():
+    """Returns True, if MGC is configured and deployed"""
+    spokes = weakget(settings)["control_plane"]["spokes"] % {}
+
+    if settings.get("standalone", False):
+        return False, "Standalone mode is enabled"
+
+    if len(spokes) == 0:
+        return False, "Spokes are not configured"
+
+    hub_openshift = settings["control_plane"]["hub"]
+    if not hub_openshift.connected:
+        return False, "Control Plane Hub Openshift is not connected"
+
+    if "managedzones" not in hub_openshift.do_action("api-resources", "--api-group=kuadrant.io").out():
+        return False, "MGC custom resources are missing on hub cluster"
+    return True, None

testsuite/config/__init__.py

@@ -41,7 +41,7 @@ def __init__(self, name, default, **kwargs) -> None:
         DefaultValueValidator("rhsso.url", default=fetch_route("no-ssl-sso")),
         DefaultValueValidator("rhsso.password", default=fetch_secret("credential-sso", "ADMIN_PASSWORD")),
         DefaultValueValidator("mockserver.url", default=fetch_route("mockserver", force_http=True)),
-        Validator("gateway_api", must_exist=False, eq=False)
+        Validator("standalone", must_exist=False, eq=True)
         | Validator("service_protection.gateway.name", must_exist=True),
     ],
     validate_only=["authorino", "kuadrant"],

testsuite/tests/conftest.py

@@ -7,6 +7,7 @@
 from dynaconf import ValidationError
 from keycloak import KeycloakAuthenticationError
 
+from testsuite.capabilities import has_kuadrant, has_mgc, is_standalone
 from testsuite.certificates import CFSSLClient
 from testsuite.config import settings
 from testsuite.mockserver import Mockserver
@@ -28,16 +29,32 @@ def pytest_addoption(parser):
     parser.addoption(
         "--performance", action="store_true", default=False, help="Run also performance tests (default: False)"
     )
-    parser.addoption("--mgc", action="store_true", default=False, help="Run also mgc tests (default: False)")
+    parser.addoption("--enforce", action="store_true", default=False, help="Fails tests instead of skip")
 
 
 def pytest_runtest_setup(item):
-    """Exclude performance tests by default, require explicit option"""
+    """
+    Skip or fail tests based on available capabilities and marks
+    First round of filtering is usually done by pytest through -m option
+    (https://docs.pytest.org/en/latest/example/markers.html#marking-test-functions-and-selecting-them-for-a-run)
+    In this function we skip or fail the tests that were selected but their capabilities are not available
+    """
     marks = [i.name for i in item.iter_markers()]
     if "performance" in marks and not item.config.getoption("--performance"):
         pytest.skip("Excluding performance tests")
-    if "mgc" in marks and not item.config.getoption("--mgc"):
-        pytest.skip("Excluding MGC tests")
+    skip_func = pytest.fail if item.config.getoption("--enforce") else pytest.skip
+    if "kuadrant_only" in marks:
+        kuadrant, error = has_kuadrant()
+        if not kuadrant:
+            skip_func(f"Unable to locate Kuadrant installation: {error}")
+    if "standalone_only" in marks:
+        status, error = is_standalone()
+        if not status:
+            skip_func(f"Unable to run Standalone tests: {error}")
+    if "mgc" in marks:
+        mgc, error = has_mgc()
+        if not mgc:
+            skip_func(f"Unable to locate MGC installation: {error}")
 
 
 @pytest.hookimpl(hookwrapper=True)
@@ -220,7 +237,7 @@ def module_label(label):
 @pytest.fixture(scope="module")
 def kuadrant(testconfig, openshift):
     """Returns Kuadrant instance if exists, or None"""
-    if not testconfig.get("gateway_api", True):
+    if testconfig.get("standalone", False):
         return None
 
     # Try if Kuadrant is deployed

testsuite/tests/kuadrant/authorino/authorization/opa/external_registry/test_auto_refresh_policy.py

@@ -10,6 +10,9 @@
 from testsuite.utils import rego_allow_header
 
 
+pytestmark = [pytest.mark.authorino]
+
+
 @pytest.fixture(scope="module")
 def updated_header():
     """Header for updated OPA policy"""

testsuite/tests/kuadrant/authorino/authorization/opa/external_registry/test_external_registry.py

@@ -1,5 +1,9 @@
 """Tests for Open Policy Agent (OPA) using Mockserver Expectations as http endpoint with Rego query"""
 
+import pytest
+
+pytestmark = [pytest.mark.authorino]
+
 
 def test_allowed_by_opa(client, auth, header):
     """Tests a request that should be authorized by OPA external registry declaration"""

testsuite/tests/kuadrant/authorino/authorization/opa/test_inline_rego.py

@@ -4,6 +4,8 @@
 
 from testsuite.utils import rego_allow_header
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def header():

testsuite/tests/kuadrant/authorino/caching/metadata/test_caching.py

@@ -8,6 +8,9 @@
 from testsuite.utils import extract_response
 
 
+pytestmark = [pytest.mark.authorino]
+
+
 @pytest.fixture(scope="module")
 def cache_ttl():
     """Returns TTL in seconds for Cached Metadata"""

testsuite/tests/kuadrant/authorino/caching/metadata/test_not_cached.py

@@ -4,6 +4,8 @@
 
 from testsuite.utils import extract_response
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def authorization(authorization, module_label, expectation_path):

testsuite/tests/kuadrant/authorino/conditions/section_conditions/test_authorization_condition.py

@@ -4,6 +4,8 @@
 
 from testsuite.policy.authorization import Pattern
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def authorization(authorization):

testsuite/tests/kuadrant/authorino/conditions/section_conditions/test_identity_condition.py

@@ -5,6 +5,8 @@
 from testsuite.policy.authorization import Pattern
 from testsuite.httpx.auth import HeaderApiKeyAuth
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def api_key(create_api_key, module_label):

testsuite/tests/kuadrant/authorino/conditions/section_conditions/test_metadata_condition.py

@@ -4,6 +4,8 @@
 
 from testsuite.policy.authorization import Pattern
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def mockserver_expectation(request, mockserver, module_label):

testsuite/tests/kuadrant/authorino/conditions/section_conditions/test_response_condition.py

@@ -5,6 +5,8 @@
 from testsuite.policy.authorization import Pattern, Value, JsonResponse
 from testsuite.utils import extract_response
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def authorization(authorization):

testsuite/tests/kuadrant/authorino/conditions/test_patternref_expressions.py

@@ -4,6 +4,8 @@
 
 from testsuite.policy.authorization import Pattern, PatternRef, AnyPattern, AllPattern
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def authorization(authorization):

testsuite/tests/kuadrant/authorino/conditions/test_top_level_condition.py

@@ -4,6 +4,8 @@
 
 from testsuite.policy.authorization import Pattern
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def authorization(authorization, module_label):

testsuite/tests/kuadrant/authorino/dinosaur/test_dinosaur.py

@@ -4,6 +4,8 @@
 
 import pytest
 
+pytestmark = [pytest.mark.authorino]
+
 ERROR_MESSAGE = {
     "kind": "Error",
     "id": "403",

testsuite/tests/kuadrant/authorino/identity/anonymous/test_anonymous_context.py

@@ -4,6 +4,8 @@
 
 from testsuite.utils import extract_response
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def authorization(authorization):

testsuite/tests/kuadrant/authorino/identity/anonymous/test_anonymous_identity.py

@@ -2,6 +2,8 @@
 
 import pytest
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def authorization(authorization, rhsso):

testsuite/tests/kuadrant/authorino/identity/api_key/test_api_key_context.py

@@ -4,6 +4,8 @@
 
 from testsuite.utils import extract_response
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def authorization(authorization, api_key):

testsuite/tests/kuadrant/authorino/identity/api_key/test_auth_credentials.py

@@ -4,6 +4,8 @@
 
 from testsuite.policy.authorization import Credentials
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module", params=["authorizationHeader", "customHeader", "queryString", "cookie"])
 def credentials(request):

testsuite/tests/kuadrant/authorino/identity/api_key/test_match_expression.py

@@ -9,6 +9,9 @@
 from testsuite.openshift import Selector, MatchExpression
 
 
+pytestmark = [pytest.mark.authorino]
+
+
 @pytest.fixture(scope="module")
 def valid_label_selectors(module_label):
     """Accepted labels for selector.matchExpressions in AuthConfig"""

testsuite/tests/kuadrant/authorino/identity/api_key/test_match_label.py

@@ -3,6 +3,9 @@
 import pytest
 
 
+pytestmark = [pytest.mark.authorino]
+
+
 @pytest.fixture(scope="module")
 def authorization(authorization, api_key):
     """Creates AuthConfig with API key identity"""

testsuite/tests/kuadrant/authorino/identity/api_key/test_reconciliation.py

@@ -5,6 +5,8 @@
 from testsuite.httpx.auth import HeaderApiKeyAuth
 from testsuite.openshift import Selector
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="function")
 def api_key(create_api_key, module_label):

testsuite/tests/kuadrant/authorino/identity/auth/test_auth_identity.py

@@ -6,6 +6,8 @@
 from testsuite.oidc import OIDCProvider
 from testsuite.oidc.rhsso import RHSSO
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def authorization(authorization, oidc_provider):

testsuite/tests/kuadrant/authorino/identity/auth/test_multiple_auth_identities.py

@@ -6,6 +6,8 @@
 
 from testsuite.httpx.auth import HttpxOidcClientAuth
 
+pytestmark = [pytest.mark.authorino]
+
 
 @pytest.fixture(scope="module")
 def auth0_auth(auth0):