feat: Allow running subsets of test cases

README.md

@@ -82,6 +82,30 @@ In order to run the tests using the `driver`:
  tox -e uats
  ```
 
+ You can also run a subset of the provided tests using the `--filter` option and passing a filter
+ that follows the same syntax as the pytest `-k` option, e.g.
+
+ ```bash
+ # run all tests containing 'kfp' or 'katib' in their name
+ tox -e uats -- --filter "kfp or katib"
+ # run any test that doesn't contain 'kserve' in its name
+ tox -e uats -- --filter "not kserve"
+ ```
+
+ This simulates the behaviour of running `pytest -k "some filter"` directly on the test suite.
+ You can read more about the options provided by Pytest in the corresponding section of the
+ [documentation](https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags).
+
+#### Run Kubeflow UATs
+
+In order to only run the Kubeflow-specific tests (i.e. no MLFlow integration) you can use the
+dedicated `kubeflow` tox test environment:
+
+```bash
+# assumes an existing `kubeflow` Juju model
+tox -e kubeflow
+```
+
 #### Developer Notes
 
 Any environment that can be used to access and configure the Charmed Kubeflow deployment is

assets/test-job.yaml.j2

@@ -22,7 +22,7 @@ spec:
  - |
  cd /tests;
  pip install -r requirements.txt >/dev/null;
- pytest;
+ {{ pytest_cmd }};
  # Kill Istio Sidecar after workload completes to have the Job status properly updated
  # https://github.com/istio/istio/issues/6324
  x=$(echo $?);

driver/conftest.py

@@ -0,0 +1,17 @@
+from _pytest.config.argparsing import Parser
+
+
+def pytest_addoption(parser: Parser):
+ """Add pytest options.
+
+ * Add a `--filter` option to (de)select test cases based on their name (see also
+ https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags)
+ """
+ parser.addoption(
+ "--filter",
+ help="Provide a filter to (de)select tests cases based on their name. The filter follows"
+ " the same syntax as the pytest `-k` option, e.g. --filter 'kfp or katib' will run all"
+ " tests containing 'kfp' or 'katib' in their name, whereas --filter 'not kserve' will run"
+ " any test that doesn't contain 'kserve' in its name. Essentially, the option simulates"
+ " the behaviour of running `pytest -k '<filter>'` directly on the test suite."
+ )

driver/test_kubeflow_workloads.py

@@ -29,6 +29,21 @@
 
 JOB_NAME = "test-kubeflow"
 
+PYTEST_CMD_BASE = "pytest"
+
+
+@pytest.fixture(scope="session")
+def pytest_filter(request):
+ """Retrieve filter from Pytest invocation."""
+ filter = request.config.getoption("filter")
+ return f"-k '{filter}'" if filter else ""
+
+
+@pytest.fixture(scope="session")
+def pytest_cmd(pytest_filter):
+ """Format the Pytest command."""
+ return f"{PYTEST_CMD_BASE} {pytest_filter}" if pytest_filter else PYTEST_CMD_BASE
+
 
 @pytest.fixture(scope="module")
 def lightkube_client():
@@ -80,13 +95,18 @@ async def test_create_profile(lightkube_client, create_profile):
  assert_namespace_active(lightkube_client, NAMESPACE)
 
 
-def test_kubeflow_workloads(lightkube_client):
+def test_kubeflow_workloads(lightkube_client, pytest_cmd):
  """Run a K8s Job to execute the notebook tests."""
  log.info(f"Starting Kubernetes Job {NAMESPACE}/{JOB_NAME} to run notebook tests...")
  resources = list(
  codecs.load_all_yaml(
  JOB_TEMPLATE_FILE.read_text(),
- context={"job_name": JOB_NAME, "test_dir": TESTS_DIR, "test_image": TESTS_IMAGE},
+ context={
+ "job_name": JOB_NAME,
+ "test_dir": TESTS_DIR,
+ "test_image": TESTS_IMAGE,
+ "pytest_cmd": pytest_cmd,
+ },
  )
  )
  assert len(resources) == 1, f"Expected 1 Job, got {len(resources)}!"

tests/README.md

@@ -25,3 +25,14 @@ pytest
 
 The above inherits the configuration set in [pytest.ini](pytest.ini). Feel free to provide any
 required extra settings either using that file or directly through CLI arguments.
+For instance, in order to select a subset of tests to run, you can filter on the notebook names
+using the [-k](https://docs.pytest.org/en/7.3.x/how-to/usage.html#specifying-which-tests-to-run)
+Pytest command-line option, e.g.
+
+```bash
+# run all tests containing 'kfp' or 'katib' in their name
+pytest -k "kfp or katib"
+
+# run any test that doesn't contain 'kserve' in its name
+pytest -k "not kserve"
+```

tox.ini

@@ -61,8 +61,23 @@ deps =
  -r requirements-lint.txt
 description = Check code against coding style standards
 
+[testenv:kubeflow]
+commands =
+ # run all tests apart from the ones that use MLFlow
+ pytest -vv --tb native {[vars]driver_path} -s --filter "not mlflow" --model kubeflow {posargs}
+deps =
+ -r requirements.txt
+description = Run UATs for Kubeflow
+
 [testenv:uats]
+# provide a filter when calling tox to (de)select test cases based on their names, e.g.
+# * run all tests containing 'kfp' or 'katib' in their name:
+# $ tox -e uats -- --filter "kfp or katib"
+# * run any test that doesn't contain 'kserve' in its name:
+# $ tox -e uats -- --filter "not kserve"
+# this simulates the behaviour of running 'pytest -k "<filter>"' directly on the test suite:
+# https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags
 commands = pytest -vv --tb native {[vars]driver_path} -s --model kubeflow {posargs}
 deps =
  -r requirements.txt
-description = Run UATs
+description = Run UATs for Kubeflow and Integrations