From 53e0389f070a7d26e97e9eb81473693e526275dd Mon Sep 17 00:00:00 2001 From: Phoevos Kalemkeris Date: Mon, 31 Jul 2023 21:16:58 +0300 Subject: [PATCH] feat: Introduce Automated UATs (#1) * Add the following test notebooks: * Test Katib Integration * Test KFP Integration * Test MinIO Integration * Test MLFlow Integration * Add a driver to automate running the test notebooks included in the 'notebooks' directory using Pytest * Add README with instructions for running the test notebooks in an automated manner and CONTRIBUTING.md with guidelines around contributing test notebooks * Add a driver to allow running the UATs included in the 'tests' directory in an automated manner * Add README at the top-level directory including general information around the design of the UATs and instructions for running them. Signed-off-by: Phoevos Kalemkeris --- .gitignore | 7 + README.md | 107 +++ assets/test-job.yaml.j2 | 37 + assets/test-profile.yaml.j2 | 8 + driver/test_kubeflow_workloads.py | 110 +++ driver/utils.py | 91 ++ pyproject.toml | 40 + requirements-fmt.in | 2 + requirements-fmt.txt | 24 + requirements-lint.in | 8 + requirements-lint.txt | 51 + requirements.in | 7 + requirements.txt | 208 ++++ tests/CONTRIBUTING.md | 59 ++ tests/README.md | 27 + tests/notebooks/katib-integration.ipynb | 892 ++++++++++++++++++ tests/notebooks/kfp-integration.ipynb | 226 +++++ .../minio-integration/minio-integration.ipynb | 522 ++++++++++ tests/notebooks/minio-integration/sample.csv | 4 + tests/notebooks/minio-integration/sample.txt | 1 + tests/notebooks/mlflow-integration.ipynb | 445 +++++++++ tests/pytest.ini | 6 + tests/requirements.txt | 2 + tests/test_notebooks.py | 52 + tests/utils.py | 41 + tox.ini | 66 ++ 26 files changed, 3043 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 assets/test-job.yaml.j2 create mode 100644 assets/test-profile.yaml.j2 create mode 100644 driver/test_kubeflow_workloads.py create mode 100644 driver/utils.py create mode 100644 pyproject.toml create mode 100644 requirements-fmt.in create mode 100644 requirements-fmt.txt create mode 100644 requirements-lint.in create mode 100644 requirements-lint.txt create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 tests/CONTRIBUTING.md create mode 100644 tests/README.md create mode 100644 tests/notebooks/katib-integration.ipynb create mode 100644 tests/notebooks/kfp-integration.ipynb create mode 100644 tests/notebooks/minio-integration/minio-integration.ipynb create mode 100644 tests/notebooks/minio-integration/sample.csv create mode 100644 tests/notebooks/minio-integration/sample.txt create mode 100644 tests/notebooks/mlflow-integration.ipynb create mode 100644 tests/pytest.ini create mode 100644 tests/requirements.txt create mode 100644 tests/test_notebooks.py create mode 100644 tests/utils.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c62198d --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.tox/ +venv/ +__pycache__/ +.pytest_cache/ +.ipynb_checkpoints/ +.idea/ +.vscode/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..8829dcb --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +# Charmed Kubeflow Automated UATs + +Automated User Acceptance Tests (UATs) are essential for evaluating the stability of Charmed +Kubeflow, as well as catching issues early, and are intended to be an invaluable testing tool both +pre-release and post-installation. They combine different components of Charmed Kubeflow in a way +that gives us confidence that everything works as expected, and are meant to be used by end-users +as well as developers alike. + +Charmed Kubeflow UATs are broken down in test scenarios implemented as Python notebooks, which are +easy to share, understand, and maintain. We provide a **standalone** test suite included in `tests` +that users can run directly from inside a Notebook with `pytest`, as well as a `driver` that +automates the execution on an existing Kubeflow cluster. More details on running the tests can be +found in the [Run the tests](#run-the-tests) section. + +## Prerequisites + +Executing the UATs requires a deployed Kubeflow cluster. That said, the deployment and +configuration steps are outside the scope of this project. In other words, the automated tests are +going to assume programmatic access to a Kubeflow installation. Such a deployment consists (at the +very least) of the following pieces: + +* A **Kubernetes cluster**, e.g. + * MicroK8s + * Charmed Kubernetes + * EKS cluster +* **Charmed Kubeflow** deployed on top of it +* **MLFlow (optional)** deployed alongside Kubeflow + +For instructions on deploying and getting started with Charmed Kubeflow, we recommend that you +start with [this guide](https://charmed-kubeflow.io/docs/get-started-with-charmed-kubeflow). + +The UATs include tests that assume MLFlow is installed alongside Kubeflow, which will otherwise +fail. For instructions on deploying MLFlow you can start with [this +guide](https://discourse.charmhub.io/t/deploying-charmed-mlflow-v2-and-kubeflow-to-eks/10973), +ignoring the EKS specific steps. + +## Run the tests + +As mentioned before, when it comes to running the tests, you've got 2 options: +* Running the `tests` suite directly with `pytest` inside a Jupyter Notebook +* Running the tests on an existing cluster using the `driver` along with the provided automation + +### Running inside a Notebook + +* Create a new Notebook using the `jupyter-scipy` image: + * Navigate to `Advanced options` > `Configurations` + * Select all available configurations in order for Kubeflow integrations to work as expected + * Launch the Notebook and wait for it to be created +* Start a new terminal session and clone this repo locally: + + ```bash + git clone https://github.com/canonical/charmed-kubeflow-uats.git + ``` +* Navigate to the `tests` directory: + + ```bash + cd charmed-kubeflow-uats/tests + ``` +* Follow the instructions of the provided [README.md](tests/README.md) to execute the test suite + with `pytest` + +### Running from a configured management environment using the `driver` + +In order to run the tests using the `driver`: +* Clone this repo locally and navigate to the repo directory: + + ```bash + git clone https://github.com/canonical/charmed-kubeflow-uats.git + cd charmed-kubeflow-uats/ + ``` +* Setup `tox`: + + ```bash + python3 -m venv venv + source venv/bin/activate + pip install tox + ``` +* Run the UATs: + + ```bash + # assumes an existing `kubeflow` Juju model + tox -e uats + ``` + +#### Developer Notes + +Any environment that can be used to access and configure the Charmed Kubeflow deployment is +considered a configured management environment. That is, essentially, any machine with `kubectl` +access to the underlying Kubernetes cluster. This is crucial, since the driver directly depends on +a Kubernetes Job to run the tests. More specifically, the `driver` executes the following steps: +1. Create a Kubeflow Profile (i.e. `test-kubeflow`) to run the tests in +2. Submit a Kubernetes Job (i.e. `test-kubeflow`) that runs `tests` + The Job performs the following: + * Mount the local `tests` directory to a Pod that uses `jupyter-scipy` as the container image + * Install python dependencies specified in the [requirements.txt](tests/requirements.txt) + * Run the test suite by executing `pytest` +3. Wait until the Job completes (regardless of the outcome) +4. Collect and report its logs, corresponding to the `pytest` execution of `tests` +5. Cleanup (remove created Job and Profile) + +##### Limitations + +With the current implementation we have to wait until the Job completes to fetch its logs. Of +course this makes for a suboptimal UX, since the user might have to wait long before they learn +about the outcome of their tests. Ideally, the Job logs should be streamed directly to the `pytest` +output, providing real-time insight. This is a known limitation that will be addressed in a future +iteration. diff --git a/assets/test-job.yaml.j2 b/assets/test-job.yaml.j2 new file mode 100644 index 0000000..f9c186e --- /dev/null +++ b/assets/test-job.yaml.j2 @@ -0,0 +1,37 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ job_name }} +spec: + backoffLimit: 0 + template: + metadata: + labels: + access-minio: "true" + access-ml-pipeline: "true" + mlflow-server-minio: "true" + spec: + serviceAccountName: default-editor + containers: + - name: {{ job_name }} + image: {{ test_image }} + command: + - bash + - -c + args: + - | + cd /tests; + pip install -r requirements.txt >/dev/null; + pytest; + # Kill Istio Sidecar after workload completes to have the Job status properly updated + # https://github.com/istio/istio/issues/6324 + x=$(echo $?); + curl -fsI -X POST http://localhost:15020/quitquitquit >/dev/null && exit $x; + volumeMounts: + - name: test-volume + mountPath: /tests + volumes: + - name: test-volume + hostPath: + path: {{ test_dir }} + restartPolicy: Never diff --git a/assets/test-profile.yaml.j2 b/assets/test-profile.yaml.j2 new file mode 100644 index 0000000..fc2b6ab --- /dev/null +++ b/assets/test-profile.yaml.j2 @@ -0,0 +1,8 @@ +apiVersion: kubeflow.org/v1 +kind: Profile +metadata: + name: {{ namespace }} +spec: + owner: + kind: User + name: {{ namespace }}@email.com diff --git a/driver/test_kubeflow_workloads.py b/driver/test_kubeflow_workloads.py new file mode 100644 index 0000000..c95985d --- /dev/null +++ b/driver/test_kubeflow_workloads.py @@ -0,0 +1,110 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +import logging +import os +from pathlib import Path + +import pytest +from lightkube import ApiError, Client, codecs +from lightkube.generic_resource import create_global_resource, load_in_cluster_generic_resources +from utils import assert_namespace_active, delete_job, fetch_job_logs, wait_for_job + +log = logging.getLogger(__name__) + +ASSETS_DIR = Path("assets") +JOB_TEMPLATE_FILE = ASSETS_DIR / "test-job.yaml.j2" +PROFILE_TEMPLATE_FILE = ASSETS_DIR / "test-profile.yaml.j2" + +TESTS_DIR = os.path.abspath(Path("tests")) +TESTS_IMAGE = "kubeflownotebookswg/jupyter-scipy:v1.7.0" + +NAMESPACE = "test-kubeflow" +PROFILE_RESOURCE = create_global_resource( + group="kubeflow.org", + version="v1", + kind="profile", + plural="profiles", +) + +JOB_NAME = "test-kubeflow" + + +@pytest.fixture(scope="module") +def lightkube_client(): + """Initialise Lightkube Client.""" + lightkube_client = Client() + load_in_cluster_generic_resources(lightkube_client) + return lightkube_client + + +@pytest.fixture(scope="module") +def create_profile(lightkube_client): + """Create Profile and handle cleanup at the end of the module tests.""" + log.info(f"Creating Profile {NAMESPACE}...") + resources = list( + codecs.load_all_yaml( + PROFILE_TEMPLATE_FILE.read_text(), + context={"namespace": NAMESPACE}, + ) + ) + assert len(resources) == 1, f"Expected 1 Profile, got {len(resources)}!" + lightkube_client.create(resources[0]) + + yield + + # delete the Profile at the end of the module tests + log.info(f"Deleting Profile {NAMESPACE}...") + lightkube_client.delete(PROFILE_RESOURCE, name=NAMESPACE) + + +@pytest.mark.abort_on_fail +async def test_create_profile(lightkube_client, create_profile): + """Test Profile creation. + + This test relies on the create_profile fixture, which handles the Profile creation and + is responsible for cleaning up at the end. + """ + try: + profile_created = lightkube_client.get( + PROFILE_RESOURCE, + name=NAMESPACE, + ) + except ApiError as e: + if e.status == 404: + profile_created = False + else: + raise + assert profile_created, f"Profile {NAMESPACE} not found!" + + assert_namespace_active(lightkube_client, NAMESPACE) + + +def test_kubeflow_workloads(lightkube_client): + """Run a K8s Job to execute the notebook tests.""" + log.info(f"Starting Kubernetes Job {NAMESPACE}/{JOB_NAME} to run notebook tests...") + resources = list( + codecs.load_all_yaml( + JOB_TEMPLATE_FILE.read_text(), + context={"job_name": JOB_NAME, "test_dir": TESTS_DIR, "test_image": TESTS_IMAGE}, + ) + ) + assert len(resources) == 1, f"Expected 1 Job, got {len(resources)}!" + lightkube_client.create(resources[0], namespace=NAMESPACE) + + try: + wait_for_job(lightkube_client, JOB_NAME, NAMESPACE) + except ValueError: + pytest.fail( + f"Something went wrong while running Job {NAMESPACE}/{JOB_NAME}. Please inspect the" + " attached logs for more info..." + ) + finally: + log.info("Fetching Job logs...") + fetch_job_logs(JOB_NAME, NAMESPACE) + + +def teardown_module(): + """Cleanup resources.""" + log.info(f"Deleting Job {NAMESPACE}/{JOB_NAME}...") + delete_job(JOB_NAME, NAMESPACE) diff --git a/driver/utils.py b/driver/utils.py new file mode 100644 index 0000000..e45e022 --- /dev/null +++ b/driver/utils.py @@ -0,0 +1,91 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +import logging +import subprocess + +import tenacity +from lightkube import Client +from lightkube.resources.batch_v1 import Job +from lightkube.resources.core_v1 import Namespace + +log = logging.getLogger(__name__) + + +@tenacity.retry( + wait=tenacity.wait_exponential(multiplier=2, min=1, max=10), + stop=tenacity.stop_after_attempt(30), + reraise=True, +) +def assert_namespace_active( + client: Client, + namespace: str, +): + """Test that the provided namespace is Active. + + Retries multiple times to allow for the K8s namespace to be created and reach Active status. + """ + # raises a 404 ApiError if the namespace doesn't exist + ns = client.get(Namespace, namespace) + phase = ns.status.phase + + log.info(f"Waiting for namespace {namespace} to become 'Active': phase == {phase}") + assert phase == "Active", f"Waited too long for namespace {namespace}!" + + +def _log_before_sleep(retry_state): + """Custom callback to log the number of seconds before the next attempt.""" + next_attempt = retry_state.attempt_number + delay = retry_state.next_action.sleep + log.info(f"Retrying in {int(delay)} seconds (attempts: {next_attempt})") + + +@tenacity.retry( + wait=tenacity.wait_exponential(multiplier=2, min=1, max=32), + retry=tenacity.retry_if_not_result(lambda result: result), + stop=tenacity.stop_after_delay(60 * 60), + before_sleep=_log_before_sleep, + reraise=True, +) +def wait_for_job( + client: Client, + job_name: str, + namespace: str, +): + """Wait for a Kubernetes Job to complete. + + Keep retrying (up to a maximum of 3600 seconds) while the Job is active or just not yet ready, + and stop once it becomes successful. This is implemented using the built-in + `retry_if_not_result` tenacity function, along with `wait_for_job` returning False or True, + respectively. + + If the Job fails or lands in an unexpected state, this function will raise a ValueError and + fail immediately. + """ + # raises a 404 ApiError if the Job doesn't exist + job = client.get(Job, name=job_name, namespace=namespace) + if job.status.succeeded: + # stop retrying, Job succeeded + log.info(f"Job {namespace}/{job_name} completed successfully!") + return True + elif job.status.failed: + raise ValueError(f"Job {namespace}/{job_name} failed!") + elif not job.status.ready or job.status.active: + # continue retrying + status = "active" if job.status.active else "not ready" + log.info(f"Waiting for Job {namespace}/{job_name} to complete (status == {status})") + return False + else: + raise ValueError(f"Unknown status {job.status} for Job {namespace}/{job_name}!") + + +def fetch_job_logs(job_name, namespace): + """Fetch the logs produced by a Kubernetes Job.""" + command = ["kubectl", "logs", "-n", namespace, f"job/{job_name}"] + subprocess.check_call(command) + + +def delete_job(job_name, namespace, lightkube_client=None): + """Delete a Kubernetes Job.""" + client = lightkube_client or Client() + client.delete(Job, name=job_name, namespace=namespace) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..308218c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +# Testing tools configuration +[tool.coverage.run] +branch = true + +[tool.coverage.report] +show_missing = true + +[tool.pytest.ini_options] +log_cli = true +log_cli_level = "INFO" + +# Formatting tools configuration +[tool.black] +line-length = 99 +target-version = ["py38"] + +[tool.isort] +line_length = 99 +profile = "black" + +# Linting tools configuration +[tool.flake8] +max-line-length = 99 +max-doc-length = 99 +max-complexity = 10 +exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"] +select = ["E", "W", "F", "C", "N", "R", "D", "H"] +# Ignore W503, E501 because using black creates errors with this +# Ignore D107 Missing docstring in __init__ +ignore = ["W503", "E501", "D107"] +# D100, D101, D102, D103: Ignore missing docstrings in tests +per-file-ignores = ["tests/*:D100,D101,D102,D103,D104"] +docstring-convention = "google" +# Check for properly formatted copyright header in each file +copyright-check = "True" +copyright-author = "Canonical Ltd." +copyright-regexp = "Copyright\\s\\d{4}([-,]\\d{4})*\\s+%(author)s" diff --git a/requirements-fmt.in b/requirements-fmt.in new file mode 100644 index 0000000..7559a40 --- /dev/null +++ b/requirements-fmt.in @@ -0,0 +1,2 @@ +black +isort diff --git a/requirements-fmt.txt b/requirements-fmt.txt new file mode 100644 index 0000000..090e6eb --- /dev/null +++ b/requirements-fmt.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile ./requirements-fmt.in +# +black==23.7.0 + # via -r ./requirements-fmt.in +click==8.1.6 + # via black +isort==5.12.0 + # via -r ./requirements-fmt.in +mypy-extensions==1.0.0 + # via black +packaging==23.1 + # via black +pathspec==0.11.2 + # via black +platformdirs==3.10.0 + # via black +tomli==2.0.1 + # via black +typing-extensions==4.7.1 + # via black diff --git a/requirements-lint.in b/requirements-lint.in new file mode 100644 index 0000000..07a4a51 --- /dev/null +++ b/requirements-lint.in @@ -0,0 +1,8 @@ +black +codespell +flake8 +flake8-builtins +flake8-copyright +isort +pep8-naming +pyproject-flake8 diff --git a/requirements-lint.txt b/requirements-lint.txt new file mode 100644 index 0000000..890bea0 --- /dev/null +++ b/requirements-lint.txt @@ -0,0 +1,51 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile ./requirements-lint.in +# +black==23.7.0 + # via -r ./requirements-lint.in +click==8.1.6 + # via black +codespell==2.2.5 + # via -r ./requirements-lint.in +flake8==6.0.0 + # via + # -r ./requirements-lint.in + # flake8-builtins + # pep8-naming + # pyproject-flake8 +flake8-builtins==2.1.0 + # via -r ./requirements-lint.in +flake8-copyright==0.2.4 + # via -r ./requirements-lint.in +isort==5.12.0 + # via -r ./requirements-lint.in +mccabe==0.7.0 + # via flake8 +mypy-extensions==1.0.0 + # via black +packaging==23.1 + # via black +pathspec==0.11.2 + # via black +pep8-naming==0.13.3 + # via -r ./requirements-lint.in +platformdirs==3.10.0 + # via black +pycodestyle==2.10.0 + # via flake8 +pyflakes==3.0.1 + # via flake8 +pyproject-flake8==6.0.0.post1 + # via -r ./requirements-lint.in +tomli==2.0.1 + # via + # black + # pyproject-flake8 +typing-extensions==4.7.1 + # via black + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..8b05587 --- /dev/null +++ b/requirements.in @@ -0,0 +1,7 @@ +# Pinning to <3.0 to ensure compatibility with the 2.9 controller version +# Note: 3.0 is not being maintained anymore +juju<3.0 +lightkube +pytest +pytest-operator +tenacity diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..447beee --- /dev/null +++ b/requirements.txt @@ -0,0 +1,208 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile ./requirements.in +# +anyio==3.7.1 + # via httpcore +asttokens==2.2.1 + # via stack-data +backcall==0.2.0 + # via ipython +bcrypt==4.0.1 + # via paramiko +cachetools==5.3.1 + # via google-auth +certifi==2023.7.22 + # via + # httpcore + # httpx + # kubernetes + # requests +cffi==1.15.1 + # via + # cryptography + # pynacl +charset-normalizer==3.2.0 + # via requests +cryptography==41.0.2 + # via paramiko +decorator==5.1.1 + # via + # ipdb + # ipython +exceptiongroup==1.1.2 + # via + # anyio + # pytest +executing==1.2.0 + # via stack-data +google-auth==2.22.0 + # via kubernetes +h11==0.14.0 + # via httpcore +httpcore==0.17.3 + # via httpx +httpx==0.24.1 + # via lightkube +idna==3.4 + # via + # anyio + # httpx + # requests +iniconfig==2.0.0 + # via pytest +ipdb==0.13.13 + # via pytest-operator +ipython==8.12.2 + # via ipdb +jedi==0.19.0 + # via ipython +jinja2==3.1.2 + # via pytest-operator +juju==2.9.44.0 + # via + # -r ./requirements.in + # pytest-operator +jujubundlelib==0.5.7 + # via theblues +kubernetes==27.2.0 + # via juju +lightkube==0.14.0 + # via -r ./requirements.in +lightkube-models==1.27.1.4 + # via lightkube +macaroonbakery==1.3.1 + # via + # juju + # theblues +markupsafe==2.1.3 + # via jinja2 +matplotlib-inline==0.1.6 + # via ipython +mypy-extensions==1.0.0 + # via typing-inspect +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib +packaging==23.1 + # via pytest +paramiko==2.12.0 + # via juju +parso==0.8.3 + # via jedi +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pluggy==1.2.0 + # via pytest +prompt-toolkit==3.0.39 + # via ipython +protobuf==3.20.3 + # via macaroonbakery +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pyasn1==0.5.0 + # via + # juju + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +pycparser==2.21 + # via cffi +pygments==2.15.1 + # via ipython +pymacaroons==0.13.0 + # via macaroonbakery +pynacl==1.5.0 + # via + # macaroonbakery + # paramiko + # pymacaroons +pyrfc3339==1.1 + # via + # juju + # macaroonbakery +pytest==7.4.0 + # via + # -r ./requirements.in + # pytest-asyncio + # pytest-operator +pytest-asyncio==0.21.1 + # via pytest-operator +pytest-operator==0.28.0 + # via -r ./requirements.in +python-dateutil==2.8.2 + # via kubernetes +pytz==2023.3 + # via pyrfc3339 +pyyaml==6.0.1 + # via + # juju + # jujubundlelib + # kubernetes + # lightkube + # pytest-operator +requests==2.31.0 + # via + # kubernetes + # macaroonbakery + # requests-oauthlib + # theblues +requests-oauthlib==1.3.1 + # via kubernetes +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # asttokens + # google-auth + # kubernetes + # macaroonbakery + # paramiko + # pymacaroons + # python-dateutil +sniffio==1.3.0 + # via + # anyio + # httpcore + # httpx +stack-data==0.6.2 + # via ipython +tenacity==8.2.2 + # via -r ./requirements.in +theblues==0.5.2 + # via juju +tomli==2.0.1 + # via + # ipdb + # pytest +toposort==1.10 + # via juju +traitlets==5.9.0 + # via + # ipython + # matplotlib-inline +typing-extensions==4.7.1 + # via + # ipython + # typing-inspect +typing-inspect==0.9.0 + # via juju +urllib3==1.26.16 + # via + # google-auth + # kubernetes + # requests +wcwidth==0.2.6 + # via prompt-toolkit +websocket-client==1.6.1 + # via kubernetes +websockets==7.0 + # via juju diff --git a/tests/CONTRIBUTING.md b/tests/CONTRIBUTING.md new file mode 100644 index 0000000..63118f3 --- /dev/null +++ b/tests/CONTRIBUTING.md @@ -0,0 +1,59 @@ +# How to Contribute + +Python notebooks are the backbone of the automated UATs mechanism, because they are easy to write +and maintain. At the same time, however, they are a great way to provide examples that can be +directly tested and easily shared. + +The `notebooks` directory will be continuously updated with new notebooks to test Kubeflow core +functionality and integrations. When adding new test notebooks you can use the existing ones for +reference in terms of style, coding conventions, and content. Below we provide some general +guidelines worth bearing in mind when contributing new notebooks. + +## Notebook Guidelines + +Notebooks added to the repository should serve both as tests and as examples, meant to be read by +users. With that in mind, they should be well-organised, including explanatory text where required. + +At the same time, however, it is important that the notebooks perform verification checks +themselves. These checks should not affect the UX of manually executing the notebook. In fact, when +everything is as expected, their execution should be transparent to the user. It is in the event of +an error that they should raise an exception. This is crucial for the notebook execution engine to +be able to pick up and report any issues. Below you can find some guidelines fundamental to the +design of the test suite. + +1. Use programmatic clients instead of CLI commands where possible. This makes programmatically + inspecting the result of any action easier: + + ```bash + # S3 CLI: hard to read, hard to parse its output + !aws --endpoint-url $MINIO_ENDPOINT_URL s3 ls s3://bpk-nb-minio/ + # MinIO client: direct programmatic verification + assert [obj for obj in mc.list_objects(BUCKET)] == [], f"Not empty!" + ``` + +2. Add logic to wait for actions/runs to be completed + * Use API calls instead of e.g. Selenium + * Use `tenacity` to implement retries, e.g. waiting for a KFP run to complete and asserting it + was successful: + + ```python + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=2, min=1, max=10), + stop=tenacity.stop_after_attempt(30), + reraise=True, + ) + def assert_run_succeeded(kfp_client, run_id): + """Wait for the run to complete successfully.""" + status = kfp_client.get_run(run_id).run.status + assert status == "Succeeded", f"KFP run in {status} state." + ``` + +3. Perform verification checks in separate cells + * Use `assert` to enforce conditions of success + * Mark verification cells with the `raises-exception` tag to instruct the execution engine to + proceed with the rest of the notebook instead of exiting immediately: + * Use the right-hand UI sidebar of Jupyterlab or + * Edit the notebook JSON directly + +4. Mark cells only destined for execution outside the automatic test suite with the `pytest-skip` + tag diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..4f54d46 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,27 @@ +# Test Kubeflow Integrations + +Run Python notebooks with Pytest inside a Notebook Server to verify the integration of Kubeflow +with different components. The notebook tests are stored in the `notebooks/` directory. + +These are regular Python notebooks, which you can view and run manually without any modifications. +They perform simple tasks using the respective APIs and programmatically verify the results. + +## Setup + +Before running the tests, make sure that the [required Python dependencies](requirements.txt) are +installed: + +``` +pip install -r requirements.txt +``` + +## Run + +You can execute the full Pytest suite by running: + +``` +pytest +``` + +The above inherits the configuration set in [pytest.ini](pytest.ini). Feel free to provide any +required extra settings either using that file or directly through CLI arguments. diff --git a/tests/notebooks/katib-integration.ipynb b/tests/notebooks/katib-integration.ipynb new file mode 100644 index 0000000..0ab2a38 --- /dev/null +++ b/tests/notebooks/katib-integration.ipynb @@ -0,0 +1,892 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test Katib Integration\n", + "\n", + "This example notebook is loosely based on [this](https://github.com/kubeflow/katib/blob/master/examples/v1beta1/sdk/cmaes-and-resume-policies.ipynb) upstream example.\n", + "\n", + "- create Katib Experiment\n", + "- monitor its execution\n", + "- get optimal HyperParameters\n", + "- get Trials\n", + "- get Suggestion\n", + "- delete Experiment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install kubeflow-katib tenacity -q" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Import required packages" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from kubeflow.katib import (\n", + " KatibClient,\n", + " V1beta1AlgorithmSpec,\n", + " V1beta1Experiment,\n", + " V1beta1ExperimentSpec,\n", + " V1beta1FeasibleSpace,\n", + " V1beta1ObjectiveSpec,\n", + " V1beta1ParameterSpec,\n", + " V1beta1TrialTemplate,\n", + " V1beta1TrialParameterSpec,\n", + ")\n", + "from kubernetes.client import V1ObjectMeta\n", + "\n", + "from tenacity import retry, stop_after_attempt, wait_exponential" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialise Katib Client\n", + "\n", + "We will be using the Katib SDK for any actions executed as part of this example." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "client = KatibClient()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define a Katib Experiment\n", + "\n", + "Define a Katib Experiment object before deploying it. This Experiment is similar to [this](https://github.com/kubeflow/katib/blob/master/examples/v1beta1/hp-tuning/cma-es.yaml) example." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "EXPERIMENT_NAME = \"cmaes-example\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "metadata = V1ObjectMeta(\n", + " name=EXPERIMENT_NAME,\n", + ")\n", + "\n", + "algorithm_spec=V1beta1AlgorithmSpec(\n", + " algorithm_name=\"cmaes\"\n", + ")\n", + "\n", + "objective_spec=V1beta1ObjectiveSpec(\n", + " type=\"maximize\",\n", + " goal= 0.99,\n", + " objective_metric_name=\"Validation-accuracy\",\n", + " additional_metric_names=[\"Train-accuracy\"]\n", + ")\n", + "\n", + "# experiment search space\n", + "# in this example we tune learning rate, number of layer, and optimizer\n", + "parameters=[\n", + " V1beta1ParameterSpec(\n", + " name=\"lr\",\n", + " parameter_type=\"double\",\n", + " feasible_space=V1beta1FeasibleSpace(\n", + " min=\"0.01\",\n", + " max=\"0.06\"\n", + " ),\n", + " ),\n", + " V1beta1ParameterSpec(\n", + " name=\"num-layers\",\n", + " parameter_type=\"int\",\n", + " feasible_space=V1beta1FeasibleSpace(\n", + " min=\"2\",\n", + " max=\"5\"\n", + " ),\n", + " ),\n", + " V1beta1ParameterSpec(\n", + " name=\"optimizer\",\n", + " parameter_type=\"categorical\",\n", + " feasible_space=V1beta1FeasibleSpace(\n", + " list=[\"sgd\", \"adam\", \"ftrl\"]\n", + " ),\n", + " ),\n", + "]\n", + "\n", + "# JSON template specification for the Trial's Worker Kubernetes Job\n", + "trial_spec={\n", + " \"apiVersion\": \"batch/v1\",\n", + " \"kind\": \"Job\",\n", + " \"spec\": {\n", + " \"template\": {\n", + " \"metadata\": {\n", + " \"annotations\": {\n", + " \"sidecar.istio.io/inject\": \"false\"\n", + " }\n", + " },\n", + " \"spec\": {\n", + " \"containers\": [\n", + " {\n", + " \"name\": \"training-container\",\n", + " \"image\": \"docker.io/kubeflowkatib/mxnet-mnist:v0.14.0\",\n", + " \"command\": [\n", + " \"python3\",\n", + " \"/opt/mxnet-mnist/mnist.py\",\n", + " \"--batch-size=64\",\n", + " \"--num-epochs=1\",\n", + " \"--lr=${trialParameters.learningRate}\",\n", + " \"--num-layers=${trialParameters.numberLayers}\",\n", + " \"--optimizer=${trialParameters.optimizer}\"\n", + " ]\n", + " }\n", + " ],\n", + " \"restartPolicy\": \"Never\"\n", + " }\n", + " }\n", + " }\n", + "}\n", + "\n", + "trial_template=V1beta1TrialTemplate(\n", + " primary_container_name=\"training-container\",\n", + " trial_parameters=[\n", + " V1beta1TrialParameterSpec(\n", + " name=\"learningRate\",\n", + " description=\"Learning rate for the training model\",\n", + " reference=\"lr\"\n", + " ),\n", + " V1beta1TrialParameterSpec(\n", + " name=\"numberLayers\",\n", + " description=\"Number of training model layers\",\n", + " reference=\"num-layers\"\n", + " ),\n", + " V1beta1TrialParameterSpec(\n", + " name=\"optimizer\",\n", + " description=\"Training model optimizer (sdg, adam or ftrl)\",\n", + " reference=\"optimizer\"\n", + " ),\n", + " ],\n", + " trial_spec=trial_spec\n", + ")\n", + "\n", + "experiment = V1beta1Experiment(\n", + " api_version=\"kubeflow.org/v1beta1\",\n", + " kind=\"Experiment\",\n", + " metadata=metadata,\n", + " spec=V1beta1ExperimentSpec(\n", + " max_trial_count=3,\n", + " parallel_trial_count=2,\n", + " max_failed_trial_count=1,\n", + " algorithm=algorithm_spec,\n", + " objective=objective_spec,\n", + " parameters=parameters,\n", + " trial_template=trial_template,\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Print the Experiment's info to verify it before submission." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: cmaes-example\n", + "Algorithm: cmaes\n", + "Objective: Validation-accuracy\n", + "Trial Parameters:\n", + "- learningRate: Learning rate for the training model\n", + "- numberLayers: Number of training model layers\n", + "- optimizer: Training model optimizer (sdg, adam or ftrl)\n", + "Max Trial Count: 3\n", + "Max Failed Trial Count: 1\n", + "Parallel Trial Count: 2\n" + ] + } + ], + "source": [ + "print(\"Name:\", experiment.metadata.name)\n", + "print(\"Algorithm:\", experiment.spec.algorithm.algorithm_name)\n", + "print(\"Objective:\", experiment.spec.objective.objective_metric_name)\n", + "print(\"Trial Parameters:\")\n", + "for param in experiment.spec.trial_template.trial_parameters:\n", + " print(f\"- {param.name}: {param.description}\")\n", + "print(\"Max Trial Count:\", experiment.spec.max_trial_count)\n", + "print(\"Max Failed Trial Count:\", experiment.spec.max_failed_trial_count)\n", + "print(\"Parallel Trial Count:\", experiment.spec.parallel_trial_count)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List existing Katib Experiments\n", + "\n", + "List Katib Experiments in the current namespace." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[exp.metadata.name for exp in client.list_experiments()]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create Katib Experiment\n", + "\n", + "Create a Katib Experiment using the SDK." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Experiment test/cmaes-example has been created\n" + ] + }, + { + "data": { + "text/html": [ + "Katib Experiment cmaes-example link here" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "client.create_experiment(experiment)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get Katib Experiment\n", + "\n", + "Get the created Katib Experiment by name and check its data. \n", + "Make sure that it completes successfully before proceeding. " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "@retry(\n", + " wait=wait_exponential(multiplier=2, min=1, max=10),\n", + " stop=stop_after_attempt(30),\n", + " reraise=True,\n", + ")\n", + "def assert_experiment_succeeded(client, experiment):\n", + " \"\"\"Wait for the Katib Experiment to complete successfully.\"\"\"\n", + " assert client.is_experiment_succeeded(name=experiment), f\"Katib Experiment was not successful.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# verify that the Experiment was created successfully\n", + "# raises an error if it doesn't exist\n", + "client.get_experiment(name=EXPERIMENT_NAME)\n", + "\n", + "# wait for the Experiment to complete successfully\n", + "assert_experiment_succeeded(client, EXPERIMENT_NAME)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Experiment: cmaes-example\n", + "\n", + "Experiment Spec:\n", + "{'algorithm': {'algorithm_name': 'cmaes', 'algorithm_settings': None},\n", + " 'early_stopping': None,\n", + " 'max_failed_trial_count': 1,\n", + " 'max_trial_count': 3,\n", + " 'metrics_collector_spec': {'collector': {'custom_collector': None,\n", + " 'kind': 'StdOut'},\n", + " 'source': None},\n", + " 'nas_config': None,\n", + " 'objective': {'additional_metric_names': ['Train-accuracy'],\n", + " 'goal': 0.99,\n", + " 'metric_strategies': [{'name': 'Validation-accuracy',\n", + " 'value': 'max'},\n", + " {'name': 'Train-accuracy',\n", + " 'value': 'max'}],\n", + " 'objective_metric_name': 'Validation-accuracy',\n", + " 'type': 'maximize'},\n", + " 'parallel_trial_count': 2,\n", + " 'parameters': [{'feasible_space': {'list': None,\n", + " 'max': '0.06',\n", + " 'min': '0.01',\n", + " 'step': None},\n", + " 'name': 'lr',\n", + " 'parameter_type': 'double'},\n", + " {'feasible_space': {'list': None,\n", + " 'max': '5',\n", + " 'min': '2',\n", + " 'step': None},\n", + " 'name': 'num-layers',\n", + " 'parameter_type': 'int'},\n", + " {'feasible_space': {'list': ['sgd', 'adam', 'ftrl'],\n", + " 'max': None,\n", + " 'min': None,\n", + " 'step': None},\n", + " 'name': 'optimizer',\n", + " 'parameter_type': 'categorical'}],\n", + " 'resume_policy': 'Never',\n", + " 'trial_template': {'config_map': None,\n", + " 'failure_condition': 'status.conditions.#(type==\"Failed\")#|#(status==\"True\")#',\n", + " 'primary_container_name': 'training-container',\n", + " 'primary_pod_labels': None,\n", + " 'retain': None,\n", + " 'success_condition': 'status.conditions.#(type==\"Complete\")#|#(status==\"True\")#',\n", + " 'trial_parameters': [{'description': 'Learning rate for '\n", + " 'the training model',\n", + " 'name': 'learningRate',\n", + " 'reference': 'lr'},\n", + " {'description': 'Number of training '\n", + " 'model layers',\n", + " 'name': 'numberLayers',\n", + " 'reference': 'num-layers'},\n", + " {'description': 'Training model '\n", + " 'optimizer (sdg, adam '\n", + " 'or ftrl)',\n", + " 'name': 'optimizer',\n", + " 'reference': 'optimizer'}],\n", + " 'trial_spec': {'apiVersion': 'batch/v1',\n", + " 'kind': 'Job',\n", + " 'spec': {'template': {'metadata': {'annotations': {'sidecar.istio.io/inject': 'false'}},\n", + " 'spec': {'containers': [{'command': ['python3',\n", + " '/opt/mxnet-mnist/mnist.py',\n", + " '--batch-size=64',\n", + " '--num-epochs=1',\n", + " '--lr=${trialParameters.learningRate}',\n", + " '--num-layers=${trialParameters.numberLayers}',\n", + " '--optimizer=${trialParameters.optimizer}'],\n", + " 'image': 'docker.io/kubeflowkatib/mxnet-mnist:v0.14.0',\n", + " 'name': 'training-container'}],\n", + " 'restartPolicy': 'Never'}}}}}}\n", + "\n", + "Experiment Status:\n", + "{'completion_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'conditions': [{'last_transition_time': datetime.datetime(2023, 7, 28, 12, 19, 37, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 19, 37, tzinfo=tzlocal()),\n", + " 'message': 'Experiment is created',\n", + " 'reason': 'ExperimentCreated',\n", + " 'status': 'True',\n", + " 'type': 'Created'},\n", + " {'last_transition_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'message': 'Experiment is running',\n", + " 'reason': 'ExperimentRunning',\n", + " 'status': 'False',\n", + " 'type': 'Running'},\n", + " {'last_transition_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'message': 'Experiment has succeeded because max trial count '\n", + " 'has reached',\n", + " 'reason': 'ExperimentMaxTrialsReached',\n", + " 'status': 'True',\n", + " 'type': 'Succeeded'}],\n", + " 'current_optimal_trial': {'best_trial_name': 'cmaes-example-sw6zbt45',\n", + " 'observation': {'metrics': [{'latest': '0.949940',\n", + " 'max': '0.949940',\n", + " 'min': '0.949940',\n", + " 'name': 'Validation-accuracy'},\n", + " {'latest': '0.924324',\n", + " 'max': '0.924324',\n", + " 'min': '0.924324',\n", + " 'name': 'Train-accuracy'}]},\n", + " 'parameter_assignments': [{'name': 'optimizer',\n", + " 'value': 'sgd'},\n", + " {'name': 'lr',\n", + " 'value': '0.04188612100654'},\n", + " {'name': 'num-layers',\n", + " 'value': '4'}]},\n", + " 'early_stopped_trial_list': None,\n", + " 'failed_trial_list': None,\n", + " 'killed_trial_list': None,\n", + " 'last_reconcile_time': None,\n", + " 'metrics_unavailable_trial_list': None,\n", + " 'pending_trial_list': None,\n", + " 'running_trial_list': None,\n", + " 'start_time': datetime.datetime(2023, 7, 28, 12, 19, 37, tzinfo=tzlocal()),\n", + " 'succeeded_trial_list': ['cmaes-example-sw6zbt45',\n", + " 'cmaes-example-vfc4ptmw',\n", + " 'cmaes-example-kjrnx4k2'],\n", + " 'trial_metrics_unavailable': None,\n", + " 'trials': 3,\n", + " 'trials_early_stopped': None,\n", + " 'trials_failed': None,\n", + " 'trials_killed': None,\n", + " 'trials_pending': None,\n", + " 'trials_running': None,\n", + " 'trials_succeeded': 3}\n", + "\n" + ] + } + ], + "source": [ + "exp = client.get_experiment(name=EXPERIMENT_NAME)\n", + "print(\"Experiment:\", exp.metadata.name, end=\"\\n\\n\")\n", + "print(\"Experiment Spec:\", exp.spec, sep=\"\\n\", end=\"\\n\\n\")\n", + "print(\"Experiment Status:\", exp.status, sep=\"\\n\", end=\"\\n\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get Experiment conditions\n", + "\n", + "Check the current Experiment conditions and verify that the last one is \"Succeeded\"." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'last_transition_time': datetime.datetime(2023, 7, 28, 12, 19, 37, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 19, 37, tzinfo=tzlocal()),\n", + " 'message': 'Experiment is created',\n", + " 'reason': 'ExperimentCreated',\n", + " 'status': 'True',\n", + " 'type': 'Created'}, {'last_transition_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'message': 'Experiment is running',\n", + " 'reason': 'ExperimentRunning',\n", + " 'status': 'False',\n", + " 'type': 'Running'}, {'last_transition_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'message': 'Experiment has succeeded because max trial count has reached',\n", + " 'reason': 'ExperimentMaxTrialsReached',\n", + " 'status': 'True',\n", + " 'type': 'Succeeded'}]\n" + ] + } + ], + "source": [ + "conditions = client.get_experiment_conditions(name=EXPERIMENT_NAME)\n", + "print(conditions)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "assert conditions[-1].type == \"Succeeded\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get the optimal HyperParameters\n", + "\n", + "Get the optimal HyperParameters at the end of the tuning Experiment. \n", + "Each metric comes with the max, min and latest value." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'best_trial_name': 'cmaes-example-sw6zbt45',\n", + " 'observation': {'metrics': [{'latest': '0.949940',\n", + " 'max': '0.949940',\n", + " 'min': '0.949940',\n", + " 'name': 'Validation-accuracy'},\n", + " {'latest': '0.924324',\n", + " 'max': '0.924324',\n", + " 'min': '0.924324',\n", + " 'name': 'Train-accuracy'}]},\n", + " 'parameter_assignments': [{'name': 'optimizer', 'value': 'sgd'},\n", + " {'name': 'lr', 'value': '0.04188612100654'},\n", + " {'name': 'num-layers', 'value': '4'}]}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.get_optimal_hyperparameters(name=EXPERIMENT_NAME)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List Katib Trials\n", + "\n", + "Get a list of the current Trials with the latest status." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Trial: cmaes-example-sw6zbt45\n", + "Trial Status:\n", + "{'last_transition_time': datetime.datetime(2023, 7, 28, 12, 20, 33, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 20, 33, tzinfo=tzlocal()),\n", + " 'message': 'Trial has succeeded',\n", + " 'reason': 'TrialSucceeded',\n", + " 'status': 'True',\n", + " 'type': 'Succeeded'}\n", + "\n", + "Trial: cmaes-example-vfc4ptmw\n", + "Trial Status:\n", + "{'last_transition_time': datetime.datetime(2023, 7, 28, 12, 20, 37, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 20, 37, tzinfo=tzlocal()),\n", + " 'message': 'Trial has succeeded',\n", + " 'reason': 'TrialSucceeded',\n", + " 'status': 'True',\n", + " 'type': 'Succeeded'}\n", + "\n", + "Trial: cmaes-example-kjrnx4k2\n", + "Trial Status:\n", + "{'last_transition_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'message': 'Trial has succeeded',\n", + " 'reason': 'TrialSucceeded',\n", + " 'status': 'True',\n", + " 'type': 'Succeeded'}\n", + "\n" + ] + } + ], + "source": [ + "trial_list = client.list_trials(experiment_name=EXPERIMENT_NAME)\n", + "for trial in trial_list:\n", + " print(\"Trial:\", trial.metadata.name)\n", + " print(\"Trial Status:\", trial.status.conditions[-1], sep=\"\\n\", end=\"\\n\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# verify that the max trial count was reached\n", + "assert len(trial_list) == experiment.spec.max_trial_count\n", + "\n", + "# verify that all trials were successful\n", + "for trial in trial_list:\n", + " assert trial.status.conditions[-1].type == \"Succeeded\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get Katib Suggestion\n", + "\n", + "Inspect the Suggestion object for more information." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Suggestion: cmaes-example\n", + "\n", + "Suggestion Spec:\n", + "{'algorithm': {'algorithm_name': 'cmaes', 'algorithm_settings': None},\n", + " 'early_stopping': None,\n", + " 'requests': 3,\n", + " 'resume_policy': 'Never'}\n", + "\n", + "Suggestion Status:\n", + "{'algorithm_settings': None,\n", + " 'completion_time': None,\n", + " 'conditions': [{'last_transition_time': datetime.datetime(2023, 7, 28, 12, 19, 37, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 19, 37, tzinfo=tzlocal()),\n", + " 'message': 'Suggestion is created',\n", + " 'reason': 'SuggestionCreated',\n", + " 'status': 'True',\n", + " 'type': 'Created'},\n", + " {'last_transition_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'message': 'Suggestion is not running',\n", + " 'reason': 'Suggestion is succeeded',\n", + " 'status': 'False',\n", + " 'type': 'Running'},\n", + " {'last_transition_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'message': 'Deployment is not ready',\n", + " 'reason': 'Suggestion is succeeded',\n", + " 'status': 'False',\n", + " 'type': 'DeploymentReady'},\n", + " {'last_transition_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'last_update_time': datetime.datetime(2023, 7, 28, 12, 21, 2, tzinfo=tzlocal()),\n", + " 'message': \"Suggestion is succeeded, can't be restarted\",\n", + " 'reason': 'Experiment is succeeded',\n", + " 'status': 'True',\n", + " 'type': 'Succeeded'}],\n", + " 'last_reconcile_time': None,\n", + " 'start_time': datetime.datetime(2023, 7, 28, 12, 19, 37, tzinfo=tzlocal()),\n", + " 'suggestion_count': 3,\n", + " 'suggestions': [{'early_stopping_rules': None,\n", + " 'labels': None,\n", + " 'name': 'cmaes-example-sw6zbt45',\n", + " 'parameter_assignments': [{'name': 'optimizer',\n", + " 'value': 'sgd'},\n", + " {'name': 'lr',\n", + " 'value': '0.04188612100654'},\n", + " {'name': 'num-layers',\n", + " 'value': '4'}]},\n", + " {'early_stopping_rules': None,\n", + " 'labels': None,\n", + " 'name': 'cmaes-example-vfc4ptmw',\n", + " 'parameter_assignments': [{'name': 'lr',\n", + " 'value': '0.04511033252270099'},\n", + " {'name': 'num-layers',\n", + " 'value': '3'},\n", + " {'name': 'optimizer',\n", + " 'value': 'adam'}]},\n", + " {'early_stopping_rules': None,\n", + " 'labels': None,\n", + " 'name': 'cmaes-example-kjrnx4k2',\n", + " 'parameter_assignments': [{'name': 'lr',\n", + " 'value': '0.02556132716757138'},\n", + " {'name': 'num-layers',\n", + " 'value': '4'},\n", + " {'name': 'optimizer',\n", + " 'value': 'ftrl'}]}]}\n", + "\n" + ] + } + ], + "source": [ + "suggestion = client.get_suggestion(name=EXPERIMENT_NAME)\n", + "print(\"Suggestion:\", suggestion.metadata.name, end=\"\\n\\n\")\n", + "print(\"Suggestion Spec:\", suggestion.spec, sep=\"\\n\", end=\"\\n\\n\")\n", + "print(\"Suggestion Status:\", suggestion.status, sep=\"\\n\", end=\"\\n\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "assert suggestion.status.conditions[-1].type == \"Succeeded\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Delete Katib Experiment\n", + "\n", + "Delete the created Experiment and check that all created resources were removed as well." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Experiment test/cmaes-example has been deleted\n" + ] + } + ], + "source": [ + "client.delete_experiment(name=EXPERIMENT_NAME)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "@retry(\n", + " wait=wait_exponential(multiplier=2, min=1, max=10),\n", + " stop=stop_after_attempt(30),\n", + " reraise=True,\n", + ")\n", + "def assert_katib_resources_removed(client, experiment_name):\n", + " \"\"\"Wait for Katib resources to be removed.\"\"\"\n", + " # fetch the existing Experiment names\n", + " # verify that the Experiment was deleted successfully\n", + " experiments = {exp.metadata.name for exp in client.list_experiments()}\n", + " assert experiment_name not in experiments, f\"Failed to delete Katib Experiment {experiment_name}!\"\n", + "\n", + " # fetch the existing Trials and retrieve the names of the Experiments these belong to\n", + " # verify that the Trials were removed successfully\n", + " trials = {tr.metadata.labels.get(\"katib.kubeflow.org/experiment\") for tr in client.list_trials()}\n", + " assert experiment_name not in trials, f\"Katib Trials of Experiment {experiment_name} were not removed!\"\n", + "\n", + " # fetch the existing Suggestion names\n", + " # verify that the Suggestion was removed successfully\n", + " suggestions = {sugg.metadata.name for sugg in client.list_suggestions()}\n", + " assert experiment_name not in suggestions, f\"Katib Suggestion {experiment_name} was not removed!\"" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# wait for Katib resources to be removed successfully\n", + "assert_katib_resources_removed(client, EXPERIMENT_NAME)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/notebooks/kfp-integration.ipynb b/tests/notebooks/kfp-integration.ipynb new file mode 100644 index 0000000..92affee --- /dev/null +++ b/tests/notebooks/kfp-integration.ipynb @@ -0,0 +1,226 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ba70a2ba-3645-419b-8f8d-3c75e5864af1", + "metadata": {}, + "source": [ + "# Test KFP Integration\n", + "\n", + "- create an experiment\n", + "- create a run\n", + "- check run passes" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "28f75e55-7bad-44e7-a65f-aedc81734a48", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install kfp==1.8.22 tenacity -q" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "4cdd7548-bae9-4430-b548-f420d72a8aec", + "metadata": {}, + "outputs": [], + "source": [ + "import kfp\n", + "import kfp.dsl as dsl\n", + "\n", + "from tenacity import retry, stop_after_attempt, wait_exponential" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fd576641-1ff4-4fbb-9b3a-122abbd281ed", + "metadata": {}, + "outputs": [], + "source": [ + "client = kfp.Client()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "af70bb9d-3fea-40d7-acb9-649007b0bde6", + "metadata": {}, + "outputs": [], + "source": [ + "EXPERIMENT_NAME = 'Simple notebook pipeline' " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "40a3a9e1-0645-474e-8451-92ccba88a122", + "metadata": {}, + "outputs": [], + "source": [ + "def add(a: float, b: float) -> float:\n", + " '''Calculates sum of two arguments'''\n", + " print(a, '+', b, '=', a + b)\n", + " return a + b" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "1d134c8b-54a7-4d10-ae2f-321ff305600a", + "metadata": {}, + "outputs": [], + "source": [ + "add_op = kfp.components.func_to_container_op(\n", + " func=add,\n", + " base_image='python:3.7',\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c8132d87-877c-4bfb-9127-e1f964fe3acb", + "metadata": {}, + "outputs": [], + "source": [ + "@dsl.pipeline(\n", + " name='Calculation pipeline',\n", + " description='A toy pipeline that performs arithmetic calculations.'\n", + ")\n", + "def calc_pipeline(a: float = 0, b: float = 7):\n", + " add_task = add_op(a, 4) \n", + " add_2_task = add_op(a, b)\n", + " add_3_task = add_op(add_task.output, add_2_task.output)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "bd297620-ff9c-4d85-82eb-10c89db09d6f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Experiment details." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run details." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "arguments = {'a': '7', 'b': '8'}\n", + "run = client.create_run_from_pipeline_func(\n", + " calc_pipeline,\n", + " arguments=arguments,\n", + " experiment_name=EXPERIMENT_NAME,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "37ebdc86-a16d-40a0-bc7e-33a2b90914f8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'created_at': datetime.datetime(2023, 7, 3, 15, 1, 11, tzinfo=tzlocal()),\n", + " 'description': None,\n", + " 'id': 'ed235fc2-79db-4324-b120-e0611afe266b',\n", + " 'name': 'Simple notebook pipeline',\n", + " 'resource_references': [{'key': {'id': 'test', 'type': 'NAMESPACE'},\n", + " 'name': None,\n", + " 'relationship': 'OWNER'}],\n", + " 'storage_state': 'STORAGESTATE_AVAILABLE'}]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.list_experiments().experiments" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "d945e7ba-dc63-46f5-93e4-a1edfe56aa81", + "metadata": {}, + "outputs": [], + "source": [ + "@retry(\n", + " wait=wait_exponential(multiplier=2, min=1, max=10),\n", + " stop=stop_after_attempt(30),\n", + " reraise=True,\n", + ")\n", + "def assert_run_succeeded(client, run_id):\n", + " \"\"\"Wait for the run to complete successfully.\"\"\"\n", + " status = client.get_run(run_id).run.status\n", + " assert status == \"Succeeded\", f\"KFP run in {status} state.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "73fae4b6-28f0-4ca2-9a5d-de608674bc1e", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# fetch KFP experiment to ensure it exists\n", + "client.get_experiment(experiment_name=EXPERIMENT_NAME)\n", + "\n", + "assert_run_succeeded(client, run.run_id)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/notebooks/minio-integration/minio-integration.ipynb b/tests/notebooks/minio-integration/minio-integration.ipynb new file mode 100644 index 0000000..bfd222c --- /dev/null +++ b/tests/notebooks/minio-integration/minio-integration.ipynb @@ -0,0 +1,522 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3246d127-6324-4554-be70-93b8dfe95ec7", + "metadata": {}, + "source": [ + "# Test Minio Integration\n", + "\n", + "- list buckets\n", + "- upload object to bucket\n", + "- download object\n", + "- retrieve data with pandas\n", + "- remove objects\n", + "- remove bucket" + ] + }, + { + "cell_type": "markdown", + "id": "50a6ee86-cc49-4399-93be-172095fd0d93", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "03ddcaea-c3b6-4fe7-830f-a1f52a812319", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "boto3 1.27.0 requires botocore<1.31.0,>=1.30.0, but you have botocore 1.29.161 which is incompatible.\n", + "awscli 1.28.0 requires botocore==1.30.0, but you have botocore 1.29.161 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!pip install minio pandas s3fs -q" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "6b2db553-bd2e-404b-9335-cb0876ab2bfd", + "metadata": {}, + "outputs": [], + "source": [ + "import filecmp\n", + "import os\n", + "\n", + "import pandas as pd\n", + "\n", + "from minio import Minio\n", + "from minio.error import BucketAlreadyOwnedByYou, NoSuchKey" + ] + }, + { + "cell_type": "markdown", + "id": "998316cb-df60-430d-ae89-177e0521cc04", + "metadata": {}, + "source": [ + "## Configure MinIO Client" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b773adce-883b-46f7-bc7a-b88ffe5e1165", + "metadata": {}, + "outputs": [], + "source": [ + "MINIO_HOST = os.environ[\"MINIO_ENDPOINT_URL\"].split(\"http://\")[1]\n", + "\n", + "# Initialize a MinIO client\n", + "mc = Minio(\n", + " endpoint=MINIO_HOST,\n", + " access_key=os.environ[\"AWS_ACCESS_KEY_ID\"],\n", + " secret_key=os.environ[\"AWS_SECRET_ACCESS_KEY\"],\n", + " secure=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "0070510c-0b52-40ee-a984-5d88ae4a3938", + "metadata": {}, + "source": [ + "## List Existing Buckets" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1204b151-4213-4398-816e-1a63bbd70634", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mlflow\n", + "\t 624471807852528081/\n", + "\t 872778786834962304/\n" + ] + } + ], + "source": [ + "# List buckets\n", + "buckets = mc.list_buckets()\n", + "for bucket in buckets:\n", + " print(bucket.name)\n", + " # List objects in bucket\n", + " objects = mc.list_objects(bucket.name)\n", + " for obj in objects:\n", + " print(\"\\t\", obj.object_name)" + ] + }, + { + "cell_type": "markdown", + "id": "aef234a3-ad96-4852-b58e-f8c9fb048abb", + "metadata": {}, + "source": [ + "## Create Bucket" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "49d525d9-7a72-4b5c-a73c-b0b41d9829bd", + "metadata": {}, + "outputs": [], + "source": [ + "BUCKET = \"kf-testing-minio\"" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e78e8267-d72b-4dbf-87f3-4468f4e25de8", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " mc.make_bucket(BUCKET)\n", + "except BucketAlreadyOwnedByYou:\n", + " print(f\"Bucket {BUCKET} already exists!\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "50788e8d-df7a-4b5f-9384-1296cd1c0673", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# check that the bucket was created successfully\n", + "assert mc.bucket_exists(BUCKET), f\"Bucket {BUCKET} does not exist!\"\n", + "# check that the new bucket is empty\n", + "assert [obj for obj in mc.list_objects(BUCKET)] == [], f\"Bucket {BUCKET} is not empty!\"" + ] + }, + { + "cell_type": "markdown", + "id": "198dcffd-9457-4d3f-8dee-614c940bf8a3", + "metadata": {}, + "source": [ + "## Upload Data to Bucket" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4cac97b9-b938-4b5d-a4ac-8744dd30c197", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "('4a507473e499735a94edc9ad9704a545', None)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "LOCAL_OBJECT = \"sample.txt\"\n", + "UPLOADED_OBJECT = \"uploaded-sample.txt\"\n", + "DOWNLOADED_OBJECT = \"downloaded-sample.txt\"\n", + "mc.fput_object(BUCKET, UPLOADED_OBJECT, LOCAL_OBJECT)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "24d1a526-f5df-4665-9d76-82535c7df1f2", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# check that the bucket only contains the uploaded object\n", + "objects = [obj for obj in mc.list_objects(BUCKET)]\n", + "assert len(objects) == 1, f\"Expected only 1 object in bucket {BUCKET}!\"\n", + "assert objects[0].object_name == UPLOADED_OBJECT, \"The uploaded and local object names do not match!\"\n", + "\n", + "# check that the size is the same\n", + "file_stat = os.stat(LOCAL_OBJECT)\n", + "assert objects[0].size == file_stat.st_size, \"The uploaded and local objects are not of the same size!\"" + ] + }, + { + "cell_type": "markdown", + "id": "ddba3ba8-a177-4965-adae-cccbb6274050", + "metadata": {}, + "source": [ + "### Download Object" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "65aaa914-7e1a-4701-8b86-5e86ecd74be5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mc.fget_object(BUCKET, UPLOADED_OBJECT, DOWNLOADED_OBJECT)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7655092d-4c60-4274-93d3-41aa1d55f1bf", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# check that the file was downloaded successfully\n", + "assert os.path.exists(DOWNLOADED_OBJECT), f\"Failed to download object {UPLOADED_OBJECT}!\"\n", + "\n", + "# check that its content matches that of the original file\n", + "assert filecmp.cmp(LOCAL_OBJECT, DOWNLOADED_OBJECT, shallow=False), f\"Downloaded object {DOWNLOADED_OBJECT} does not match the original!\"" + ] + }, + { + "cell_type": "markdown", + "id": "4d91d5eb-f45e-423d-a6b9-18eba066e11b", + "metadata": {}, + "source": [ + "### Download Data with Pandas" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "39290b93-eaa6-45d0-9c23-ebe3d7e79e3d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "('c886b0a6971427fc0faf293423e7a320', None)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "LOCAL_CSV = \"sample.csv\"\n", + "UPLOADED_CSV = \"uploaded-sample.csv\"\n", + "DOWNLOADED_CSV = \"downloaded-sample.csv\"\n", + "mc.fput_object(BUCKET, UPLOADED_CSV, LOCAL_CSV)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "04ac9362-d840-46a4-a1e1-ed3d3e6dad81", + "metadata": {}, + "outputs": [], + "source": [ + "local = pd.read_csv(LOCAL_CSV, delimiter=\";\")\n", + "uploaded = pd.read_csv(f\"s3://{BUCKET}/{UPLOADED_CSV}\", delimiter=\";\",storage_options={\n", + " \"key\": os.environ[\"AWS_ACCESS_KEY_ID\"],\n", + " \"secret\": os.environ[\"AWS_SECRET_ACCESS_KEY\"],\n", + " \"client_kwargs\":{\n", + " \"endpoint_url\": os.environ[\"MINIO_ENDPOINT_URL\"]\n", + " }\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "f2b0746c-00e1-4d99-b86d-c1db19d7e087", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
col1col2col3
0123
1345
2345
\n", + "
" + ], + "text/plain": [ + " col1 col2 col3\n", + "0 1 2 3\n", + "1 3 4 5\n", + "2 3 4 5" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# inspect contents of uploaded CSV\n", + "uploaded" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "58f95acc-f8fc-4ee4-afbe-66adc97ee8aa", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "assert local.equals(uploaded), \"Uploaded and local CSV contents do not match!\"" + ] + }, + { + "cell_type": "markdown", + "id": "18e5adf5-6de1-469f-93ac-d82212b6ccd2", + "metadata": {}, + "source": [ + "## Clean Up" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "c5c3ddbc-93d7-4758-a933-6cf8bfa378a5", + "metadata": {}, + "outputs": [], + "source": [ + "mc.remove_object(BUCKET, UPLOADED_OBJECT)\n", + "mc.remove_object(BUCKET, UPLOADED_CSV)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "0ef62dc0-a6f7-426d-b580-8b7567bc2ae4", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# check that the bucket is now empty\n", + "assert [obj for obj in mc.list_objects(BUCKET)] == [], f\"Bucket {BUCKET} is not empty!\"" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "712cceec-0394-4fd6-9c95-ae1de1ea94dd", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# check that attempting to retrieve a deleted object raises an error\n", + "try:\n", + " res = None\n", + " res = mc.get_object(BUCKET, UPLOADED_OBJECT)\n", + "except Exception as e:\n", + " if not isinstance(e, NoSuchKey):\n", + " raise\n", + " \n", + "assert not res, f\"Failed to delete {UPLOADED_OBJECT}!\"" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "394b7a65-c622-49f9-a9ad-3d0ebfda08de", + "metadata": {}, + "outputs": [], + "source": [ + "mc.remove_bucket(BUCKET)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "1896c46b-f83c-4152-b6c2-bbfde1242368", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "assert BUCKET not in {b.name for b in mc.list_buckets()}, f\"Failed to delete bucket {BUCKET}!\"" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "88e95d3b-62f1-497c-b837-fde1b1b226dd", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " os.remove(DOWNLOADED_OBJECT)\n", + "except FileNotFoundError:\n", + " print(f\"File {DOWNLOADED_OBJECT} already deleted!\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/notebooks/minio-integration/sample.csv b/tests/notebooks/minio-integration/sample.csv new file mode 100644 index 0000000..eed4fec --- /dev/null +++ b/tests/notebooks/minio-integration/sample.csv @@ -0,0 +1,4 @@ +col1;col2;col3 +1;2;3 +3;4;5 +3;4;5 \ No newline at end of file diff --git a/tests/notebooks/minio-integration/sample.txt b/tests/notebooks/minio-integration/sample.txt new file mode 100644 index 0000000..2ce4ba2 --- /dev/null +++ b/tests/notebooks/minio-integration/sample.txt @@ -0,0 +1 @@ +Hello in the World of MLOps! \ No newline at end of file diff --git a/tests/notebooks/mlflow-integration.ipynb b/tests/notebooks/mlflow-integration.ipynb new file mode 100644 index 0000000..ee198cf --- /dev/null +++ b/tests/notebooks/mlflow-integration.ipynb @@ -0,0 +1,445 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c6894075-6c1d-462a-a715-51746ef066d6", + "metadata": {}, + "source": [ + "# Test MLFlow Integration\n", + "\n", + "- start experiment \n", + "- train model\n", + "- save metrics\n", + "- save artifact" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "opening-plate", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "!pip install minio mlflow==2.1.1 boto3 tenacity -q" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "built-jacksonville", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import warnings\n", + "\n", + "import pandas as pd\n", + "import mlflow\n", + "import numpy as np\n", + "\n", + "from minio import Minio\n", + "from minio.error import BucketAlreadyOwnedByYou\n", + "from mlflow.models.signature import infer_signature\n", + "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.linear_model import ElasticNet\n", + "from tenacity import retry, stop_after_attempt, wait_exponential\n", + "\n", + "# suppress warnings\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "08f9a28d-5399-438b-98c2-c4665d9d8ea8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
fixed acidityvolatile aciditycitric acidresidual sugarchloridesfree sulfur dioxidetotal sulfur dioxidedensitypHsulphatesalcoholquality
07.40.700.001.90.07611.034.00.99783.510.569.45
17.80.880.002.60.09825.067.00.99683.200.689.85
27.80.760.042.30.09215.054.00.99703.260.659.85
311.20.280.561.90.07517.060.00.99803.160.589.86
47.40.700.001.90.07611.034.00.99783.510.569.45
\n", + "
" + ], + "text/plain": [ + " fixed acidity volatile acidity citric acid residual sugar chlorides \\\n", + "0 7.4 0.70 0.00 1.9 0.076 \n", + "1 7.8 0.88 0.00 2.6 0.098 \n", + "2 7.8 0.76 0.04 2.3 0.092 \n", + "3 11.2 0.28 0.56 1.9 0.075 \n", + "4 7.4 0.70 0.00 1.9 0.076 \n", + "\n", + " free sulfur dioxide total sulfur dioxide density pH sulphates \\\n", + "0 11.0 34.0 0.9978 3.51 0.56 \n", + "1 25.0 67.0 0.9968 3.20 0.68 \n", + "2 15.0 54.0 0.9970 3.26 0.65 \n", + "3 17.0 60.0 0.9980 3.16 0.58 \n", + "4 11.0 34.0 0.9978 3.51 0.56 \n", + "\n", + " alcohol quality \n", + "0 9.4 5 \n", + "1 9.8 5 \n", + "2 9.8 5 \n", + "3 9.8 6 \n", + "4 9.4 5 " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = pd.read_csv(\"http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv\", sep=\";\")\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "3a2511a7-1f49-4e6d-ad5f-aaf94f2ec23d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1599, 12)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "thick-begin", + "metadata": {}, + "outputs": [], + "source": [ + "TARGET_COLUMN = \"quality\"\n", + "train, test = train_test_split(data)\n", + "\n", + "train_x = train.drop([TARGET_COLUMN], axis=1)\n", + "test_x = test.drop([TARGET_COLUMN], axis=1)\n", + "train_y = train[[TARGET_COLUMN]]\n", + "test_y = test[[TARGET_COLUMN]]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e67bb23d-65e1-4b65-93e3-8c5aec7a9b7a", + "metadata": {}, + "outputs": [], + "source": [ + "MINIO_HOST = os.environ[\"MINIO_ENDPOINT_URL\"].split(\"http://\")[1]\n", + "MINIO_BUCKET = \"mlflow\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "cfa5af5f-f2fe-4b21-839d-384073a8aefe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Bucket mlflow already exists!\n" + ] + } + ], + "source": [ + "# Initialize a MinIO client\n", + "mc = Minio(\n", + " endpoint=MINIO_HOST,\n", + " access_key=os.environ[\"AWS_ACCESS_KEY_ID\"],\n", + " secret_key=os.environ[\"AWS_SECRET_ACCESS_KEY\"],\n", + " secure=False,\n", + ")\n", + "\n", + "try:\n", + " mc.make_bucket(MINIO_BUCKET)\n", + "except BucketAlreadyOwnedByYou:\n", + " print(f\"Bucket {MINIO_BUCKET} already exists!\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f99648d1-080b-4fc0-9627-14959eef3630", + "metadata": {}, + "outputs": [], + "source": [ + "wine_experiment_name = \"My Wine Experiment\"\n", + "experiment = mlflow.get_experiment_by_name(wine_experiment_name)\n", + "experiment_id = (\n", + " mlflow.create_experiment(name=wine_experiment_name)\n", + " if experiment is None\n", + " else experiment.experiment_id\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "e6fe6371-2439-48c0-bdcf-cbf1fc20e164", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "# check that the experiment was created successfully\n", + "assert mlflow.get_experiment(experiment_id).name == wine_experiment_name, f\"Failed to create experiment {wine_experiment_name}!\"" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "c3d74d1e-5f79-4cd0-8b8d-6b8afa0d8e97", + "metadata": {}, + "outputs": [], + "source": [ + "def experiment(alpha, l1_ratio):\n", + " mlflow.sklearn.autolog()\n", + " with mlflow.start_run(run_name='wine_models', experiment_id=experiment_id) as run:\n", + " mlflow.set_tag(\"author\", \"kf-testing\")\n", + " lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42)\n", + " lr.fit(train_x, train_y)\n", + "\n", + " pred_y = lr.predict(test_x)\n", + " mlflow.log_metric(\"rmse\", np.sqrt(mean_squared_error(test_y, pred_y)))\n", + " mlflow.log_metric(\"r2\", r2_score(test_y, pred_y))\n", + " mlflow.log_metric(\"mae\", mean_absolute_error(test_y, pred_y))\n", + "\n", + " signature = infer_signature(test_x, pred_y)\n", + " mlflow.sklearn.log_model(lr, \"model\", registered_model_name=\"wine-elasticnet\", signature=signature)\n", + " \n", + " return run" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a4295ea1-93fb-4100-acac-9191010caddc", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Registered model 'wine-elasticnet' already exists. Creating a new version of this model...\n", + "2023/07/06 08:39:06 INFO mlflow.tracking._model_registry.client: Waiting up to 300 seconds for model version to finish creation. Model name: wine-elasticnet, version 7\n", + "Created version '7' of model 'wine-elasticnet'.\n", + "Registered model 'wine-elasticnet' already exists. Creating a new version of this model...\n", + "2023/07/06 08:39:14 INFO mlflow.tracking._model_registry.client: Waiting up to 300 seconds for model version to finish creation. Model name: wine-elasticnet, version 8\n", + "Created version '8' of model 'wine-elasticnet'.\n", + "Registered model 'wine-elasticnet' already exists. Creating a new version of this model...\n", + "2023/07/06 08:39:25 INFO mlflow.tracking._model_registry.client: Waiting up to 300 seconds for model version to finish creation. Model name: wine-elasticnet, version 9\n", + "Created version '9' of model 'wine-elasticnet'.\n" + ] + } + ], + "source": [ + "# run experiments\n", + "runs = [\n", + " experiment(0.5, 0.5),\n", + " experiment(1, 0),\n", + " experiment(0, 1),\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "01a262ec-6a1f-476b-b392-6d27425f920b", + "metadata": {}, + "outputs": [], + "source": [ + "@retry(\n", + " wait=wait_exponential(multiplier=2, min=1, max=10),\n", + " stop=stop_after_attempt(30),\n", + " reraise=True,\n", + ")\n", + "def assert_run_finished(client, run_id):\n", + " \"\"\"Wait for the run to complete successfully.\"\"\"\n", + " status = client.get_run(run_id).info.status\n", + " assert status == \"FINISHED\", f\"MLFlow run in {status} state.\"\n", + "\n", + "\n", + "def assert_has_metrics(client, run_id, metrics):\n", + " \"\"\"Assert that the run contains the specified metrics.\"\"\"\n", + " run = client.get_run(run_id)\n", + " for m in metrics:\n", + " assert m in run.data.metrics, f\"Metric {m} not found in logged data!\"\n", + "\n", + "\n", + "def assert_model(client, run_id):\n", + " \"\"\"Assert Model exists.\"\"\"\n", + " model = client.sklearn.load_model(f\"runs:/{run_id}/model\")\n", + " assert isinstance(model, ElasticNet), f\"Model {model} is not of type ElasticNet!\"" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "1df491c5-0b99-4dd6-b41c-ad3198b39dd7", + "metadata": { + "tags": [ + "raises-exception" + ] + }, + "outputs": [], + "source": [ + "METRICS = [\"rmse\", \"r2\", \"mae\"]\n", + "\n", + "for run in runs:\n", + " run_id = run.info.run_id\n", + " assert_run_finished(mlflow, run_id)\n", + " assert_has_metrics(mlflow, run_id, METRICS)\n", + " assert_model(mlflow, run_id)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 0000000..7eb32d3 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +addopts = -vv -p no:warnings +log_cli = true +log_level = INFO +tb_style = native +markers = ipynb diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..61fd1a1 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +nbconvert +pytest diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py new file mode 100644 index 0000000..44777af --- /dev/null +++ b/tests/test_notebooks.py @@ -0,0 +1,52 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +import logging +import os + +import nbformat +import pytest +from nbclient.exceptions import CellExecutionError +from nbconvert.preprocessors import ExecutePreprocessor +from utils import discover_notebooks, format_error_message, save_notebook + +EXAMPLES_DIR = "notebooks" +NOTEBOOKS = discover_notebooks(EXAMPLES_DIR) + +log = logging.getLogger(__name__) + + +@pytest.mark.ipynb +@pytest.mark.parametrize( + # notebook - ipynb file to execute + "test_notebook", + NOTEBOOKS.values(), + ids=NOTEBOOKS.keys(), +) +def test_notebook(test_notebook): + """Test Notebook Generic Wrapper.""" + os.chdir(os.path.dirname(test_notebook)) + + with open(test_notebook) as nb: + notebook = nbformat.read(nb, as_version=nbformat.NO_CONVERT) + + ep = ExecutePreprocessor(timeout=-1, kernel_name="python3") + ep.skip_cells_with_tag = "pytest-skip" + + try: + log.info(f"Running {os.path.basename(test_notebook)}...") + output_notebook, _ = ep.preprocess(notebook, {"metadata": {"path": "./"}}) + # persist the notebook output to the original file for debugging purposes + save_notebook(output_notebook, test_notebook) + except CellExecutionError as e: + # handle underlying error + pytest.fail(f"Notebook execution failed with {e.ename}: {e.evalue}") + + for cell in output_notebook.cells: + metadata = cell.get("metadata", dict) + if "raises-exception" in metadata.get("tags", []): + for cell_output in cell.outputs: + if cell_output.output_type == "error": + # extract the error message from the cell output + log.error(format_error_message(cell_output.traceback)) + pytest.fail(cell_output.traceback[-1]) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..8443548 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,41 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +import os + +import nbformat + + +def format_error_message(traceback: list): + """Format error message.""" + return "".join(traceback[-2:]) + + +def discover_notebooks(directory): + """Return a dictionary of notebooks in the provided directory. + + The dictionary contains a mapping between the notebook names (in alphabetical order) and the + absolute paths to the notebook files. Directories containing IPYNB execution checkpoints are + ignored. + + Args: + directory: The directory to search. + + Returns: + A dictionary of notebook name - notebook file absolute path pairs. + """ + notebooks = {} + for root, dirs, files in os.walk(directory): + # exclude .ipynb_checkpoints directories from the search + dirs[:] = [d for d in dirs if d != ".ipynb_checkpoints"] + for file in files: + if file.endswith(".ipynb"): + # file name - absolute file path + notebooks[file.split(".ipynb")[0]] = os.path.abspath(os.path.join(root, file)) + return dict(sorted(notebooks.items())) + + +def save_notebook(notebook, file_path): + """Save notebook to a file.""" + with open(file_path, "w", encoding="utf-8") as nb_file: + nbformat.write(notebook, nb_file) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..6df5a14 --- /dev/null +++ b/tox.ini @@ -0,0 +1,66 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +[tox] +skipsdist = True +skip_missing_interpreters = True +envlist = fmt, lint, uats + +[vars] +all_path = {[vars]driver_path} {[vars]tst_path} +driver_path = {toxinidir}/driver/ +tst_path = {toxinidir}/tests/ + +[testenv] +passenv = + PYTHONPATH + CHARM_BUILD_DIR + MODEL_SETTINGS + KUBECONFIG +setenv = + PYTHONPATH = {toxinidir} + PYTHONBREAKPOINT=ipdb.set_trace + PY_COLORS=1 + +[testenv:update-requirements] +allowlist_externals = + bash + find + pip-compile + xargs +commands = +; uses 'bash -c' because piping didn't work in regular tox commands + bash -c 'find . -type f -name "requirements*.in" | xargs --replace=\{\} pip-compile -U \{\}' +deps = + pip-tools +description = Update requirements files by executing pip-compile on all requirements*.in files, including those in subdirs. + +[testenv:fmt] +commands = + isort {[vars]all_path} + black {[vars]all_path} +deps = + -r requirements-fmt.txt +description = Apply coding style standards to code + +[testenv:lint] +commands = + # uncomment the following line if this charm owns a lib + # codespell {[vars]lib_path} + codespell {toxinidir}/. --skip {toxinidir}/./.git --skip {toxinidir}/./.tox \ + --skip {toxinidir}/./build --skip {toxinidir}/./lib --skip {toxinidir}/./venv \ + --skip {toxinidir}/./.mypy_cache \ + --skip {toxinidir}/./icon.svg --skip *.json.tmpl + # pflake8 wrapper supports config from pyproject.toml + pflake8 {[vars]all_path} + isort --check-only --diff {[vars]all_path} + black --check --diff {[vars]all_path} +deps = + -r requirements-lint.txt +description = Check code against coding style standards + +[testenv:uats] +commands = pytest -vv --tb native {[vars]driver_path} -s --model kubeflow {posargs} +deps = + -r requirements.txt +description = Run UATs