Merge KF-5945-configure-proxy-dev-branch into main

README.md

@@ -139,6 +139,77 @@ tox -e kubeflow-remote
 tox -e kubeflow-local
 ```
 
+### Run behind proxy
+
+#### Running using Notebook
+
+##### Prerequistes
+
+Edit the PodDefault `tests/proxy-poddefault.yaml` to replace the placeholders for:
+
+* `http_proxy` and `https_proxy` - The address and port of your proxy server, format should be `<proxy_address>:<proxy_port>`
+* `no_proxy` - A comma separated list of items that should not be proxied. It is recommended to include the following:
+
+`<cluster cidr>,<service cluster ip range>,127.0.0.1,localhost,<nodes internal ip(s)>/24,<cluster hostname>,.svc,.local`
+
+where,
+
+ * `<cluster cidr>`: you can get this value by running:
+
+ ```
+ cat /var/snap/microk8s/current/args/kube-proxy | grep cluster-cidr
+ ```
+
+ * `<service cluster ip range>`: you can get this value by running:
+
+ ```
+ cat /var/snap/microk8s/current/args/kube-apiserver | grep service-cluster-ip-range
+ ```
+
+ * `<nodes internal ip(s)>`: the Internal IP of the nodes where your cluster is running, you can get this value by running:
+
+ ```
+ microk8s kubectl get nodes -o wide
+ ```
+ It is the `INTERNAL-IP` value
+
+ * `<hostname>`: the name of your host on which the cluster is deployed, you can use the `hostname` command to get it
+
+ * `localhost` and `127.0.0.1` are recommended to avoid proxying requests to `localhost`
+
+
+To run the tests behind proxy using Notebook:
+1. Login to the Dashboard and Create a Profile
+2. Apply the PodDefault to your Profile's namespace, make sure you already followed the Prerequisites
+ section to modify the PodDefault. Apply it with:
+ ```
+ kubectl apply -f ./tests/proxy-poddefault.yaml -n <your_namespace>
+ ```
+3. Create a Notebook and from the `Advanced Options > Configurations` select `Add proxy settings`,
+ then click `Launch` to start the Notebook.
+ Wait for the Notebook to be Ready, then Connect to it.
+4. From inside the Notebook, start a new terminal session and clone this repo:
+
+ ```bash
+ git clone https://github.com/canonical/charmed-kubeflow-uats.git
+ ```
+ Open the `charmed-kubeflow-uats/tests` directory and for each `.ipynb` test file there, open it
+ and run the Notebook.
+
+ Currently, the following tests are supported to run behind proxy:
+ * katib
+ * kserve
+ * kfp_v2
+ * training (except TFJob due to https://github.com/canonical/training-operator/issues/182)
+
+#### Running using `driver`
+
+You can pass the `--proxy` flag and set the values for proxies to the tox command and this should automatically apply the required changes to run behind proxy.
+
+```bash
+tox -e kubeflow-<local|remote> -- --proxy http_proxy="http_proxy:port" https_proxy="https_proxy:port" no_proxy="<cluster cidr>,<service cluster ip range>,127.0.0.1,localhost,<nodes internal ip(s)>/24,<cluster hostname>,.svc,.local"
+```
+
 #### Developer Notes
 
 Any environment that can be used to access and configure the Charmed Kubeflow deployment is

assets/test-job.yaml.j2

@@ -7,6 +7,9 @@ spec:
  template:
  metadata:
  labels:
+ {% if proxy %}
+ notebook-proxy: "true"
+ {% endif %}
  access-minio: "true"
  access-ml-pipeline: "true"
  mlflow-server-minio: "true"

driver/conftest.py

@@ -10,6 +10,17 @@ def pytest_addoption(parser: Parser):
  * Add a `--filter` option to (de)select test cases based on their name (see also
  https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags)
  """
+ parser.addoption(
+ "--proxy",
+ nargs=3,
+ metavar=("http_proxy", "https_proxy", "no_proxy"),
+ help="Set a number of key-value pairs for the proxy environment variables."
+ " Example: "
+ "--proxy http_proxy='proxy:port' https_proxy='proxy:port' no_proxy=<comma separated of no proxy>'"
+ " If used, a PodDefault will be rendered and applied to the Kubernetes deployment."
+ " It is not used by default.",
+ action="store",
+ )
  parser.addoption(
  "--filter",
  help="Provide a filter to (de)select tests cases based on their name. The filter follows"

driver/test_kubeflow_workloads.py

@@ -5,10 +5,15 @@
 import os
 import subprocess
 from pathlib import Path
+from typing import Dict
 
 import pytest
 from lightkube import ApiError, Client, codecs
-from lightkube.generic_resource import create_global_resource, load_in_cluster_generic_resources
+from lightkube.generic_resource import (
+ create_global_resource,
+ create_namespaced_resource,
+ load_in_cluster_generic_resources,
+)
 from utils import assert_namespace_active, delete_job, fetch_job_logs, wait_for_job
 
 log = logging.getLogger(__name__)
@@ -34,6 +39,14 @@
 
 PYTEST_CMD_BASE = "pytest"
 
+PODDEFAULT_RESOURCE = create_namespaced_resource(
+ group="kubeflow.org",
+ version="v1alpha1",
+ kind="poddefault",
+ plural="poddefaults",
+)
+PODDEFAULT_WITH_PROXY_PATH = Path("tests") / "proxy-poddefault.yaml.j2"
+
 
 @pytest.fixture(scope="session")
 def pytest_filter(request):
@@ -83,6 +96,33 @@ def create_profile(lightkube_client):
  lightkube_client.delete(PROFILE_RESOURCE, name=NAMESPACE)
 
 
+@pytest.fixture(scope="function")
+def create_poddefaults_on_proxy(request, lightkube_client):
+ """Create PodDefault with proxy env variables for the Notebook inside the Job."""
+ # Simply yield if the proxy flag is not set
+ if not request.config.getoption("proxy"):
+ yield
+ else:
+ log.info("Adding PodDefault with proxy settings.")
+ poddefault_resource = codecs.load_all_yaml(
+ PODDEFAULT_WITH_PROXY_PATH.read_text(),
+ context=proxy_context(request),
+ )
+ # Using the first item of the list of poddefault_resource. It is a one item list.
+ lightkube_client.create(poddefault_resource[0], namespace=NAMESPACE)
+
+ yield
+
+ # delete the PodDefault at the end of the module tests
+ log.info("Deleting PodDefault...")
+ poddefault_resource = codecs.load_all_yaml(
+ PODDEFAULT_WITH_PROXY_PATH.read_text(),
+ context=proxy_context(request),
+ )
+ poddefault_name = poddefault_resource[0].metadata.name
+ lightkube_client.delete(PODDEFAULT_RESOURCE, name=poddefault_name, namespace=NAMESPACE)
+
+
 @pytest.mark.abort_on_fail
 async def test_create_profile(lightkube_client, create_profile):
  """Test Profile creation.
@@ -105,7 +145,9 @@ async def test_create_profile(lightkube_client, create_profile):
  assert_namespace_active(lightkube_client, NAMESPACE)
 
 
-def test_kubeflow_workloads(lightkube_client, pytest_cmd, tests_checked_out_commit):
+def test_kubeflow_workloads(
+ lightkube_client, pytest_cmd, tests_checked_out_commit, request, create_poddefaults_on_proxy
+):
  """Run a K8s Job to execute the notebook tests."""
  log.info(f"Starting Kubernetes Job {NAMESPACE}/{JOB_NAME} to run notebook tests...")
  resources = list(
@@ -118,9 +160,11 @@ def test_kubeflow_workloads(lightkube_client, pytest_cmd, tests_checked_out_comm
  "tests_image": TESTS_IMAGE,
  "tests_remote_commit": tests_checked_out_commit,
  "pytest_cmd": pytest_cmd,
+ "proxy": True if request.config.getoption("proxy") else False,
  },
  )
  )
+
  assert len(resources) == 1, f"Expected 1 Job, got {len(resources)}!"
  lightkube_client.create(resources[0], namespace=NAMESPACE)
 
@@ -140,3 +184,12 @@ def teardown_module():
  """Cleanup resources."""
  log.info(f"Deleting Job {NAMESPACE}/{JOB_NAME}...")
  delete_job(JOB_NAME, NAMESPACE)
+
+
+def proxy_context(request) -> Dict[str, str]:
+ """Return a dictionary with proxy environment variables from user input."""
+ proxy_context = {}
+ for proxy in request.config.getoption("proxy"):
+ key, value = proxy.split("=")
+ proxy_context[key] = value
+ return proxy_context