diff --git a/README.md b/README.md index 837f93e..c5af1ad 100644 --- a/README.md +++ b/README.md @@ -140,30 +140,44 @@ tox -e kubeflow-local ``` ### Run behind proxy -#### Prerequistes -**To run the tests behind proxy using Notebook or using the driver, the following step is necessary:** + +#### Running using Notebook + +##### Prerequistes Edit the PodDefault `tests/proxy-poddefault.yaml` to replace the placeholders for: - * `:`: The address and port of your proxy server - * ``: you can get this value by running: - ``` - cat /var/snap/microk8s/current/args/kube-proxy | grep cluster-cidr - ``` - * ``: you can get this value by running: - ``` - cat /var/snap/microk8s/current/args/kube-apiserver | grep service-cluster-ip-range - ``` + +* `http_proxy` and `https_proxy` - The address and port of your proxy server, format should be `:` +* `no_proxy` - A comma separated list of items that should not be proxied. It is recommended to include the following: + +`,,127.0.0.1,localhost,/24,,.svc,.local` + +where, + + * ``: you can get this value by running: + + ``` + cat /var/snap/microk8s/current/args/kube-proxy | grep cluster-cidr + ``` + + * ``: you can get this value by running: + + ``` + cat /var/snap/microk8s/current/args/kube-apiserver | grep service-cluster-ip-range + ``` - * ``: the Internal IP of the nodes where your cluster is running, you can - get this value by running: - ``` - microk8s kubectl get nodes -o wide - ``` - It is the `INTERNAL-IP` value - * ``: the name of your host on which the cluster is deployed, you can use the - `hostname` command to get it + * ``: the Internal IP of the nodes where your cluster is running, you can get this value by running: + + ``` + microk8s kubectl get nodes -o wide + ``` + It is the `INTERNAL-IP` value + + * ``: the name of your host on which the cluster is deployed, you can use the `hostname` command to get it + + * `localhost` and `127.0.0.1` are recommended to avoid proxying requests to `localhost` + -#### Running using Notebook To run the tests behind proxy using Notebook: 1. Login to the Dashboard and Create a Profile 2. Apply the PodDefault to your Profile's namespace, make sure you already followed the Prerequisites @@ -188,6 +202,14 @@ To run the tests behind proxy using Notebook: * kfp_v2 * training (except TFJob due to https://github.com/canonical/training-operator/issues/182) +#### Running using `driver` + +You can pass the `--proxy` flag and set the values for proxies to the tox command and this should automatically apply the required changes to run behind proxy. + +```bash +tox -e kubeflow- -- --proxy http_proxy="http_proxy:port" https_proxy="https_proxy:port" no_proxy=",,127.0.0.1,localhost,/24,,.svc,.local" +``` + #### Developer Notes Any environment that can be used to access and configure the Charmed Kubeflow deployment is diff --git a/assets/test-job.yaml.j2 b/assets/test-job.yaml.j2 index 89d254f..2dd22ca 100644 --- a/assets/test-job.yaml.j2 +++ b/assets/test-job.yaml.j2 @@ -7,6 +7,9 @@ spec: template: metadata: labels: + {% if proxy %} + notebook-proxy: "true" + {% endif %} access-minio: "true" access-ml-pipeline: "true" mlflow-server-minio: "true" diff --git a/driver/conftest.py b/driver/conftest.py index aabb9fc..cf66f58 100644 --- a/driver/conftest.py +++ b/driver/conftest.py @@ -10,6 +10,17 @@ def pytest_addoption(parser: Parser): * Add a `--filter` option to (de)select test cases based on their name (see also https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags) """ + parser.addoption( + "--proxy", + nargs=3, + metavar=("http_proxy", "https_proxy", "no_proxy"), + help="Set a number of key-value pairs for the proxy environment variables." + " Example: " + "--proxy http_proxy='proxy:port' https_proxy='proxy:port' no_proxy='" + " If used, a PodDefault will be rendered and applied to the Kubernetes deployment." + " It is not used by default.", + action="store", + ) parser.addoption( "--filter", help="Provide a filter to (de)select tests cases based on their name. The filter follows" diff --git a/driver/test_kubeflow_workloads.py b/driver/test_kubeflow_workloads.py index 203a995..5d9dd22 100644 --- a/driver/test_kubeflow_workloads.py +++ b/driver/test_kubeflow_workloads.py @@ -5,10 +5,15 @@ import os import subprocess from pathlib import Path +from typing import Dict import pytest from lightkube import ApiError, Client, codecs -from lightkube.generic_resource import create_global_resource, load_in_cluster_generic_resources +from lightkube.generic_resource import ( + create_global_resource, + create_namespaced_resource, + load_in_cluster_generic_resources, +) from utils import assert_namespace_active, delete_job, fetch_job_logs, wait_for_job log = logging.getLogger(__name__) @@ -34,6 +39,14 @@ PYTEST_CMD_BASE = "pytest" +PODDEFAULT_RESOURCE = create_namespaced_resource( + group="kubeflow.org", + version="v1alpha1", + kind="poddefault", + plural="poddefaults", +) +PODDEFAULT_WITH_PROXY_PATH = Path("tests") / "proxy-poddefault.yaml.j2" + @pytest.fixture(scope="session") def pytest_filter(request): @@ -83,6 +96,33 @@ def create_profile(lightkube_client): lightkube_client.delete(PROFILE_RESOURCE, name=NAMESPACE) +@pytest.fixture(scope="function") +def create_poddefaults_on_proxy(request, lightkube_client): + """Create PodDefault with proxy env variables for the Notebook inside the Job.""" + # Simply yield if the proxy flag is not set + if not request.config.getoption("proxy"): + yield + else: + log.info("Adding PodDefault with proxy settings.") + poddefault_resource = codecs.load_all_yaml( + PODDEFAULT_WITH_PROXY_PATH.read_text(), + context=proxy_context(request), + ) + # Using the first item of the list of poddefault_resource. It is a one item list. + lightkube_client.create(poddefault_resource[0], namespace=NAMESPACE) + + yield + + # delete the PodDefault at the end of the module tests + log.info("Deleting PodDefault...") + poddefault_resource = codecs.load_all_yaml( + PODDEFAULT_WITH_PROXY_PATH.read_text(), + context=proxy_context(request), + ) + poddefault_name = poddefault_resource[0].metadata.name + lightkube_client.delete(PODDEFAULT_RESOURCE, name=poddefault_name, namespace=NAMESPACE) + + @pytest.mark.abort_on_fail async def test_create_profile(lightkube_client, create_profile): """Test Profile creation. @@ -105,7 +145,9 @@ async def test_create_profile(lightkube_client, create_profile): assert_namespace_active(lightkube_client, NAMESPACE) -def test_kubeflow_workloads(lightkube_client, pytest_cmd, tests_checked_out_commit): +def test_kubeflow_workloads( + lightkube_client, pytest_cmd, tests_checked_out_commit, request, create_poddefaults_on_proxy +): """Run a K8s Job to execute the notebook tests.""" log.info(f"Starting Kubernetes Job {NAMESPACE}/{JOB_NAME} to run notebook tests...") resources = list( @@ -118,9 +160,11 @@ def test_kubeflow_workloads(lightkube_client, pytest_cmd, tests_checked_out_comm "tests_image": TESTS_IMAGE, "tests_remote_commit": tests_checked_out_commit, "pytest_cmd": pytest_cmd, + "proxy": True if request.config.getoption("proxy") else False, }, ) ) + assert len(resources) == 1, f"Expected 1 Job, got {len(resources)}!" lightkube_client.create(resources[0], namespace=NAMESPACE) @@ -140,3 +184,12 @@ def teardown_module(): """Cleanup resources.""" log.info(f"Deleting Job {NAMESPACE}/{JOB_NAME}...") delete_job(JOB_NAME, NAMESPACE) + + +def proxy_context(request) -> Dict[str, str]: + """Return a dictionary with proxy environment variables from user input.""" + proxy_context = {} + for proxy in request.config.getoption("proxy"): + key, value = proxy.split("=") + proxy_context[key] = value + return proxy_context diff --git a/tests/proxy-poddefault.yaml b/tests/proxy-poddefault.yaml deleted file mode 100644 index a1f7300..0000000 --- a/tests/proxy-poddefault.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: kubeflow.org/v1alpha1 -kind: PodDefault -metadata: - name: notebook-proxy -spec: - desc: Add proxy settings - env: - - name: HTTP_PROXY - value: : - - name: http_proxy - value: : - - name: HTTPS_PROXY - value: : - - name: https_proxy - value: : - - name: NO_PROXY - value: ,,127.0.0.1,/24,,.svc,.local - - name: no_proxy - value: ,,127.0.0.1,/24,,.svc,.local - selector: - matchLabels: - notebook-proxy: "true" diff --git a/tests/proxy-poddefault.yaml.j2 b/tests/proxy-poddefault.yaml.j2 new file mode 100644 index 0000000..c5a5868 --- /dev/null +++ b/tests/proxy-poddefault.yaml.j2 @@ -0,0 +1,45 @@ +apiVersion: kubeflow.org/v1alpha1 +kind: PodDefault +metadata: + name: notebook-proxy +spec: + desc: Add proxy settings + env: + - name: HTTP_PROXY + value: {{ http_proxy }} + - name: http_proxy + value: {{ http_proxy }} + - name: HTTPS_PROXY + value: {{ https_proxy }} + - name: https_proxy + value: {{ https_proxy }} + - name: NO_PROXY + value: {{ no_proxy }} + - name: no_proxy + value: {{ no_proxy }} + _example_env: + ################################ + # # + # EXAMPLE CONFIGURATION # + # # + ################################ + + # This is not actually functional, just serves as an example for how to configure + # the values of proxy and which ones have to be included to make things work properly. + # If you are running the UATs directly in a Notebook, please modify the above env block + # with the values that fit your specific configuration. + - name: HTTP_PROXY + value: : + - name: http_proxy + value: : + - name: HTTPS_PROXY + value: : + - name: https_proxy + value: : + - name: NO_PROXY + value: ,,127.0.0.1,localhost,/24,,.svc,.local + - name: no_proxy + value: ,,127.0.0.1,localhost,/24,,.svc,.local + selector: + matchLabels: + notebook-proxy: "true"