diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index b83afb4b8..6aede88e4 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -79,7 +79,7 @@ jobs: - name: Set up specific Python version uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.9.18' cache: 'pip' # caching pip dependencies - name: Setup and start KinD cluster @@ -98,39 +98,14 @@ jobs: kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager cd .. - - name: Add user to KinD - uses: ./common/github-actions/kind-add-user - with: - user-name: sdk-user - - name: Add kueue resources run: kubectl apply --server-side -f "https://github.com/kubernetes-sigs/kueue/releases/download/v0.6.2/manifests.yaml" - - name: Configure RBAC for sdk user with limited permissions - run: | - kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses - kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user - kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces - kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user - kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch --resource=rayclusters - kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user - kubectl create clusterrole appwrapper-creator --verb=get,list,create,delete,patch --resource=appwrappers - kubectl create clusterrolebinding sdk-user-appwrapper-creator --clusterrole=appwrapper-creator --user=sdk-user - kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors - kubectl create clusterrolebinding sdk-user-resourceflavor-creator --clusterrole=resourceflavor-creator --user=sdk-user - kubectl create clusterrole clusterqueue-creator --verb=get,list,create,delete,patch --resource=clusterqueues - kubectl create clusterrolebinding sdk-user-clusterqueue-creator --clusterrole=clusterqueue-creator --user=sdk-user - kubectl create clusterrole localqueue-creator --verb=get,list,create,delete,patch --resource=localqueues - kubectl create clusterrolebinding sdk-user-localqueue-creator --clusterrole=localqueue-creator --user=sdk-user - kubectl create clusterrole list-secrets --verb=get,list --resource=secrets - kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user - kubectl config use-context sdk-user - - name: Run e2e tests run: | export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - + python --version set -euo pipefail pip install poetry poetry install --with test,docs diff --git a/tests/e2e/local_interactive_sdk_kind_test.py b/tests/e2e/local_interactive_sdk_kind_test.py index 647ac4727..066bce38c 100644 --- a/tests/e2e/local_interactive_sdk_kind_test.py +++ b/tests/e2e/local_interactive_sdk_kind_test.py @@ -8,6 +8,7 @@ import pytest import ray import math +from time import sleep from support import * @@ -45,11 +46,51 @@ def run_local_interactives(self): max_memory=2, num_gpus=0, image=ray_image, - write_to_file=True, + write_to_file=False, verify_tls=False, ) ) cluster.up() + sleep(60) + api_instance = client.CustomObjectsApi() + rcs = api_instance.list_namespaced_custom_object( + group="ray.io", + version="v1", + namespace=self.namespace, + plural="rayclusters", + ) + print("------------------ Ray Cluster ------------------") + for rc in rcs["items"]: + print(rc) + print("------------------ EVENTS ------------------") + v1 = client.CoreV1Api() + print(f"Events in namespace: {self.namespace}") + try: + events = v1.list_namespaced_event(namespace=self.namespace).items + for event in events: + print( + f"Event: {event.metadata.name}, Reason: {event.reason}, Message: {event.message}, Timestamp: {event.last_timestamp}" + ) + except client.exceptions.ApiException as e: + print(f"Exception when calling CoreV1Api->list_namespaced_event: {e}") + + print("------------------ Workloads ------------------") + api_instance = client.CustomObjectsApi() + try: + workloads = api_instance.list_namespaced_custom_object( + "kueue.x-k8s.io", "v1beta1", self.namespace, "workloads" + ) + for workload in workloads.get("items", []): + name = workload["metadata"]["name"] + status = workload.get("status", {}) + print( + f"Workload: {name}, Namespace: {self.namespace}, Status: {status}" + ) + except client.exceptions.ApiException as e: + print( + f"Exception when calling CustomObjectsApi->list_namespaced_custom_object: {e}" + ) + cluster.wait_ready() generate_cert.generate_tls_cert(cluster_name, self.namespace) diff --git a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py index 2aa5da16d..b560a1a8a 100644 --- a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py @@ -8,6 +8,7 @@ import pytest from support import * +from time import sleep # This test creates an AppWrapper containing a Ray Cluster and covers the Ray Job submission functionality on Kind Cluster @@ -49,7 +50,45 @@ def run_mnist_raycluster_sdk_kind(self): ) cluster.up() - + sleep(45) + api_instance = client.CustomObjectsApi() + rcs = api_instance.list_namespaced_custom_object( + group="ray.io", + version="v1", + namespace=self.namespace, + plural="rayclusters", + ) + print("------------------ Ray Cluster ------------------") + for rc in rcs["items"]: + print(rc) + print("------------------ EVENTS ------------------") + v1 = client.CoreV1Api() + print(f"Events in namespace: {self.namespace}") + try: + events = v1.list_namespaced_event(namespace=self.namespace).items + for event in events: + print( + f"Event: {event.metadata.name}, Reason: {event.reason}, Message: {event.message}, Timestamp: {event.last_timestamp}" + ) + except client.exceptions.ApiException as e: + print(f"Exception when calling CoreV1Api->list_namespaced_event: {e}") + + print("------------------ Workloads ------------------") + api_instance = client.CustomObjectsApi() + try: + workloads = api_instance.list_namespaced_custom_object( + "kueue.x-k8s.io", "v1beta1", self.namespace, "workloads" + ) + for workload in workloads.get("items", []): + name = workload["metadata"]["name"] + status = workload.get("status", {}) + print( + f"Workload: {name}, Namespace: {self.namespace}, Status: {status}" + ) + except client.exceptions.ApiException as e: + print( + f"Exception when calling CustomObjectsApi->list_namespaced_custom_object: {e}" + ) cluster.status() cluster.wait_ready() diff --git a/tests/e2e/mnist_raycluster_sdk_kind_test.py b/tests/e2e/mnist_raycluster_sdk_kind_test.py index 630bc5df4..e9f09b99a 100644 --- a/tests/e2e/mnist_raycluster_sdk_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_kind_test.py @@ -8,6 +8,7 @@ import pytest from support import * +from time import sleep # This test creates a Ray Cluster and covers the Ray Job submission functionality on Kind Cluster @@ -49,7 +50,45 @@ def run_mnist_raycluster_sdk_kind(self): ) cluster.up() - + sleep(45) + api_instance = client.CustomObjectsApi() + rcs = api_instance.list_namespaced_custom_object( + group="ray.io", + version="v1", + namespace=self.namespace, + plural="rayclusters", + ) + print("------------------ Ray Cluster ------------------") + for rc in rcs["items"]: + print(rc) + print("------------------ EVENTS ------------------") + v1 = client.CoreV1Api() + print(f"Events in namespace: {self.namespace}") + try: + events = v1.list_namespaced_event(namespace=self.namespace).items + for event in events: + print( + f"Event: {event.metadata.name}, Reason: {event.reason}, Message: {event.message}, Timestamp: {event.last_timestamp}" + ) + except client.exceptions.ApiException as e: + print(f"Exception when calling CoreV1Api->list_namespaced_event: {e}") + + print("------------------ Workloads ------------------") + api_instance = client.CustomObjectsApi() + try: + workloads = api_instance.list_namespaced_custom_object( + "kueue.x-k8s.io", "v1beta1", self.namespace, "workloads" + ) + for workload in workloads.get("items", []): + name = workload["metadata"]["name"] + status = workload.get("status", {}) + print( + f"Workload: {name}, Namespace: {self.namespace}, Status: {status}" + ) + except client.exceptions.ApiException as e: + print( + f"Exception when calling CustomObjectsApi->list_namespaced_custom_object: {e}" + ) cluster.status() cluster.wait_ready()