TEST Python ver

updated python ver

print rc

Added debug helpers

Added debug helpers to all scripts

Added exception for event lister

test

Removed sdk-user for testing purposes

.github/workflows/e2e_tests.yaml

@@ -79,7 +79,7 @@ jobs:
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.9.18'
           cache: 'pip' # caching pip dependencies
 
       - name: Setup and start KinD cluster
@@ -98,39 +98,14 @@ jobs:
           kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager
           cd ..
 
-      - name: Add user to KinD
-        uses: ./common/github-actions/kind-add-user
-        with:
-          user-name: sdk-user
-
       - name: Add kueue resources
         run: kubectl apply --server-side -f "https://github.com/kubernetes-sigs/kueue/releases/download/v0.6.2/manifests.yaml"
 
-      - name: Configure RBAC for sdk user with limited permissions
-        run: |
-          kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses
-          kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user
-          kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces
-          kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
-          kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch --resource=rayclusters
-          kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user
-          kubectl create clusterrole appwrapper-creator --verb=get,list,create,delete,patch --resource=appwrappers
-          kubectl create clusterrolebinding sdk-user-appwrapper-creator --clusterrole=appwrapper-creator --user=sdk-user
-          kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors
-          kubectl create clusterrolebinding sdk-user-resourceflavor-creator --clusterrole=resourceflavor-creator --user=sdk-user
-          kubectl create clusterrole clusterqueue-creator --verb=get,list,create,delete,patch --resource=clusterqueues
-          kubectl create clusterrolebinding sdk-user-clusterqueue-creator --clusterrole=clusterqueue-creator --user=sdk-user
-          kubectl create clusterrole localqueue-creator --verb=get,list,create,delete,patch --resource=localqueues
-          kubectl create clusterrolebinding sdk-user-localqueue-creator --clusterrole=localqueue-creator --user=sdk-user
-          kubectl create clusterrole list-secrets --verb=get,list --resource=secrets
-          kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user
-          kubectl config use-context sdk-user
-
       - name: Run e2e tests
         run: |
           export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
-
+          python --version
           set -euo pipefail
           pip install poetry
           poetry install --with test,docs

tests/e2e/local_interactive_sdk_kind_test.py

@@ -8,6 +8,7 @@
 import pytest
 import ray
 import math
+from time import sleep
 
 from support import *
 
@@ -45,11 +46,51 @@ def run_local_interactives(self):
                 max_memory=2,
                 num_gpus=0,
                 image=ray_image,
-                write_to_file=True,
+                write_to_file=False,
                 verify_tls=False,
             )
         )
         cluster.up()
+        sleep(60)
+        api_instance = client.CustomObjectsApi()
+        rcs = api_instance.list_namespaced_custom_object(
+            group="ray.io",
+            version="v1",
+            namespace=self.namespace,
+            plural="rayclusters",
+        )
+        print("------------------ Ray Cluster ------------------")
+        for rc in rcs["items"]:
+            print(rc)
+        print("------------------ EVENTS ------------------")
+        v1 = client.CoreV1Api()
+        print(f"Events in namespace: {self.namespace}")
+        try:
+            events = v1.list_namespaced_event(namespace=self.namespace).items
+            for event in events:
+                print(
+                    f"Event: {event.metadata.name}, Reason: {event.reason}, Message: {event.message}, Timestamp: {event.last_timestamp}"
+                )
+        except client.exceptions.ApiException as e:
+            print(f"Exception when calling CoreV1Api->list_namespaced_event: {e}")
+
+        print("------------------ Workloads ------------------")
+        api_instance = client.CustomObjectsApi()
+        try:
+            workloads = api_instance.list_namespaced_custom_object(
+                "kueue.x-k8s.io", "v1beta1", self.namespace, "workloads"
+            )
+            for workload in workloads.get("items", []):
+                name = workload["metadata"]["name"]
+                status = workload.get("status", {})
+                print(
+                    f"Workload: {name}, Namespace: {self.namespace}, Status: {status}"
+                )
+        except client.exceptions.ApiException as e:
+            print(
+                f"Exception when calling CustomObjectsApi->list_namespaced_custom_object: {e}"
+            )
+
         cluster.wait_ready()
 
         generate_cert.generate_tls_cert(cluster_name, self.namespace)

tests/e2e/mnist_raycluster_sdk_aw_kind_test.py

@@ -8,6 +8,7 @@
 import pytest
 
 from support import *
+from time import sleep
 
 # This test creates an AppWrapper containing a Ray Cluster and covers the Ray Job submission functionality on Kind Cluster
 
@@ -49,7 +50,45 @@ def run_mnist_raycluster_sdk_kind(self):
         )
 
         cluster.up()
-
+        sleep(45)
+        api_instance = client.CustomObjectsApi()
+        rcs = api_instance.list_namespaced_custom_object(
+            group="ray.io",
+            version="v1",
+            namespace=self.namespace,
+            plural="rayclusters",
+        )
+        print("------------------ Ray Cluster ------------------")
+        for rc in rcs["items"]:
+            print(rc)
+        print("------------------ EVENTS ------------------")
+        v1 = client.CoreV1Api()
+        print(f"Events in namespace: {self.namespace}")
+        try:
+            events = v1.list_namespaced_event(namespace=self.namespace).items
+            for event in events:
+                print(
+                    f"Event: {event.metadata.name}, Reason: {event.reason}, Message: {event.message}, Timestamp: {event.last_timestamp}"
+                )
+        except client.exceptions.ApiException as e:
+            print(f"Exception when calling CoreV1Api->list_namespaced_event: {e}")
+
+        print("------------------ Workloads ------------------")
+        api_instance = client.CustomObjectsApi()
+        try:
+            workloads = api_instance.list_namespaced_custom_object(
+                "kueue.x-k8s.io", "v1beta1", self.namespace, "workloads"
+            )
+            for workload in workloads.get("items", []):
+                name = workload["metadata"]["name"]
+                status = workload.get("status", {})
+                print(
+                    f"Workload: {name}, Namespace: {self.namespace}, Status: {status}"
+                )
+        except client.exceptions.ApiException as e:
+            print(
+                f"Exception when calling CustomObjectsApi->list_namespaced_custom_object: {e}"
+            )
         cluster.status()
 
         cluster.wait_ready()

tests/e2e/mnist_raycluster_sdk_kind_test.py

@@ -8,6 +8,7 @@
 import pytest
 
 from support import *
+from time import sleep
 
 # This test creates a Ray Cluster and covers the Ray Job submission functionality on Kind Cluster
 
@@ -49,7 +50,45 @@ def run_mnist_raycluster_sdk_kind(self):
         )
 
         cluster.up()
-
+        sleep(45)
+        api_instance = client.CustomObjectsApi()
+        rcs = api_instance.list_namespaced_custom_object(
+            group="ray.io",
+            version="v1",
+            namespace=self.namespace,
+            plural="rayclusters",
+        )
+        print("------------------ Ray Cluster ------------------")
+        for rc in rcs["items"]:
+            print(rc)
+        print("------------------ EVENTS ------------------")
+        v1 = client.CoreV1Api()
+        print(f"Events in namespace: {self.namespace}")
+        try:
+            events = v1.list_namespaced_event(namespace=self.namespace).items
+            for event in events:
+                print(
+                    f"Event: {event.metadata.name}, Reason: {event.reason}, Message: {event.message}, Timestamp: {event.last_timestamp}"
+                )
+        except client.exceptions.ApiException as e:
+            print(f"Exception when calling CoreV1Api->list_namespaced_event: {e}")
+
+        print("------------------ Workloads ------------------")
+        api_instance = client.CustomObjectsApi()
+        try:
+            workloads = api_instance.list_namespaced_custom_object(
+                "kueue.x-k8s.io", "v1beta1", self.namespace, "workloads"
+            )
+            for workload in workloads.get("items", []):
+                name = workload["metadata"]["name"]
+                status = workload.get("status", {})
+                print(
+                    f"Workload: {name}, Namespace: {self.namespace}, Status: {status}"
+                )
+        except client.exceptions.ApiException as e:
+            print(
+                f"Exception when calling CustomObjectsApi->list_namespaced_custom_object: {e}"
+            )
         cluster.status()
 
         cluster.wait_ready()