From af3529aac1e267326745c94c4bb337d59f78aa96 Mon Sep 17 00:00:00 2001 From: Venkat Date: Tue, 6 Aug 2024 23:30:58 -0400 Subject: [PATCH 1/7] updated --- ray_provider/decorators/ray.py | 4 ++-- tests/decorators/test_ray_decorators.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ray_provider/decorators/ray.py b/ray_provider/decorators/ray.py index 5dcc4e7..610e812 100644 --- a/ray_provider/decorators/ray.py +++ b/ray_provider/decorators/ray.py @@ -128,9 +128,9 @@ def _extract_function_name(self) -> str: return self.python_callable.__name__ -class task: +class ray: @staticmethod - def ray( + def task( python_callable: Callable[..., Any] | None = None, multiple_outputs: bool | None = None, **kwargs: Any, diff --git a/tests/decorators/test_ray_decorators.py b/tests/decorators/test_ray_decorators.py index 545197f..07b7be5 100644 --- a/tests/decorators/test_ray_decorators.py +++ b/tests/decorators/test_ray_decorators.py @@ -5,7 +5,7 @@ from airflow.exceptions import AirflowException from airflow.utils.context import Context -from ray_provider.decorators.ray import _RayDecoratedOperator, task +from ray_provider.decorators.ray import _RayDecoratedOperator, ray from ray_provider.operators.ray import SubmitRayJob DEFAULT_DATE = "2023-01-01" @@ -169,14 +169,14 @@ def dummy_callable(): class TestRayTaskDecorator: def test_ray_task_decorator(self): - @task.ray() + @ray.task() def dummy_function(): return "dummy" assert isinstance(dummy_function, _TaskDecorator) def test_ray_task_decorator_with_multiple_outputs(self): - @task.ray(multiple_outputs=True) + @ray.task(multiple_outputs=True) def dummy_function(): return {"key": "value"} @@ -189,7 +189,7 @@ def test_ray_task_decorator_with_config(self): "memory": 1024, } - @task.ray(**config) + @ray.task(**config) def dummy_function(): return "dummy" From 1bcde5ecfc8b16a72c3617a143725cc2b2162539 Mon Sep 17 00:00:00 2001 From: Venkat Date: Wed, 7 Aug 2024 00:27:37 -0400 Subject: [PATCH 2/7] cluster_name changes --- .github/workflows/test.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 096294e..f2fba0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -109,9 +109,13 @@ jobs: - name: 'Set GCP Project ID' run: gcloud config set project ${{ secrets.PROJECT_ID }} + - name: Generate unique cluster name + id: cluster-name + run: echo "name=integration-test-cluster-${GITHUB_RUN_ID}-$(uuidgen)" >> $GITHUB_OUTPUT + - name: Create GKE cluster run: | - gcloud container clusters create integration-test-cluster \ + gcloud container clusters create ${{ steps.cluster-name.outputs.name }} \ --zone us-central1-a \ --num-nodes 2 \ --machine-type e2-standard-8 \ @@ -123,7 +127,7 @@ jobs: - name: Setup gcloud and get kubeconfig run: | gcloud components install gke-gcloud-auth-plugin - gcloud container clusters get-credentials integration-test-cluster --zone us-central1-a + gcloud container clusters get-credentials ${{ steps.cluster-name.outputs.name }} --zone us-central1-a kubectl config view --raw > kubeconfig.yaml echo "KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml" >> $GITHUB_ENV echo "USE_GKE_GCLOUD_AUTH_PLUGIN=True" >> $GITHUB_ENV @@ -139,4 +143,4 @@ jobs: - name: Delete GKE cluster if: always() run: | - gcloud container clusters delete integration-test-cluster --zone us-central1-a --quiet + gcloud container clusters delete ${{ steps.cluster-name.outputs.name }} --zone us-central1-a --quiet From 6cc2d5aa3ed9ed23a0bff1866c36fb4ad0a5ba64 Mon Sep 17 00:00:00 2001 From: Venkat Date: Wed, 7 Aug 2024 00:30:50 -0400 Subject: [PATCH 3/7] name to less than 40 characters --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f2fba0f..4650950 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -111,7 +111,7 @@ jobs: - name: Generate unique cluster name id: cluster-name - run: echo "name=integration-test-cluster-${GITHUB_RUN_ID}-$(uuidgen)" >> $GITHUB_OUTPUT + run: echo "name=test-${GITHUB_RUN_ID:0:12}" >> $GITHUB_OUTPUT - name: Create GKE cluster run: | From e30d7e88bd10d7f733fc307cb5b1225aaba778c4 Mon Sep 17 00:00:00 2001 From: Venkat Date: Wed, 7 Aug 2024 00:51:52 -0400 Subject: [PATCH 4/7] lower cluster spec --- .github/workflows/test.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4650950..c04fe48 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,10 +117,9 @@ jobs: run: | gcloud container clusters create ${{ steps.cluster-name.outputs.name }} \ --zone us-central1-a \ - --num-nodes 2 \ - --machine-type e2-standard-8 \ - --no-enable-autoupgrade \ - --no-enable-autorepair \ + --num-nodes 1 \ + --machine-type e2-standard-4 \ + --enable-autoscaling --min-nodes=1 --max-nodes=3 \ --no-enable-ip-alias \ --service-account=${{ secrets.SERVICE_ACCOUNT_EMAIL }} From b4fba59b50815a948b99ebd41fb821d71d5b1b30 Mon Sep 17 00:00:00 2001 From: Venkat Date: Wed, 7 Aug 2024 01:20:20 -0400 Subject: [PATCH 5/7] scale up cluster assignment --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c04fe48..4d35ba8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,9 +117,9 @@ jobs: run: | gcloud container clusters create ${{ steps.cluster-name.outputs.name }} \ --zone us-central1-a \ - --num-nodes 1 \ + --num-nodes 2 \ --machine-type e2-standard-4 \ - --enable-autoscaling --min-nodes=1 --max-nodes=3 \ + --enable-autoscaling --min-nodes=1 --max-nodes=4 \ --no-enable-ip-alias \ --service-account=${{ secrets.SERVICE_ACCOUNT_EMAIL }} From 3d0a98c8cece377ad07170a3cbca9d749cb4ae23 Mon Sep 17 00:00:00 2001 From: Venkat Date: Wed, 7 Aug 2024 01:27:24 -0400 Subject: [PATCH 6/7] underpowering the ray cluster --- .github/workflows/test.yml | 4 ++-- example_dags/scripts/ray.yaml | 26 +++++++++++--------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d35ba8..c04fe48 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,9 +117,9 @@ jobs: run: | gcloud container clusters create ${{ steps.cluster-name.outputs.name }} \ --zone us-central1-a \ - --num-nodes 2 \ + --num-nodes 1 \ --machine-type e2-standard-4 \ - --enable-autoscaling --min-nodes=1 --max-nodes=4 \ + --enable-autoscaling --min-nodes=1 --max-nodes=3 \ --no-enable-ip-alias \ --service-account=${{ secrets.SERVICE_ACCOUNT_EMAIL }} diff --git a/example_dags/scripts/ray.yaml b/example_dags/scripts/ray.yaml index 1eb8e70..ac93f6e 100644 --- a/example_dags/scripts/ray.yaml +++ b/example_dags/scripts/ray.yaml @@ -20,11 +20,11 @@ spec: image: rayproject/ray-ml:latest resources: limits: - cpu: 4 - memory: 8Gi + cpu: 2 + memory: 4Gi requests: - cpu: 4 - memory: 8Gi + cpu: 2 + memory: 4Gi lifecycle: preStop: exec: @@ -40,15 +40,11 @@ spec: name: serve - containerPort: 8080 name: metrics - - containerPort: 44217 - name: as-metrics # autoscaler - - containerPort: 44227 - name: dash-metrics # dashboard workerGroupSpecs: - groupName: small-group - replicas: 2 - minReplicas: 2 - maxReplicas: 5 + replicas: 1 + minReplicas: 1 + maxReplicas: 2 rayStartParams: block: "true" template: @@ -59,8 +55,8 @@ spec: image: rayproject/ray-ml:latest resources: limits: - cpu: 2 - memory: 4Gi + cpu: 1 + memory: 1Gi requests: - cpu: 2 - memory: 4Gi + cpu: 1 + memory: 1Gi From 44962d0362b3dc2599e7957219ee6d31aa70ef7b Mon Sep 17 00:00:00 2001 From: Venkat Date: Wed, 7 Aug 2024 02:16:16 -0400 Subject: [PATCH 7/7] fewer resources --- example_dags/scripts/ray.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example_dags/scripts/ray.yaml b/example_dags/scripts/ray.yaml index ac93f6e..0c4313f 100644 --- a/example_dags/scripts/ray.yaml +++ b/example_dags/scripts/ray.yaml @@ -20,11 +20,11 @@ spec: image: rayproject/ray-ml:latest resources: limits: - cpu: 2 - memory: 4Gi + cpu: 1 + memory: 3Gi requests: - cpu: 2 - memory: 4Gi + cpu: 1 + memory: 3Gi lifecycle: preStop: exec: