From 3c92d12ce6925c0b169c469c6f028b3c6b56c72f Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 16 Oct 2023 18:02:42 -0700 Subject: [PATCH 01/14] Add integration tests Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 6 +-- .github/workflows/docs_build.yml | 6 +-- .github/workflows/end2end.yml | 38 +++++++++++++++++++ examples/integration_test.py | 37 ++++++++++++++++++ examples/ray_plugin/ray_plugin/ray_example.py | 2 +- 5 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/end2end.yml create mode 100644 examples/integration_test.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 81b8840f3..625506676 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -6,9 +6,9 @@ on: - master paths-ignore: - "docs/**" - pull_request: - paths-ignore: - - "docs/**" +# pull_request: +# paths-ignore: +# - "docs/**" jobs: lint: runs-on: ubuntu-latest diff --git a/.github/workflows/docs_build.yml b/.github/workflows/docs_build.yml index cc418bc1e..c121d9c43 100644 --- a/.github/workflows/docs_build.yml +++ b/.github/workflows/docs_build.yml @@ -4,9 +4,9 @@ on: push: branches: - master - pull_request: - branches: - - master +# pull_request: +# branches: +# - master jobs: docs_warnings: name: Docs Warnings diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml new file mode 100644 index 000000000..e85aef2a7 --- /dev/null +++ b/.github/workflows/end2end.yml @@ -0,0 +1,38 @@ +name: End to End tests + +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + endtoend: + name: End to End tests + runs-on: ubuntu-latest + env: + FLYTESNACKS_VERSION: "" + steps: + - name: Checkout + uses: actions/checkout@v3 + - uses: unionai/flytectl-setup-action@v0.0.1 + name: Setup flytectl + - uses: actions/setup-python@v3 + with: + python-version: 3.10 + - name: Create Sandbox Cluster + run: | + flytectl config init + flytectl demo start + kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v0.5.2" + kubectl rollout restart deployment flyte-sandbox -n flyte + - name: Setup Flytekit + run: | + python -m pip install --upgrade pip + pip install flytekit flytekitplugins-deck-standard flytekitplugins-ray + pip freeze + - name: execute flyte workflows + working-directory: examples + run: | + pyflyte run --remote integration_test.py integration_test diff --git a/examples/integration_test.py b/examples/integration_test.py new file mode 100644 index 000000000..a94a382d9 --- /dev/null +++ b/examples/integration_test.py @@ -0,0 +1,37 @@ +from basics.basics.documenting_workflows import google_docstring_wf, numpy_docstring_wf, sphinx_docstring_wf +from basics.basics.hello_world import hello_world_wf +from basics.basics.imperative_workflow import imperative_wf +from basics.basics.launch_plan import simple_wf_lp_fixed_inputs +from basics.basics.named_outputs import simple_wf_with_named_outputs +from basics.basics.shell_task import shell_task_wf +from basics.basics.task import slope +from basics.basics.workflow import simple_wf_with_partial +from ray_plugin.ray_plugin.ray_example import ray_workflow +from flytekit import workflow, task + + +@workflow +def integration_test(): + # Test Basic Workflows + google_docstring_wf() + numpy_docstring_wf() + sphinx_docstring_wf() + + hello_world_wf() + imperative_wf(x=[-3, 0, 3], y=[7, 4, -2]) + + simple_wf_lp_fixed_inputs(y=[-3, 2, -2]) + simple_wf_with_named_outputs() + + shell_task_wf() + slope(x=[-3, 0, 3], y=[7, 4, -2]) + simple_wf_with_partial(x=[-3, 0, 3], y=[7, 4, -2]) + + # Test Plugins + ray_workflow(n=5) + + # TODO: Add more plugins here, like spark, tensorflow, torch, Dask. + + +if __name__ == '__main__': + integration_test() diff --git a/examples/ray_plugin/ray_plugin/ray_example.py b/examples/ray_plugin/ray_plugin/ray_example.py index 9f3a2fc41..95687e6fc 100644 --- a/examples/ray_plugin/ray_plugin/ray_example.py +++ b/examples/ray_plugin/ray_plugin/ray_example.py @@ -84,7 +84,7 @@ def f(x): # %% @task( task_config=ray_config, - requests=Resources(mem="2Gi", cpu="2"), + requests=Resources(mem="800Mi", cpu="1"), container_image=custom_image, ) def ray_task(n: int) -> typing.List[int]: From 634583d462424ee416d231b9ebf7a25b641a1dcf Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 16 Oct 2023 18:06:22 -0700 Subject: [PATCH 02/14] Add integration tests locally Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index e85aef2a7..e10d66667 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -32,7 +32,11 @@ jobs: python -m pip install --upgrade pip pip install flytekit flytekitplugins-deck-standard flytekitplugins-ray pip freeze - - name: execute flyte workflows + - name: execute flyte workflows locally + working-directory: examples + run: | + pyflyte run --remote integration_test.py integration_test + - name: execute flyte workflows on demo cluster working-directory: examples run: | pyflyte run --remote integration_test.py integration_test From ba58ef3032c040b9ef16cfada7d3465bfb9aacea Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 16 Oct 2023 18:08:51 -0700 Subject: [PATCH 03/14] Add concurrency Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index e10d66667..c4ce20157 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -1,5 +1,9 @@ name: End to End tests +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + on: push: branches: From d85e1b8dd0a63064e5a5516ebf2c4231c7357ae3 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 16 Oct 2023 18:11:39 -0700 Subject: [PATCH 04/14] Add concurrency Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 4 ++++ .github/workflows/docs_build.yml | 4 ++++ .github/workflows/end2end.yml | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 625506676..9ff18a4b9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,5 +1,9 @@ name: Master +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + on: push: branches: diff --git a/.github/workflows/docs_build.yml b/.github/workflows/docs_build.yml index c121d9c43..ae852fb30 100644 --- a/.github/workflows/docs_build.yml +++ b/.github/workflows/docs_build.yml @@ -1,5 +1,9 @@ name: Docs Build +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + on: push: branches: diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index c4ce20157..faafcbc9c 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@v3 - uses: unionai/flytectl-setup-action@v0.0.1 name: Setup flytectl - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v4 with: python-version: 3.10 - name: Create Sandbox Cluster From f18b5f2d8a5b584761ce4055e81ce66cb49b71c3 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 16 Oct 2023 18:15:06 -0700 Subject: [PATCH 05/14] update python version Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index faafcbc9c..3ea2016a4 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -24,7 +24,7 @@ jobs: name: Setup flytectl - uses: actions/setup-python@v4 with: - python-version: 3.10 + python-version: "3.10" - name: Create Sandbox Cluster run: | flytectl config init From 3dda97793499a40652dd3de5f5bcca6317fcf5d5 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 16 Oct 2023 18:19:57 -0700 Subject: [PATCH 06/14] remove end-to-end tests from check.yaml Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 68 ------------------------------------ 1 file changed, 68 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9ff18a4b9..6a04098eb 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -171,71 +171,3 @@ jobs: "tag_name": "'${{ needs.bump-version.outputs.version }}'", "prerelease": false }' - - e2e-tests: - runs-on: ubuntu-latest - env: - FLYTESNACKS_PRIORITIES: 'P0' - FLYTESNACKS_VERSION: '' - timeout-minutes: 30 - steps: - - name: Set latest Flytesnacks release - if: ${{ env.FLYTESNACKS_VERSION == '' }} - run: | - FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" - echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - uses: unionai/flytectl-setup-action@v0.0.1 - - name: Setup sandbox - run: | - mkdir -p ~/.flyte/sandbox - cat << EOF > ~/.flyte/sandbox/config.yaml - task_resources: - defaults: - cpu: "0" - memory: "0" - limits: - cpu: "0" - memory: "0" - EOF - flytectl demo start --imagePullPolicy Never - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install flytekit flytekitplugins-deck-standard - pip freeze - - name: Checkout flytesnacks - uses: actions/checkout@v3 - with: - repository: flyteorg/flytesnacks - path: flytesnacks - - name: Register specific tests - run: | - for f in \ - basics/basics/hello_world.py \ - basics/basics/workflow.py \ - basics/basics/named_outputs.py \ - advanced_composition/advanced_composition/chain_entities.py \ - advanced_composition/advanced_composition/dynamics.py \ - advanced_composition/advanced_composition/map_task.py \ - advanced_composition/advanced_composition/subworkflows.py \ - data_types_and_io/data_types_and_io/custom_objects.py \ - data_types_and_io/data_types_and_io/schema.py \ - data_types_and_io/data_types_and_io/typed_schema.py ; - do - pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ - register \ - --project flytesnacks \ - --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ - --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/examples/$f; - done - - name: End2End - run: | - make end2end_execute From c009d2fbe5a6205c7cfb6360a77d915471d0b1cc Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 16 Oct 2023 20:46:44 -0700 Subject: [PATCH 07/14] update console endpoint Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 3ea2016a4..348f4b9da 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -27,7 +27,7 @@ jobs: python-version: "3.10" - name: Create Sandbox Cluster run: | - flytectl config init + flytectl config init --console localhost:30080 flytectl demo start kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v0.5.2" kubectl rollout restart deployment flyte-sandbox -n flyte @@ -39,7 +39,7 @@ jobs: - name: execute flyte workflows locally working-directory: examples run: | - pyflyte run --remote integration_test.py integration_test + pyflyte run integration_test.py integration_test - name: execute flyte workflows on demo cluster working-directory: examples run: | From f912d89c22d59d693e505ca6f2bbcd22457ebf9c Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 16 Oct 2023 21:22:28 -0700 Subject: [PATCH 08/14] update pyflyte run command Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 348f4b9da..ef5b31cbe 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -43,4 +43,4 @@ jobs: - name: execute flyte workflows on demo cluster working-directory: examples run: | - pyflyte run --remote integration_test.py integration_test + pyflyte --config ~/.flyte/config.yaml run --remote integration_test.py integration_test From 0fccd8f2eee9ffbced6dbb84ecd84c0e656885e6 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 17 Oct 2023 00:33:32 -0700 Subject: [PATCH 09/14] update pyflyte run command Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index ef5b31cbe..97bb1358b 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -27,7 +27,7 @@ jobs: python-version: "3.10" - name: Create Sandbox Cluster run: | - flytectl config init --console localhost:30080 + flytectl config init --host localhost:30080 flytectl demo start kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v0.5.2" kubectl rollout restart deployment flyte-sandbox -n flyte From 8bd82979f455213fe15faeadd26287fa91a7abc8 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 17 Oct 2023 00:44:26 -0700 Subject: [PATCH 10/14] update default config path Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 97bb1358b..96728ab32 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -43,4 +43,4 @@ jobs: - name: execute flyte workflows on demo cluster working-directory: examples run: | - pyflyte --config ~/.flyte/config.yaml run --remote integration_test.py integration_test + pyflyte --config /home/runner/.flyte/config-sandbox.yaml run --remote integration_test.py integration_test From c7d3370c5e567ebdad9a25b7b1090abca8102585 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 17 Oct 2023 01:01:54 -0700 Subject: [PATCH 11/14] install envd Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 96728ab32..41cd28016 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -34,7 +34,7 @@ jobs: - name: Setup Flytekit run: | python -m pip install --upgrade pip - pip install flytekit flytekitplugins-deck-standard flytekitplugins-ray + pip install envd==0.3.35 flytekitplugins-deck-standard flytekitplugins-ray flytekitplugins-envd pip freeze - name: execute flyte workflows locally working-directory: examples From 745b10b3b5daff4f668ba22990f46f182ec90bf0 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 10 Nov 2023 11:40:24 -0800 Subject: [PATCH 12/14] Add spark Signed-off-by: Kevin Su --- examples/integration_test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/integration_test.py b/examples/integration_test.py index a94a382d9..9774d9197 100644 --- a/examples/integration_test.py +++ b/examples/integration_test.py @@ -6,8 +6,10 @@ from basics.basics.shell_task import shell_task_wf from basics.basics.task import slope from basics.basics.workflow import simple_wf_with_partial +from flytekit import workflow +from k8s_spark_plugin.k8s_spark_plugin.dataframe_passing import spark_to_pandas_wf +from k8s_spark_plugin.k8s_spark_plugin.pyspark_pi import my_spark from ray_plugin.ray_plugin.ray_example import ray_workflow -from flytekit import workflow, task @workflow @@ -29,9 +31,10 @@ def integration_test(): # Test Plugins ray_workflow(n=5) + my_spark() + spark_to_pandas_wf() + # TODO: Add more plugins here, like tensorflow, torch, Dask. - # TODO: Add more plugins here, like spark, tensorflow, torch, Dask. - -if __name__ == '__main__': +if __name__ == "__main__": integration_test() From 1ac26698560d8572646e01ee1c030e3fb53175d9 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 10 Nov 2023 11:46:12 -0800 Subject: [PATCH 13/14] wip Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 41cd28016..1704d58d7 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -29,6 +29,7 @@ jobs: run: | flytectl config init --host localhost:30080 flytectl demo start + # TODO: Build a single binary image that enables and install all the plugins kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v0.5.2" kubectl rollout restart deployment flyte-sandbox -n flyte - name: Setup Flytekit @@ -44,3 +45,4 @@ jobs: working-directory: examples run: | pyflyte --config /home/runner/.flyte/config-sandbox.yaml run --remote integration_test.py integration_test + # TODO: Check workflow status \ No newline at end of file From 0b3fce0dfc79d5ee2bdd0d6c824a96f9b113a3ad Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 10 Nov 2023 11:54:50 -0800 Subject: [PATCH 14/14] test Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 1704d58d7..646aaeb77 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -35,7 +35,7 @@ jobs: - name: Setup Flytekit run: | python -m pip install --upgrade pip - pip install envd==0.3.35 flytekitplugins-deck-standard flytekitplugins-ray flytekitplugins-envd + pip install flytekitplugins-deck-standard flytekitplugins-ray flytekitplugins.spark flytekitplugins-envd pip freeze - name: execute flyte workflows locally working-directory: examples