diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 60fa7fb77..3907bf1da 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,14 +1,18 @@ name: Master +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + on: push: branches: - master paths-ignore: - "docs/**" - pull_request: - paths-ignore: - - "docs/**" +# pull_request: +# paths-ignore: +# - "docs/**" jobs: lint: runs-on: ubuntu-latest @@ -168,69 +172,69 @@ jobs: "prerelease": false }' - e2e-tests: - runs-on: ubuntu-latest - env: - FLYTESNACKS_PRIORITIES: "P0" - FLYTESNACKS_VERSION: "" - timeout-minutes: 30 - steps: - - name: Set latest Flytesnacks release - if: ${{ env.FLYTESNACKS_VERSION == '' }} - run: | - FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" - echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - uses: unionai/flytectl-setup-action@v0.0.1 - - name: Setup sandbox - run: | - mkdir -p ~/.flyte/sandbox - cat << EOF > ~/.flyte/sandbox/config.yaml - task_resources: - defaults: - cpu: "0" - memory: "0" - limits: - cpu: "0" - memory: "0" - EOF - flytectl demo start --imagePullPolicy Never - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install flytekit flytekitplugins-deck-standard - pip freeze - - name: Checkout flytesnacks - uses: actions/checkout@v3 - with: - repository: flyteorg/flytesnacks - path: flytesnacks - - name: Register specific tests - run: | - for f in \ - basics/basics/hello_world.py \ - basics/basics/workflow.py \ - basics/basics/named_outputs.py \ - advanced_composition/advanced_composition/chain_entities.py \ - advanced_composition/advanced_composition/dynamics.py \ - advanced_composition/advanced_composition/map_task.py \ - advanced_composition/advanced_composition/subworkflows.py \ - data_types_and_io/data_types_and_io/dataclass.py \ - data_types_and_io/data_types_and_io/structured_dataset.py ; - do - pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ - register \ - --project flytesnacks \ - --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ - --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/examples/$f; - done - - name: End2End - run: | - make end2end_execute +# e2e-tests: +# runs-on: ubuntu-latest +# env: +# FLYTESNACKS_PRIORITIES: "P0" +# FLYTESNACKS_VERSION: "" +# timeout-minutes: 30 +# steps: +# - name: Set latest Flytesnacks release +# if: ${{ env.FLYTESNACKS_VERSION == '' }} +# run: | +# FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" +# echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} +# - name: Checkout +# uses: actions/checkout@v3 +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: "3.11" +# - uses: unionai/flytectl-setup-action@v0.0.1 +# - name: Setup sandbox +# run: | +# mkdir -p ~/.flyte/sandbox +# cat << EOF > ~/.flyte/sandbox/config.yaml +# task_resources: +# defaults: +# cpu: "0" +# memory: "0" +# limits: +# cpu: "0" +# memory: "0" +# EOF +# flytectl demo start --imagePullPolicy Never +# - name: Install Python dependencies +# run: | +# python -m pip install --upgrade pip +# pip install flytekit flytekitplugins-deck-standard +# pip freeze +# - name: Checkout flytesnacks +# uses: actions/checkout@v3 +# with: +# repository: flyteorg/flytesnacks +# path: flytesnacks +# - name: Register specific tests +# run: | +# for f in \ +# basics/basics/hello_world.py \ +# basics/basics/workflow.py \ +# basics/basics/named_outputs.py \ +# advanced_composition/advanced_composition/chain_entities.py \ +# advanced_composition/advanced_composition/dynamics.py \ +# advanced_composition/advanced_composition/map_task.py \ +# advanced_composition/advanced_composition/subworkflows.py \ +# data_types_and_io/data_types_and_io/dataclass.py \ +# data_types_and_io/data_types_and_io/structured_dataset.py ; +# do +# pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ +# register \ +# --project flytesnacks \ +# --domain development \ +# --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ +# --version ${{ env.FLYTESNACKS_VERSION }} \ +# flytesnacks/examples/$f; +# done +# - name: End2End +# run: | +# make end2end_execute diff --git a/.github/workflows/docs_build.yml b/.github/workflows/docs_build.yml index cc418bc1e..ae852fb30 100644 --- a/.github/workflows/docs_build.yml +++ b/.github/workflows/docs_build.yml @@ -1,12 +1,16 @@ name: Docs Build +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + on: push: branches: - master - pull_request: - branches: - - master +# pull_request: +# branches: +# - master jobs: docs_warnings: name: Docs Warnings diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml new file mode 100644 index 000000000..646aaeb77 --- /dev/null +++ b/.github/workflows/end2end.yml @@ -0,0 +1,48 @@ +name: End to End tests + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + endtoend: + name: End to End tests + runs-on: ubuntu-latest + env: + FLYTESNACKS_VERSION: "" + steps: + - name: Checkout + uses: actions/checkout@v3 + - uses: unionai/flytectl-setup-action@v0.0.1 + name: Setup flytectl + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Create Sandbox Cluster + run: | + flytectl config init --host localhost:30080 + flytectl demo start + # TODO: Build a single binary image that enables and install all the plugins + kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v0.5.2" + kubectl rollout restart deployment flyte-sandbox -n flyte + - name: Setup Flytekit + run: | + python -m pip install --upgrade pip + pip install flytekitplugins-deck-standard flytekitplugins-ray flytekitplugins.spark flytekitplugins-envd + pip freeze + - name: execute flyte workflows locally + working-directory: examples + run: | + pyflyte run integration_test.py integration_test + - name: execute flyte workflows on demo cluster + working-directory: examples + run: | + pyflyte --config /home/runner/.flyte/config-sandbox.yaml run --remote integration_test.py integration_test + # TODO: Check workflow status \ No newline at end of file diff --git a/examples/integration_test.py b/examples/integration_test.py new file mode 100644 index 000000000..9774d9197 --- /dev/null +++ b/examples/integration_test.py @@ -0,0 +1,40 @@ +from basics.basics.documenting_workflows import google_docstring_wf, numpy_docstring_wf, sphinx_docstring_wf +from basics.basics.hello_world import hello_world_wf +from basics.basics.imperative_workflow import imperative_wf +from basics.basics.launch_plan import simple_wf_lp_fixed_inputs +from basics.basics.named_outputs import simple_wf_with_named_outputs +from basics.basics.shell_task import shell_task_wf +from basics.basics.task import slope +from basics.basics.workflow import simple_wf_with_partial +from flytekit import workflow +from k8s_spark_plugin.k8s_spark_plugin.dataframe_passing import spark_to_pandas_wf +from k8s_spark_plugin.k8s_spark_plugin.pyspark_pi import my_spark +from ray_plugin.ray_plugin.ray_example import ray_workflow + + +@workflow +def integration_test(): + # Test Basic Workflows + google_docstring_wf() + numpy_docstring_wf() + sphinx_docstring_wf() + + hello_world_wf() + imperative_wf(x=[-3, 0, 3], y=[7, 4, -2]) + + simple_wf_lp_fixed_inputs(y=[-3, 2, -2]) + simple_wf_with_named_outputs() + + shell_task_wf() + slope(x=[-3, 0, 3], y=[7, 4, -2]) + simple_wf_with_partial(x=[-3, 0, 3], y=[7, 4, -2]) + + # Test Plugins + ray_workflow(n=5) + my_spark() + spark_to_pandas_wf() + # TODO: Add more plugins here, like tensorflow, torch, Dask. + + +if __name__ == "__main__": + integration_test() diff --git a/examples/ray_plugin/ray_plugin/ray_example.py b/examples/ray_plugin/ray_plugin/ray_example.py index fa6bc10fe..8036b7a1d 100644 --- a/examples/ray_plugin/ray_plugin/ray_example.py +++ b/examples/ray_plugin/ray_plugin/ray_example.py @@ -85,7 +85,7 @@ def f(x): # %% @task( task_config=ray_config, - requests=Resources(mem="2Gi", cpu="2"), + requests=Resources(mem="800Mi", cpu="1"), container_image=custom_image, ) def ray_task(n: int) -> typing.List[int]: