From 08b199bdf73a76c2769e3785eb68847ffabd0068 Mon Sep 17 00:00:00 2001 From: Raymond Kim <109366641+tt-rkim@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:04:11 -0400 Subject: [PATCH] #8764: Add single card WH demo tests (#9058) #8764: Add single-card demo tests and move nightly scripts into new single_card folder --- ...-dispatch-full-regressions-and-models.yaml | 10 ++-- .github/workflows/single-card-demo-tests.yaml | 59 +++++++++++++++++++ tests/scripts/run_tests.sh | 10 ++++ .../nightly/run_common_models.sh | 0 .../{ => single_card}/nightly/run_gs_only.sh | 0 .../{ => single_card}/nightly/run_ttnn.sh | 0 .../nightly/run_wh_b0_only.sh | 0 .../run_demos_single_card_tests.sh | 27 +++++++++ 8 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/single-card-demo-tests.yaml rename tests/scripts/{ => single_card}/nightly/run_common_models.sh (100%) rename tests/scripts/{ => single_card}/nightly/run_gs_only.sh (100%) rename tests/scripts/{ => single_card}/nightly/run_ttnn.sh (100%) rename tests/scripts/{ => single_card}/nightly/run_wh_b0_only.sh (100%) create mode 100755 tests/scripts/single_card/run_demos_single_card_tests.sh diff --git a/.github/workflows/fast-dispatch-full-regressions-and-models.yaml b/.github/workflows/fast-dispatch-full-regressions-and-models.yaml index 28773caae65..152463c20b1 100644 --- a/.github/workflows/fast-dispatch-full-regressions-and-models.yaml +++ b/.github/workflows/fast-dispatch-full-regressions-and-models.yaml @@ -19,11 +19,11 @@ jobs: matrix: test-group: [ - { name: "Common models GS", arch: grayskull, cmd: tests/scripts/nightly/run_common_models.sh, timeout: 40 }, - { name: "Common models N300 WH B0", arch: wormhole_b0, cmd: tests/scripts/nightly/run_common_models.sh, timeout: 40 }, - { name: "GS-only ttnn nightly", arch: grayskull, cmd: tests/scripts/nightly/run_ttnn.sh, timeout: 40 }, - { name: "GS-only models", arch: grayskull, cmd: tests/scripts/nightly/run_gs_only.sh, timeout: 40 }, - { name: "N300 WH-only models", arch: wormhole_b0, cmd: tests/scripts/nightly/run_wh_b0_only.sh, timeout: 60 }, + { name: "Common models GS", arch: grayskull, cmd: tests/scripts/single_chip/nightly/run_common_models.sh, timeout: 40 }, + { name: "Common models N300 WH B0", arch: wormhole_b0, cmd: tests/scripts/single_chip/nightly/run_common_models.sh, timeout: 40 }, + { name: "GS-only ttnn nightly", arch: grayskull, cmd: tests/scripts/single_chip/nightly/run_ttnn.sh, timeout: 40 }, + { name: "GS-only models", arch: grayskull, cmd: tests/scripts/single_chip/nightly/run_gs_only.sh, timeout: 40 }, + { name: "N300 WH-only models", arch: wormhole_b0, cmd: tests/scripts/single_chip/nightly/run_wh_b0_only.sh, timeout: 60 }, { name: "API tests GS", arch: grayskull, cmd: ./tests/scripts/run_tests.sh --tt-arch grayskull --pipeline-type frequent_api --dispatch-mode fast, timeout: 40 }, { name: "API tests N300 WH B0", arch: wormhole_b0, cmd: ./tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type frequent_api --dispatch-mode fast, timeout: 40 }, ] diff --git a/.github/workflows/single-card-demo-tests.yaml b/.github/workflows/single-card-demo-tests.yaml new file mode 100644 index 00000000000..6b3808a1bc5 --- /dev/null +++ b/.github/workflows/single-card-demo-tests.yaml @@ -0,0 +1,59 @@ +name: "[Single-card] Demo tests" + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * 1,2,3,4,5" + - cron: "0 */4 * * 0,6" + +jobs: + build-artifact: + uses: ./.github/workflows/build-artifact.yaml + with: + arch: '["wormhole_b0"]' + secrets: inherit + t3000-demo-tests: + needs: build-artifact + strategy: + fail-fast: false + matrix: + test-group: [ + { + name: "N150", + arch: wormhole_b0, + runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-1"], + cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type demos_single_card --dispatch-mode ""' + }, + { + name: "N300", + arch: wormhole_b0, + runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"], + cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type demos_single_card --dispatch-mode ""' + } + ] + name: ${{ matrix.test-group.name }} + env: + TT_METAL_ENV: ${{ vars.TT_METAL_ENV }} + ARCH_NAME: ${{ matrix.test-group.arch }} + LOGURU_LEVEL: INFO + LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib + environment: dev + runs-on: ${{ matrix.test-group.runs-on }} + steps: + - uses: tenstorrent-metal/metal-workflows/.github/actions/checkout-with-submodule-lfs@v2.0.0 + - name: Set up dynamic env vars for build + run: | + echo "TT_METAL_HOME=$(pwd)" >> $GITHUB_ENV + - uses: actions/download-artifact@v4 + with: + name: TTMetal_build_${{ matrix.test-group.arch }} + - name: Extract files + run: tar -xvf ttm_${{ matrix.test-group.arch }}.tar + - uses: ./.github/actions/install-python-deps + - name: Run demo regression tests + timeout-minutes: 150 + run: | + source ${{ github.workspace }}/python_env/bin/activate + cd $TT_METAL_HOME + export PYTHONPATH=$TT_METAL_HOME + ${{ matrix.test-group.cmd }} diff --git a/tests/scripts/run_tests.sh b/tests/scripts/run_tests.sh index 4e540e67461..46cc54764b5 100755 --- a/tests/scripts/run_tests.sh +++ b/tests/scripts/run_tests.sh @@ -185,6 +185,14 @@ run_ttnn_sweeps_pipeline_tests() { ./tests/scripts/run_ttnn_sweeps.sh } +run_demos_single_card_tests() { + local tt_arch=$1 + local pipeline_type=$2 + local dispatch_mode=$3 + + ./tests/scripts/single_card/run_demos_single_card_tests.sh +} + ##########################T3000########################## # Run t3000 unit tests unit_t3000_device() { @@ -325,6 +333,8 @@ run_pipeline_tests() { run_microbenchmarks_pipeline_tests "$tt_arch" "$pipeline_type" "$dispatch_mode" elif [[ $pipeline_type == "ttnn_sweeps" ]]; then run_ttnn_sweeps_pipeline_tests "$tt_arch" "$pipeline_type" "$dispatch_mode" + elif [[ $pipeline_type == "demos_single_card" ]]; then + run_demos_single_card_tests "$tt_arch" "$pipeline_type" "$dispatch_mode" # T3000 pipelines elif [[ $pipeline_type == "unit_t3000_device" ]]; then unit_t3000_device "$tt_arch" "$pipeline_type" "$dispatch_mode" diff --git a/tests/scripts/nightly/run_common_models.sh b/tests/scripts/single_card/nightly/run_common_models.sh similarity index 100% rename from tests/scripts/nightly/run_common_models.sh rename to tests/scripts/single_card/nightly/run_common_models.sh diff --git a/tests/scripts/nightly/run_gs_only.sh b/tests/scripts/single_card/nightly/run_gs_only.sh similarity index 100% rename from tests/scripts/nightly/run_gs_only.sh rename to tests/scripts/single_card/nightly/run_gs_only.sh diff --git a/tests/scripts/nightly/run_ttnn.sh b/tests/scripts/single_card/nightly/run_ttnn.sh similarity index 100% rename from tests/scripts/nightly/run_ttnn.sh rename to tests/scripts/single_card/nightly/run_ttnn.sh diff --git a/tests/scripts/nightly/run_wh_b0_only.sh b/tests/scripts/single_card/nightly/run_wh_b0_only.sh similarity index 100% rename from tests/scripts/nightly/run_wh_b0_only.sh rename to tests/scripts/single_card/nightly/run_wh_b0_only.sh diff --git a/tests/scripts/single_card/run_demos_single_card_tests.sh b/tests/scripts/single_card/run_demos_single_card_tests.sh new file mode 100755 index 00000000000..55d7b03704e --- /dev/null +++ b/tests/scripts/single_card/run_demos_single_card_tests.sh @@ -0,0 +1,27 @@ +#/bin/bash + +set -eo pipefail + +if [[ -z "$TT_METAL_HOME" ]]; then + echo "Must provide TT_METAL_HOME in environment" 1>&2 + exit 1 +fi + +export WH_ARCH_YAML=wormhole_b0_80_arch_eth_dispatch.yaml + +# working on both +pytest --disable-warnings -q -s --input-method=cli --cli-input="YOUR PROMPT GOES HERE!" models/demos/wormhole/falcon7b/demo_wormhole.py::test_demo[user_input0-default_mode_stochastic] + +# working on both +pytest --disable-warnings -q -s --input-method=cli --cli-input="YOUR PROMPT GOES HERE!" models/demos/wormhole/mistral7b/demo/demo.py + +# working on both +pytest --disable-warnings -q -s --input-method=cli --cli-input="YOUR PROMPT GOES HERE!" models/demos/mamba/demo/demo.py + +# working on both +pytest --disable-warnings --input-path="models/demos/wormhole/stable_diffusion/demo/input_data.json" models/demos/wormhole/stable_diffusion/demo/demo.py::test_demo + +# Not working on N150, working on N300 +unset WH_ARCH_YAML +rm -rf built +pytest --disable-warnings models/demos/metal_BERT_large_11/demo/demo.py::test_demo[models/demos/metal_BERT_large_11/demo/input_data.json-1-batch_7]