#8764: Separate n150/n300 demo tests to not run BERT 11 on N150 (#9073)

* #8764: Separate n150/n300 demo tests to not run BERT 11 on N150

* #8764: Add N150/N300 notes on Front page README

.github/workflows/single-card-demo-tests.yaml

@@ -22,13 +22,13 @@ jobs:
             name: "N150",
             arch: wormhole_b0,
             runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-1"],
-            cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type demos_single_card --dispatch-mode ""'
+            cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type demos_single_card_n150 --dispatch-mode ""'
           },
           {
             name: "N300",
             arch: wormhole_b0,
             runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"],
-            cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type demos_single_card --dispatch-mode ""'
+            cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type demos_single_card_n300 --dispatch-mode ""'
           }
         ]
     name: ${{ matrix.test-group.name }}

README.md

@@ -40,18 +40,24 @@
 
 > [!NOTE]
 >
-> We currently test our demo models for Wormhole on N300 boards only.
+> All model demos in this table function on both N150 and N300 Wormhole cards, unless otherwise stated.
 
 | Model                                                       | Gen. Token [3]     |  Batch               | End-to-end throughput [1]    | Device throughput [2]       | Target         |
 |-------------------------------------------------------------|--------------------|----------------------|------------------------------|-----------------------------|----------------|
 | [Falcon7B-decode](./models/demos/wormhole/falcon7b)         | 129th              | 32                   | 11.6 t/s/u - 371 t/s         | 15.4 t/s/u - 493 t/s        | 21 t/s/u       |
 | [Mistral-7B-decode](./models/demos/wormhole/mistral7b)      |  33rd              | 32                   | 10.9 t/s/u - 349 t/s         | 13.3 t/s/u - 426 t/s        | 21 t/s/u       |
 | [Mamba-2.8B-decode](./models/demos/mamba)                   |  any               | 32                   | 9.2 t/s/u -  295 t/s         | 13.1 t/s/u - 419 t/s        | 22 t/s/u       |
-| [BERT-Large](./models/demos/metal_BERT_large_11/) (sen/s)   | any                |  8                   | 270                          | 340                         | 400            |
+| [BERT-Large](./models/demos/metal_BERT_large_11/) (sen/s) [4] | any                |  8                   | 270                          | 340                         | 400            |
 | [Stable Diffusion 1.4](./models/demos/wormhole/stable_diffusion) 512x512  (sec/img)                              |          | 1                    |      8s                        |    5s                         |                |
 
+[1] - Observed from the host. Includes dispatch overhead and kernel execution time.
+
+[2] - Ignoring host overhead. Kernel execution time only.
+
 [3] - Generating the `i`'th token in a sequence while the kv_cache is filled with `i-1` rows.
 
+[4] - This model demo does not work on N150. It does work on N300.
+
 ## T3000 (2x4 mesh of WHs) Models
 
 | Model                                                     |   Technique        | Gen. Token [3]      |  Batch                | End-to-end throughput [1]    | Device throughput [2]        | Target          |

tests/scripts/run_tests.sh

@@ -185,12 +185,20 @@ run_ttnn_sweeps_pipeline_tests() {
     ./tests/scripts/run_ttnn_sweeps.sh
 }
 
-run_demos_single_card_tests() {
+run_demos_single_card_n150_tests() {
     local tt_arch=$1
     local pipeline_type=$2
     local dispatch_mode=$3
 
-    ./tests/scripts/single_card/run_demos_single_card_tests.sh
+    ./tests/scripts/single_card/run_demos_single_card_n150_tests.sh
+}
+
+run_demos_single_card_n300_tests() {
+    local tt_arch=$1
+    local pipeline_type=$2
+    local dispatch_mode=$3
+
+    ./tests/scripts/single_card/run_demos_single_card_n300_tests.sh
 }
 
 ##########################T3000##########################
@@ -333,8 +341,10 @@ run_pipeline_tests() {
         run_microbenchmarks_pipeline_tests "$tt_arch" "$pipeline_type" "$dispatch_mode"
     elif [[ $pipeline_type == "ttnn_sweeps" ]]; then
         run_ttnn_sweeps_pipeline_tests "$tt_arch" "$pipeline_type" "$dispatch_mode"
-    elif [[ $pipeline_type == "demos_single_card" ]]; then
-        run_demos_single_card_tests "$tt_arch" "$pipeline_type" "$dispatch_mode"
+    elif [[ $pipeline_type == "demos_single_card_n150" ]]; then
+        run_demos_single_card_n150_tests "$tt_arch" "$pipeline_type" "$dispatch_mode"
+    elif [[ $pipeline_type == "demos_single_card_n300" ]]; then
+        run_demos_single_card_n300_tests "$tt_arch" "$pipeline_type" "$dispatch_mode"
     # T3000 pipelines
     elif [[ $pipeline_type == "unit_t3000_device" ]]; then
         unit_t3000_device "$tt_arch" "$pipeline_type" "$dispatch_mode"

tests/scripts/single_card/run_demos_single_card_tests.sh → tests/scripts/single_card/run_demos_single_card_n150_tests.sh

@@ -20,8 +20,3 @@ pytest --disable-warnings -q -s --input-method=cli --cli-input="YOUR PROMPT GOES
 
 # working on both
 pytest --disable-warnings --input-path="models/demos/wormhole/stable_diffusion/demo/input_data.json" models/demos/wormhole/stable_diffusion/demo/demo.py::test_demo
-
-# Not working on N150, working on N300
-unset WH_ARCH_YAML
-rm -rf built
-pytest --disable-warnings models/demos/metal_BERT_large_11/demo/demo.py::test_demo[models/demos/metal_BERT_large_11/demo/input_data.json-1-batch_7]

tests/scripts/single_card/run_demos_single_card_n300_tests.sh

@@ -0,0 +1,17 @@
+#/bin/bash
+
+set -eo pipefail
+
+if [[ -z "$TT_METAL_HOME" ]]; then
+  echo "Must provide TT_METAL_HOME in environment" 1>&2
+  exit 1
+fi
+
+export WH_ARCH_YAML=wormhole_b0_80_arch_eth_dispatch.yaml
+
+source tests/scripts/single_card/run_demos_single_card_n150_tests.sh
+
+# Not working on N150, working on N300
+unset WH_ARCH_YAML
+rm -rf built
+pytest --disable-warnings models/demos/metal_BERT_large_11/demo/demo.py::test_demo[models/demos/metal_BERT_large_11/demo/input_data.json-1-batch_7]