#8558: Refactor t3000, tg and tgg pipelines, workflows and run test s…

…cripts
tenstorrent · May 16, 2024 · dfd8986 · dfd8986
1 parent e85e468
commit dfd8986
Show file tree

Hide file tree

Showing 26 changed files with 769 additions and 334 deletions.
diff --git a/.github/workflows/multi-device-build-and-unit-tests-frequent.yaml b/.github/workflows/multi-device-build-and-unit-tests-frequent.yaml
diff --git a/.github/workflows/run-profiler-regression.yaml b/.github/workflows/run-profiler-regression.yaml
@@ -18,8 +18,6 @@ jobs:
           {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-1"]},
           # N300
           {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"]},
-          # N300 2x4
-          {name: "n300-2x4", arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-4", "multi-chip-num-chips-8"]},
         ]
     env:
       TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}

diff --git a/.github/workflows/t3000-demo-tests.yaml b/.github/workflows/t3000-demo-tests.yaml
@@ -0,0 +1,54 @@
+name: "[T3K] T3000 demo tests"
+
+on:
+  push:
+    branches:
+      - tpatel/issue-8558
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *' # This cron schedule runs the workflow every day at 12am UTC
+
+jobs:
+  build-artifact:
+    uses: ./.github/workflows/build-artifact.yaml
+    secrets: inherit
+  t3000-demo-tests:
+    needs: build-artifact
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          {
+            name: "T3000 demo tests",
+            arch: wormhole_b0,
+            runs-on: [arch-wormhole_b0, "config-t3000", "in-service", "runner-test", "bare-metal", "pipeline-perf"],
+            cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type demos_t3000_device --dispatch-mode ""'
+          },
+        ]
+    name: ${{ matrix.test-group.name }} ${{ matrix.test-group.arch }}
+    env:
+      TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}
+      ARCH_NAME: ${{ matrix.test-group.arch }}
+      LOGURU_LEVEL: INFO
+      LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
+    environment: dev
+    runs-on: ${{ matrix.test-group.runs-on }}
+    steps:
+      - uses: tenstorrent-metal/metal-workflows/.github/actions/[email protected]
+      - name: Set up dynamic env vars for build
+        run: |
+          echo "TT_METAL_HOME=$(pwd)" >> $GITHUB_ENV
+      - uses: actions/download-artifact@v4
+        with:
+          name: TTMetal_build_${{ matrix.runner-info.arch }}
+      - name: Extract files
+        run: tar -xvf ttm_${{ matrix.runner-info.arch }}.tar
+      - uses: ./.github/actions/install-python-deps
+      - name: ${{ matrix.test-group.name }} tests
+      - name: Run demo regression tests
+        timeout-minutes: 180
+        run: |
+          source ${{ github.workspace }}/python_env/bin/activate
+          cd $TT_METAL_HOME
+          export PYTHONPATH=$TT_METAL_HOME
+          ${{ matrix.test-group.cmd }}
diff --git a/.github/workflows/t3000-frequent-tests.yaml b/.github/workflows/t3000-frequent-tests.yaml
@@ -0,0 +1,53 @@
+name: "[T3K] T3000 frequent tests"
+
+on:
+  push:
+    branches:
+      - tpatel/issue-8558
+  workflow_dispatch:
+  schedule:
+    - cron: "0 */8 * * *" # This cron schedule runs the workflow every 8 hours
+
+jobs:
+  build-artifact:
+    uses: ./.github/workflows/build-artifact.yaml
+    secrets: inherit
+  t3000-frequent-tests:
+    needs: build-artifact
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          {
+            name: "T3000 frequent tests",
+            arch: wormhole_b0,
+            runs-on: [arch-wormhole_b0, "config-t3000", "in-service", "runner-test", "bare-metal", "pipeline-functional"],
+            cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type frequent_t3000_device --dispatch-mode ""'
+          },
+        ]
+    name: ${{ matrix.test-group.name }} ${{ matrix.test-group.arch }}
+    env:
+      TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}
+      ARCH_NAME: ${{ matrix.test-group.arch }}
+      LOGURU_LEVEL: INFO
+      LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
+    environment: dev
+    runs-on: ${{ matrix.test-group.runs-on }}
+    steps:
+      - uses: tenstorrent-metal/metal-workflows/.github/actions/[email protected]
+      - name: Set up dynamic env vars for build
+        run: |
+          echo "TT_METAL_HOME=$(pwd)" >> $GITHUB_ENV
+      - uses: actions/download-artifact@v4
+        with:
+          name: TTMetal_build_${{ matrix.runner-info.arch }}
+      - name: Extract files
+        run: tar -xvf ttm_${{ matrix.runner-info.arch }}.tar
+      - uses: ./.github/actions/install-python-deps
+      - name: Run frequent regression tests
+        timeout-minutes: 60
+        run: |
+          source ${{ github.workspace }}/python_env/bin/activate
+          cd $TT_METAL_HOME
+          export PYTHONPATH=$TT_METAL_HOME
+          ${{ matrix.test-group.cmd }}
diff --git a/...b/workflows/multi-device-perf-models.yaml → ...hub/workflows/t3000-model-perf-tests.yaml b/...b/workflows/multi-device-perf-models.yaml → ...hub/workflows/t3000-model-perf-tests.yaml
@@ -1,35 +1,49 @@
-name: "Multi-Nebula model perf regressions and output report"
+name: "[T3K] T3000 model perf tests"
 
 on:
+  push:
+    branches:
+      - tpatel/issue-8558
   workflow_dispatch:
   schedule:
     - cron: "0 */12 * * *" # This cron schedule runs the workflow every 12 hours
 
 jobs:
-  multi-device-models-perf:
+  build-artifact:
+    uses: ./.github/workflows/build-artifact.yaml
+    secrets: inherit
+  t3000-model-perf-tests:
+    needs: build-artifact
     strategy:
-      # Do not fail-fast because we need to ensure all tests go to completion
-      # so we try not to get hanging machines
       fail-fast: false
       matrix:
-        runner-info: [
-          # N300 2x4
-          # NOTE: Never use arch-wormhole_b0 tags, however we're using it here because this machine is used by devs during the day
-          # We don't want other CI runs to interrupt dev flows. However, we need to fix this once we have more 2x4 machines dedicated to CI
-          {name: "n300-2x4", arch: wormhole_b0, runs-on: ["perf-t3000", "arch-wormhole_b0", "multi-chip-num-pcie-4", "multi-chip-num-chips-8"], machine-type: "bare_metal"},
+        test-group: [
+          {
+            name: "T3000 LLM model perf tests", 
+            model-type: "LLM",
+            arch: wormhole_b0, 
+            runs-on: [arch-wormhole_b0, "config-t3000", "in-service", "runner-test", "bare-metal", "pipeline-perf"],
+            cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type llm_model_perf_t3000_device --dispatch-mode ""'
+          },
+          {
+            name: "T3000 CNN model perf tests", 
+            model-type: "CNN",
+            arch: wormhole_b0, 
+            runs-on: [arch-wormhole_b0, "config-t3000", "in-service", "runner-test", "bare-metal", "pipeline-perf"],
+            cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type cnn_model_perf_t3000_device --dispatch-mode ""'
+          },
         ]
-        model-type: [llm_javelin, cnn_javelin, other]
-    name: "${{ matrix.model-type }} ${{ matrix.runner-info.arch }}"
+    name: ${{ matrix.test-group.name }} ${{ matrix.test-group.arch }}
     env:
       TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}
-      ARCH_NAME: ${{ matrix.runner-info.arch }}
+      ARCH_NAME: ${{ matrix.test-group.arch }}
       LOGURU_LEVEL: INFO
-      TTNN_CONFIG_OVERRIDES: '{"enable_fast_runtime_mode": true}'
+      LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
     environment: dev
-    runs-on: ${{ matrix.runner-info.runs-on }}
+    runs-on: ${{ matrix.test-group.runs-on }}
     steps:
       - uses: tenstorrent-metal/metal-workflows/.github/actions/[email protected]
-      - name: Enable Performance mode
+      - name: Enable performance mode
         run: |
           sudo cpupower frequency-set -g performance
       - name: Ensure weka mount is active
@@ -43,11 +57,13 @@ jobs:
           echo "PYTHONPATH=$(pwd)" >> $GITHUB_ENV
       - name: Build tt-metal and libs
         run: PYTHON_ENV_DIR=$(pwd)/build/python_env ./build_metal.sh
-      - name: Run performance regressions
+      - name: Run model perf regression tests
         timeout-minutes: 60
         run: |
-          source build/python_env/bin/activate
-          ./tests/scripts/run_tests.sh --tt-arch $ARCH_NAME --pipeline-type ${{ matrix.model-type }}_models_performance_${{ matrix.runner-info.machine-type }}_multi_device
+          source ${{ github.workspace }}/python_env/bin/activate
+          cd $TT_METAL_HOME
+          export PYTHONPATH=$TT_METAL_HOME
+          ${{ matrix.test-group.cmd }}
       - name: Check perf report exists
         id: check-perf-report
         if: ${{ !cancelled() }}
@@ -62,7 +78,7 @@ jobs:
         with:
           name: perf-report-csv-${{ matrix.model-type }}-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.machine-type }}
           path: "${{ steps.check-perf-report.outputs.perf_report_filename }}"
-      - name: Disable Performance mode
+      - name: Disable performance mode
         if: always()
         run: |
           sudo cpupower frequency-set -g ondemand
diff --git a/.github/workflows/t3000-profiler-tests.yaml b/.github/workflows/t3000-profiler-tests.yaml
@@ -0,0 +1,43 @@
+name: "[T3K] T3000 profiler tests"
+
+on:
+  push:
+    branches:
+      - tpatel/issue-8558
+  workflow_dispatch:
+  workflow_call:
+  schedule:
+    - cron: "0 */8 * * *" # This cron schedule runs the workflow every 8 hours
+
+jobs:
+  t3000-profiler-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          {
+            name: "T3000 unit tests",
+            arch: wormhole_b0,
+            runs-on: [arch-wormhole_b0, "config-t3000", "in-service", "runner-test", "bare-metal", "pipeline-perf"],
+            cmd: './tests/scripts/run_profiler_regressions.sh'
+          },
+        ]
+    env:
+      TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}
+      ARCH_NAME: ${{ matrix.test-group.arch }}
+      LOGURU_LEVEL: INFO
+      LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
+    environment: dev
+    runs-on: ${{ matrix.runner-info.runs-on }}
+    steps:
+      - uses: tenstorrent-metal/metal-workflows/.github/actions/[email protected]
+      - name: Set up dynamic env vars for build
+        run: |
+          echo "TT_METAL_HOME=$(pwd)" >> $GITHUB_ENV
+      - name: Build tt-metal and libs
+        run: |
+          ./scripts/build_scripts/build_with_profiler_opt.sh
+      - name: Run profiler regression tests
+        timeout-minutes: 30
+        run: |
+          ./tests/scripts/run_profiler_regressions.sh
diff --git a/.github/workflows/t3000-unit-tests.yaml b/.github/workflows/t3000-unit-tests.yaml
@@ -0,0 +1,53 @@
+name: "[T3K] T3000 unit tests"
+
+on:
+  push:
+    branches:
+      - tpatel/issue-8558
+  workflow_dispatch:
+  schedule:
+    - cron: "0 */3 * * *" # This cron schedule runs the workflow every 3 hours
+
+jobs:
+  build-artifact:
+    uses: ./.github/workflows/build-artifact.yaml
+    secrets: inherit
+  t3000-unit-tests:
+    needs: build-artifact
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          {
+            name: "T3000 unit tests",
+            arch: wormhole_b0,
+            runs-on: [arch-wormhole_b0, "config-t3000", "in-service", "runner-test", "bare-metal", "pipeline-functional"],
+            cmd: './tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type unit_t3000_device --dispatch-mode ""'
+          },
+        ]
+    name: ${{ matrix.test-group.name }} ${{ matrix.test-group.arch }}
+    env:
+      TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}
+      ARCH_NAME: ${{ matrix.test-group.arch }}
+      LOGURU_LEVEL: INFO
+      LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
+    environment: dev
+    runs-on: ${{ matrix.test-group.runs-on }}
+    steps:
+      - uses: tenstorrent-metal/metal-workflows/.github/actions/[email protected]
+      - name: Set up dynamic env vars for build
+        run: |
+          echo "TT_METAL_HOME=$(pwd)" >> $GITHUB_ENV
+      - uses: actions/download-artifact@v4
+        with:
+          name: TTMetal_build_${{ matrix.runner-info.arch }}
+      - name: Extract files
+        run: tar -xvf ttm_${{ matrix.runner-info.arch }}.tar
+      - uses: ./.github/actions/install-python-deps
+      - name: Run unit regression tests
+        timeout-minutes: 120
+        run: |
+          source ${{ github.workspace }}/python_env/bin/activate
+          cd $TT_METAL_HOME
+          export PYTHONPATH=$TT_METAL_HOME
+          ${{ matrix.test-group.cmd }}