#12177: Add ability to run single sweeps in CI and related static ver…

…ification
tenstorrent · Sep 5, 2024 · 2d5fa12 · 2d5fa12
1 parent ca39cb9
commit 2d5fa12
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 4 deletions.
diff --git a/.github/workflows/all-static-checks.yaml b/.github/workflows/all-static-checks.yaml
@@ -49,3 +49,12 @@ jobs:
         run: |
           if (( $(grep -Rnw 'tests/tt_metal' -e 'tt_lib' | wc -l ) > 0 )); then exit 1; fi
           if (( $(grep -Rnw 'tests/tt_metal' -e 'tt_eager' | wc -l ) > 10 )); then exit 1; fi
+  check-sweeps-workflow:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/[email protected]
+      - name: Check sweeps workflow option count against sweep file count
+        run: |
+          pip install pyyaml
+          python tests/sweep_framework/sweeps_workflow_verification.py
diff --git a/.github/workflows/ttnn-run-sweeps.yaml b/.github/workflows/ttnn-run-sweeps.yaml
@@ -2,8 +2,33 @@ name: "ttnn - Run sweeps"
 
 on:
   workflow_dispatch:
-    schedule:
-     - cron: "0 21 * * *" # This cron schedule runs the workflow at 9:00pm UTC nightly
+    inputs:
+      sweep_name:
+        type: choice
+        description: "Which sweep module to run?"
+        required: true
+        default: "ALL SWEEPS (Nightly)"
+        options:
+          - ALL SWEEPS (Nightly)
+          - add
+          - line_all_gather
+          - logical_and_
+          - matmul.full.matmul_default_block_sharded
+          - matmul.full.matmul_default_height_sharded
+          - matmul.full.matmul_default_interleaved
+          - matmul.full.matmul_default_width_sharded
+          - matmul.short.matmul_create_program_config
+          - matmul.short.matmul_default_sharded
+          - matmul.short.matmul_default
+          - matmul.short.matmul_user_program_config_mcast_1d
+          - matmul.short.matmul_user_program_config_mcast_2d
+          - matmul.short.matmul_user_program_config
+          - matmul.short.matmul
+          - data_movement.concat.concat_interleaved_n_tensors
+          - data_movement.concat.concat_interleaved
+          - data_movement.concat.concat_sharded
+  schedule:
+    - cron: "0 21 * * *" # This cron schedule runs the workflow at 9:00pm UTC nightly
 
 jobs:
   build-artifact:
@@ -29,7 +54,15 @@ jobs:
       - uses: ./.github/actions/prepare-metal-run
         with:
           arch: wormhole_b0
-      - name: Run ttnn sweeps generation
+      - name: Run ttnn sweeps generation (single sweep)
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.sweep_name != 'ALL SWEEPS (Nightly)' }}
+        run: |
+          source ${{ github.workspace }}/python_env/bin/activate
+          cd $TT_METAL_HOME
+          export PYTHONPATH=$TT_METAL_HOME
+          python tests/sweep_framework/parameter_generator.py --module-name ${{ github.event.inputs.sweep_name }} --elastic cloud --tag ci-main --explicit
+      - name: Run ttnn sweeps generation (all sweeps)
+        if: ${{ github.event_name == 'schedule' || github.event.inputs.sweep_name == 'ALL SWEEPS (Nightly)' }}
         run: |
           source ${{ github.workspace }}/python_env/bin/activate
           cd $TT_METAL_HOME
@@ -82,7 +115,15 @@ jobs:
       - uses: ./.github/actions/prepare-metal-run
         with:
           arch: ${{ matrix.test-group.arch }}
-      - name: Run ttnn sweeps
+      - name: Run ttnn sweeps (single sweep)
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.sweep_name != 'ALL SWEEPS (Nightly)' }}
+        run: |
+          source ${{ github.workspace }}/python_env/bin/activate
+          cd $TT_METAL_HOME
+          export PYTHONPATH=$TT_METAL_HOME
+          python tests/sweep_framework/runner.py --module-name ${{ github.event.inputs.sweep_name }} --elastic cloud --tag ci-main
+      - name: Run ttnn sweeps (all sweeps, nightly)
+        if: ${{ github.event_name == 'schedule' || github.event.inputs.sweep_name == 'ALL SWEEPS (Nightly)' }}
         run: |
           source ${{ github.workspace }}/python_env/bin/activate
           cd $TT_METAL_HOME

diff --git a/tests/sweep_framework/sweeps_workflow_verification.py b/tests/sweep_framework/sweeps_workflow_verification.py
@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: © 2024 Tenstorrent Inc.
+
+# SPDX-License-Identifier: Apache-2.0
+
+from yaml import load, Loader
+import pathlib
+
+with open(".github/workflows/ttnn-run-sweeps.yaml") as file:
+    workflow = load(file, Loader)
+    workflow_count = len(workflow[True]["workflow_dispatch"]["inputs"]["sweep_name"]["options"])
+
+    sweeps_path = pathlib.Path(__file__).parent / "sweeps"
+    file_count = len(list(sweeps_path.glob("**/*.py")))
+    assert (
+        file_count + 1 == workflow_count
+    ), f"Sweeps workflow options does not match expected number of sweep files ({workflow_count} exist, expected {file_count + 1}). If you added a new sweep file, please add it to the options in the .github/workflows/ttnn-run-sweeps.yaml workflow file."
+    print("Sweeps workflow options match expected number of sweep files.")