From 1ee8d8a65417d08d009d4e08b9dfb6ea51447c3c Mon Sep 17 00:00:00 2001 From: archana-ramalingam Date: Wed, 8 Jan 2025 02:00:00 +0000 Subject: [PATCH] Temporarily disable IREE perplexity tests --- .github/workflows/ci_eval.yaml | 116 +++++++++++++-------------- .github/workflows/ci_eval_short.yaml | 8 +- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci_eval.yaml b/.github/workflows/ci_eval.yaml index fe29f54d5..40e379854 100644 --- a/.github/workflows/ci_eval.yaml +++ b/.github/workflows/ci_eval.yaml @@ -21,64 +21,64 @@ concurrency: cancel-in-progress: true jobs: - test_perplexity_iree: - if: ${{ github.repository_owner == 'nod-ai' || github.event_name != 'schedule' }} - timeout-minutes: 1000 - name: "IREE Perplexity" - strategy: - matrix: - version: [3.11] - runs-on: [llama-mi300x-3] - fail-fast: false - runs-on: ${{matrix.runs-on}} - defaults: - run: - shell: bash - env: - VENV_DIR: ${{ github.workspace }}/.venv - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: "Setting up Python" - id: setup_python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: ${{matrix.version}} - - name: Create Python venv - run: python -m venv ${VENV_DIR} - - - name: Install sharktank deps - run: | - source ${VENV_DIR}/bin/activate - python -m pip install --no-compile --upgrade pip - - # Note: We install in three steps in order to satisfy requirements - # from non default locations first. Installing the PyTorch CPU - # wheels saves multiple minutes and a lot of bandwidth on runner setup. - pip install --no-compile -r pytorch-cpu-requirements.txt - - # Install nightly IREE packages. - # We could also pin to a known working or stable version. - pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \ - iree-base-compiler \ - iree-base-runtime \ - iree-turbine - - pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/ - pip freeze - - - name: Run perplexity test with IREE - run: | - source ${VENV_DIR}/bin/activate - pytest -n 8 -v -s sharktank/tests/evaluate/perplexity_iree_test.py --run-nightly-llama-tests --bs=100 --iree-device=hip://0 --iree-hip-target=gfx942 --iree-hal-target-device=hip --llama3-8b-f16-model-path=/data/llama3.1/8b/llama8b_f16.irpa --llama3-8b-tokenizer-path=/data/llama3.1/8b/tokenizer_config.json --html=out/llm/llama/perplexity/iree_perplexity/index.html - - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 - with: - github_token: ${{ secrets.SHARK_PLATFORM_GH_TOKEN }} - publish_dir: ./out/llm/llama/perplexity/iree_perplexity - destination_dir: ./llm/llama/perplexity/iree_perplexity - keep_files: true + # test_perplexity_iree: + # if: ${{ github.repository_owner == 'nod-ai' || github.event_name != 'schedule' }} + # timeout-minutes: 1000 + # name: "IREE Perplexity" + # strategy: + # matrix: + # version: [3.11] + # runs-on: [llama-mi300x-3] + # fail-fast: false + # runs-on: ${{matrix.runs-on}} + # defaults: + # run: + # shell: bash + # env: + # VENV_DIR: ${{ github.workspace }}/.venv + # steps: + # - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + # - name: "Setting up Python" + # id: setup_python + # uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + # with: + # python-version: ${{matrix.version}} + # - name: Create Python venv + # run: python -m venv ${VENV_DIR} + + # - name: Install sharktank deps + # run: | + # source ${VENV_DIR}/bin/activate + # python -m pip install --no-compile --upgrade pip + + # # Note: We install in three steps in order to satisfy requirements + # # from non default locations first. Installing the PyTorch CPU + # # wheels saves multiple minutes and a lot of bandwidth on runner setup. + # pip install --no-compile -r pytorch-cpu-requirements.txt + + # # Install nightly IREE packages. + # # We could also pin to a known working or stable version. + # pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \ + # iree-base-compiler \ + # iree-base-runtime \ + # iree-turbine + + # pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/ + # pip freeze + + # - name: Run perplexity test with IREE + # run: | + # source ${VENV_DIR}/bin/activate + # pytest -n 8 -v -s sharktank/tests/evaluate/perplexity_iree_test.py --run-nightly-llama-tests --bs=100 --iree-device=hip://0 --iree-hip-target=gfx942 --iree-hal-target-device=hip --llama3-8b-f16-model-path=/data/llama3.1/8b/llama8b_f16.irpa --llama3-8b-tokenizer-path=/data/llama3.1/8b/tokenizer_config.json --html=out/llm/llama/perplexity/iree_perplexity/index.html + + # - name: Deploy to GitHub Pages + # uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 + # with: + # github_token: ${{ secrets.SHARK_PLATFORM_GH_TOKEN }} + # publish_dir: ./out/llm/llama/perplexity/iree_perplexity + # destination_dir: ./llm/llama/perplexity/iree_perplexity + # keep_files: true test_perplexity_torch: if: ${{ github.repository_owner == 'nod-ai' || github.event_name != 'schedule' }} diff --git a/.github/workflows/ci_eval_short.yaml b/.github/workflows/ci_eval_short.yaml index a02d7bf14..b55f81341 100644 --- a/.github/workflows/ci_eval_short.yaml +++ b/.github/workflows/ci_eval_short.yaml @@ -8,10 +8,10 @@ name: CI - sharktank perplexity short on: workflow_dispatch: - pull_request: - push: - branches: - - main + # pull_request: + # push: + # branches: + # - main concurrency: # A PR number if a pull request and otherwise the commit hash. This cancels