From 8822d806a9b84cc076579b588899e5fe387e73db Mon Sep 17 00:00:00 2001 From: Archana Ramalingam <98564406+archana-ramalingam@users.noreply.github.com> Date: Thu, 24 Oct 2024 20:28:25 -0700 Subject: [PATCH] [sharktank] Evaluation - Update timeout for Perplexity CI test (#316) Update default CI timeout from 10 to 20 hrs for the first run and adjust later. --- .github/workflows/ci_eval.yaml | 4 ++-- sharktank/sharktank/evaluate/perplexity.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_eval.yaml b/.github/workflows/ci_eval.yaml index 9181d5b72..7288ed8ac 100644 --- a/.github/workflows/ci_eval.yaml +++ b/.github/workflows/ci_eval.yaml @@ -16,7 +16,7 @@ concurrency: jobs: test_perplexity: - timeout-minutes: 600 + timeout-minutes: 1000 name: "Evaluation Tests - perplexity" strategy: matrix: @@ -59,4 +59,4 @@ jobs: pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/ - name: Run perplexity test - run: pytest sharktank/tests/evaluate/perplexity_test.py --longrun + run: pytest -n 4 -v -s sharktank/tests/evaluate/perplexity_test.py --longrun diff --git a/sharktank/sharktank/evaluate/perplexity.py b/sharktank/sharktank/evaluate/perplexity.py index 2c76a76ad..aa9d35dcc 100644 --- a/sharktank/sharktank/evaluate/perplexity.py +++ b/sharktank/sharktank/evaluate/perplexity.py @@ -177,6 +177,7 @@ def get_logits(self): start = 0 for i in tqdm( range(start, self.max_prompt_length - 1), + mininterval=300, desc="eval: Calculating logits", ): logger.debug(f"Iteration: {i}")