From 9d35b3d6f6c21c0c8f37de20fce17709a8cc88d4 Mon Sep 17 00:00:00 2001 From: Xing Han Lu <21180505+xhluca@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:45:44 +0000 Subject: [PATCH] Add initial tests --- .github/workflows/run-tests.yml | 53 +++++++++++++++++++++++++++++++ .gitignore | 5 ++- docs/CONTRIBUTING.md | 9 ++++++ docs/README.md | 4 +++ tests/requirements.txt | 1 + tests/test_web_turn_processor.py | 54 ++++++++++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/run-tests.yml create mode 100644 docs/CONTRIBUTING.md create mode 100644 tests/requirements.txt create mode 100644 tests/test_web_turn_processor.py diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..041c9eb --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,53 @@ +name: Run Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.9' # Specify your required Python version + + - name: Cache Python dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('tests/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r tests/requirements.txt # Assumes you have a requirements.txt file + + - name: Cache test assets + uses: actions/cache@v2 + with: + path: tests/demonstrations + key: assets-${{ github.sha }} + restore-keys: | + assets- + + - name: Download test demos from release URL into `tests/demonstrations` + run: | + mkdir -p tests/demonstrations + curl -L -o tests/demonstrations/aaabtsd.zip https://github.com/McGill-NLP/weblinx/releases/download/tests-assets/aaabtsd.zip + unzip -u tests/demonstrations/aaabtsd.zip -d tests/demonstrations + curl -L -o tests/demonstrations/aajfwoq.zip https://github.com/McGill-NLP/weblinx/releases/download/tests-assets/aajfwoq.zip + unzip -u tests/demonstrations/aajfwoq.zip -d tests/demonstrations + + - name: Run tests + run: | + python -m unittest discover -s tests \ No newline at end of file diff --git a/.gitignore b/.gitignore index aa22073..7b9a319 100644 --- a/.gitignore +++ b/.gitignore @@ -172,4 +172,7 @@ modeling/wl_data app/data/inputs.json modeling/logs/ venv*/ -.python-version \ No newline at end of file +.python-version + +# TESTS +tests/demonstrations \ No newline at end of file diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 0000000..d60845e --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,9 @@ +# Contributing Instructions + +## Running tests + +To run the unit tests, run: + +```bash +python -m unittest discover -s tests +``` \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 8f2a5d2..820ba8f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -386,3 +386,7 @@ To be added. ### Build `webllama.experimental.classes.State` To be added. + +## Contributing + +For more information on contributing, please check out the [contributing docs](docs/CONTRIBUTING.md). \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..54ae57f --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1 @@ +-e .[all] \ No newline at end of file diff --git a/tests/test_web_turn_processor.py b/tests/test_web_turn_processor.py new file mode 100644 index 0000000..c53dbf8 --- /dev/null +++ b/tests/test_web_turn_processor.py @@ -0,0 +1,54 @@ +from functools import partial +import time +import logging +import unittest + +from sentence_transformers import SentenceTransformer +from transformers import AutoTokenizer, pipeline +import weblinx as wl +import webllama.experimental as wa + +logging.getLogger("urllib3").setLevel(logging.WARNING) + + +class TestWebTurnProcessor(unittest.TestCase): + def setUp(self): + + demos = wl.list_demonstrations("tests/demonstrations") + replay = wl.Replay.from_demonstration(demos[0]) + turn = replay[26] + + self.turn = turn + self.replay = replay + self.action_model_name = "McGill-NLP/Sheared-LLaMA-2.7B-weblinx" + + self.tokenizer = AutoTokenizer.from_pretrained(self.action_model_name) + + format_intent_input_dmr, format_intent_out_dmr = ( + wa.formatting.build_formatters_dmr() + ) + format_intent_am = partial( + wa.formatting.build_formatters_action_model(), return_as=dict + ) + self.action_history = wa.functions.create_action_history_from_replay( + replay, format_intent_input_dmr, format_intent_am, stop_at_idx=turn.index + ) + self.state = wa.classes.State( + index=turn.index, + html=turn.html, + bboxes=turn.bboxes, + viewport_height=turn.viewport_height, + viewport_width=turn.viewport_width, + type=turn.type, + ) + + def test_prepare_dmr_query(self): + # We will initialize our processor, which helps us prepare the input for action model + proc = wa.processing.WebTurnProcessor(tokenizer=self.tokenizer) + + # Step 1: prepare query, run DMR and prepare retrieved candidates + query_dmr = proc.prepare_dmr_query(self.action_history, self.state) + + CORRECT_RESULT = 'Viewport(height=746, width=1536) ---- Instructor Utterances: [00:07] Hello [00:13] Open independent ie Website. [01:30] Go to life and send me some life related news [04:00] Open second one and Summarize the first three paragraphs in a few words ---- Previous Turns:tabswitch(origin=102465633, target=102465635, timestamp="04:19") ; load(url="https://search.yahoo.com/search?fr=mcafee&type=E211US714G0&p=chatgpt", timestamp="04:23") ; click(x=268, y=201, tag="a", attrs={}, timestamp="04:24") ; tabcreate(target=102465636, timestamp="04:25") ; tabswitch(origin=102465635, target=102465636, timestamp="04:25")' + self.assertIsInstance(query_dmr, str) + self.assertEqual(query_dmr, CORRECT_RESULT)