diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml new file mode 100644 index 000000000..aaf051794 --- /dev/null +++ b/.github/workflows/sanity.yml @@ -0,0 +1,63 @@ +name: Install Local Version of Opik + +on: + workflow_dispatch: + +jobs: + test_installation: + runs-on: ubuntu-20.04 + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install Opik + run: pip install ${{ github.workspace }}/sdks/python + + - name: Install Test Dependencies + run: | + pip install -r ${{ github.workspace }}/tests_end_to_end/test_requirements.txt + playwright install + + - name: Install Opik + env: + OPIK_USAGE_REPORT_ENABLED: false + run: | + cd ${{ github.workspace }}/deployment/docker-compose + docker compose up -d --build + + - name: Check Docker pods are up + run: | + chmod +x ./tests_end_to_end/installer/check_docker_compose_pods.sh + ./tests_end_to_end/installer/check_docker_compose_pods.sh + shell: bash + + - name: Check backend health + run: | + chmod +x ./tests_end_to_end/installer/check_backend.sh + ./tests_end_to_end/installer/check_backend.sh + shell: bash + + - name: Check app is up via the UI + run: | + pytest -v -s ${{ github.workspace }}/tests_end_to_end/installer/test_app_status.py + + - name: Run sanity suite + run: | + cd ${{ github.workspace }}/tests_end_to_end + export PYTHONPATH='.' + pytest -s application_sanity/test_sanity.py --browser chromium --base-url http://localhost:5173 --setup-show + + - name: Stop Opik server + if: always() + run: | + cd ${{ github.workspace }}/deployment/docker-compose + docker compose down + cd - diff --git a/tests_end_to_end/application_sanity/conftest.py b/tests_end_to_end/application_sanity/conftest.py index affe999d9..6ef87c913 100644 --- a/tests_end_to_end/application_sanity/conftest.py +++ b/tests_end_to_end/application_sanity/conftest.py @@ -20,16 +20,17 @@ def config(): @pytest.fixture(scope='session', autouse=True) def configure_local(config): - configure(use_local=True) + os.environ['OPIK_URL_OVERRIDE'] = "http://localhost:5173/api" + os.environ['OPIK_WORKSPACE'] = 'default' os.environ['OPIK_PROJECT_NAME'] = config['project']['name'] @pytest.fixture(scope='session', autouse=True) def client(config): - return opik.Opik(project_name=config['project']['name']) + return opik.Opik(project_name=config['project']['name'], host='http://localhost:5173/api') -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def log_traces_and_spans_low_level(client, config): """ Log 5 traces with spans and subspans using the low level Opik client @@ -37,7 +38,7 @@ def log_traces_and_spans_low_level(client, config): """ trace_config = { - 'count': config['traces']['client']['count'], + 'count': config['traces']['count'], 'prefix': config['traces']['client']['prefix'], 'tags': config['traces']['client']['tags'], 'metadata': config['traces']['client']['metadata'], @@ -45,7 +46,7 @@ def log_traces_and_spans_low_level(client, config): } span_config = { - 'count': config['spans']['client']['count'], + 'count': config['spans']['count'], 'prefix': config['spans']['client']['prefix'], 'tags': config['spans']['client']['tags'], 'metadata': config['spans']['client']['metadata'], @@ -55,8 +56,8 @@ def log_traces_and_spans_low_level(client, config): for trace_index in range(trace_config['count']): client_trace = client.trace( name=trace_config['prefix'] + str(trace_index), - input=f'input-{trace_index}', - output=f'output-{trace_index}', + input={'input': f'input-{trace_index}'}, + output={'output': f'output-{trace_index}'}, tags=trace_config['tags'], metadata=trace_config['metadata'], feedback_scores=trace_config['feedback_scores'] @@ -64,8 +65,8 @@ def log_traces_and_spans_low_level(client, config): for span_index in range(span_config['count']): client_span = client_trace.span( name=span_config['prefix'] + str(span_index), - input=f'input-{span_index}', - output=f'output-{span_index}', + input={'input': f'input-{span_index}'}, + output={'output': f'output-{span_index}'}, tags=span_config['tags'], metadata=span_config['metadata'] ) @@ -73,7 +74,7 @@ def log_traces_and_spans_low_level(client, config): client_span.log_feedback_score(name=score['name'], value=score['value']) -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def log_traces_and_spans_decorator(config): """ Log 5 traces with spans and subspans using the low level Opik client @@ -81,7 +82,7 @@ def log_traces_and_spans_decorator(config): """ trace_config = { - 'count': config['traces']['decorator']['count'], + 'count': config['traces']['count'], 'prefix': config['traces']['decorator']['prefix'], 'tags': config['traces']['decorator']['tags'], 'metadata': config['traces']['decorator']['metadata'], @@ -89,7 +90,7 @@ def log_traces_and_spans_decorator(config): } span_config = { - 'count': config['spans']['decorator']['count'], + 'count': config['spans']['count'], 'prefix': config['spans']['decorator']['prefix'], 'tags': config['spans']['decorator']['tags'], 'metadata': config['spans']['decorator']['metadata'], @@ -100,12 +101,12 @@ def log_traces_and_spans_decorator(config): def make_span(x): opik_context.update_current_span( name=span_config['prefix'] + str(x), - input=f'input-{x}', + input={'input': f'input-{x}'}, metadata=span_config['metadata'], tags=span_config['tags'], feedback_scores=span_config['feedback_scores'] ) - return f'output-{x}' + return {'output': f'output-{x}'} @track() def make_trace(x): @@ -114,12 +115,12 @@ def make_trace(x): opik_context.update_current_trace( name=trace_config['prefix'] + str(x), - input=f'input-{x}', + input={'input': f'input-{x}'}, metadata=trace_config['metadata'], tags=trace_config['tags'], feedback_scores=trace_config['feedback_scores'] ) - return f'output-{x}' + return {'output': f'output-{x}'} for x in range(trace_config['count']): make_trace(x) diff --git a/tests_end_to_end/application_sanity/sanity_config.yaml b/tests_end_to_end/application_sanity/sanity_config.yaml index 9d53bda6b..7efebfab0 100644 --- a/tests_end_to_end/application_sanity/sanity_config.yaml +++ b/tests_end_to_end/application_sanity/sanity_config.yaml @@ -2,9 +2,9 @@ project: name: "test-project" traces: + count: 5 client: prefix: "client-trace-" - count: 5 tags: ["c-tag1", "c-tag2"] feedback-scores: c-score1: 0.5 @@ -15,7 +15,6 @@ traces: decorator: prefix: "decorator-trace-" - count: 5 tags: ["d-tag1", "d-tag2"] feedback-scores: d-score1: 0.1 @@ -25,9 +24,9 @@ traces: d-md2: "val2" spans: + count: 2 client: prefix: "client-span-" - count: 2 tags: ["c-span1", "c-span2"] feedback-scores: s-score1: 0.2 @@ -38,7 +37,6 @@ spans: decorator: prefix: "decorator-span-" - count: 2 tags: ["d-span1", "d-span2"] feedback-scores: s-score1: 0.93 diff --git a/tests_end_to_end/application_sanity/test_sanity.py b/tests_end_to_end/application_sanity/test_sanity.py new file mode 100644 index 000000000..d940228f1 --- /dev/null +++ b/tests_end_to_end/application_sanity/test_sanity.py @@ -0,0 +1,56 @@ +import pytest +from playwright.sync_api import Page, expect +from page_objects.ProjectsPage import ProjectsPage +from page_objects.TracesPage import TracesPage +from page_objects.TracesPageSpansMenu import TracesPageSpansMenu + + +def test_project_name(page: Page, log_traces_and_spans_decorator, log_traces_and_spans_low_level): + projects_page = ProjectsPage(page) + projects_page.go_to_page() + projects_page.check_project_exists('test-project') + + +def test_traces_created(page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator): + #navigate to project + projects_page = ProjectsPage(page) + projects_page.go_to_page() + + #wait for data to actually arrive to the frontend + #TODO: replace this with a smarter waiting mechanism + page.wait_for_timeout(5000) + projects_page.click_project(config['project']['name']) + + #grab all traces of project + traces_page = TracesPage(page) + trace_names = traces_page.get_all_trace_names() + + client_prefix = config['traces']['client']['prefix'] + decorator_prefix = config['traces']['decorator']['prefix'] + + for count in range(config['traces']['count']): + for prefix in [client_prefix, decorator_prefix]: + assert prefix+str(count) in trace_names + + +def test_spans_of_traces(page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator): + projects_page = ProjectsPage(page) + projects_page.go_to_page() + + #wait for data to actually arrive to the frontend + #TODO: replace this with a smarter waiting mechanism + projects_page.click_project(config['project']['name']) + + #grab all traces of project + traces_page = TracesPage(page) + trace_names = traces_page.get_all_trace_names() + + for trace in trace_names: + page.get_by_text(trace).click() + spans_menu = TracesPageSpansMenu(page) + trace_type = trace.split('-')[0] # 'client' or 'decorator' + for count in range(config['spans']['count']): + prefix = config['spans'][trace_type]['prefix'] + spans_menu.check_span_exists_by_name(f'{prefix}{count}') + + diff --git a/tests_end_to_end/page_objects/ProjectsPage.py b/tests_end_to_end/page_objects/ProjectsPage.py new file mode 100644 index 000000000..c06e87a58 --- /dev/null +++ b/tests_end_to_end/page_objects/ProjectsPage.py @@ -0,0 +1,16 @@ +from playwright.sync_api import Page, expect + +class ProjectsPage: + def __init__(self, page: Page): + self.page = page + self.url = '/projects' + self.projects_table = page.get_by_role('table') + + def go_to_page(self): + self.page.goto(self.url) + + def click_project(self, project_name): + self.page.get_by_role('cell', name=project_name).click() + + def check_project_exists(self, project_name): + expect(self.projects_table.get_by_role('cell', name=project_name)).to_be_visible() \ No newline at end of file diff --git a/tests_end_to_end/page_objects/TracesPage.py b/tests_end_to_end/page_objects/TracesPage.py new file mode 100644 index 000000000..2df04eab0 --- /dev/null +++ b/tests_end_to_end/page_objects/TracesPage.py @@ -0,0 +1,13 @@ +from playwright.sync_api import Page, expect + +class TracesPage: + def __init__(self, page: Page): + self.page = page + self.traces_table = self.page.get_by_role('table') + self.trace_names_selector = 'tr td:nth-child(2) div span' + + def get_all_trace_names(self): + self.page.wait_for_selector(self.trace_names_selector) + + names = self.page.locator(self.trace_names_selector).all_inner_texts() + return names \ No newline at end of file diff --git a/tests_end_to_end/page_objects/TracesPageSpansMenu.py b/tests_end_to_end/page_objects/TracesPageSpansMenu.py new file mode 100644 index 000000000..a0a495ebc --- /dev/null +++ b/tests_end_to_end/page_objects/TracesPageSpansMenu.py @@ -0,0 +1,8 @@ +from playwright.sync_api import Page, expect + +class TracesPageSpansMenu: + def __init__(self, page: Page): + self.page = page + + def check_span_exists_by_name(self, name): + expect(self.page.get_by_role('button', name=name)).to_be_visible() \ No newline at end of file diff --git a/tests_end_to_end/page_objects/__init__.py b/tests_end_to_end/page_objects/__init__.py new file mode 100644 index 000000000..e69de29bb