diff --git a/.github/workflows/run-e2e-ui-starter.yml b/.github/workflows/run-e2e-ui-starter.yml new file mode 100644 index 0000000..fff356f --- /dev/null +++ b/.github/workflows/run-e2e-ui-starter.yml @@ -0,0 +1,65 @@ +name: run-e2e-ui-starter +run-name: Checking UI Starter +on: + push: + branches: + - '**' + workflow_dispatch: + branches: + - main + schedule: + # Run at 00:25 and 12:25 on saturday and sunday + - cron: '25 0,12 * * 0,6' +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true +jobs: + cypress-run: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + # Run ui-starter + - run: | + git clone https://github.com/nuclia/ui-starter.git + cd ./ui-starter + npm install + npm run missdev + npm run build + npm run preview & + # Install npm dependencies, cache them correctly and run all Cypress tests + - name: Cypress run + id: cypress + uses: cypress-io/github-action@v6 + with: + install-command: yarn install + spec: cypress/e2e/5-ui-starter/ui-starter.cy.js + env: + CYPRESS_BASE_URL: https://stashify.cloud + CYPRESS_BEARER_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + CYPRESS_NUA_KEY: ${{ secrets.NUA_KEY }} + CYPRESS_USER_NAME: ${{ secrets.USER_NAME }} + CYPRESS_USER_PWD: ${{ secrets.USER_PWD }} + CYPRESS_RUNNING_ENV: stage + CYPRESS_UI_STARTER: true + # after the test run completes store reports and any screenshots + - name: Cypress reports + id: report + uses: actions/upload-artifact@v4 + if: ${{ failure() && steps.cypress.conclusion == 'failure' }} + with: + name: cypress-reports + path: cypress/reports + if-no-files-found: ignore # 'warn' or 'error' are also available, defaults to `warn` + - name: Slack notification + id: slack + uses: slackapi/slack-github-action@v1.25.0 + if: ${{ failure() && steps.cypress.conclusion == 'failure' && github.ref_name == 'main' }} + with: + payload: | + { + "text": "⚠️ UI Starter E2E failed\nTriggered by ${{ github.triggering_actor }} \nRun #${{ github.run_number }}: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\nCypress report: ${{ steps.report.outputs.artifact-url }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_HOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.gitignore b/.gitignore index 55f5540..2a45f30 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ cypress.env.*.json /cypress/videos /cypress/reports /cypress/downloads +/ui-starter # dependencies node_modules diff --git a/cypress/e2e/5-ui-starter/ui-starter.cy.js b/cypress/e2e/5-ui-starter/ui-starter.cy.js new file mode 100644 index 0000000..dac73fb --- /dev/null +++ b/cypress/e2e/5-ui-starter/ui-starter.cy.js @@ -0,0 +1,23 @@ +/// + +import { UI_STARTER } from '../../support/common'; + +describe('UI Starter', () => { + it('should build UI Starter', () => { + if (!UI_STARTER) { + cy.log('Not the UI Starter environment'); + cy.on('fail', (err, runnable) => { + cy.log(err.message); + return false; + }); + cy.request({ url: 'http://localhost:4173', failOnStatusCode: false }).should('be.false'); + } else { + cy.visit('http://localhost:4173'); + cy.get('.nuclia-widget .sw-search-input').should('exist'); + cy.get('.sw-textarea textarea').click(); + cy.get('.sw-textarea textarea').type(`Gold price evolution in 2023\n`, { force: true }); + cy.get('.sw-initial-answer').should('exist'); + cy.get('h3.result-title').should('exist'); + } + }); +}); diff --git a/cypress/support/common.js b/cypress/support/common.js index cba0bb7..7a9167a 100644 --- a/cypress/support/common.js +++ b/cypress/support/common.js @@ -134,6 +134,8 @@ export const user = { password: `${Cypress.env('USER_PWD')}`, }; +export const UI_STARTER = !!Cypress.env('UI_STARTER'); + export function getAuthHeader(synchronous = true) { const headers = { Authorization: `Bearer ${Cypress.env('BEARER_TOKEN')}` }; if (synchronous) { diff --git a/nua/e2e/assets/financial-new-kb.arrow b/nua/e2e/assets/financial-new-kb.arrow deleted file mode 100644 index 7f5c6fa..0000000 Binary files a/nua/e2e/assets/financial-new-kb.arrow and /dev/null differ diff --git a/nua/e2e/assets/financial-news-kb.arrow b/nua/e2e/assets/financial-news-kb.arrow new file mode 100644 index 0000000..95bd046 Binary files /dev/null and b/nua/e2e/assets/financial-news-kb.arrow differ diff --git a/nua/e2e/assets/legal-text-kb.arrow b/nua/e2e/assets/legal-text-kb.arrow index 18d0c76..ab09687 100644 Binary files a/nua/e2e/assets/legal-text-kb.arrow and b/nua/e2e/assets/legal-text-kb.arrow differ diff --git a/nua/e2e/conftest.py b/nua/e2e/conftest.py index fa75f60..fe1a8e6 100644 --- a/nua/e2e/conftest.py +++ b/nua/e2e/conftest.py @@ -4,9 +4,10 @@ import nuclia import pytest -from nuclia.config import reset_config_file, set_config_file from nuclia.sdk.auth import NucliaAuth +from nuclia.lib.nua import AsyncNuaClient from dataclasses import dataclass +from typing import AsyncGenerator logger = logging.getLogger("e2e") @@ -22,17 +23,17 @@ class Tokens: PROD_ACCOUNT_ID = "5cec111b-ea23-4b0c-a82a-d1a666dd1fd2" TOKENS: dict[str, Tokens] = { - "europe-1.stashify.cloud": Tokens( + "https://europe-1.stashify.cloud": Tokens( nua_key=os.environ.get("TEST_EUROPE1_STASHIFY_NUA"), pat_key=os.environ.get("STAGE_PERMAMENT_ACCOUNT_OWNER_PAT_TOKEN"), account_id=os.environ.get("TEST_EUROPE1_STASHIFY_ACCOUNT", STAGE_ACCOUNT_ID), ), - "europe-1.nuclia.cloud": Tokens( + "https://europe-1.nuclia.cloud": Tokens( nua_key=os.environ.get("TEST_EUROPE1_NUCLIA_NUA"), pat_key=os.environ.get("PROD_PERMAMENT_ACCOUNT_OWNER_PAT_TOKEN"), account_id=PROD_ACCOUNT_ID, ), - "aws-us-east-2-1.nuclia.cloud": Tokens( + "https://aws-us-east-2-1.nuclia.cloud": Tokens( nua_key=os.environ.get("TEST_AWS_US_EAST_2_1_NUCLIA_NUA"), pat_key=os.environ.get("PROD_PERMAMENT_ACCOUNT_OWNER_PAT_TOKEN"), account_id=PROD_ACCOUNT_ID, @@ -43,12 +44,12 @@ class Tokens: @pytest.fixture( scope="function", params=[ - "europe-1.stashify.cloud", - "europe-1.nuclia.cloud", - "aws-us-east-2-1.nuclia.cloud", + "https://europe-1.stashify.cloud", + "https://europe-1.nuclia.cloud", + "https://aws-us-east-2-1.nuclia.cloud", ], ) -def nua_config(request): +async def nua_config(request) -> AsyncGenerator[AsyncNuaClient, None]: if ( os.environ.get("TEST_ENV") == "stage" and "stashify.cloud" not in request.param # noqa ): # noqa @@ -63,14 +64,6 @@ def nua_config(request): assert token.pat_key assert token.account_id - with tempfile.NamedTemporaryFile() as temp_file: - temp_file.write(b"{}") - temp_file.flush() - set_config_file(temp_file.name) - nuclia_auth = NucliaAuth() - client_id = nuclia_auth.nua(token.nua_key) - assert client_id - nuclia_auth._config.set_default_nua(client_id) - - yield request.param - reset_config_file() + yield AsyncNuaClient( + region=request.param, account=token.account_id, token=token.nua_key + ) diff --git a/nua/e2e/regional/models.py b/nua/e2e/regional/models.py index 889f335..6a4cb21 100644 --- a/nua/e2e/regional/models.py +++ b/nua/e2e/regional/models.py @@ -1,16 +1,17 @@ ALL_LLMS = [ # Extracted from learning/libraries/learning_models/generative.py "chatgpt-azure-4o", "chatgpt-azure-4-turbo", + "chatgpt-azure-4o-mini", "claude-3", "claude-3-fast", "claude-3-5-fast", + "claude-3-5-small", "gemini-1-5-pro", "gemini-1-5-pro-vision", "gemini-1-5-flash", "gemini-1-5-flash-vision", "mistral", "azure-mistral", - "chatgpt-vision", "chatgpt4", "chatgpt4o", "chatgpt4o-mini", @@ -31,12 +32,15 @@ LLM_WITH_JSON_OUTPUT_SUPPORT = [ "chatgpt-azure-4-turbo", "chatgpt-azure-4o", + "chatgpt-azure-4o-mini", "claude-3", "claude-3-fast", "claude-3-5-fast", + "claude-3-5-small", "chatgpt4", "chatgpt4o", "chatgpt4o-mini", "gemini-1-5-pro", + "gemini-1-5-flash", "azure-mistral", ] diff --git a/nua/e2e/regional/test_da_tasks.py b/nua/e2e/regional/test_da_tasks.py index e20e3a4..7a70593 100644 --- a/nua/e2e/regional/test_da_tasks.py +++ b/nua/e2e/regional/test_da_tasks.py @@ -21,13 +21,14 @@ from conftest import TOKENS from regional.utils import define_path from typing import Callable, AsyncGenerator -from httpx import AsyncClient import asyncio import aiofiles from nucliadb_protos.writer_pb2 import BrokerMessage from dataclasses import dataclass import base64 from typing import Optional +import aiohttp +from nuclia.lib.nua import AsyncNuaClient @dataclass @@ -38,79 +39,89 @@ class TestInput: validate_output: Callable[[BrokerMessage], None] -@pytest.fixture -def httpx_client() -> Callable[[str, str], AsyncGenerator[AsyncClient, None]]: - async def create_httpx_client( +@pytest.fixture(scope="session") +def aiohttp_client() -> ( + Callable[[str, str], AsyncGenerator[aiohttp.ClientSession, None]] +): + async def create_aiohttp_client( base_url: str, nua_key: Optional[str] = None, pat_key: Optional[str] = None, - timeout: int = 5, - ) -> AsyncGenerator[AsyncClient, None]: - client = AsyncClient() - async with AsyncClient( - base_url=base_url, - headers={"X-NUCLIA-NUAKEY": f"Bearer {nua_key}"} + timeout: int = 30, + ) -> AsyncGenerator[aiohttp.ClientSession, None]: + headers = ( + {"X-NUCLIA-NUAKEY": f"Bearer {nua_key}"} if nua_key - else {"Authorization": f"Bearer {pat_key}"}, - timeout=timeout, - ) as client: - yield client + else {"Authorization": f"Bearer {pat_key}"} + ) + timeout_config = aiohttp.ClientTimeout(total=timeout) + + async with aiohttp.ClientSession( + base_url=base_url, + headers=headers, + timeout=timeout_config, + ) as session: + yield session - return create_httpx_client + return create_aiohttp_client async def create_nua_key( - client: AsyncClient, account_id: str, title: str + client: aiohttp.ClientSession, account_id: str, title: str ) -> tuple[str, str]: body = { "title": title, "contact": "temporal key, safe to delete", } resp = await client.post(f"/api/v1/account/{account_id}/nua_clients", json=body) - assert resp.status_code == 201, resp.text - nua_response = resp.json() + assert resp.status == 201, await resp.text() + nua_response = await resp.json() return nua_response["client_id"], nua_response["token"] -async def delete_nua_key(client: AsyncClient, account_id: str, nua_client_id: str): +async def delete_nua_key( + client: aiohttp.ClientSession, account_id: str, nua_client_id: str +): resp = await client.delete( f"/api/v1/account/{account_id}/nua_client/{nua_client_id}" ) - assert resp.status_code == 204, resp.text + assert resp.status == 204, await resp.text() def task_done(task_request: dict) -> bool: return task_request["failed"] or task_request["completed"] -async def create_dataset(client: AsyncClient) -> str: +async def create_dataset(client: aiohttp.ClientSession) -> str: dataset_body = { "name": "e2e-test-dataset", "filter": {"labels": []}, "type": "FIELD_STREAMING", } resp = await client.post("/api/v1/datasets", json=dataset_body) - assert resp.status_code == 201, resp.text - return resp.json()["id"] + assert resp.status == 201, await resp.text() + return (await resp.json())["id"] -async def delete_dataset(client: AsyncClient, dataset_id: str): +async def delete_dataset(client: aiohttp.ClientSession, dataset_id: str): resp = await client.delete(f"/api/v1/dataset/{dataset_id}") - assert resp.status_code == 204, resp.text + assert resp.status == 204, await resp.text() -async def push_data_to_dataset(client: AsyncClient, dataset_id: str, filename: str): +async def push_data_to_dataset( + client: aiohttp.ClientSession, dataset_id: str, filename: str +): async with aiofiles.open(define_path(filename), "rb") as f: content = await f.read() resp = await client.put( f"/api/v1/dataset/{dataset_id}/partition/1", data=content, ) - assert resp.status_code == 204, resp.text + assert resp.status == 204, await resp.text() async def start_task( - client: AsyncClient, + client: aiohttp.ClientSession, dataset_id: str, task_name: str, parameters: PARAMETERS_TYPING, @@ -119,22 +130,22 @@ async def start_task( f"/api/v1/dataset/{dataset_id}/task/start", json=TaskStart(name=task_name, parameters=parameters).model_dump(), ) - assert resp.status_code == 200, resp.text - return resp.json()["id"] + assert resp.status == 200, await resp.text() + return (await resp.json())["id"] -async def stop_task(client: AsyncClient, dataset_id: str, task_id: str): +async def stop_task(client: aiohttp.ClientSession, dataset_id: str, task_id: str): resp = await client.post(f"/api/v1/dataset/{dataset_id}/task/{task_id}/stop") - assert resp.status_code == 200, resp.text + assert resp.status == 200, await resp.text() -async def delete_task(client: AsyncClient, dataset_id: str, task_id: str): +async def delete_task(client: aiohttp.ClientSession, dataset_id: str, task_id: str): resp = await client.delete(f"/api/v1/dataset/{dataset_id}/task/{task_id}") - assert resp.status_code == 200, resp.text + assert resp.status == 200, await resp.text() async def wait_for_task_completion( - client: AsyncClient, + client: aiohttp.ClientSession, dataset_id: str, task_id: str, max_duration: int = 2300, @@ -150,8 +161,8 @@ async def wait_for_task_completion( resp = await client.get( f"/api/v1/dataset/{dataset_id}/task/{task_id}/inspect", ) - assert resp.status_code == 200, resp.text - task_request = resp.json() + assert resp.status == 200, await resp.text() + task_request = await resp.json() if task_done(task_request): return task_request @@ -159,15 +170,15 @@ async def wait_for_task_completion( async def validate_task_output( - client: AsyncClient, validation: Callable[[BrokerMessage], None] + client: aiohttp.ClientSession, validation: Callable[[BrokerMessage], None] ): max_retries = 5 for _ in range(max_retries): resp = await client.get( "/api/v1/processing/pull", params={"from_cursor": 0, "limit": 1} ) - assert resp.status_code == 200, resp.text - pull_response = resp.json() + assert resp.status == 200, await resp.text() + pull_response = await resp.json() if pull_response["payloads"]: assert len(pull_response["payloads"]) == 1 message = BrokerMessage() @@ -291,7 +302,7 @@ def validate_labeler_output_text_block(msg: BrokerMessage): DA_TEST_INPUTS: list[TestInput] = [ TestInput( - filename="financial-new-kb.arrow", + filename="financial-news-kb.arrow", task_name=TaskName.LABELER, parameters=DataAugmentation( name="e2e-test-labeler", @@ -435,7 +446,7 @@ def validate_labeler_output_text_block(msg: BrokerMessage): validate_output=validate_synthetic_questions_output, ), TestInput( - filename="financial-new-kb.arrow", + filename="financial-news-kb.arrow", task_name=TaskName.LABELER, parameters=DataAugmentation( name="e2e-test-labeler-text-block", @@ -499,16 +510,21 @@ def validate_labeler_output_text_block(msg: BrokerMessage): @pytest.fixture -async def nua_key(nua_config: str, httpx_client: AsyncGenerator[AsyncClient, None]): - account_id = TOKENS[nua_config].account_id - pat_client_generator = httpx_client( - base_url=f"https://{nua_config}", pat_key=TOKENS[nua_config].pat_key, timeout=5 +async def tmp_nua_key( + nua_config: AsyncNuaClient, + aiohttp_client: AsyncGenerator[aiohttp.ClientSession, None], +) -> AsyncGenerator[str, None]: + account_id = TOKENS[nua_config.region].account_id + pat_client_generator = aiohttp_client( + base_url=nua_config.url, + pat_key=TOKENS[nua_config.url].pat_key, + timeout=30, ) pat_client = await anext(pat_client_generator) nua_client_id, nua_key = await create_nua_key( client=pat_client, account_id=account_id, - title=f"E2E DA AGENTS - {nua_config}", + title=f"E2E DA AGENTS - {nua_config.region}", ) try: yield nua_key @@ -523,21 +539,21 @@ async def nua_key(nua_config: str, httpx_client: AsyncGenerator[AsyncClient, Non "test_input", DA_TEST_INPUTS, ids=lambda test_input: test_input.parameters.name ) async def test_da_agent_tasks( - nua_config: str, - httpx_client: AsyncGenerator[AsyncClient, None], - nua_key: str, + nua_config: AsyncNuaClient, + aiohttp_client: AsyncGenerator[aiohttp.ClientSession, None], + tmp_nua_key: str, test_input: TestInput, ): dataset_id = None task_id = None + start_time = asyncio.get_event_loop().time() try: - nua_client_generator = httpx_client( - base_url=f"https://{nua_config}", nua_key=nua_key, timeout=30 + nua_client_generator = aiohttp_client( + base_url=nua_config.url, nua_key=tmp_nua_key, timeout=30 ) nua_client = await anext(nua_client_generator) dataset_id = await create_dataset(client=nua_client) - print(f"{test_input.parameters.name} dataset_id: {dataset_id}") await push_data_to_dataset( client=nua_client, dataset_id=dataset_id, filename=test_input.filename ) @@ -569,3 +585,8 @@ async def test_da_agent_tasks( client=nua_client, dataset_id=dataset_id, task_id=task_id ) await delete_dataset(client=nua_client, dataset_id=dataset_id) + end_time = asyncio.get_event_loop().time() + elapsed_time = end_time - start_time + print( + f"Test {test_input.parameters.name} completed in {elapsed_time:.2f} seconds." + ) diff --git a/nua/e2e/regional/test_llm_chat.py b/nua/e2e/regional/test_llm_chat.py index 9d566cf..c5d4f1b 100644 --- a/nua/e2e/regional/test_llm_chat.py +++ b/nua/e2e/regional/test_llm_chat.py @@ -1,15 +1,17 @@ from nuclia.lib.nua_responses import ChatModel, UserPrompt -from nuclia.sdk.predict import NucliaPredict - +from nuclia.sdk.predict import AsyncNucliaPredict +from nuclia.lib.nua import AsyncNuaClient from regional.models import ALL_LLMS +import pytest -def test_llm_chat(nua_config): +@pytest.mark.asyncio_cooperative +async def test_llm_chat(nua_config: AsyncNuaClient): # Validate that other features such as # * citations # * custom prompts # * reranking (TODO once supported by the SDK) - np = NucliaPredict() + np = AsyncNucliaPredict() chat_model = ChatModel( question="Which is the CEO of Nuclia?", retrieval=False, @@ -24,9 +26,10 @@ def test_llm_chat(nua_config): }, citations=True, ) - generated = np.generate( + generated = await np.generate( text=chat_model, model=ALL_LLMS[0], + nc=nua_config, ) # Check that system + user prompt worked assert generated.answer.startswith("ITALIAN") diff --git a/nua/e2e/regional/test_llm_citation.py b/nua/e2e/regional/test_llm_citation.py deleted file mode 100644 index 0b0cff3..0000000 --- a/nua/e2e/regional/test_llm_citation.py +++ /dev/null @@ -1,12 +0,0 @@ -# def test_llm_citations_everest(nua_config): -# np = NucliaPredict() -# embed = np.generate("Who is the CEO of Nuclia?", model="everest") -# assert embed.time > 0 -# assert len(embed.tokens) == 1 - - -# def test_llm_citations_azure_chatgpt(nua_config): -# np = NucliaPredict() -# embed = np.generate("Who is the CEO of Nuclia?", model="chatgpt") -# assert embed.time > 0 -# assert len(embed.tokens) == 1 diff --git a/nua/e2e/regional/test_llm_config.py b/nua/e2e/regional/test_llm_config.py index 2ff22f9..531ad09 100644 --- a/nua/e2e/regional/test_llm_config.py +++ b/nua/e2e/regional/test_llm_config.py @@ -1,24 +1,26 @@ import pytest from nuclia.exceptions import NuaAPIException from nuclia.lib.nua_responses import LearningConfigurationCreation -from nuclia.sdk.predict import NucliaPredict +from nuclia.lib.nua import AsyncNuaClient +from nuclia.sdk.predict import AsyncNucliaPredict -def test_llm_config_nua(nua_config): - np = NucliaPredict() +@pytest.mark.asyncio_cooperative +async def test_llm_config_nua(nua_config: AsyncNuaClient): + np = AsyncNucliaPredict() try: - np.del_config("kbid") + await np.del_config("kbid", nc=nua_config) except NuaAPIException: pass with pytest.raises(NuaAPIException): - config = np.config("kbid") + config = await np.config("kbid", nc=nua_config) lcc = LearningConfigurationCreation() - np.set_config("kbid", lcc) + await np.set_config("kbid", lcc, nc=nua_config) - config = np.config("kbid") + config = await np.config("kbid", nc=nua_config) assert config.resource_labelers_models is None assert config.ner_model == "multilingual" diff --git a/nua/e2e/regional/test_llm_generate.py b/nua/e2e/regional/test_llm_generate.py index 26acf68..143b6e4 100644 --- a/nua/e2e/regional/test_llm_generate.py +++ b/nua/e2e/regional/test_llm_generate.py @@ -1,11 +1,14 @@ import pytest -from nuclia.sdk.predict import NucliaPredict - +from nuclia.sdk.predict import AsyncNucliaPredict +from nuclia.lib.nua import AsyncNuaClient from regional.models import ALL_LLMS +@pytest.mark.asyncio_cooperative @pytest.mark.parametrize("model", ALL_LLMS) -def test_llm_generate(nua_config, model): - np = NucliaPredict() - generated = np.generate("Which is the capital of Catalonia?", model=model) +async def test_llm_generate(nua_config: AsyncNuaClient, model): + np = AsyncNucliaPredict() + generated = await np.generate( + "Which is the capital of Catalonia?", model=model, nc=nua_config + ) assert "Barcelona" in generated.answer diff --git a/nua/e2e/regional/test_llm_json.py b/nua/e2e/regional/test_llm_json.py index 7531960..2c5af5f 100644 --- a/nua/e2e/regional/test_llm_json.py +++ b/nua/e2e/regional/test_llm_json.py @@ -1,7 +1,7 @@ import pytest from nuclia.lib.nua_responses import ChatModel, UserPrompt -from nuclia.sdk.predict import NucliaPredict - +from nuclia.sdk.predict import AsyncNucliaPredict +from nuclia.lib.nua import AsyncNuaClient from regional.models import LLM_WITH_JSON_OUTPUT_SUPPORT SCHEMA = { @@ -34,10 +34,11 @@ TEXT = """"Many football players have existed. Messi is by far the greatest. Messi was born in Rosario, 24th of June 1987""" +@pytest.mark.asyncio_cooperative @pytest.mark.parametrize("model_name", LLM_WITH_JSON_OUTPUT_SUPPORT) -def test_llm_json(nua_config, model_name): - np = NucliaPredict() - results = np.generate( +async def test_llm_json(nua_config: AsyncNuaClient, model_name): + np = AsyncNucliaPredict() + results = await np.generate( text=ChatModel( question="", retrieval=False, @@ -46,5 +47,6 @@ def test_llm_json(nua_config, model_name): json_schema=SCHEMA, ), model=model_name, + nc=nua_config, ) assert "SPORTS" in results.object["document_type"] diff --git a/nua/e2e/regional/test_llm_rag.py b/nua/e2e/regional/test_llm_rag.py index 38fb6b2..cc5222d 100644 --- a/nua/e2e/regional/test_llm_rag.py +++ b/nua/e2e/regional/test_llm_rag.py @@ -1,19 +1,21 @@ import pytest -from nuclia.sdk.predict import NucliaPredict - +from nuclia.sdk.predict import AsyncNucliaPredict +from nuclia.lib.nua import AsyncNuaClient from regional.models import ALL_LLMS +@pytest.mark.asyncio_cooperative @pytest.mark.parametrize("model", ALL_LLMS) -def test_llm_rag(nua_config, model): - np = NucliaPredict() - generated = np.rag( +async def test_llm_rag(nua_config: AsyncNuaClient, model): + np = AsyncNucliaPredict() + generated = await np.rag( question="Which is the CEO of Nuclia?", context=[ "Nuclia CTO is Ramon Navarro", "Eudald Camprubí is CEO at the same company as Ramon Navarro", ], model=model, + nc=nua_config, ) assert "Eudald" in generated.answer diff --git a/nua/e2e/regional/test_llm_schema.py b/nua/e2e/regional/test_llm_schema.py index b68cc43..616a174 100644 --- a/nua/e2e/regional/test_llm_schema.py +++ b/nua/e2e/regional/test_llm_schema.py @@ -1,16 +1,20 @@ -from nuclia.sdk.predict import NucliaPredict +from nuclia.sdk.predict import AsyncNucliaPredict +import pytest +from nuclia.lib.nua import AsyncNuaClient -def test_llm_schema_nua(nua_config): - np = NucliaPredict() - config = np.schema() +@pytest.mark.asyncio_cooperative +async def test_llm_schema_nua(nua_config: AsyncNuaClient): + np = AsyncNucliaPredict() + config = await np.schema(nc=nua_config) assert len(config.ner_model.options) == 1 assert len(config.generative_model.options) >= 5 -def test_llm_schema_kbid(nua_config): - np = NucliaPredict() - config = np.schema("fake_kbid") +@pytest.mark.asyncio_cooperative +async def test_llm_schema_kbid(nua_config: AsyncNuaClient): + np = AsyncNucliaPredict() + config = await np.schema("fake_kbid", nc=nua_config) assert len(config.ner_model.options) == 1 assert len(config.generative_model.options) >= 5 diff --git a/nua/e2e/regional/test_llm_summarize.py b/nua/e2e/regional/test_llm_summarize.py index 483f4a8..75cfd95 100644 --- a/nua/e2e/regional/test_llm_summarize.py +++ b/nua/e2e/regional/test_llm_summarize.py @@ -1,4 +1,6 @@ -from nuclia.sdk.predict import NucliaPredict +from nuclia.sdk.predict import AsyncNucliaPredict +import pytest +from nuclia.lib.nua import AsyncNuaClient DATA = { "barcelona": "Barcelona (pronunciat en català central, [bərsəˈlonə]) és una ciutat i metròpoli a la costa mediterrània de la península Ibèrica. És la capital de Catalunya,[1] així com de la comarca del Barcelonès i de la província de Barcelona, i la segona ciutat en població i pes econòmic de la península Ibèrica,[2][3] després de Madrid. El municipi creix sobre una plana encaixada entre la serralada Litoral, el mar Mediterrani, el riu Besòs i la muntanya de Montjuïc. La ciutat acull les seus de les institucions d'autogovern més importants de la Generalitat de Catalunya: el Parlament de Catalunya, el President i el Govern de la Generalitat. Pel fet d'haver estat capital del Comtat de Barcelona, rep sovint el sobrenom de Ciutat Comtal. També, com que ha estat la ciutat més important del Principat de Catalunya des d'època medieval, rep sovint el sobrenom o títol de cap i casal.[4]", # noqa @@ -25,23 +27,26 @@ } -def test_summarize_chatgpt(nua_config): - np = NucliaPredict() - embed = np.summarize(DATA, model="chatgpt4o") +@pytest.mark.asyncio_cooperative +async def test_summarize_chatgpt(nua_config: AsyncNuaClient): + np = AsyncNucliaPredict() + embed = await np.summarize(DATA, model="chatgpt4o", nc=nua_config) assert "Manresa" in embed.summary assert "Barcelona" in embed.summary -def test_summarize_azure_chatgpt(nua_config): - np = NucliaPredict() - embed = np.summarize(DATA, model="chatgpt-azure-4o") +@pytest.mark.asyncio_cooperative +async def test_summarize_azure_chatgpt(nua_config: AsyncNuaClient): + np = AsyncNucliaPredict() + embed = await np.summarize(DATA, model="chatgpt-azure-4o", nc=nua_config) assert "Manresa" in embed.summary assert "Barcelona" in embed.summary -def test_summarize_claude(nua_config): - np = NucliaPredict() - embed = np.summarize(DATA_COFFEE, model="claude-3-fast") +@pytest.mark.asyncio_cooperative +async def test_summarize_claude(nua_config: AsyncNuaClient): + np = AsyncNucliaPredict() + embed = await np.summarize(DATA_COFFEE, model="claude-3-fast", nc=nua_config) # changed to partial summaries since anthropic is not consistent in the global summary at all assert "flat white" in embed.resources["Flat white"].summary.lower() assert "macchiato" in embed.resources["Macchiato"].summary.lower() diff --git a/nua/e2e/regional/test_predict.py b/nua/e2e/regional/test_predict.py index adfd3ac..07da311 100644 --- a/nua/e2e/regional/test_predict.py +++ b/nua/e2e/regional/test_predict.py @@ -1,13 +1,16 @@ import pytest -from nuclia.sdk.predict import NucliaPredict +from nuclia.sdk.predict import AsyncNucliaPredict from regional.models import ALL_ENCODERS, ALL_LLMS +from nuclia_models.predict.remi import RemiRequest +from nuclia.lib.nua import AsyncNuaClient +@pytest.mark.asyncio_cooperative @pytest.mark.parametrize("model", ALL_ENCODERS.keys()) -def test_predict_sentence(nua_config, model): - np = NucliaPredict() - embed = np.sentence(text="This is my text", model=model) +async def test_predict_sentence(nua_config: AsyncNuaClient, model): + np = AsyncNucliaPredict() + embed = await np.sentence(text="This is my text", model=model, nc=nua_config) assert embed.time > 0 # Deprecated field (data) assert len(embed.data) == ALL_ENCODERS[model] @@ -17,9 +20,10 @@ def test_predict_sentence(nua_config, model): # TODO: Add test for predict sentence with multiple models in one request (vectorsets) -def test_predict_query(nua_config): - np = NucliaPredict() - embed = np.query(text="I love Barcelona") +@pytest.mark.asyncio_cooperative +async def test_predict_query(nua_config: AsyncNuaClient): + np = AsyncNucliaPredict() + embed = await np.query(text="I love Barcelona", nc=nua_config) # Semantic assert embed.semantic_threshold > 0 assert len(embed.sentence.data) > 128 @@ -35,19 +39,47 @@ def test_predict_query(nua_config): # TODO: Add test for predict rerank once SDK supports rerank -def test_predict_tokens(nua_config): - np = NucliaPredict() - embed = np.tokens(text="I love Barcelona") +@pytest.mark.asyncio_cooperative +async def test_predict_tokens(nua_config: AsyncNuaClient): + np = AsyncNucliaPredict() + embed = await np.tokens(text="I love Barcelona", nc=nua_config) assert embed.tokens[0].text == "Barcelona" assert embed.tokens[0].start == 7 assert embed.tokens[0].end == 16 assert embed.time > 0 +@pytest.mark.asyncio_cooperative @pytest.mark.parametrize("model", ALL_LLMS) -def test_predict_rephrase(nua_config, model): +async def test_predict_rephrase(nua_config: AsyncNuaClient, model): # Check that rephrase is working for all models - np = NucliaPredict() + np = AsyncNucliaPredict() # TODO: Test that custom rephrase prompt works once SDK supports it - rephrased = np.rephrase(question="Barcelona best coffe", model=model) + rephrased = await np.rephrase( + question="Barcelona best coffe", model=model, nc=nua_config + ) assert rephrased != "Barcelona best coffe" and rephrased != "" + + +@pytest.mark.asyncio_cooperative +async def test_predict_remi(nua_config: AsyncNuaClient): + np = AsyncNucliaPredict() + results = await np.remi( + RemiRequest( + user_id="NUA E2E", + question="What is the capital of France?", + answer="Paris is the capital of france!", + contexts=[ + "Paris is the capital of France.", + "Berlin is the capital of Germany.", + ], + ), + nc=nua_config, + ) + assert results.answer_relevance.score >= 4 + + assert results.context_relevance[0] >= 4 + assert results.groundedness[0] >= 4 + + assert results.context_relevance[1] < 2 + assert results.groundedness[1] < 2 diff --git a/nua/e2e/regional/test_processor.py b/nua/e2e/regional/test_processor.py index 45b4ac3..4a3e25c 100644 --- a/nua/e2e/regional/test_processor.py +++ b/nua/e2e/regional/test_processor.py @@ -1,13 +1,15 @@ import pytest -from nuclia.sdk.process import NucliaProcessing +from nuclia.sdk.process import AsyncNucliaProcessing from regional.utils import define_path +from nuclia.lib.nua import AsyncNuaClient +@pytest.mark.asyncio_cooperative @pytest.mark.timeout(660) -def test_pdf(nua_config): +async def test_pdf(nua_config: AsyncNuaClient): path = define_path("2310.14587.pdf") - nc = NucliaProcessing() - payload = nc.process_file(path, kbid="kbid", timeout=300) + nc = AsyncNucliaProcessing() + payload = await nc.process_file(path, kbid="kbid", timeout=300, nc=nua_config) assert payload assert ( "As the training data of LLMs often contains undesirable" @@ -15,11 +17,12 @@ def test_pdf(nua_config): ) +@pytest.mark.asyncio_cooperative @pytest.mark.timeout(360) -def test_video(nua_config): +async def test_video(nua_config: AsyncNuaClient): path = define_path("simple_video.mp4") - nc = NucliaProcessing() - payload = nc.process_file(path, kbid="kbid", timeout=300) + nc = AsyncNucliaProcessing() + payload = await nc.process_file(path, kbid="kbid", timeout=300, nc=nua_config) assert payload assert ( "This is one of the most reflective mirrors" @@ -27,18 +30,20 @@ def test_video(nua_config): ) +@pytest.mark.asyncio_cooperative @pytest.mark.timeout(660) -def test_vude_1(nua_config): +async def test_vude_1(nua_config: AsyncNuaClient): path = define_path( "y2mate_is_Stone_1_Minute_Short_Film_Hot_Shot_5hPtU8Jbpg0_720p_1701938639.mp4" # noqa ) - nc = NucliaProcessing() - payload = nc.process_file(path, kbid="kbid", timeout=300) + nc = AsyncNucliaProcessing() + payload = await nc.process_file(path, kbid="kbid", timeout=300, nc=nua_config) assert payload print(payload.extracted_text[0].body) assert "harmful" in payload.extracted_text[0].body.text -def test_activity(nua_config): - nc = NucliaProcessing() - nc.status() +@pytest.mark.asyncio_cooperative +async def test_activity(nua_config: AsyncNuaClient): + nc = AsyncNucliaProcessing() + await nc.status(nc=nua_config) diff --git a/nua/e2e/regional/utils.py b/nua/e2e/regional/utils.py index 4e9331a..d0a6884 100644 --- a/nua/e2e/regional/utils.py +++ b/nua/e2e/regional/utils.py @@ -1,7 +1,14 @@ from pathlib import Path +from dataclasses import dataclass FILE_PATH = f"{Path(__file__).parent.parent}/assets/" def define_path(file: str): return FILE_PATH + file + + +@dataclass +class TestConfig: + domain: str + config_path: str diff --git a/nua/requirements-test.txt b/nua/requirements-test.txt index f65efb5..4f2799f 100644 --- a/nua/requirements-test.txt +++ b/nua/requirements-test.txt @@ -1,5 +1,3 @@ ruff==0.4.5 mypy==1.8.0 mypy-extensions==1.0.0 -pytest-asyncio==0.20.3 -pytest==7.2.1 \ No newline at end of file diff --git a/nua/requirements.txt b/nua/requirements.txt index b9abc95..5ccf2ba 100644 --- a/nua/requirements.txt +++ b/nua/requirements.txt @@ -1,8 +1,8 @@ -nuclia>=3.1.2 +nuclia>=4.4.2 pytest pytest-timeout aiofiles nuclia-models -httpx nucliadb_protos pytest-asyncio-cooperative # not compatible with pytest-asyncio +aiohttp