From ad5a762a2c00eb8bc59c355b326c316fb9a47254 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:08:15 +0000 Subject: [PATCH 1/5] fix: Remove test string --- src/writer/blocks/logmessage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/writer/blocks/logmessage.py b/src/writer/blocks/logmessage.py index 6b1be0c39..c88ba5eb5 100644 --- a/src/writer/blocks/logmessage.py +++ b/src/writer/blocks/logmessage.py @@ -56,8 +56,6 @@ def run(self): return self.runner.state.add_log_entry(type, "Workflows message", message) - import time - self.runner.state.add_log_entry("error", "Workflows message", f"gato {time.time()}") self.result = None self.outcome = "success" except BaseException as e: From e43b97fc402fb6c9618a78b0e28ae43a95eadb39 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:42:16 +0000 Subject: [PATCH 2/5] test: Tests for Workflows blocks --- tests/backend/blocks/conftest.py | 51 +++++++ tests/backend/blocks/test_addtostatelist.py | 37 ++++++ tests/backend/blocks/test_base_block.py | 44 +++++++ tests/backend/blocks/test_calleventhandler.py | 48 +++++++ tests/backend/blocks/test_foreach.py | 52 ++++++++ tests/backend/blocks/test_httprequest.py | 120 +++++++++++++++++ tests/backend/blocks/test_logmessage.py | 73 +++++++++++ tests/backend/blocks/test_parsejson.py | 24 +++- tests/backend/blocks/test_returnvalue.py | 29 ++++ tests/backend/blocks/test_runworkflow.py | 30 +++++ tests/backend/blocks/test_setstate.py | 61 +++++++++ .../blocks/test_writeraddchatmessage.py | 50 +++++++ tests/backend/blocks/test_writerchat.py | 124 ++++++++++++++++++ .../blocks/test_writerclassification.py | 40 ++++++ tests/backend/blocks/test_writercompletion.py | 42 ++++++ tests/backend/blocks/test_writerinitchat.py | 33 +++++ tests/backend/blocks/test_writernocodeapp.py | 41 ++++++ 17 files changed, 896 insertions(+), 3 deletions(-) create mode 100644 tests/backend/blocks/conftest.py create mode 100644 tests/backend/blocks/test_addtostatelist.py create mode 100644 tests/backend/blocks/test_base_block.py create mode 100644 tests/backend/blocks/test_calleventhandler.py create mode 100644 tests/backend/blocks/test_foreach.py create mode 100644 tests/backend/blocks/test_httprequest.py create mode 100644 tests/backend/blocks/test_logmessage.py create mode 100644 tests/backend/blocks/test_returnvalue.py create mode 100644 tests/backend/blocks/test_runworkflow.py create mode 100644 tests/backend/blocks/test_setstate.py create mode 100644 tests/backend/blocks/test_writeraddchatmessage.py create mode 100644 tests/backend/blocks/test_writerchat.py create mode 100644 tests/backend/blocks/test_writerclassification.py create mode 100644 tests/backend/blocks/test_writercompletion.py create mode 100644 tests/backend/blocks/test_writerinitchat.py create mode 100644 tests/backend/blocks/test_writernocodeapp.py diff --git a/tests/backend/blocks/conftest.py b/tests/backend/blocks/conftest.py new file mode 100644 index 000000000..93bd7a6f5 --- /dev/null +++ b/tests/backend/blocks/conftest.py @@ -0,0 +1,51 @@ +from typing import Dict +import pytest +from writer.core import WriterSession, WriterState +from writer.core_ui import Branch, Component, ComponentTree, ComponentTreeBranch +from writer.workflows import WorkflowRunner + + +class BlockTesterMockSession(WriterSession): + + def __init__(self): + self.session_state = WriterState({}) + self.bmc_branch = ComponentTreeBranch(Branch.bmc) + component_tree = ComponentTree([self.bmc_branch]) + self.session_component_tree = component_tree + + def add_fake_component(self, content={}, id="fake_id", type="fake_type"): + self.bmc_branch.attach(Component(id=id, type=type, content=content)) + + +class BlockTesterMockWorkflowRunner(WorkflowRunner): + + def __init__(self, session): + super().__init__(session) + + def run_branch(self, component_id: str, base_outcome_id: str, execution_environment: Dict, title: str): + return f"Branch run {component_id} {base_outcome_id}" + + def run_workflow_by_key(self, workflow_key: str, execution_environment: Dict): + if "env_injection_test" in execution_environment: + return execution_environment.get("env_injection_test") + if workflow_key == "workflow1": + return 1 + if workflow_key == "workflowDict": + return { "a": "b" } + if workflow_key == "duplicator": + return execution_environment.get("item") * 2 + if workflow_key == "showId": + return execution_environment.get("itemId") + if workflow_key == "boom": + return 1/0 + raise ValueError("Workflow not found.") + + +@pytest.fixture +def session(): + yield BlockTesterMockSession() + + +@pytest.fixture +def runner(session): + yield BlockTesterMockWorkflowRunner(session) diff --git a/tests/backend/blocks/test_addtostatelist.py b/tests/backend/blocks/test_addtostatelist.py new file mode 100644 index 000000000..ffc222548 --- /dev/null +++ b/tests/backend/blocks/test_addtostatelist.py @@ -0,0 +1,37 @@ +import pytest +from writer.blocks.addtostatelist import AddToStateList +from writer.workflows import WorkflowRunner + + +def test_empty_list(session, runner): + session.add_fake_component({ + "element": "my_list", + "value": "my_value" + }) + block = AddToStateList("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert session.session_state["my_list"] == ["my_value"] + +def test_non_empty_list(session, runner): + session.session_state["my_list"] = ["a"] + session.add_fake_component({ + "element": "my_list", + "value": "b" + }) + block = AddToStateList("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert session.session_state["my_list"] == ["a", "b"] + + +def test_non_list_element(session, runner): + session.session_state["my_element"] = "dog" + session.add_fake_component({ + "element": "my_element", + "value": "cat" + }) + block = AddToStateList("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() + assert block.outcome == "error" \ No newline at end of file diff --git a/tests/backend/blocks/test_base_block.py b/tests/backend/blocks/test_base_block.py new file mode 100644 index 000000000..6e25a2a96 --- /dev/null +++ b/tests/backend/blocks/test_base_block.py @@ -0,0 +1,44 @@ +from writer.blocks.base_block import WorkflowBlock +from writer.core import WriterState + + +def test_get_field(session, runner): + session.session_state = WriterState({ + "animal": "rat", + "my_list": "[1,2,3]", + "my_dict": '{ "a": "b" }' + }) + session.add_fake_component({ + "my_element": "@{animal}", + "my_list": "@{my_list}", + "my_dict": "@{my_dict}" + }) + block = WorkflowBlock("fake_id", runner, {}) + assert "rat" == block._get_field("my_element", as_json=False, default_field_value="elephant") + assert [1, 2, 3] == block._get_field("my_list", as_json=True, default_field_value=None) + assert "b" == block._get_field("my_dict", as_json=True, default_field_value=None).get("a") + assert "ok" == block._get_field("ghost_field", as_json=False, default_field_value="ok") + assert {} == block._get_field("ghost_field", as_json=True) + assert "ok" == block._get_field("ghost_field_json", as_json=True, default_field_value='{ "ok": "ok" }').get("ok") + block.run() + assert block.outcome == None + + +def test_set_state(session, runner): + session.session_state = WriterState({ + "animal": "rat", + "my_list": [1,2,4], + "my_dict": { "animal": "dog"}, + "unchanged": "unchanged" + }) + block = WorkflowBlock("fake_id", runner, {}) + block._set_state("animal", "cat") + block._set_state("animal", "bat") + block._set_state("my_list", [1, 2]) + block._set_state("my_dict", { "animal": "cat"}) + block.run() + assert session.session_state["animal"] == "bat" + assert session.session_state["my_list"] == [1, 2] + assert session.session_state["my_dict"]["animal"] == "cat" + assert session.session_state["unchanged"] == "unchanged" + assert block.outcome == None \ No newline at end of file diff --git a/tests/backend/blocks/test_calleventhandler.py b/tests/backend/blocks/test_calleventhandler.py new file mode 100644 index 000000000..4f3cf52db --- /dev/null +++ b/tests/backend/blocks/test_calleventhandler.py @@ -0,0 +1,48 @@ +import pytest +from writer.blocks.calleventhandler import CallEventHandler +import writer.core + + +def valid_handler(state): + state["animal"] = "duck" + return 1 + +def invalid_handler(state): + state["animal"] = "cat" + return 1/0 + +class MockHandlerRegistry(): + + def find_handler_callable(self, handler_name: str): + if handler_name == "valid_handler": + return valid_handler + elif handler_name == "invalid_handler": + return invalid_handler + raise None + +class MockAppProcess(): + + def __init__(self): + self.handler_registry = MockHandlerRegistry() + +def test_call_event_handler(session, runner): + writer.core.get_app_process = lambda: MockAppProcess() + session.add_fake_component({ + "name": "valid_handler" + }) + block = CallEventHandler("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result == 1 + assert session.session_state["animal"] == "duck" + +def test_invalid_json(session, runner): + writer.core.get_app_process = lambda: MockAppProcess() + session.add_fake_component({ + "name": "invalid_handler" + }) + block = CallEventHandler("fake_id", runner, {}) + with pytest.raises(BaseException): + block.run() + assert block.outcome == "error" + assert session.session_state["animal"] == "cat" \ No newline at end of file diff --git a/tests/backend/blocks/test_foreach.py b/tests/backend/blocks/test_foreach.py new file mode 100644 index 000000000..c81e4e5b6 --- /dev/null +++ b/tests/backend/blocks/test_foreach.py @@ -0,0 +1,52 @@ +import pytest +from writer.blocks.foreach import ForEach + +def test_basic_list(session, runner): + session.add_fake_component({ + "workflowKey": "workflow1", + "items": "[2, 2, 2, 2]" + }) + block = ForEach("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result == [1, 1, 1, 1] + +def test_list_of_dict(session, runner): + session.add_fake_component({ + "workflowKey": "workflowDict", + "items": "[2, 2]" + }) + block = ForEach("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result == [{"a": "b"}, {"a": "b"}] + +def test_basic_dict(session, runner): + session.add_fake_component({ + "workflowKey": "workflow1", + "items": '{"a": "zzz", "b": "zzz"}' + }) + block = ForEach("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result == {"a": 1, "b": 1} + +def test_workflow_that_does_not_exist(session, runner): + session.add_fake_component({ + "workflowKey": "workflowThatDoesNotExist", + "items": '{"a": "zzz", "b": "zzz"}' + }) + block = ForEach("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() + assert block.outcome == "error" + +def test_duplicator(session, runner): + session.add_fake_component({ + "workflowKey": "duplicator", + "items": '{"a": 1, "b": 2, "c": 11}' + }) + block = ForEach("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result == {"a": 2, "b": 4, "c": 22} \ No newline at end of file diff --git a/tests/backend/blocks/test_httprequest.py b/tests/backend/blocks/test_httprequest.py new file mode 100644 index 000000000..6679f6589 --- /dev/null +++ b/tests/backend/blocks/test_httprequest.py @@ -0,0 +1,120 @@ +import json +import pytest +import requests +from writer.blocks.httprequest import HTTPRequest + + +class FakeResponse(): + def __init__(self, status_code=200, ok=True, headers={}, text=None): + self.status_code = status_code + self.ok = ok + self.headers = headers + self.text = text + self.json = lambda: json.loads(text) + + +def fake_request(method, url, headers={}, data=""): + if not headers.get("TestHeader", "not-a-sec-ret"): + raise RuntimeError("Test header not present.") + if method == "GET" and url == "https://www.duck.com": + return FakeResponse( + headers={"Content-Type": "text/plain"}, + text="Ducks are birds." + ) + if method == "POST" and url == "https://www.elephant.com": + return FakeResponse( + headers={"Content-Type": "application/json"}, + text='{ "elephant_name": "Momo", "request_body": "' + data + '" }' + ) + if method == "POST" and url == "https://www.elephant.com/history": + return FakeResponse( + status_code=404, + ok=False, + headers={"Content-Type": "application/json"}, + text='{ "error_message": "Page not found." }' + ) + + raise requests.ConnectionError() + + +@pytest.mark.explicit +def test_actual_request(session, runner): + session.add_fake_component({ + "url": 'https://www.example.com' + }) + block = HTTPRequest("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result.get("headers") is not None + + +@pytest.mark.explicit +def test_actual_failing_request(session, runner): + session.add_fake_component({ + "url": 'https://www.site-that-does-not-exist-3017673369.com' + }) + block = HTTPRequest("fake_id", runner, {}) + with pytest.raises(requests.ConnectionError): + block.run() + assert block.outcome == "connectionError" + + +@pytest.mark.explicit +def test_actual_request_with_bad_path(session, runner): + session.add_fake_component({ + "url": 'https://www.writer.com/3017673369' + }) + block = HTTPRequest("fake_id", runner, {}) + with pytest.raises(RuntimeError): + block.run() + assert block.outcome == "responseError" + + +def test_patched_request(session, runner): + requests.request = fake_request + session.add_fake_component({ + "url": "https://www.duck.com" + }) + block = HTTPRequest("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result.get("body") == "Ducks are birds." + + +def test_patched_request_to_nowhere(session, runner): + requests.request = fake_request + session.add_fake_component({ + "url": "https://www.cat.com" + }) + block = HTTPRequest("fake_id", runner, {}) + with pytest.raises(requests.ConnectionError): + block.run() + assert block.outcome == "connectionError" + + +def test_patched_request_with_json(session, runner): + requests.request = fake_request + session.add_fake_component({ + "url": "https://www.elephant.com", + "method": "POST", + "body": "Posting the elephant." + }) + block = HTTPRequest("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result.get("body").get("elephant_name") == "Momo" + assert block.result.get("body").get("request_body") == "Posting the elephant." + + +def test_patched_request_with_json_and_bad_path(session, runner): + requests.request = fake_request + session.add_fake_component({ + "url": "https://www.elephant.com/history", + "method": "POST", + "body": "Posting the elephant." + }) + block = HTTPRequest("fake_id", runner, {}) + with pytest.raises(RuntimeError): + block.run() + assert block.outcome == "responseError" # due to not "ok" + assert block.result.get("body").get("error_message") == "Page not found." diff --git a/tests/backend/blocks/test_logmessage.py b/tests/backend/blocks/test_logmessage.py new file mode 100644 index 000000000..82cc425ab --- /dev/null +++ b/tests/backend/blocks/test_logmessage.py @@ -0,0 +1,73 @@ +from writer.blocks.logmessage import LogMessage +from writer.core import WriterState +import writer.core +import pytest + + +def test_log_message(session, runner): + session.session_state = WriterState({ + "animal": "rat", + }, [{"type": "test", "payload": "Just a test"}]) + session.add_fake_component({ + "message": "The quick brown fox is under the table." + }) + writer.core.Config.is_mail_enabled_for_log = True + block = LogMessage("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + latest_mail = session.session_state.mail[0] + assert latest_mail.get("type") == "logEntry" + assert latest_mail.get("payload").get("type") == "info" + assert latest_mail.get("payload").get("message") == "The quick brown fox is under the table." + + +def test_log_message_with_template(session, runner): + session.session_state = WriterState({ + "animal": "rat", + }) + session.add_fake_component({ + "message": "The quick brown @{animal} is under the @{object}." + }) + writer.core.Config.is_mail_enabled_for_log = True + block = LogMessage("fake_id", runner, { + "object": "tent" + }) + block.run() + assert block.outcome == "success" + latest_mail = session.session_state.mail[0] + assert latest_mail.get("type") == "logEntry" + assert latest_mail.get("payload").get("type") == "info" + assert latest_mail.get("payload").get("message") == "The quick brown rat is under the tent." + + +def test_log_error_message(session, runner): + session.session_state = WriterState({ + "animal": "squirrel", + }) + session.add_fake_component({ + "type": "error", + "message": "The quick brown @{animal} has escaped." + }) + writer.core.Config.is_mail_enabled_for_log = True + block = LogMessage("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + latest_mail = session.session_state.mail[0] + assert latest_mail.get("type") == "logEntry" + assert latest_mail.get("payload").get("type") == "error" + assert latest_mail.get("payload").get("message") == "The quick brown squirrel has escaped." + + +def test_empty_message(session, runner): + session.session_state = WriterState({ + "animal": None, + }) + session.add_fake_component({ + "type": "error", + "message": "@{animals}" + }) + writer.core.Config.is_mail_enabled_for_log = True + block = LogMessage("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() + assert block.outcome == "error" \ No newline at end of file diff --git a/tests/backend/blocks/test_parsejson.py b/tests/backend/blocks/test_parsejson.py index e7eefca62..3a91e90fd 100644 --- a/tests/backend/blocks/test_parsejson.py +++ b/tests/backend/blocks/test_parsejson.py @@ -1,6 +1,24 @@ +import json +import pytest from writer.blocks.parsejson import ParseJSON +from writer.workflows import WorkflowRunner -def test_valid_json(): - assert 1 == 1 - # tool = ParseJSON() \ No newline at end of file +def test_valid_json(session, runner): + session.add_fake_component({ + "plainText": '{ "hi": "yes" }' + }) + block = ParseJSON("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result.get("hi") == "yes" + +def test_invalid_json(session, runner): + session.add_fake_component({ + "plainText": '{ "hi": yes }' + }) + block = ParseJSON("fake_id", runner, {}) + with pytest.raises(json.JSONDecodeError): + block.run() + + assert block.outcome == "error" \ No newline at end of file diff --git a/tests/backend/blocks/test_returnvalue.py b/tests/backend/blocks/test_returnvalue.py new file mode 100644 index 000000000..6215d7b08 --- /dev/null +++ b/tests/backend/blocks/test_returnvalue.py @@ -0,0 +1,29 @@ +import pytest +from writer.blocks.returnvalue import ReturnValue +from writer.core import WriterState + +def test_basic_return(session, runner): + session.session_state = WriterState({ + "animal": "marmot", + }) + session.add_fake_component({ + "value": "@{animal}" + }) + block = ReturnValue("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result == block.return_value + assert block.return_value == "marmot" + +def test_empty_return(session, runner): + session.session_state = WriterState({ + "animal": None, + }) + session.add_fake_component({ + "value": "@{animal}" + }) + block = ReturnValue("fake_id", runner, {}) + + with pytest.raises(ValueError): + block.run() + assert block.outcome == "error" \ No newline at end of file diff --git a/tests/backend/blocks/test_runworkflow.py b/tests/backend/blocks/test_runworkflow.py new file mode 100644 index 000000000..e74b59ad5 --- /dev/null +++ b/tests/backend/blocks/test_runworkflow.py @@ -0,0 +1,30 @@ +import pytest +from writer.blocks.runworkflow import RunWorkflow + +def test_workflow_that_does_not_exist(session, runner): + session.add_fake_component({ + "workflowKey": "workflowThatDoesNotExist", + }) + block = RunWorkflow("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() + assert block.outcome == "error" + +def test_duplicator(session, runner): + session.add_fake_component({ + "workflowKey": "duplicator", + "executionEnv": '{"item": 23}' + }) + block = RunWorkflow("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert block.result == 46 + +def test_bad_workflow(session, runner): + session.add_fake_component({ + "workflowKey": "boom" + }) + block = RunWorkflow("fake_id", runner, {}) + with pytest.raises(BaseException): + block.run() + assert block.outcome == "error" \ No newline at end of file diff --git a/tests/backend/blocks/test_setstate.py b/tests/backend/blocks/test_setstate.py new file mode 100644 index 000000000..82bfbdee3 --- /dev/null +++ b/tests/backend/blocks/test_setstate.py @@ -0,0 +1,61 @@ +import pytest +from writer.blocks.setstate import SetState +from writer.workflows import WorkflowRunner + + +def test_basic_assignment(session): + session.add_fake_component({ + "element": "my_element", + "value": "my_value" + }) + runner = WorkflowRunner(session) + block = SetState("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert session.session_state["my_element"] == "my_value" + +def test_nested_assignment_without_parent(session): + session.add_fake_component({ + "element": "parent_element.my_element", + "value": "my_value" + }) + runner = WorkflowRunner(session) + block = SetState("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() + assert block.outcome == "error" + +def test_nested_assignment_with_parent(session, runner): + session.session_state["parent_element"] = { + "sibling_element": "yes" + } + session.add_fake_component({ + "element": "parent_element.my_element", + "value": "my_value" + }) + block = SetState("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + assert session.session_state["parent_element"]["sibling_element"] == "yes" + assert session.session_state["parent_element"]["my_element"] == "my_value" + +def test_assignment_with_empty_element(session): + session.add_fake_component({ + "element": "", + "value": "my_value" + }) + runner = WorkflowRunner(session) + block = SetState("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() + assert block.outcome == "error" + +def test_assignment_with_empty_value(session): + session.add_fake_component({ + "element": "my_element", + "value": "" + }) + runner = WorkflowRunner(session) + block = SetState("fake_id", runner, {}) + block.run() + assert block.outcome == "success" \ No newline at end of file diff --git a/tests/backend/blocks/test_writeraddchatmessage.py b/tests/backend/blocks/test_writeraddchatmessage.py new file mode 100644 index 000000000..1bd067b18 --- /dev/null +++ b/tests/backend/blocks/test_writeraddchatmessage.py @@ -0,0 +1,50 @@ +import pytest +from writer.blocks.writeraddchatmessage import WriterAddChatMessage +import writer.ai + + +def test_add_chat_message(session, runner): + session.session_state["convo"] = writer.ai.Conversation() + session.add_fake_component({ + "conversationStateElement": "convo", + "message": '{ "role": "user", "content": "hi" }' + }) + block = WriterAddChatMessage("fake_id", runner, {}) + assert len(session.session_state["convo"].messages) == 0 + block.run() + assert len(session.session_state["convo"].messages) == 1 + + +def test_add_chat_message_bad_template(session, runner): + session.session_state["convo"] = writer.ai.Conversation() + session.add_fake_component({ + "conversationStateElement": "@{convo}", # Should be convo not @{convo} + "message": '{ "role": "user", "content": "hi" }' + }) + block = WriterAddChatMessage("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() + + +def test_add_chat_message_bad_message(session, runner): + session.session_state["convo"] = writer.ai.Conversation() + session.add_fake_component({ + "conversationStateElement": "convo", # Should be convo not @{convo} + "message": '{ "x": "user", "content": "hi" }' + }) + block = WriterAddChatMessage("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() + + +def test_add_chat_message_missing_convo(session, runner): + session.session_state["convo"] = None + session.add_fake_component({ + "conversationStateElement": "@{convo}", + "message": '{ "role": "user", "content": "hi" }' + }) + block = WriterAddChatMessage("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() + + diff --git a/tests/backend/blocks/test_writerchat.py b/tests/backend/blocks/test_writerchat.py new file mode 100644 index 000000000..03e6c59d8 --- /dev/null +++ b/tests/backend/blocks/test_writerchat.py @@ -0,0 +1,124 @@ +import json +import pytest +from writer.blocks.writerchat import WriterChat +from writer.ai import Conversation + + +class MockConversation(Conversation): + + def __init__(self): + super().__init__() + + def _check_tools(self, tools): + if tools is None or tools == []: + return + if len(tools) != 2: + raise RuntimeError("Invalid number of tools.") + function_tool = tools[0] + + assert function_tool.get("type") == "function" + assert function_tool.get("name") == "bat_locator" + assert function_tool.get("description") == "Locates bats." + assert function_tool.get("parameters").get("color") == { + "type": "string", + "description": "The color of the bat you're looking for." + } + graph_tool = tools[1] + assert graph_tool.get("type") == "graph" + assert graph_tool.get("graph_ids") == [111, 112, 113] + + def complete(self, tools=None): + self._check_tools(tools) + return { + "role": "assistant", + "content": "Next to the grill." + } + + def stream_complete(self, tools=None): + self._check_tools(tools) + yield { + "role": "assistant", + "content": "On " + } + yield { + "role": "assistant", + "content": "the ", + "chunk": True + } + yield { + "role": "assistant", + "content": "car's ", + "chunk": True + } + yield { + "role": "assistant", + "content": "roof.", + "chunk": True + } + +@pytest.fixture +def conversation(): + return MockConversation() + +def test_chat_complete(session, runner, conversation): + conversation.add("user", "Hi, where's the bat?") + session.session_state["convo"] = conversation + session.add_fake_component({ + "conversationStateElement": "convo", + "useStreaming": "no" + }) + block = WriterChat("fake_id", runner, {}) + block.run() + assert conversation.messages[1].get("content") == "Next to the grill." + + +def test_chat_stream_complete(session, runner, conversation): + conversation.add("user", "Hi, where's the bat?") + session.session_state["convo"] = conversation + session.add_fake_component({ + "conversationStateElement": "convo" + # streaming should be default + }) + block = WriterChat("fake_id", runner, {}) + block.run() + assert conversation.messages[1].get("content") == "On the car's roof." + + +def test_chat_stream_complete_with_tools(session, runner, conversation): + conversation.add("user", "Hi, where's the bat?") + session.session_state["convo"] = conversation + session.add_fake_component({ + "conversationStateElement": "convo", + "tools": json.dumps({ + "bat_locator": { + "type": "function", + "description": "Locates bats.", + "parameters": { + "color": { + "type": "string", + "description": "The color of the bat you're looking for." + } + } + }, + "known_bat_spots": { + "type": "graph", + "graph_ids": [111, 112, 113] + } + }), + "useStreaming": "yes" + }) + block = WriterChat("fake_id", runner, {}) + block.run() + assert conversation.messages[1].get("content") == "On the car's roof." + + +def test_chat_stream_complete_no_conversation(session, runner, conversation): + conversation.add("user", "Hi, where's the bat?") + session.session_state["convo"] = "not_a_conversation" + session.add_fake_component({ + "conversationStateElement": "convo" + # streaming should be default + }) + block = WriterChat("fake_id", runner, {}) + with pytest.raises(ValueError): + block.run() \ No newline at end of file diff --git a/tests/backend/blocks/test_writerclassification.py b/tests/backend/blocks/test_writerclassification.py new file mode 100644 index 000000000..66148b81c --- /dev/null +++ b/tests/backend/blocks/test_writerclassification.py @@ -0,0 +1,40 @@ +import json +import pytest +from writer.blocks.writerclassification import WriterClassification +import writer.ai + +def fake_complete(prompt, config): + additional_context = "It's about animal classification." + if "canine" in prompt and additional_context in prompt: + return "dog" + if "feline" in prompt and additional_context in prompt: + return "cat" + return "other" + + +def test_classify(session, runner): + writer.ai.complete = fake_complete + session.add_fake_component({ + "text": "canine", + "categories": json.dumps({ + "cat": "Pertaining to cats.", + "dog": "Pertaining to dogs." + }), + "additionalContext": "It's about animal classification." + }) + block = WriterClassification("fake_id", runner, {}) + block.run() + assert block.result == "dog" + assert block.outcome == "category_dog" + + +def test_classify_missing_categories(session, runner): + writer.ai.complete = fake_complete + session.add_fake_component({ + "text": "canine", + "categories": json.dumps({}) + }) + block = WriterClassification("fake_id", runner, {}) + + with pytest.raises(ValueError): + block.run() \ No newline at end of file diff --git a/tests/backend/blocks/test_writercompletion.py b/tests/backend/blocks/test_writercompletion.py new file mode 100644 index 000000000..daae69795 --- /dev/null +++ b/tests/backend/blocks/test_writercompletion.py @@ -0,0 +1,42 @@ +from writer.blocks.writercompletion import WriterCompletion +import writer.ai + + +def test_complete(session, runner): + def fake_complete(prompt, config): + assert config.get("temperature") == 0.9 + assert config.get("model") == "buenos-aires-x-004" + assert prompt == "What color is the sea?" + return "Blue." + + writer.ai.complete = fake_complete + session.add_fake_component({ + "prompt": "What color is the sea?", + "modelId": "buenos-aires-x-004", + "temperature": "0.9" + }) + block = WriterCompletion("fake_id", runner, {}) + block.run() + assert block.result == "Blue." + assert block.outcome == "success" + +def test_complete_missing_text(session, runner): + def fake_complete(prompt, config): + assert config.get("temperature") == 0.9 + assert config.get("model") == "buenos-aires-x-004" + assert not prompt + return "Plants are usually green." + + writer.ai.complete = fake_complete + session.add_fake_component({ + "prompt": "", + "modelId": "buenos-aires-x-004", + "temperature": "0.9" + }) + block = WriterCompletion("fake_id", runner, {}) + + # Not expected to fail, just hallucinate + + block.run() + assert block.result == "Plants are usually green." + assert block.outcome == "success" \ No newline at end of file diff --git a/tests/backend/blocks/test_writerinitchat.py b/tests/backend/blocks/test_writerinitchat.py new file mode 100644 index 000000000..a4e15af7f --- /dev/null +++ b/tests/backend/blocks/test_writerinitchat.py @@ -0,0 +1,33 @@ +import pytest +from writer.blocks.writerinitchat import WriterInitChat +from writer.ai import Conversation + + +def test_init_chat_already_initialized(session, runner): + session.session_state["convo"] = Conversation() + session.add_fake_component({ + "conversationStateElement": "convo", + }) + block = WriterInitChat("fake_id", runner, {}) + block.run() + assert block.outcome == "success" + + +def test_init_chat_already_initialized_with_rubbish(session, runner): + session.session_state["convo"] = "-hello -hello there. This is a conversation but not the right kind." + session.add_fake_component({ + "conversationStateElement": "convo", + }) + block = WriterInitChat("fake_id", runner, {}) + + with pytest.raises(ValueError): + block.run() + +def test_init_chat_from_scratch(session, runner): + session.add_fake_component({ + "conversationStateElement": "convo", + }) + block = WriterInitChat("fake_id", runner, {}) + block.run() + assert isinstance(session.session_state["convo"], Conversation) + assert block.outcome == "success" \ No newline at end of file diff --git a/tests/backend/blocks/test_writernocodeapp.py b/tests/backend/blocks/test_writernocodeapp.py new file mode 100644 index 000000000..b708f4ea2 --- /dev/null +++ b/tests/backend/blocks/test_writernocodeapp.py @@ -0,0 +1,41 @@ +import json +import pytest +from writer.blocks.writernocodeapp import WriterNoCodeApp +import writer.ai + +def fake_generate_content(application_id, app_inputs): + assert application_id == "123" + + name = app_inputs.get("name") + animal = app_inputs.get("animal") + + return f"{name} the {animal} " + +def test_call_nocode_app(session, runner): + writer.ai.apps.generate_content = fake_generate_content + session.add_fake_component({ + "appId": "123", + "appInputs": json.dumps({ + "name": "Koko", + "animal": "Hamster" + }) + }) + block = WriterNoCodeApp("fake_id", runner, {}) + block.run() + assert block.result == "Koko the Hamster" + assert block.outcome == "success" + + +def test_call_nocode_app_missing_appid(session, runner): + writer.ai.apps.generate_content = fake_generate_content + session.add_fake_component({ + "appId": "", + "appInputs": json.dumps({ + "name": "Momo", + "animal": "Squirrel" + }) + }) + block = WriterNoCodeApp("fake_id", runner, {}) + + with pytest.raises(ValueError): + block.run() \ No newline at end of file From 2f72d1282aea7adbcbfe65c6d493a1a01bf1d280 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:42:54 +0000 Subject: [PATCH 3/5] chore: Refactoring, logging improvements --- .../builder/BuilderLogWorkflowExecution.vue | 21 +++++- src/ui/src/builder/builderManager.ts | 1 + src/writer/blocks/addtostatelist.py | 6 +- src/writer/blocks/base_block.py | 23 +++++- src/writer/blocks/calleventhandler.py | 2 +- src/writer/blocks/httprequest.py | 7 +- src/writer/blocks/logmessage.py | 9 +-- src/writer/blocks/returnvalue.py | 4 +- src/writer/blocks/setstate.py | 3 +- src/writer/blocks/writeraddchatmessage.py | 30 ++------ src/writer/blocks/writerchat.py | 46 +++++------- src/writer/blocks/writerclassification.py | 14 ++-- src/writer/blocks/writerinitchat.py | 10 +-- src/writer/blocks/writernocodeapp.py | 2 +- src/writer/evaluator.py | 16 ++-- src/writer/ss_types.py | 6 +- src/writer/workflows.py | 73 ++++++++++--------- 17 files changed, 141 insertions(+), 132 deletions(-) diff --git a/src/ui/src/builder/BuilderLogWorkflowExecution.vue b/src/ui/src/builder/BuilderLogWorkflowExecution.vue index 62171723c..133fa97a2 100644 --- a/src/ui/src/builder/BuilderLogWorkflowExecution.vue +++ b/src/ui/src/builder/BuilderLogWorkflowExecution.vue @@ -119,12 +119,18 @@
{{ formatExecutionTime(item.executionTimeInSeconds) }}
+