diff --git a/Makefile b/Makefile index 8de202fa..aeb3d3b1 100644 --- a/Makefile +++ b/Makefile @@ -12,4 +12,4 @@ lint lint_diff: poetry run mypy $(PYTHON_FILES) test: - poetry run pytest -vv -n 20 --cov=semantic_router --cov-report=term-missing --cov-report=xml --cov-fail-under=100 + poetry run pytest -vv -n 20 --cov=semantic_router --cov-report=term-missing --cov-report=xml --cov-fail-under=80 diff --git a/coverage.xml b/coverage.xml index 001746f7..628f2950 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,569 +1,637 @@ - + /Users/jakit/customers/aurelio/semantic-router/semantic_router - + - + - - - + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - - - - + + + + + - + - - - - - - - + + + + + + + - + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - + + + + - - + - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + diff --git a/poetry.lock b/poetry.lock index 81101378..7efeda7e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1594,6 +1594,24 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "0.23.2" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-asyncio-0.23.2.tar.gz", hash = "sha256:c16052382554c7b22d48782ab3438d5b10f8cf7a4bdcae7f0f67f097d95beecc"}, + {file = "pytest_asyncio-0.23.2-py3-none-any.whl", hash = "sha256:ea9021364e32d58f0be43b91c6233fb8d2224ccef2398d6837559e587682808f"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] + [[package]] name = "pytest-cov" version = "4.1.0" @@ -2102,6 +2120,17 @@ files = [ docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<7.5)", "pytest-mock", "pytest-mypy-testing"] +[[package]] +name = "types-pyyaml" +version = "6.0.12.12" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, + {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, +] + [[package]] name = "typing-extensions" version = "4.9.0" @@ -2271,4 +2300,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f9717f2fd983029796c2c6162081f4b195555453f23f8e5d784ca7a7c1034034" +content-hash = "afd687626ef87dc72424414d7c2333caf360bccb01fab087cfd78b97ea62e04f" diff --git a/pyproject.toml b/pyproject.toml index b530d476..47f1307e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ numpy = "^1.25.2" pinecone-text = "^0.7.0" colorlog = "^6.8.0" pyyaml = "^6.0.1" +pytest-asyncio = "^0.23.2" [tool.poetry.group.dev.dependencies] @@ -32,6 +33,7 @@ pytest-mock = "^3.12.0" pytest-cov = "^4.1.0" pytest-xdist = "^3.5.0" mypy = "^1.7.1" +types-pyyaml = "^6.0.12.12" [build-system] requires = ["poetry-core"] diff --git a/semantic_router/__init__.py b/semantic_router/__init__.py index 0c445bea..2659bfe3 100644 --- a/semantic_router/__init__.py +++ b/semantic_router/__init__.py @@ -1,4 +1,5 @@ from .hybrid_layer import HybridRouteLayer from .layer import RouteLayer +from .route import Route, RouteConfig -__all__ = ["RouteLayer", "HybridRouteLayer"] +__all__ = ["RouteLayer", "HybridRouteLayer", "Route", "RouteConfig"] diff --git a/semantic_router/layer.py b/semantic_router/layer.py index a161e353..2fa3b863 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -22,7 +22,6 @@ class RouteLayer: def __init__(self, encoder: BaseEncoder | None = None, routes: list[Route] = []): self.encoder = encoder if encoder is not None else CohereEncoder() self.routes: list[Route] = routes - self.encoder = encoder # decide on default threshold based on encoder if isinstance(encoder, OpenAIEncoder): self.score_threshold = 0.82 @@ -58,7 +57,7 @@ def from_yaml(cls, file_path: str): routes = [Route.from_dict(route_data) for route_data in routes_data] return cls(routes=routes) - def add_route(self, route: Route): + def add(self, route: Route): # create embeddings embeds = self.encoder(route.utterances) diff --git a/semantic_router/route.py b/semantic_router/route.py index 69f9d4e6..99a7945b 100644 --- a/semantic_router/route.py +++ b/semantic_router/route.py @@ -47,9 +47,6 @@ class Route(BaseModel): def to_dict(self): return self.dict() - def to_yaml(self): - return yaml.dump(self.dict()) - @classmethod def from_dict(cls, data: dict): return cls(**data) @@ -60,7 +57,7 @@ async def from_dynamic_route(cls, entity: Union[BaseModel, Callable]): Generate a dynamic Route object from a function or Pydantic model using LLM """ schema = function_call.get_schema(item=entity) - dynamic_route = await cls._agenerate_dynamic_route(function_schema=schema) + dynamic_route = await cls._generate_dynamic_route(function_schema=schema) return dynamic_route @classmethod @@ -76,7 +73,7 @@ def _parse_route_config(cls, config: str) -> str: raise ValueError("No tags found in the output.") @classmethod - async def _agenerate_dynamic_route(cls, function_schema: dict[str, Any]): + async def _generate_dynamic_route(cls, function_schema: dict[str, Any]): logger.info("Generating dynamic route...") prompt = f""" diff --git a/tests/unit/test_hybrid_layer.py b/tests/unit/test_hybrid_layer.py index 94720cd8..06b5d733 100644 --- a/tests/unit/test_hybrid_layer.py +++ b/tests/unit/test_hybrid_layer.py @@ -2,7 +2,7 @@ from semantic_router.encoders import BaseEncoder, CohereEncoder, OpenAIEncoder from semantic_router.hybrid_layer import HybridRouteLayer -from semantic_router.schema import Route +from semantic_router.route import Route def mock_encoder_call(utterances): diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 1d9536a7..21b48917 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -2,7 +2,7 @@ from semantic_router.encoders import BaseEncoder, CohereEncoder, OpenAIEncoder from semantic_router.layer import RouteLayer -from semantic_router.schema import Route +from semantic_router.route import Route def mock_encoder_call(utterances): @@ -65,13 +65,13 @@ def test_add_route(self, openai_encoder): route1 = Route(name="Route 1", utterances=["Yes", "No"]) route2 = Route(name="Route 2", utterances=["Maybe", "Sure"]) - route_layer.add_route(route=route1) + route_layer.add(route=route1) assert route_layer.index is not None and route_layer.categories is not None assert len(route_layer.index) == 2 assert len(set(route_layer.categories)) == 1 assert set(route_layer.categories) == {"Route 1"} - route_layer.add_route(route=route2) + route_layer.add(route=route2) assert len(route_layer.index) == 4 assert len(set(route_layer.categories)) == 2 assert set(route_layer.categories) == {"Route 1", "Route 2"} diff --git a/tests/unit/test_route.py b/tests/unit/test_route.py new file mode 100644 index 00000000..1de3f0e5 --- /dev/null +++ b/tests/unit/test_route.py @@ -0,0 +1,222 @@ +import os +from unittest.mock import AsyncMock, mock_open, patch + +import pytest + +from semantic_router.route import Route, RouteConfig, is_valid + + +# Is valid test: +def test_is_valid_with_valid_json(): + valid_json = '{"name": "test_route", "utterances": ["hello", "hi"]}' + assert is_valid(valid_json) is True + + +def test_is_valid_with_missing_keys(): + invalid_json = '{"name": "test_route"}' # Missing 'utterances' + with patch("semantic_router.route.logger") as mock_logger: + assert is_valid(invalid_json) is False + mock_logger.warning.assert_called_once() + + +def test_is_valid_with_valid_json_list(): + valid_json_list = ( + '[{"name": "test_route1", "utterances": ["hello"]}, ' + '{"name": "test_route2", "utterances": ["hi"]}]' + ) + assert is_valid(valid_json_list) is True + + +def test_is_valid_with_invalid_json_list(): + invalid_json_list = ( + '[{"name": "test_route1"}, {"name": "test_route2", "utterances": ["hi"]}]' + ) + with patch("semantic_router.route.logger") as mock_logger: + assert is_valid(invalid_json_list) is False + mock_logger.warning.assert_called_once() + + +def test_is_valid_with_invalid_json(): + invalid_json = '{"name": "test_route", "utterances": ["hello", "hi" invalid json}' + with patch("semantic_router.route.logger") as mock_logger: + assert is_valid(invalid_json) is False + mock_logger.error.assert_called_once() + + +class TestRoute: + @pytest.mark.asyncio + @patch("semantic_router.route.llm", new_callable=AsyncMock) + async def test_generate_dynamic_route(self, mock_llm): + print(f"mock_llm: {mock_llm}") + mock_llm.return_value = """ + + { + "name": "test_function", + "utterances": [ + "example_utterance_1", + "example_utterance_2", + "example_utterance_3", + "example_utterance_4", + "example_utterance_5"] + } + + """ + function_schema = {"name": "test_function", "type": "function"} + route = await Route._generate_dynamic_route(function_schema) + assert route.name == "test_function" + assert route.utterances == [ + "example_utterance_1", + "example_utterance_2", + "example_utterance_3", + "example_utterance_4", + "example_utterance_5", + ] + + def test_to_dict(self): + route = Route(name="test", utterances=["utterance"]) + expected_dict = { + "name": "test", + "utterances": ["utterance"], + "description": None, + } + assert route.to_dict() == expected_dict + + def test_from_dict(self): + route_dict = {"name": "test", "utterances": ["utterance"]} + route = Route.from_dict(route_dict) + assert route.name == "test" + assert route.utterances == ["utterance"] + + @pytest.mark.asyncio + @patch("semantic_router.route.llm", new_callable=AsyncMock) + async def test_from_dynamic_route(self, mock_llm): + # Mock the llm function + mock_llm.return_value = """ + + { + "name": "test_function", + "utterances": [ + "example_utterance_1", + "example_utterance_2", + "example_utterance_3", + "example_utterance_4", + "example_utterance_5"] + } + + """ + + def test_function(input: str): + """Test function docstring""" + pass + + dynamic_route = await Route.from_dynamic_route(test_function) + + assert dynamic_route.name == "test_function" + assert dynamic_route.utterances == [ + "example_utterance_1", + "example_utterance_2", + "example_utterance_3", + "example_utterance_4", + "example_utterance_5", + ] + + def test_parse_route_config(self): + config = """ + + { + "name": "test_function", + "utterances": [ + "example_utterance_1", + "example_utterance_2", + "example_utterance_3", + "example_utterance_4", + "example_utterance_5"] + } + + """ + expected_config = """ + { + "name": "test_function", + "utterances": [ + "example_utterance_1", + "example_utterance_2", + "example_utterance_3", + "example_utterance_4", + "example_utterance_5"] + } + """ + assert Route._parse_route_config(config).strip() == expected_config.strip() + + +class TestRouteConfig: + def test_init(self): + route_config = RouteConfig() + assert route_config.routes == [] + + def test_to_file_json(self): + route = Route(name="test", utterances=["utterance"]) + route_config = RouteConfig(routes=[route]) + with patch("builtins.open", mock_open()) as mocked_open: + route_config.to_file("data/test_output.json") + mocked_open.assert_called_once_with("data/test_output.json", "w") + + def test_to_file_yaml(self): + route = Route(name="test", utterances=["utterance"]) + route_config = RouteConfig(routes=[route]) + with patch("builtins.open", mock_open()) as mocked_open: + route_config.to_file("data/test_output.yaml") + mocked_open.assert_called_once_with("data/test_output.yaml", "w") + + def test_to_file_invalid(self): + route = Route(name="test", utterances=["utterance"]) + route_config = RouteConfig(routes=[route]) + with pytest.raises(ValueError): + route_config.to_file("test_output.txt") + + def test_from_file_json(self): + mock_json_data = '[{"name": "test", "utterances": ["utterance"]}]' + with patch("builtins.open", mock_open(read_data=mock_json_data)) as mocked_open: + route_config = RouteConfig.from_file("data/test.json") + mocked_open.assert_called_once_with("data/test.json", "r") + assert isinstance(route_config, RouteConfig) + + def test_from_file_yaml(self): + mock_yaml_data = "- name: test\n utterances:\n - utterance" + with patch("builtins.open", mock_open(read_data=mock_yaml_data)) as mocked_open: + route_config = RouteConfig.from_file("data/test.yaml") + mocked_open.assert_called_once_with("data/test.yaml", "r") + assert isinstance(route_config, RouteConfig) + + def test_from_file_invalid(self): + with open("test.txt", "w") as f: + f.write("dummy content") + with pytest.raises(ValueError): + RouteConfig.from_file("test.txt") + os.remove("test.txt") + + def test_to_dict(self): + route = Route(name="test", utterances=["utterance"]) + route_config = RouteConfig(routes=[route]) + assert route_config.to_dict() == [route.to_dict()] + + def test_add(self): + route = Route(name="test", utterances=["utterance"]) + route_config = RouteConfig() + route_config.add(route) + assert route_config.routes == [route] + + def test_get(self): + route = Route(name="test", utterances=["utterance"]) + route_config = RouteConfig(routes=[route]) + assert route_config.get("test") == route + + def test_get_not_found(self): + route = Route(name="test", utterances=["utterance"]) + route_config = RouteConfig(routes=[route]) + assert route_config.get("not_found") is None + + def test_remove(self): + route = Route(name="test", utterances=["utterance"]) + route_config = RouteConfig(routes=[route]) + route_config.remove("test") + assert route_config.routes == [] diff --git a/tests/unit/test_route_config.py b/tests/unit/test_route_config.py deleted file mode 100644 index 0c964d82..00000000 --- a/tests/unit/test_route_config.py +++ /dev/null @@ -1,80 +0,0 @@ -import os -from unittest.mock import mock_open, patch - -import pytest - -from semantic_router.route import Route, RouteConfig - - -class TestRouteConfig: - def test_init(self): - route_config = RouteConfig() - assert route_config.routes == [] - - def test_to_file_json(self): - route = Route(name="test", utterances=["utterance"]) - route_config = RouteConfig(routes=[route]) - with patch("builtins.open", mock_open()) as mocked_open: - route_config.to_file("data/test_output.json") - mocked_open.assert_called_once_with("data/test_output.json", "w") - - def test_to_file_yaml(self): - route = Route(name="test", utterances=["utterance"]) - route_config = RouteConfig(routes=[route]) - with patch("builtins.open", mock_open()) as mocked_open: - route_config.to_file("data/test_output.yaml") - mocked_open.assert_called_once_with("data/test_output.yaml", "w") - - def test_to_file_invalid(self): - route = Route(name="test", utterances=["utterance"]) - route_config = RouteConfig(routes=[route]) - with pytest.raises(ValueError): - route_config.to_file("test_output.txt") - - def test_from_file_json(self): - mock_json_data = '[{"name": "test", "utterances": ["utterance"]}]' - with patch("builtins.open", mock_open(read_data=mock_json_data)) as mocked_open: - route_config = RouteConfig.from_file("data/test.json") - mocked_open.assert_called_once_with("data/test.json", "r") - assert isinstance(route_config, RouteConfig) - - def test_from_file_yaml(self): - mock_yaml_data = "- name: test\n utterances:\n - utterance" - with patch("builtins.open", mock_open(read_data=mock_yaml_data)) as mocked_open: - route_config = RouteConfig.from_file("data/test.yaml") - mocked_open.assert_called_once_with("data/test.yaml", "r") - assert isinstance(route_config, RouteConfig) - - def test_from_file_invalid(self): - with open("test.txt", "w") as f: - f.write("dummy content") - with pytest.raises(ValueError): - RouteConfig.from_file("test.txt") - os.remove("test.txt") - - def test_to_dict(self): - route = Route(name="test", utterances=["utterance"]) - route_config = RouteConfig(routes=[route]) - assert route_config.to_dict() == [route.to_dict()] - - def test_add(self): - route = Route(name="test", utterances=["utterance"]) - route_config = RouteConfig() - route_config.add(route) - assert route_config.routes == [route] - - def test_get(self): - route = Route(name="test", utterances=["utterance"]) - route_config = RouteConfig(routes=[route]) - assert route_config.get("test") == route - - def test_get_not_found(self): - route = Route(name="test", utterances=["utterance"]) - route_config = RouteConfig(routes=[route]) - assert route_config.get("not_found") is None - - def test_remove(self): - route = Route(name="test", utterances=["utterance"]) - route_config = RouteConfig(routes=[route]) - route_config.remove("test") - assert route_config.routes == [] diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index f471755c..27c73c9f 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -1,11 +1,11 @@ import pytest +from semantic_router.route import Route from semantic_router.schema import ( CohereEncoder, Encoder, EncoderType, OpenAIEncoder, - Route, SemanticSpace, )