diff --git a/pyproject.toml b/pyproject.toml index 873a2dfd..1b1c481c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "semantic-router" -version = "0.0.36" +version = "0.0.37" description = "Super fast semantic router for AI decision making" authors = [ "James Briggs ", diff --git a/semantic_router/__init__.py b/semantic_router/__init__.py index 39506cc5..cd0a624b 100644 --- a/semantic_router/__init__.py +++ b/semantic_router/__init__.py @@ -4,4 +4,4 @@ __all__ = ["RouteLayer", "HybridRouteLayer", "Route", "LayerConfig"] -__version__ = "0.0.36" +__version__ = "0.0.37" diff --git a/semantic_router/encoders/openai.py b/semantic_router/encoders/openai.py index d56a1e71..3cfa2a70 100644 --- a/semantic_router/encoders/openai.py +++ b/semantic_router/encoders/openai.py @@ -114,7 +114,8 @@ def __call__(self, docs: List[str], truncate: bool = True) -> List[List[float]]: return embeddings def _truncate(self, text: str) -> str: - tokens = self._token_encoder.encode(text) + # we use encode_ordinary as faster equivalent to encode(text, disallowed_special=()) + tokens = self._token_encoder.encode_ordinary(text) if len(tokens) > self.token_limit: logger.warning( f"Document exceeds token limit: {len(tokens)} > {self.token_limit}"