Skip to content

Commit

Permalink
fix: sync issues
Browse files Browse the repository at this point in the history
  • Loading branch information
jamescalam committed Nov 28, 2024
1 parent 362e98e commit 7a6e823
Show file tree
Hide file tree
Showing 10 changed files with 137 additions and 36 deletions.
10 changes: 5 additions & 5 deletions docs/indexes/pinecone-sync-routes.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,14 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## RouteLayer"
"## SemanticRouter"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The `RouteLayer` class supports both sync and async operations by default, so we initialize as usual:"
"The `SemanticRouter` class supports both sync and async operations by default, so we initialize as usual:"
]
},
{
Expand Down Expand Up @@ -160,10 +160,10 @@
}
],
"source": [
"from semantic_router.routers import RouteLayer\n",
"from semantic_router.routers import SemanticRouter\n",
"import time\n",
"\n",
"rl = RouteLayer(encoder=encoder, routes=routes, index=pc_index, auto_sync=\"local\")\n",
"rl = SemanticRouter(encoder=encoder, routes=routes, index=pc_index, auto_sync=\"local\")\n",
"# due to pinecone indexing latency we wait 3 seconds\n",
"time.sleep(3)"
]
Expand Down Expand Up @@ -210,7 +210,7 @@
"source": [
"del rl\n",
"\n",
"rl = RouteLayer(encoder=encoder, routes=[politics], index=pc_index)\n",
"rl = SemanticRouter(encoder=encoder, routes=[politics], index=pc_index)\n",
"time.sleep(3)"
]
},
Expand Down
82 changes: 70 additions & 12 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ boto3 = { version = "^1.34.98", optional = true }
botocore = {version = "^1.34.110", optional = true}
aiohttp = "^3.10.11"
fastembed = {version = "^0.3.0", optional = true}
psycopg2 = {version = "^2.9.9", optional = true}
psycopg2-binary = {version = "^2.9.9", optional = true}
sphinx = {version = "^7.0.0", optional = true}
sphinxawesome-theme = {version = "^5.2.0", optional = true}
tornado = {version = "^6.4.2", optional = true}
Expand Down
1 change: 0 additions & 1 deletion semantic_router/encoders/bm25.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ def __call__(self, docs: List[str]) -> list[SparseEmbedding]:
if len(docs) == 1:
sparse_dicts = self.model.encode_queries(docs)
elif len(docs) > 1:
print(docs)
sparse_dicts = self.model.encode_documents(docs)
else:
raise ValueError("No documents to encode.")
Expand Down
2 changes: 2 additions & 0 deletions semantic_router/index/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from semantic_router.index.hybrid_local import HybridLocalIndex
from semantic_router.index.local import LocalIndex
from semantic_router.index.pinecone import PineconeIndex
from semantic_router.index.postgres import PostgresIndex
from semantic_router.index.qdrant import QdrantIndex

__all__ = [
Expand All @@ -10,4 +11,5 @@
"LocalIndex",
"QdrantIndex",
"PineconeIndex",
"PostgresIndex",
]
1 change: 0 additions & 1 deletion semantic_router/index/pinecone.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,6 @@ def query(
top_k: int = 5,
route_filter: Optional[List[str]] = None,
sparse_vector: dict[int, float] | SparseEmbedding | None = None,
**kwargs: Any,
) -> Tuple[np.ndarray, List[str]]:
"""Search the index for the query vector and return the top_k results.
Expand Down
17 changes: 13 additions & 4 deletions semantic_router/index/postgres.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
import os
import uuid
from enum import Enum
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING, Union

import numpy as np
import psycopg2
from pydantic import BaseModel
from pydantic import BaseModel, Field

from semantic_router.index.base import BaseIndex
from semantic_router.schema import ConfigParameter, Metric, SparseEmbedding
from semantic_router.utils.logger import logger

if TYPE_CHECKING:
import psycopg2


class MetricPgVecOperatorMap(Enum):
"""
Expand Down Expand Up @@ -104,8 +106,9 @@ class PostgresIndex(BaseIndex):
dimensions: int = 1536
metric: Metric = Metric.COSINE
namespace: Optional[str] = ""
conn: Optional[psycopg2.extensions.connection] = None
conn: Optional["psycopg2.extensions.connection"] = None
type: str = "postgres"
pg2: Any = Field(default=None, exclude=True)

def __init__(
self,
Expand Down Expand Up @@ -133,6 +136,12 @@ def __init__(
:type namespace: Optional[str]
"""
super().__init__()
# try and import psycopg2
try:
import psycopg2
except ImportError:
raise ImportError("Please install psycopg2 to use PostgresIndex. "
"You can install it with: `pip install 'semantic-router[postgres]'`")
if connection_string:
self.connection_string = connection_string
else:
Expand Down
4 changes: 0 additions & 4 deletions semantic_router/routers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -497,14 +497,10 @@ def retrieve_multiple_routes(
vector_arr = self._encode(text=[text])
else:
vector_arr = np.array(vector)
print(f"{text=}")
print(f"{vector_arr}")
# get relevant utterances
results = self._retrieve(xq=vector_arr)
print(f"{results=}")
# decide most relevant routes
categories_with_scores = self._semantic_classify_multiple_routes(results)
print(f"{categories_with_scores=}")
return [
RouteChoice(name=category, similarity_score=score)
for category, score in categories_with_scores
Expand Down
3 changes: 0 additions & 3 deletions semantic_router/routers/semantic.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,6 @@ def __init__(
aggregation=aggregation,
auto_sync=auto_sync,
)
# run initialize index now if auto sync is active
if self.auto_sync:
self._init_index_state()

def _encode(self, text: list[str]) -> Any:
"""Given some text, encode it."""
Expand Down
Loading

0 comments on commit 7a6e823

Please sign in to comment.