fix: update encoder ref

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "semantic-chunkers"
-version = "0.1.0.dev0"
+version = "0.1.0.dev1"
 description = "Super advanced chunking methods for AI"
 authors = ["Aurelio AI <[email protected]>"]
 readme = "README.md"
@@ -19,7 +19,7 @@ regex = "^2023.12.25"
 tiktoken = ">=0.7.0,<1.0.0"
 matplotlib = { version = "^3.8.3", optional = true}
 requests-mock = "^1.12.1"
-semantic-router = ">=0.1.0.dev0"
+semantic-router = ">=0.1.0.dev2"
 
 [tool.poetry.extras]
 stats = ["matplotlib"]

semantic_chunkers/__init__.py

@@ -17,4 +17,4 @@
     "RegexChunker",
 ]
 
-__version__ = "0.1.0.dev0"
+__version__ = "0.1.0.dev1"

semantic_chunkers/chunkers/base.py

@@ -2,15 +2,15 @@
 
 from colorama import Fore, Style
 from pydantic.v1 import BaseModel, Extra
-from semantic_router.encoders.base import BaseEncoder
+from semantic_router.encoders.base import DenseEncoder
 
 from semantic_chunkers.schema import Chunk
 from semantic_chunkers.splitters.base import BaseSplitter
 
 
 class BaseChunker(BaseModel):
     name: str
-    encoder: Optional[BaseEncoder]
+    encoder: Optional[DenseEncoder]
     splitter: BaseSplitter
 
     class Config:

semantic_chunkers/chunkers/consecutive.py

@@ -1,7 +1,7 @@
 from typing import Any, List
 
 import numpy as np
-from semantic_router.encoders.base import BaseEncoder
+from semantic_router.encoders.base import DenseEncoder
 from tqdm.auto import tqdm
 
 from semantic_chunkers.chunkers.base import BaseChunker
@@ -15,11 +15,11 @@ class ConsecutiveChunker(BaseChunker):
     Called "consecutive sim chunker" because we check the similarities of consecutive document embeddings (compare ith to i+1th document embedding).
     """
 
-    encoder: BaseEncoder
+    encoder: DenseEncoder
 
     def __init__(
         self,
-        encoder: BaseEncoder,
+        encoder: DenseEncoder,
         splitter: BaseSplitter = RegexSplitter(),
         name: str = "consecutive_chunker",
         score_threshold: float = 0.45,

semantic_chunkers/chunkers/cumulative.py

@@ -1,7 +1,7 @@
 from typing import Any, List
 
 import numpy as np
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 from tqdm.auto import tqdm
 
 from semantic_chunkers.chunkers.base import BaseChunker
@@ -16,11 +16,11 @@ class CumulativeChunker(BaseChunker):
     embeddings of cumulative concatenated documents with the next document.
     """
 
-    encoder: BaseEncoder
+    encoder: DenseEncoder
 
     def __init__(
         self,
-        encoder: BaseEncoder,
+        encoder: DenseEncoder,
         splitter: BaseSplitter = RegexSplitter(),
         name: str = "cumulative_chunker",
         score_threshold: float = 0.45,

semantic_chunkers/chunkers/statistical.py

@@ -3,7 +3,7 @@
 from typing import Any, List, Optional
 
 import numpy as np
-from semantic_router.encoders.base import BaseEncoder
+from semantic_router.encoders.base import DenseEncoder
 from tqdm.auto import tqdm
 
 from semantic_chunkers.chunkers.base import BaseChunker
@@ -44,11 +44,11 @@ def __str__(self):
 
 
 class StatisticalChunker(BaseChunker):
-    encoder: BaseEncoder
+    encoder: DenseEncoder
 
     def __init__(
         self,
-        encoder: BaseEncoder,
+        encoder: DenseEncoder,
         splitter: BaseSplitter = RegexSplitter(),
         name="statistical_chunker",
         threshold_adjustment=0.01,

tests/unit/test_chunkers.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 import pytest
-from semantic_router.encoders.base import BaseEncoder
+from semantic_router.encoders.base import DenseEncoder
 from semantic_router.encoders.openai import OpenAIEncoder
 
 from semantic_chunkers import (
@@ -82,7 +82,7 @@ async def async_return(*args, **kwargs):
 
 
 def test_cumulative_sim_splitter():
-    # Mock the BaseEncoder
+    # Mock the DenseEncoder
     mock_encoder = Mock()
     # Adjust the side_effect to simulate the encoder's behavior for cumulative document comparisons
     # This simplistic simulation assumes binary embeddings for demonstration purposes
@@ -118,7 +118,7 @@ def test_cumulative_sim_splitter():
 
 @pytest.mark.asyncio
 async def test_async_cumulative_sim_splitter():
-    # Mock the BaseEncoder
+    # Mock the DenseEncoder
     mock_encoder = AsyncMock()
     # Adjust the side_effect to simulate the encoder's behavior for cumulative document comparisons
     # This simplistic simulation assumes binary embeddings for demonstration purposes
@@ -153,7 +153,7 @@ async def test_async_cumulative_sim_splitter():
 
 
 def test_consecutive_similarity_splitter_single_doc():
-    mock_encoder = create_autospec(BaseEncoder)
+    mock_encoder = create_autospec(DenseEncoder)
     # Assuming any return value since it should not reach the point of using the encoder
     mock_encoder.return_value = np.array([[0.5, 0]])
 
@@ -166,7 +166,7 @@ def test_consecutive_similarity_splitter_single_doc():
 
 
 def test_cumulative_similarity_splitter_single_doc():
-    mock_encoder = create_autospec(BaseEncoder)
+    mock_encoder = create_autospec(DenseEncoder)
     # Assuming any return value since it should not reach the point of using the encoder
     mock_encoder.return_value = np.array([[0.5, 0]])
 
@@ -241,7 +241,7 @@ async def test_async_statistical_chunker():
 @pytest.fixture
 def base_splitter_instance():
     # Now MockEncoder includes default values for required fields
-    mock_encoder = Mock(spec=BaseEncoder)
+    mock_encoder = Mock(spec=DenseEncoder)
     mock_encoder.name = "mock_encoder"
     mock_encoder.score_threshold = 0.5
     mock_splitter = Mock(spec=BaseSplitter)