From 2d3ec8b52575276d643b3c6565170f672c36df76 Mon Sep 17 00:00:00 2001 From: Luca Mannini Date: Thu, 1 Feb 2024 17:49:43 +0100 Subject: [PATCH 01/26] Multiple routes added --- coverage.xml | 1067 ++++++++++++++++++++++++------------ docs/00-introduction.ipynb | 80 +-- semantic_router/layer.py | 91 +++ 3 files changed, 846 insertions(+), 392 deletions(-) diff --git a/coverage.xml b/coverage.xml index 5850def2..f6013daa 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,12 +1,12 @@ - - + + - /Users/jakit/customers/aurelio/semantic-router/semantic_router + /Users/johnny_silicio/aurelio_ai/semantic-router/semantic_router - + @@ -15,310 +15,438 @@ + - + - + - - - - - + + + + - - + + - - - + + + - - + + + - - - + + - + + + - - - + + - - - + + - + - + - + + + + + - - - + + - - + - - + - + - + - + - - - + + + + + - - - + + - - + + + + + + + - + - + + - - - + + - + - - - + + + + - - - - + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + + - + - - - + + - - - - - + + + + + + + - - + + - - - + + - + - + - + - + + - - + - - - + + - - + + + - + - + + - - - + - - - - - - - + + + + + + + - - + + + + - + - - - - + - - - - - - - + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + + - + - - - - - - + + + + + - + - + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -339,7 +467,7 @@ - + @@ -350,26 +478,26 @@ - + - + - + - + - - + + - + @@ -380,54 +508,56 @@ + - - - - - + + + - - + + + - - + - + - - + + - - - - - - + + + + + + - + - - + + + + + - + - + - - + + @@ -437,44 +567,48 @@ - - + + + - - + - + - - - - - + + + + + + - + - - + + - - - + + + + + + - + @@ -485,21 +619,24 @@ + + - - + + - + - + + @@ -575,17 +712,17 @@ - - + + - + @@ -610,12 +747,12 @@ - + - - - - + + + + @@ -624,10 +761,8 @@ - - - + @@ -636,76 +771,131 @@ + - + - - - + + - + + - + - - - + + + - - + + - - + + - - - - - - + + + + + + - + - - - - + + + - - + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + @@ -715,7 +905,8 @@ - + + @@ -723,19 +914,114 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -744,21 +1030,54 @@ - + + - + - - - - + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -782,25 +1101,70 @@ - - - + + - + - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -819,28 +1183,28 @@ - - + - - + + - - + + + - - + + @@ -861,98 +1225,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + + - + - + - - + + - + - + - + - - + + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - + + @@ -996,16 +1376,13 @@ + - - - - - - - - - + + + + + @@ -1014,17 +1391,27 @@ - + - - + + + + + + - + + + + + + + diff --git a/docs/00-introduction.ipynb b/docs/00-introduction.ipynb index 2ff5a985..7d4fed5d 100644 --- a/docs/00-introduction.ipynb +++ b/docs/00-introduction.ipynb @@ -53,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -81,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -108,7 +108,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -136,17 +136,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2024-01-07 18:08:29 INFO semantic_router.utils.logger Initializing RouteLayer\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "from semantic_router.layer import RouteLayer\n", "\n", @@ -162,40 +154,18 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RouteChoice(name='politics', function_call=None)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rl(\"don't you love politics?\")" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RouteChoice(name='chitchat', function_call=None)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rl(\"how's the weather today?\")" ] @@ -209,24 +179,30 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RouteChoice(name=None, function_call=None)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rl(\"I'm interested in learning about llama 2\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also retrieve multiple routes with its associated score:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "rl.retrieve_multiple_routes(\"Hi! How are you doing in politics??\")" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 08afccfa..31c7cfa0 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -242,6 +242,32 @@ def __call__( # if no route passes threshold, return empty route choice return RouteChoice() + def retrieve_multiple_routes( + self, + text: Optional[str] = None, + vector: Optional[List[float]] = None, + ) -> List[RouteChoice]: + if vector is None: + if text is None: + raise ValueError("Either text or vector must be provided") + vector_arr = self._encode(text=text) + else: + vector_arr = np.array(vector) + # get relevant utterances + results = self._retrieve(xq=vector_arr) + + # decide most relevant routes + categories_with_scores = self._semantic_classify_multiple_routes(results, self.score_threshold) + + route_choices = [] + for category, score in categories_with_scores: + route = self.check_for_matching_routes(category) + if route: + route_choice = RouteChoice(name=route.name, similarity_score=score, route=route) + route_choices.append(route_choice) + + return route_choices + def __str__(self): return ( f"RouteLayer(encoder={self.encoder}, " @@ -375,6 +401,27 @@ def _semantic_classify(self, query_results: List[dict]) -> Tuple[str, List[float logger.warning("No classification found for semantic classifier.") return "", [] + def _semantic_classify_multiple_routes(self, query_results: List[dict], threshold: float) -> List[Tuple[str, float]]: + scores_by_class: Dict[str, List[float]] = {} + for result in query_results: + score = result["score"] + route = result["route"] + if route in scores_by_class: + scores_by_class[route].append(score) + else: + scores_by_class[route] = [score] + + + # Filter classes based on threshold and find max score for each + classes_above_threshold = [] + for route, scores in scores_by_class.items(): + if self._pass_threshold(scores, threshold): + max_score = max(scores) + classes_above_threshold.append((route, max_score)) + + return classes_above_threshold + + def _pass_threshold(self, scores: List[float], threshold: float) -> bool: if scores: return max(scores) > threshold @@ -490,3 +537,47 @@ def threshold_random_search( for i, route in enumerate(route_names) } return score_thresholds + + +if __name__ == "__main__": + from semantic_router import Route + from semantic_router.layer import RouteLayer + from semantic_router.encoders import OpenAIEncoder + + # Define routes with example phrases + politics = Route( + name="politics", + utterances=[ + "isn't politics the best thing ever", + "why don't you tell me about your political opinions", + "don't you just love the president", + "don't you just hate the president", + "they're going to destroy this country!", + "they will save the country!", + ], + ) + + chitchat = Route( + name="chitchat", + utterances=[ + "how's the weather today?", + "how are things going?", + "lovely weather today", + "the weather is horrendous", + "let's go to the chippy", + ], + ) + + routes = [politics, chitchat] + + # Initialize the encoder + encoder = OpenAIEncoder() + + # Initialize the RouteLayer with the encoder and routes + rl = RouteLayer(encoder=encoder, routes=routes) + + # Test the RouteLayer with example queries + print(rl.retrieve_multiple_routes("how's the weather today?")) # Expected to match the chitchat route + print(rl.retrieve_multiple_routes("don't you love politics?")) # Expected to match the politics route + print(rl.retrieve_multiple_routes("I'm interested in learning about llama 2")) # Expected to return None since it doesn't match any route + print(rl.retrieve_multiple_routes("Hi! How are you doing in politics??")) \ No newline at end of file From 0cef1bdead3b5c4cafd7f112694d93f9fabe2b14 Mon Sep 17 00:00:00 2001 From: Luca Mannini Date: Thu, 1 Feb 2024 17:50:51 +0100 Subject: [PATCH 02/26] Lint --- docs/00-introduction.ipynb | 1 - semantic_router/layer.py | 32 +++++++++++++++++++++----------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/00-introduction.ipynb b/docs/00-introduction.ipynb index 7d4fed5d..5ec13702 100644 --- a/docs/00-introduction.ipynb +++ b/docs/00-introduction.ipynb @@ -199,7 +199,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "rl.retrieve_multiple_routes(\"Hi! How are you doing in politics??\")" ] }, diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 31c7cfa0..6f1b6a46 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -257,13 +257,17 @@ def retrieve_multiple_routes( results = self._retrieve(xq=vector_arr) # decide most relevant routes - categories_with_scores = self._semantic_classify_multiple_routes(results, self.score_threshold) + categories_with_scores = self._semantic_classify_multiple_routes( + results, self.score_threshold + ) route_choices = [] for category, score in categories_with_scores: route = self.check_for_matching_routes(category) if route: - route_choice = RouteChoice(name=route.name, similarity_score=score, route=route) + route_choice = RouteChoice( + name=route.name, similarity_score=score, route=route + ) route_choices.append(route_choice) return route_choices @@ -401,7 +405,9 @@ def _semantic_classify(self, query_results: List[dict]) -> Tuple[str, List[float logger.warning("No classification found for semantic classifier.") return "", [] - def _semantic_classify_multiple_routes(self, query_results: List[dict], threshold: float) -> List[Tuple[str, float]]: + def _semantic_classify_multiple_routes( + self, query_results: List[dict], threshold: float + ) -> List[Tuple[str, float]]: scores_by_class: Dict[str, List[float]] = {} for result in query_results: score = result["score"] @@ -411,7 +417,6 @@ def _semantic_classify_multiple_routes(self, query_results: List[dict], threshol else: scores_by_class[route] = [score] - # Filter classes based on threshold and find max score for each classes_above_threshold = [] for route, scores in scores_by_class.items(): @@ -420,8 +425,7 @@ def _semantic_classify_multiple_routes(self, query_results: List[dict], threshol classes_above_threshold.append((route, max_score)) return classes_above_threshold - - + def _pass_threshold(self, scores: List[float], threshold: float) -> bool: if scores: return max(scores) > threshold @@ -541,8 +545,8 @@ def threshold_random_search( if __name__ == "__main__": from semantic_router import Route - from semantic_router.layer import RouteLayer from semantic_router.encoders import OpenAIEncoder + from semantic_router.layer import RouteLayer # Define routes with example phrases politics = Route( @@ -577,7 +581,13 @@ def threshold_random_search( rl = RouteLayer(encoder=encoder, routes=routes) # Test the RouteLayer with example queries - print(rl.retrieve_multiple_routes("how's the weather today?")) # Expected to match the chitchat route - print(rl.retrieve_multiple_routes("don't you love politics?")) # Expected to match the politics route - print(rl.retrieve_multiple_routes("I'm interested in learning about llama 2")) # Expected to return None since it doesn't match any route - print(rl.retrieve_multiple_routes("Hi! How are you doing in politics??")) \ No newline at end of file + print( + rl.retrieve_multiple_routes("how's the weather today?") + ) # Expected to match the chitchat route + print( + rl.retrieve_multiple_routes("don't you love politics?") + ) # Expected to match the politics route + print( + rl.retrieve_multiple_routes("I'm interested in learning about llama 2") + ) # Expected to return None since it doesn't match any route + print(rl.retrieve_multiple_routes("Hi! How are you doing in politics??")) From 809aa14eac9b82d4aca3def77688bbb32f996846 Mon Sep 17 00:00:00 2001 From: Luca Mannini Date: Thu, 1 Feb 2024 17:52:11 +0100 Subject: [PATCH 03/26] fix to comment --- docs/00-introduction.ipynb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/00-introduction.ipynb b/docs/00-introduction.ipynb index 5ec13702..e84105e0 100644 --- a/docs/00-introduction.ipynb +++ b/docs/00-introduction.ipynb @@ -201,13 +201,6 @@ "source": [ "rl.retrieve_multiple_routes(\"Hi! How are you doing in politics??\")" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this case, we return `None` because no matches were identified." - ] } ], "metadata": { From 954fc926786583a3a95de8e8eca1fd123587b41b Mon Sep 17 00:00:00 2001 From: Luca Mannini Date: Thu, 1 Feb 2024 17:54:48 +0100 Subject: [PATCH 04/26] Deleted main in layer.py --- semantic_router/layer.py | 50 ---------------------------------------- 1 file changed, 50 deletions(-) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 6f1b6a46..895e90a9 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -541,53 +541,3 @@ def threshold_random_search( for i, route in enumerate(route_names) } return score_thresholds - - -if __name__ == "__main__": - from semantic_router import Route - from semantic_router.encoders import OpenAIEncoder - from semantic_router.layer import RouteLayer - - # Define routes with example phrases - politics = Route( - name="politics", - utterances=[ - "isn't politics the best thing ever", - "why don't you tell me about your political opinions", - "don't you just love the president", - "don't you just hate the president", - "they're going to destroy this country!", - "they will save the country!", - ], - ) - - chitchat = Route( - name="chitchat", - utterances=[ - "how's the weather today?", - "how are things going?", - "lovely weather today", - "the weather is horrendous", - "let's go to the chippy", - ], - ) - - routes = [politics, chitchat] - - # Initialize the encoder - encoder = OpenAIEncoder() - - # Initialize the RouteLayer with the encoder and routes - rl = RouteLayer(encoder=encoder, routes=routes) - - # Test the RouteLayer with example queries - print( - rl.retrieve_multiple_routes("how's the weather today?") - ) # Expected to match the chitchat route - print( - rl.retrieve_multiple_routes("don't you love politics?") - ) # Expected to match the politics route - print( - rl.retrieve_multiple_routes("I'm interested in learning about llama 2") - ) # Expected to return None since it doesn't match any route - print(rl.retrieve_multiple_routes("Hi! How are you doing in politics??")) From 1fc4eb52b894b8b2989144b21ab08428e32a9ff9 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Tue, 2 Apr 2024 16:06:57 +0400 Subject: [PATCH 05/26] Moving repeated code into a separate method. Created the group_scores_by_class method. --- semantic_router/layer.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 895e90a9..f7c534d9 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -385,14 +385,7 @@ def _retrieve(self, xq: Any, top_k: int = 5) -> List[dict]: return [] def _semantic_classify(self, query_results: List[dict]) -> Tuple[str, List[float]]: - scores_by_class: Dict[str, List[float]] = {} - for result in query_results: - score = result["score"] - route = result["route"] - if route in scores_by_class: - scores_by_class[route].append(score) - else: - scores_by_class[route] = [score] + scores_by_class = self.group_scores_by_class(query_results) # Calculate total score for each class total_scores = {route: sum(scores) for route, scores in scores_by_class.items()} @@ -408,14 +401,7 @@ def _semantic_classify(self, query_results: List[dict]) -> Tuple[str, List[float def _semantic_classify_multiple_routes( self, query_results: List[dict], threshold: float ) -> List[Tuple[str, float]]: - scores_by_class: Dict[str, List[float]] = {} - for result in query_results: - score = result["score"] - route = result["route"] - if route in scores_by_class: - scores_by_class[route].append(score) - else: - scores_by_class[route] = [score] + scores_by_class = self.group_scores_by_class(query_results) # Filter classes based on threshold and find max score for each classes_above_threshold = [] @@ -425,6 +411,18 @@ def _semantic_classify_multiple_routes( classes_above_threshold.append((route, max_score)) return classes_above_threshold + + def group_scores_by_class(self, query_results: List[dict]) -> Dict[str, List[float]]: + scores_by_class: Dict[str, List[float]] = {} + for result in query_results: + score = result["score"] + route = result["route"] + if route in scores_by_class: + scores_by_class[route].append(score) + else: + scores_by_class[route] = [score] + return scores_by_class + def _pass_threshold(self, scores: List[float], threshold: float) -> bool: if scores: From e21beba6383657d3a23b30906ffd17d64f642aca Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Tue, 2 Apr 2024 16:23:03 +0400 Subject: [PATCH 06/26] Now using Route thresholds if they exist. Updated _semantic_classify_multiple_routes to use the Route thresholds if they exist. --- semantic_router/layer.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index f7c534d9..04d611ad 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -257,9 +257,7 @@ def retrieve_multiple_routes( results = self._retrieve(xq=vector_arr) # decide most relevant routes - categories_with_scores = self._semantic_classify_multiple_routes( - results, self.score_threshold - ) + categories_with_scores = self._semantic_classify_multiple_routes(results) route_choices = [] for category, score in categories_with_scores: @@ -397,18 +395,30 @@ def _semantic_classify(self, query_results: List[dict]) -> Tuple[str, List[float else: logger.warning("No classification found for semantic classifier.") return "", [] + + def get(self, name: str) -> Optional[Route]: + for route in self.routes: + if route.name == name: + return route + logger.error(f"Route `{name}` not found") + return None def _semantic_classify_multiple_routes( - self, query_results: List[dict], threshold: float + self, query_results: List[dict] ) -> List[Tuple[str, float]]: scores_by_class = self.group_scores_by_class(query_results) # Filter classes based on threshold and find max score for each classes_above_threshold = [] - for route, scores in scores_by_class.items(): - if self._pass_threshold(scores, threshold): - max_score = max(scores) - classes_above_threshold.append((route, max_score)) + for route_name, scores in scores_by_class.items(): + # Use the get method to find the Route object by its name + route_obj = self.get(route_name) + if route_obj is not None: + # Use the Route object's threshold if it exists, otherwise use the provided threshold + _threshold = route_obj.score_threshold if route_obj.score_threshold is not None else self.score_threshold + if self._pass_threshold(scores, _threshold): + max_score = max(scores) + classes_above_threshold.append((route_name, max_score)) return classes_above_threshold From 9b98205c41000e7cb5f64a741d5d85f384e3be16 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Tue, 2 Apr 2024 22:31:57 +0400 Subject: [PATCH 07/26] Checking that multi-routes work. --- docs/00-introduction.ipynb | 108 ++++++++++++++++++++++++++++++------- 1 file changed, 90 insertions(+), 18 deletions(-) diff --git a/docs/00-introduction.ipynb b/docs/00-introduction.ipynb index e84105e0..97b2ce33 100644 --- a/docs/00-introduction.ipynb +++ b/docs/00-introduction.ipynb @@ -37,11 +37,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[notice] A new release of pip is available: 23.1.2 -> 24.0\n", + "[notice] To update, run: python.exe -m pip install --upgrade pip\n" + ] + } + ], "source": [ - "!pip install -qU semantic-router==0.0.20" + "!pip install -qU semantic-router==0.0.21" ] }, { @@ -53,9 +63,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "from semantic_router import Route\n", "\n", @@ -81,7 +100,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -108,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -136,9 +155,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2024-04-02 22:18:59 INFO semantic_router.utils.logger Initializing RouteLayer\u001b[0m\n" + ] + } + ], "source": [ "from semantic_router.layer import RouteLayer\n", "\n", @@ -154,18 +181,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RouteChoice(name='politics', function_call=None, similarity_score=None, trigger=None)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rl(\"don't you love politics?\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RouteChoice(name='chitchat', function_call=None, similarity_score=None, trigger=None)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rl(\"how's the weather today?\")" ] @@ -179,9 +228,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RouteChoice(name=None, function_call=None, similarity_score=None, trigger=None)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rl(\"I'm interested in learning about llama 2\")" ] @@ -195,9 +255,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[RouteChoice(name='politics', function_call=None, similarity_score=0.8596186767854479, trigger=None),\n", + " RouteChoice(name='chitchat', function_call=None, similarity_score=0.8356239688161818, trigger=None)]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rl.retrieve_multiple_routes(\"Hi! How are you doing in politics??\")" ] @@ -219,7 +291,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.11.4" } }, "nbformat": 4, From 884e15e117a7a647abb295e3525f8aedf0a25582 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Tue, 2 Apr 2024 22:49:39 +0400 Subject: [PATCH 08/26] Linting. --- docs/00-introduction.ipynb | 2 +- semantic_router/layer.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/00-introduction.ipynb b/docs/00-introduction.ipynb index 97b2ce33..dfd85f43 100644 --- a/docs/00-introduction.ipynb +++ b/docs/00-introduction.ipynb @@ -162,7 +162,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2024-04-02 22:18:59 INFO semantic_router.utils.logger Initializing RouteLayer\u001b[0m\n" + "\u001b[32m2024-04-02 22:45:21 INFO semantic_router.utils.logger Initializing RouteLayer\u001b[0m\n" ] } ], diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 04d611ad..77e6713e 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -264,7 +264,7 @@ def retrieve_multiple_routes( route = self.check_for_matching_routes(category) if route: route_choice = RouteChoice( - name=route.name, similarity_score=score, route=route + name=route.name, similarity_score=score ) route_choices.append(route_choice) @@ -395,7 +395,7 @@ def _semantic_classify(self, query_results: List[dict]) -> Tuple[str, List[float else: logger.warning("No classification found for semantic classifier.") return "", [] - + def get(self, name: str) -> Optional[Route]: for route in self.routes: if route.name == name: @@ -415,14 +415,20 @@ def _semantic_classify_multiple_routes( route_obj = self.get(route_name) if route_obj is not None: # Use the Route object's threshold if it exists, otherwise use the provided threshold - _threshold = route_obj.score_threshold if route_obj.score_threshold is not None else self.score_threshold + _threshold = ( + route_obj.score_threshold + if route_obj.score_threshold is not None + else self.score_threshold + ) if self._pass_threshold(scores, _threshold): max_score = max(scores) classes_above_threshold.append((route_name, max_score)) return classes_above_threshold - - def group_scores_by_class(self, query_results: List[dict]) -> Dict[str, List[float]]: + + def group_scores_by_class( + self, query_results: List[dict] + ) -> Dict[str, List[float]]: scores_by_class: Dict[str, List[float]] = {} for result in query_results: score = result["score"] @@ -433,7 +439,6 @@ def group_scores_by_class(self, query_results: List[dict]) -> Dict[str, List[flo scores_by_class[route] = [score] return scores_by_class - def _pass_threshold(self, scores: List[float], threshold: float) -> bool: if scores: return max(scores) > threshold From 1e48125e23c646ce3eb33521aaf88c15990c8d66 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Tue, 2 Apr 2024 23:55:14 +0400 Subject: [PATCH 09/26] Update converage.xml to be same as main. --- coverage.xml | 1823 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 1379 insertions(+), 444 deletions(-) diff --git a/coverage.xml b/coverage.xml index f6013daa..321f6c5c 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,12 +1,12 @@ - - + + - /Users/johnny_silicio/aurelio_ai/semantic-router/semantic_router + /Users/andreped/workspace/semantic-router/semantic_router - + @@ -18,7 +18,7 @@ - + @@ -33,151 +33,165 @@ - - + - - - + + + - - + + + + + + - - - - - + + - + + - + + - + - - - - - - + + + + - + + - + + + - - - + + + - + + + - - - - - - - + - + - + + - + - - - + + + + - - - - + + + + + + + + + + + + + + + + + + - + - + - + + + - - - - - - + + + + + - - + - - - + + + - - - - - - + + + + + + + + - + - - - - - - - + + + + @@ -187,266 +201,247 @@ + - - + - - + + + + - + + - - + - + + - - - - - + - - - - - + + - - + + + + + - - - + + + + - - - - - - + - - + + + + + + - - - + + + - - + + + - - - + + - - - - - + + + + + - + + + - - - - - - + + + + + - - - - - - - - + - - + - - - - - - - - + + + - - - + + - + + + - + - - + - - + + + - - + + + - - - - - - - - - - + + + + - - - - - - - - + + + - - - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - + - + + + + + - - - - - + + + + + + - - - + - - - - - + + + + + + - - - + - - - - + + + + + - + - + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + @@ -467,7 +462,7 @@ - + @@ -478,93 +473,101 @@ + - - - + + - + + - - + + + - - + - - + - - - + + + + + + + - - - - - - + + - - + + - - - - - + + + - - - + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + @@ -585,30 +588,117 @@ + - - - - + + + + - + - + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -621,7 +711,10 @@ + + + @@ -691,38 +784,128 @@ - + - + + + + + + + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -747,12 +930,12 @@ - + - - - - + + + + @@ -822,55 +1005,103 @@ - + - - + - - + - + - - + - + - + - + + + + - - + + - + + + + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -942,7 +1173,79 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -953,7 +1256,7 @@ - + @@ -961,8 +1264,8 @@ - - + + @@ -974,135 +1277,380 @@ - + + + - - + + - - + + - - - + + + - - - + - + - + - - + + - - - + + + + - + - + - - - + - + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + - - - - - + + + + + + + + + + + + + - - - + - - - - + + + + + + + + + + + + - + + + - + + - - - - + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1159,7 +1707,7 @@ - + @@ -1168,35 +1716,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - - - - + + + + + + @@ -1241,7 +1865,7 @@ - + @@ -1250,48 +1874,402 @@ - + - - + + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + @@ -1371,49 +2349,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 247cd9b268747ee379ece2923b0dd073e1b7d8ae Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Tue, 2 Apr 2024 23:59:12 +0400 Subject: [PATCH 10/26] Linting. --- semantic_router/layer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index c4a471df..8c01e5b6 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -292,9 +292,7 @@ def retrieve_multiple_routes( for category, score in categories_with_scores: route = self.check_for_matching_routes(category) if route: - route_choice = RouteChoice( - name=route.name, similarity_score=score - ) + route_choice = RouteChoice(name=route.name, similarity_score=score) route_choices.append(route_choice) return route_choices From 4a16405f3c3fe70d76571dab44f3064390e20b87 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 00:30:33 +0400 Subject: [PATCH 11/26] Added pytests for new _semantic_classify_multiple_routes() method. --- tests/unit/test_layer.py | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 59830da1..bbc67f08 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -504,6 +504,52 @@ def test_get_thresholds(self, openai_encoder, routes, index_cls): assert route_layer.get_thresholds() == {"Route 1": 0.82, "Route 2": 0.82} + @pytest.fixture + def route_layer(self, routes, index_cls): + # Initialize RouteLayer with actual routes and a mock index class + route_layer = RouteLayer(encoder=None, routes=routes, index=index_cls()) + # Manually set the score_threshold for testing + route_layer.score_threshold = 0.5 + return route_layer + + def test_with_multiple_routes_passing_threshold(self, route_layer, index_cls): + # Assuming route_layer is already set up with routes "Route 1" and "Route 2" + query_results = [ + {"route": "Route 1", "score": 0.6}, + {"route": "Route 2", "score": 0.7}, + {"route": "Route 1", "score": 0.8}, + ] + # Override _pass_threshold to always return True for this test + route_layer._pass_threshold = lambda scores, threshold: True + expected = [("Route 1", 0.8), ("Route 2", 0.7)] + results = route_layer._semantic_classify_multiple_routes(query_results) + assert sorted(results) == sorted(expected), "Should classify and return routes above their thresholds" + + def test_with_no_routes_passing_threshold(self, route_layer, index_cls): + # Override _pass_threshold to always return False for this test + route_layer._pass_threshold = lambda scores, threshold: False + query_results = [ + {"route": "Route 1", "score": 0.3}, + {"route": "Route 2", "score": 0.2}, + ] + expected = [] + results = route_layer._semantic_classify_multiple_routes(query_results) + assert results == expected, "Should return an empty list when no routes pass their thresholds" + + def test_with_no_query_results(self, route_layer, index_cls): + query_results = [] + expected = [] + results = route_layer._semantic_classify_multiple_routes(query_results) + assert results == expected, "Should return an empty list when there are no query results" + + def test_with_unrecognized_route(self, route_layer, index_cls): + # Test with a route name that does not exist in the route_layer's routes + query_results = [{"route": "UnrecognizedRoute", "score": 0.9}] + expected = [] + results = route_layer._semantic_classify_multiple_routes(query_results) + assert results == expected, "Should ignore and not return unrecognized routes" + + class TestLayerFit: def test_eval(self, openai_encoder, routes, test_data): route_layer = RouteLayer(encoder=openai_encoder, routes=routes) From 129dcea65fc764e9c9d3406eb213afe0c28a4586 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 00:31:09 +0400 Subject: [PATCH 12/26] Linting. --- tests/unit/test_layer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index bbc67f08..31a7f64d 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -503,7 +503,6 @@ def test_get_thresholds(self, openai_encoder, routes, index_cls): ) assert route_layer.get_thresholds() == {"Route 1": 0.82, "Route 2": 0.82} - @pytest.fixture def route_layer(self, routes, index_cls): # Initialize RouteLayer with actual routes and a mock index class @@ -523,7 +522,9 @@ def test_with_multiple_routes_passing_threshold(self, route_layer, index_cls): route_layer._pass_threshold = lambda scores, threshold: True expected = [("Route 1", 0.8), ("Route 2", 0.7)] results = route_layer._semantic_classify_multiple_routes(query_results) - assert sorted(results) == sorted(expected), "Should classify and return routes above their thresholds" + assert sorted(results) == sorted( + expected + ), "Should classify and return routes above their thresholds" def test_with_no_routes_passing_threshold(self, route_layer, index_cls): # Override _pass_threshold to always return False for this test @@ -534,13 +535,17 @@ def test_with_no_routes_passing_threshold(self, route_layer, index_cls): ] expected = [] results = route_layer._semantic_classify_multiple_routes(query_results) - assert results == expected, "Should return an empty list when no routes pass their thresholds" + assert ( + results == expected + ), "Should return an empty list when no routes pass their thresholds" def test_with_no_query_results(self, route_layer, index_cls): query_results = [] expected = [] results = route_layer._semantic_classify_multiple_routes(query_results) - assert results == expected, "Should return an empty list when there are no query results" + assert ( + results == expected + ), "Should return an empty list when there are no query results" def test_with_unrecognized_route(self, route_layer, index_cls): # Test with a route name that does not exist in the route_layer's routes From 358185d70e8a760329f94d0cb32cc1233adaf271 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 00:39:56 +0400 Subject: [PATCH 13/26] Fix pytests so that they use the mock OpenAI encoder. --- tests/unit/test_layer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 31a7f64d..1158bb08 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -504,9 +504,9 @@ def test_get_thresholds(self, openai_encoder, routes, index_cls): assert route_layer.get_thresholds() == {"Route 1": 0.82, "Route 2": 0.82} @pytest.fixture - def route_layer(self, routes, index_cls): - # Initialize RouteLayer with actual routes and a mock index class - route_layer = RouteLayer(encoder=None, routes=routes, index=index_cls()) + def route_layer(self, openai_encoder, routes, index_cls): + # Initialize RouteLayer with mocked routes and a mock index class + route_layer = RouteLayer(encoder=openai_encoder, routes=routes, index=index_cls()) # Manually set the score_threshold for testing route_layer.score_threshold = 0.5 return route_layer From 900410923c26fba01eaa1975f0f6d96232ad24d5 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 00:40:10 +0400 Subject: [PATCH 14/26] Linting. --- tests/unit/test_layer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 1158bb08..9d2ad73f 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -506,7 +506,9 @@ def test_get_thresholds(self, openai_encoder, routes, index_cls): @pytest.fixture def route_layer(self, openai_encoder, routes, index_cls): # Initialize RouteLayer with mocked routes and a mock index class - route_layer = RouteLayer(encoder=openai_encoder, routes=routes, index=index_cls()) + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) # Manually set the score_threshold for testing route_layer.score_threshold = 0.5 return route_layer From 52770a550b80c18db91270616eeac9195792f38a Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 00:46:52 +0400 Subject: [PATCH 15/26] Simplified tests. --- tests/unit/test_layer.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 9d2ad73f..ce2820fa 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -503,17 +503,13 @@ def test_get_thresholds(self, openai_encoder, routes, index_cls): ) assert route_layer.get_thresholds() == {"Route 1": 0.82, "Route 2": 0.82} - @pytest.fixture - def route_layer(self, openai_encoder, routes, index_cls): - # Initialize RouteLayer with mocked routes and a mock index class + def test_with_multiple_routes_passing_threshold( + self, openai_encoder, routes, index_cls + ): route_layer = RouteLayer( encoder=openai_encoder, routes=routes, index=index_cls() ) - # Manually set the score_threshold for testing - route_layer.score_threshold = 0.5 - return route_layer - - def test_with_multiple_routes_passing_threshold(self, route_layer, index_cls): + route_layer.score_threshold = 0.5 # Set the score_threshold if needed # Assuming route_layer is already set up with routes "Route 1" and "Route 2" query_results = [ {"route": "Route 1", "score": 0.6}, @@ -528,7 +524,11 @@ def test_with_multiple_routes_passing_threshold(self, route_layer, index_cls): expected ), "Should classify and return routes above their thresholds" - def test_with_no_routes_passing_threshold(self, route_layer, index_cls): + def test_with_no_routes_passing_threshold(self, openai_encoder, routes, index_cls): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) + route_layer.score_threshold = 0.5 # Override _pass_threshold to always return False for this test route_layer._pass_threshold = lambda scores, threshold: False query_results = [ @@ -541,7 +541,11 @@ def test_with_no_routes_passing_threshold(self, route_layer, index_cls): results == expected ), "Should return an empty list when no routes pass their thresholds" - def test_with_no_query_results(self, route_layer, index_cls): + def test_with_no_query_results(self, openai_encoder, routes, index_cls): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) + route_layer.score_threshold = 0.5 query_results = [] expected = [] results = route_layer._semantic_classify_multiple_routes(query_results) @@ -549,7 +553,11 @@ def test_with_no_query_results(self, route_layer, index_cls): results == expected ), "Should return an empty list when there are no query results" - def test_with_unrecognized_route(self, route_layer, index_cls): + def test_with_unrecognized_route(self, openai_encoder, routes, index_cls): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) + route_layer.score_threshold = 0.5 # Test with a route name that does not exist in the route_layer's routes query_results = [{"route": "UnrecognizedRoute", "score": 0.9}] expected = [] From a0e6e0fad3cac3288b0942a6ea95b3270f37c29e Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 00:57:04 +0400 Subject: [PATCH 16/26] More tests for retrieve_multiple_routes. --- tests/unit/test_layer.py | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index ce2820fa..a40cfee3 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -564,6 +564,66 @@ def test_with_unrecognized_route(self, openai_encoder, routes, index_cls): results = route_layer._semantic_classify_multiple_routes(query_results) assert results == expected, "Should ignore and not return unrecognized routes" + @pytest.fixture + def mock_route_layer(self, openai_encoder, index_cls): + routes = [ + Route(name="Route 1", utterances=["Hello", "Hi"]), + Route(name="Route 2", utterances=["Goodbye", "Bye"]), + ] + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) + return route_layer + + def test_retrieve_with_text(self, mock_route_layer): + with patch.object( + mock_route_layer, "_encode", return_value=[0.1, 0.2, 0.3] + ) as mock_encode, patch.object( + mock_route_layer, + "_retrieve", + return_value=[{"route": "Route 1", "score": 0.9}], + ) as mock_retrieve: + results = mock_route_layer.retrieve_multiple_routes(text="Hello") + assert len(results) == 1 + assert results[0].name == "Route 1" + mock_encode.assert_called_once_with(text="Hello") + mock_retrieve.assert_called() + + def test_retrieve_with_vector(self, mock_route_layer): + vector = [0.1, 0.2, 0.3] + with patch.object( + mock_route_layer, + "_retrieve", + return_value=[{"route": "Route 2", "score": 0.8}], + ) as mock_retrieve: + results = mock_route_layer.retrieve_multiple_routes(vector=vector) + assert len(results) == 1 + assert results[0].name == "Route 2" + mock_retrieve.assert_called() + + def test_retrieve_no_matches(self, mock_route_layer): + with patch.object( + mock_route_layer, "_retrieve", return_value=[] + ) as mock_retrieve: + results = mock_route_layer.retrieve_multiple_routes(text="Unknown") + assert len(results) == 0 + mock_retrieve.assert_called() + + def test_retrieve_multiple_matches(self, mock_route_layer): + with patch.object( + mock_route_layer, + "_retrieve", + return_value=[ + {"route": "Route 1", "score": 0.7}, + {"route": "Route 2", "score": 0.6}, + ], + ) as mock_retrieve: + results = mock_route_layer.retrieve_multiple_routes(text="Mixed") + assert len(results) == 2 + assert results[0].name in ["Route 1", "Route 2"] + assert results[1].name in ["Route 1", "Route 2"] + mock_retrieve.assert_called() + class TestLayerFit: def test_eval(self, openai_encoder, routes, test_data): From 4089c316d7c9da8854657d0f953a9a9076cb0417 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 01:00:05 +0400 Subject: [PATCH 17/26] Linting. --- tests/unit/test_layer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index a40cfee3..2dd002af 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -564,6 +564,7 @@ def test_with_unrecognized_route(self, openai_encoder, routes, index_cls): results = route_layer._semantic_classify_multiple_routes(query_results) assert results == expected, "Should ignore and not return unrecognized routes" + @pytest.fixture def mock_route_layer(self, openai_encoder, index_cls): routes = [ From 694cf2b7a2c2c1366c5cc8fb6aa180aedca8d152 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 01:00:27 +0400 Subject: [PATCH 18/26] Linting. --- tests/unit/test_layer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 2dd002af..a40cfee3 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -564,7 +564,6 @@ def test_with_unrecognized_route(self, openai_encoder, routes, index_cls): results = route_layer._semantic_classify_multiple_routes(query_results) assert results == expected, "Should ignore and not return unrecognized routes" - @pytest.fixture def mock_route_layer(self, openai_encoder, index_cls): routes = [ From af111b4f8d26046ec5532ab7a51e93f657adad3c Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 01:02:20 +0400 Subject: [PATCH 19/26] Linting. --- docs/09-route-filter.ipynb | 7 ++++--- docs/indexes/qdrant.ipynb | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/09-route-filter.ipynb b/docs/09-route-filter.ipynb index bfddcad4..ad2ccda6 100644 --- a/docs/09-route-filter.ipynb +++ b/docs/09-route-filter.ipynb @@ -41,7 +41,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install -qU semantic-router==0.0.29\n" + "!pip install -qU semantic-router==0.0.29" ] }, { @@ -124,6 +124,7 @@ "import os\n", "from getpass import getpass\n", "from semantic_router.encoders import CohereEncoder, OpenAIEncoder\n", + "\n", "os.environ[\"COHERE_API_KEY\"] = os.getenv(\"COHERE_API_KEY\") or getpass(\n", " \"Enter Cohere API Key: \"\n", ")\n", @@ -259,7 +260,7 @@ } ], "source": [ - "rl(\"don't you love politics?\", route_filter=[\"chitchat\"])\n" + "rl(\"don't you love politics?\", route_filter=[\"chitchat\"])" ] }, { @@ -288,7 +289,7 @@ } ], "source": [ - "rl(\"how's the weather today?\", route_filter=[\"politics\"])\n" + "rl(\"how's the weather today?\", route_filter=[\"politics\"])" ] }, { diff --git a/docs/indexes/qdrant.ipynb b/docs/indexes/qdrant.ipynb index 56427753..9f32fb87 100644 --- a/docs/indexes/qdrant.ipynb +++ b/docs/indexes/qdrant.ipynb @@ -55,6 +55,7 @@ "import os\n", "from getpass import getpass\n", "from semantic_router.encoders import CohereEncoder\n", + "\n", "os.environ[\"COHERE_API_KEY\"] = os.environ.get(\"COHERE_API_KEY\") or getpass(\n", " \"Enter COHERE API key: \"\n", ")\n", From cbd67917c393f9e64862d115c43c2952ff4a30d2 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 01:14:18 +0400 Subject: [PATCH 20/26] Use existing mock_encoder_call in pytest. --- tests/unit/test_layer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index a40cfee3..c69d296c 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -577,7 +577,7 @@ def mock_route_layer(self, openai_encoder, index_cls): def test_retrieve_with_text(self, mock_route_layer): with patch.object( - mock_route_layer, "_encode", return_value=[0.1, 0.2, 0.3] + mock_route_layer, "_encode", side_effect=mock_encoder_call ) as mock_encode, patch.object( mock_route_layer, "_retrieve", @@ -586,6 +586,7 @@ def test_retrieve_with_text(self, mock_route_layer): results = mock_route_layer.retrieve_multiple_routes(text="Hello") assert len(results) == 1 assert results[0].name == "Route 1" + # Ensure "Hello" is encoded to [0.1, 0.2, 0.3] as per mock_encoder_call mock_encode.assert_called_once_with(text="Hello") mock_retrieve.assert_called() From f6aa88fc5e7575cec1359687cc44fe8a104831a0 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 02:47:52 +0400 Subject: [PATCH 21/26] Fixed pytests for retrieve_multiple_routes. --- tests/unit/test_layer.py | 118 ++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 56 deletions(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index c69d296c..98529380 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -96,6 +96,19 @@ def routes(): Route(name="Route 2", utterances=["Goodbye", "Bye", "Au revoir"]), ] +@pytest.fixture +def routes_2(): + return [ + Route(name="Route 1", utterances=["Hello"]), + Route(name="Route 2", utterances=["Hello"]), + ] + +@pytest.fixture +def routes_3(): + return [ + Route(name="Route 1", utterances=["Hello"]), + Route(name="Route 2", utterances=["Asparagus"]), + ] @pytest.fixture def dynamic_routes(): @@ -482,6 +495,9 @@ def test_from_file_with_llm(self, tmp_path, index_cls): layer_config.routes[0].llm.name == "fake-model-v1" ), "LLM instance should have the 'name' attribute set correctly" + def debug_test(): + assert 1 == 0 + def test_config(self, openai_encoder, routes, index_cls): os.environ["OPENAI_API_KEY"] = "test_api_key" route_layer = RouteLayer( @@ -564,67 +580,57 @@ def test_with_unrecognized_route(self, openai_encoder, routes, index_cls): results = route_layer._semantic_classify_multiple_routes(query_results) assert results == expected, "Should ignore and not return unrecognized routes" - @pytest.fixture - def mock_route_layer(self, openai_encoder, index_cls): - routes = [ - Route(name="Route 1", utterances=["Hello", "Hi"]), - Route(name="Route 2", utterances=["Goodbye", "Bye"]), - ] + def test_retrieve_with_text(self, openai_encoder, routes, index_cls): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) + text = "Hello" + results = route_layer.retrieve_multiple_routes(text=text) + assert len(results) >= 1, "Expected at least one result" + assert any( + result.name in ["Route 1", "Route 2"] for result in results + ), "Expected the result to be either 'Route 1' or 'Route 2'" + + def test_retrieve_with_vector(self, openai_encoder, routes, index_cls): route_layer = RouteLayer( encoder=openai_encoder, routes=routes, index=index_cls() ) - return route_layer - - def test_retrieve_with_text(self, mock_route_layer): - with patch.object( - mock_route_layer, "_encode", side_effect=mock_encoder_call - ) as mock_encode, patch.object( - mock_route_layer, - "_retrieve", - return_value=[{"route": "Route 1", "score": 0.9}], - ) as mock_retrieve: - results = mock_route_layer.retrieve_multiple_routes(text="Hello") - assert len(results) == 1 - assert results[0].name == "Route 1" - # Ensure "Hello" is encoded to [0.1, 0.2, 0.3] as per mock_encoder_call - mock_encode.assert_called_once_with(text="Hello") - mock_retrieve.assert_called() - - def test_retrieve_with_vector(self, mock_route_layer): vector = [0.1, 0.2, 0.3] - with patch.object( - mock_route_layer, - "_retrieve", - return_value=[{"route": "Route 2", "score": 0.8}], - ) as mock_retrieve: - results = mock_route_layer.retrieve_multiple_routes(vector=vector) - assert len(results) == 1 - assert results[0].name == "Route 2" - mock_retrieve.assert_called() - - def test_retrieve_no_matches(self, mock_route_layer): - with patch.object( - mock_route_layer, "_retrieve", return_value=[] - ) as mock_retrieve: - results = mock_route_layer.retrieve_multiple_routes(text="Unknown") - assert len(results) == 0 - mock_retrieve.assert_called() - - def test_retrieve_multiple_matches(self, mock_route_layer): - with patch.object( - mock_route_layer, - "_retrieve", - return_value=[ - {"route": "Route 1", "score": 0.7}, - {"route": "Route 2", "score": 0.6}, - ], - ) as mock_retrieve: - results = mock_route_layer.retrieve_multiple_routes(text="Mixed") - assert len(results) == 2 - assert results[0].name in ["Route 1", "Route 2"] - assert results[1].name in ["Route 1", "Route 2"] - mock_retrieve.assert_called() + results = route_layer.retrieve_multiple_routes(vector=vector) + assert len(results) >= 1, "Expected at least one result" + assert any( + result.name in ["Route 1", "Route 2"] for result in results + ), "Expected the result to be either 'Route 1' or 'Route 2'" + + + def test_retrieve_no_matches(self, openai_encoder, routes, index_cls): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) + text = "Asparagus" + results = route_layer.retrieve_multiple_routes(text=text) + assert len(results) == 0, f"Expected no results, but got {len(results)}" + def test_retrieve_one_match(self, openai_encoder, routes_3, index_cls): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes_3, index=index_cls() + ) + text = "Hello" + results = route_layer.retrieve_multiple_routes(text=text) + assert len(results) == 1, f"Expected one result, and got {len(results)}" + matched_routes = [result.name for result in results] + assert "Route 1" in matched_routes, "Expected 'Route 1' to be a match" + + def test_retrieve_with_text_for_multiple_matches(self, openai_encoder, routes_2, index_cls): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes_2, index=index_cls() + ) + text = "Hello" + results = route_layer.retrieve_multiple_routes(text=text) + assert len(results) == 2, "Expected two results" + matched_routes = [result.name for result in results] + assert "Route 1" in matched_routes, "Expected 'Route 1' to be a match" + assert "Route 2" in matched_routes, "Expected 'Route 2' to be a match" class TestLayerFit: def test_eval(self, openai_encoder, routes, test_data): From 61654091e8336277baec9036301b4b19a7e9e9d2 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 02:48:12 +0400 Subject: [PATCH 22/26] Linting. --- tests/unit/test_layer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 98529380..0ffbf69d 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -96,6 +96,7 @@ def routes(): Route(name="Route 2", utterances=["Goodbye", "Bye", "Au revoir"]), ] + @pytest.fixture def routes_2(): return [ @@ -103,6 +104,7 @@ def routes_2(): Route(name="Route 2", utterances=["Hello"]), ] + @pytest.fixture def routes_3(): return [ @@ -110,6 +112,7 @@ def routes_3(): Route(name="Route 2", utterances=["Asparagus"]), ] + @pytest.fixture def dynamic_routes(): return [ @@ -602,7 +605,6 @@ def test_retrieve_with_vector(self, openai_encoder, routes, index_cls): result.name in ["Route 1", "Route 2"] for result in results ), "Expected the result to be either 'Route 1' or 'Route 2'" - def test_retrieve_no_matches(self, openai_encoder, routes, index_cls): route_layer = RouteLayer( encoder=openai_encoder, routes=routes, index=index_cls() @@ -621,7 +623,9 @@ def test_retrieve_one_match(self, openai_encoder, routes_3, index_cls): matched_routes = [result.name for result in results] assert "Route 1" in matched_routes, "Expected 'Route 1' to be a match" - def test_retrieve_with_text_for_multiple_matches(self, openai_encoder, routes_2, index_cls): + def test_retrieve_with_text_for_multiple_matches( + self, openai_encoder, routes_2, index_cls + ): route_layer = RouteLayer( encoder=openai_encoder, routes=routes_2, index=index_cls() ) @@ -632,6 +636,7 @@ def test_retrieve_with_text_for_multiple_matches(self, openai_encoder, routes_2, assert "Route 1" in matched_routes, "Expected 'Route 1' to be a match" assert "Route 2" in matched_routes, "Expected 'Route 2' to be a match" + class TestLayerFit: def test_eval(self, openai_encoder, routes, test_data): route_layer = RouteLayer(encoder=openai_encoder, routes=routes) From 2852a7c7021c312b6a94ab8e21e0aaa1a0df82ca Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 02:56:24 +0400 Subject: [PATCH 23/26] Removed debugging test. --- tests/unit/test_layer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 0ffbf69d..63fe31d7 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -498,9 +498,6 @@ def test_from_file_with_llm(self, tmp_path, index_cls): layer_config.routes[0].llm.name == "fake-model-v1" ), "LLM instance should have the 'name' attribute set correctly" - def debug_test(): - assert 1 == 0 - def test_config(self, openai_encoder, routes, index_cls): os.environ["OPENAI_API_KEY"] = "test_api_key" route_layer = RouteLayer( From c5d6d0220b0891fcb0f2a38be1325254fdf4804d Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 03:05:56 +0400 Subject: [PATCH 24/26] More test coverage. --- tests/unit/test_layer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 63fe31d7..dd4167fa 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -602,6 +602,13 @@ def test_retrieve_with_vector(self, openai_encoder, routes, index_cls): result.name in ["Route 1", "Route 2"] for result in results ), "Expected the result to be either 'Route 1' or 'Route 2'" + def test_retrieve_without_text_or_vector(self, openai_encoder, routes, index_cls): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) + with pytest.raises(ValueError, match="Either text or vector must be provided"): + route_layer.retrieve_multiple_routes() + def test_retrieve_no_matches(self, openai_encoder, routes, index_cls): route_layer = RouteLayer( encoder=openai_encoder, routes=routes, index=index_cls() From c438c622d117bfebba4c15f8d825b7fa5f2951cd Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Wed, 3 Apr 2024 03:15:42 +0400 Subject: [PATCH 25/26] More test coverage. --- tests/unit/test_layer.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index dd4167fa..8f4833f0 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -640,6 +640,28 @@ def test_retrieve_with_text_for_multiple_matches( assert "Route 1" in matched_routes, "Expected 'Route 1' to be a match" assert "Route 2" in matched_routes, "Expected 'Route 2' to be a match" + def test_set_aggregation_method_with_unsupported_value( + self, openai_encoder, routes, index_cls + ): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) + unsupported_aggregation = "unsupported_aggregation_method" + with pytest.raises( + ValueError, + match=f"Unsupported aggregation method chosen: {unsupported_aggregation}. Choose either 'SUM', 'MEAN', or 'MAX'.", + ): + route_layer._set_aggregation_method(unsupported_aggregation) + + def test_refresh_routes_not_implemented(self, openai_encoder, routes, index_cls): + route_layer = RouteLayer( + encoder=openai_encoder, routes=routes, index=index_cls() + ) + with pytest.raises( + NotImplementedError, match="This method has not yet been implemented." + ): + route_layer._refresh_routes() + class TestLayerFit: def test_eval(self, openai_encoder, routes, test_data): From f403c257811b74351dfcd4af3154d5eb3c88896e Mon Sep 17 00:00:00 2001 From: theanupllm Date: Fri, 19 Apr 2024 18:52:22 +0530 Subject: [PATCH 26/26] Mergining with main --- docs/00-introduction.ipynb | 78 +++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/docs/00-introduction.ipynb b/docs/00-introduction.ipynb index f75774eb..9c68cd0c 100644 --- a/docs/00-introduction.ipynb +++ b/docs/00-introduction.ipynb @@ -51,7 +51,7 @@ } ], "source": [ - "!pip install -qU semantic-router==0.0.29" + "!pip install -qU semantic-router==0.0.35" ] }, { @@ -63,18 +63,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "from semantic_router import Route\n", "\n", @@ -100,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -127,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -155,14 +146,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2024-04-02 22:45:21 INFO semantic_router.utils.logger Initializing RouteLayer\u001b[0m\n" + "\u001b[32m2024-04-19 18:34:06 INFO semantic_router.utils.logger local\u001b[0m\n" ] } ], @@ -181,16 +172,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "RouteChoice(name='politics', function_call=None, similarity_score=None, trigger=None)" + "RouteChoice(name='politics', function_call=None, similarity_score=None)" ] }, - "execution_count": 6, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -201,16 +192,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "RouteChoice(name='chitchat', function_call=None, similarity_score=None, trigger=None)" + "RouteChoice(name='chitchat', function_call=None, similarity_score=None)" ] }, - "execution_count": 7, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -228,16 +219,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "RouteChoice(name=None, function_call=None, similarity_score=None, trigger=None)" + "RouteChoice(name=None, function_call=None, similarity_score=None)" ] }, - "execution_count": 8, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -255,17 +246,17 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[RouteChoice(name='politics', function_call=None, similarity_score=0.8596186767854479, trigger=None),\n", - " RouteChoice(name='chitchat', function_call=None, similarity_score=0.8356239688161818, trigger=None)]" + "[RouteChoice(name='politics', function_call=None, similarity_score=0.8596186767854487),\n", + " RouteChoice(name='chitchat', function_call=None, similarity_score=0.8356239688161808)]" ] }, - "execution_count": 9, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -273,6 +264,33 @@ "source": [ "rl.retrieve_multiple_routes(\"Hi! How are you doing in politics??\")" ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rl.retrieve_multiple_routes(\"I'm interested in learning about llama 2\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -291,7 +309,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.12.2" } }, "nbformat": 4,