update on rag

examples/conversation_with_RAG_agents/README.md

@@ -13,7 +13,7 @@ capability can be used to build easily.
 * **Cloning repo:** This example requires cloning the whole AgentScope repo to local.
 * **Packages:** This example is built on the LlamaIndex package. Thus, some packages need to be installed before running the example.
     ```bash
-    pip install llama-index tree_sitter tree-sitter-languages
+    pip install llama-index llama-index-readers-docstring-walker tree_sitter tree-sitter-languages
     ```
 * **Model APIs:** This example uses Dashscope APIs. Thus, we also need an API key for DashScope.
   ```bash

examples/conversation_with_RAG_agents/configs/agent_config.json

@@ -0,0 +1,77 @@
+[
+  {
+    "class": "LlamaIndexAgent",
+    "args": {
+      "name": "Tutorial-Assistant",
+      "description": "Tutorial-Assistant is an agent that can provide answer based on English tutorial material, mainly the markdown files. It can answer general questions about AgentScope.",
+      "sys_prompt": "You're an assistant helping new users to use AgentScope. The language style is helpful and cheerful. You generate answers based on the provided context. The answer is expected to be no longer than 100 words. If the key words of the question can be found in the provided context, the answer should contain the section name which contains the answer. For example, 'You may refer to SECTION_NAME for more details.'",
+      "model_config_name": "qwen_config",
+      "emb_model_config_name": "qwen_emb_config",
+      "rag_config": {
+            "knowledge_id": "agentscope_tutorial_rag",
+            "similarity_top_k": 5,
+            "log_retrieval": false,
+            "recent_n_mem": 1
+      }
+    }
+  },
+  {
+    "class": "LlamaIndexAgent",
+    "args": {
+      "name": "Code-Search-Assistant",
+      "description": "Code-Search-Assistant is an agent that can provide answer based on AgentScope code base. It can answer questions about specific modules in AgentScope.",
+      "sys_prompt": "You're a coding assistant of AgentScope. The answer starts with appreciation for the question, then provide details regarding the functionality and features of the modules mentioned in the question. The language should be in a professional and simple style. The answer is limited to be less than 100 words.",
+      "model_config_name": "qwen_config",
+      "emb_model_config_name": "qwen_emb_config",
+      "rag_config": {
+            "knowledge_id": "agentscope_code_rag",
+            "similarity_top_k": 5,
+            "log_retrieval": false,
+            "recent_n_mem": 1
+      }
+    }
+  },
+  {
+    "class": "LlamaIndexAgent",
+    "args": {
+      "name": "API-Assistant",
+      "description": "API-Assistant is an agent that can answer questions about APIs in AgentScope. It can answer general questions about AgentScope.",
+      "sys_prompt": "You're an assistant providing answers to the questions related to APIs (functions and classes) in AgentScope. The language style is helpful and cheerful. You generate answers based on the provided context. The answer is expected to be no longer than 200 words. If the key words of the question can be found in the provided context, the answer should contain the module of the API. For example, 'You may refer to MODULE_NAME for more details.'",
+      "model_config_name": "qwen_config",
+      "emb_model_config_name": "qwen_emb_config",
+      "rag_config": {
+          "knowledge_id": "agentscope_api_rag",
+          "similarity_top_k": 3,
+          "log_retrieval": true,
+          "recent_n_mem": 1
+      }
+    }
+  },
+  {
+    "class": "LlamaIndexAgent",
+    "args": {
+      "name": "Searching-Assistant",
+      "description": "Search-Assistant is an agent that can provide answer based on AgentScope code and tutorial. It can answer questions about everything in AgentScope codes and tutorials.",
+      "sys_prompt": "You're a helpful assistant of AgentScope. The answer starts with appreciation for the question, then provide output the location of the code or section that the most relevant to the question. The answer is limited to be less than 50 words.",
+      "model_config_name": "qwen_config",
+      "emb_model_config_name": "qwen_emb_config",
+      "rag_config": {
+            "knowledge_id": "agentscope_global_rag",
+            "similarity_top_k": 5,
+            "log_retrieval": false,
+            "recent_n_mem": 1,
+            "persist_dir": "./rag_storage/searching_assist"
+          }
+    }
+  },
+  {
+    "class": "DialogAgent",
+    "args": {
+      "name": "Agent-Guiding-Assistant",
+      "description": "Agent-Guiding-Assistant is an agent that decide which agent should provide the answer next. It can answer questions about specific functions and classes in AgentScope.",
+      "sys_prompt": "You're an assistant guiding the user to specific agent for help. The answer is in a cheerful styled language. The output starts with appreciation for the question. Next, rephrase the question in a simple declarative Sentence for example, 'I think you are asking...'. Last, if the question is about detailed code or example in AgentScope Framework, output '@ Code-Search-Assistant you might be suitable for answering the question'; if the question is about API or function calls (Example: 'Is there function related...' or 'how can I initialize ...' ) in AgentScope, output '@ API-Assistant, I think you are more suitable for the question, please tell us more about it'; if question is about where to find some context (Example:'where can I find...'), output '@ Searching-Assistant, we need your help', otherwise, output '@ Tutorial-Assistant, I think you are more suitable for the question, can you tell us more about it?'. The answer is expected to be only one sentence",
+      "model_config_name": "qwen_config",
+      "use_memory": false
+    }
+  }
+]

examples/conversation_with_RAG_agents/configs/detailed_rag_config_example.json

@@ -0,0 +1,114 @@
+[
+  {
+    "knowledge_id": "agentscope_code_rag",
+    "persist_dir": "./rag_storage/code_assist",
+    "chunk_size": 2048,
+    "chunk_overlap": 40,
+    "data_processing": [
+      {
+        "load_data": {
+          "loader": {
+            "create_object": true,
+            "module": "llama_index.core",
+            "class": "SimpleDirectoryReader",
+            "init_args": {
+              "input_dir": "../../src/agentscope",
+              "recursive": true,
+              "required_exts": [
+                ".py"
+              ]
+            }
+          }
+        },
+        "store_and_index": {
+          "transformations": [
+            {
+              "create_object": true,
+              "module": "llama_index.core.node_parser",
+              "class": "CodeSplitter",
+              "init_args": {
+                "language": "python",
+                "chunk_lines": 100
+              }
+            }
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "knowledge_id": "agentscope_api_rag",
+    "persist_dir": "./rag_storage/api_assist",
+    "chunk_size": 2048,
+    "chunk_overlap": 40,
+    "data_processing": [
+      {
+        "load_data": {
+          "loader": {
+            "create_object": true,
+            "module": "llama_index.core",
+            "class": "SimpleDirectoryReader",
+            "init_args": {
+              "input_dir": "../../docs/docstring_html/",
+              "required_exts": [
+                ".html"
+              ]
+            }
+          }
+        }
+      }
+    ]
+  },
+  {
+    "knowledge_id": "agentscope_global_rag",
+    "persist_dir": "./rag_storage/searching_assist",
+    "chunk_size": 2048,
+    "chunk_overlap": 40,
+    "data_processing": [
+      {
+        "load_data": {
+          "loader": {
+            "create_object": true,
+            "module": "llama_index.core",
+            "class": "SimpleDirectoryReader",
+            "init_args": {
+              "input_dir": "../../docs/sphinx_doc/en/source/tutorial",
+              "required_exts": [
+                ".md"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "load_data": {
+          "loader": {
+            "create_object": true,
+            "module": "llama_index.core",
+            "class": "SimpleDirectoryReader",
+            "init_args": {
+              "input_dir": "../../src/agentscope",
+              "recursive": true,
+              "required_exts": [
+                ".py"
+              ]
+            }
+          }
+        },
+        "store_and_index": {
+          "transformations": [
+            {
+              "create_object": true,
+              "module": "llama_index.core.node_parser",
+              "class": "CodeSplitter",
+              "init_args": {
+                "language": "python",
+                "chunk_lines": 100
+              }
+            }
+          ]
+        }
+      }
+    ]
+  }
+]

examples/conversation_with_RAG_agents/configs/model_config.json

@@ -0,0 +1,27 @@
+[
+  {
+    "model_type": "post_api_chat",
+    "config_name": "gpt_postapi_config",
+    "api_url": "http://47.88.8.18:8088/api/ask",
+    "headers": {
+        "Content-Type": "application/json",
+        "Authorization": ""
+    },
+    "messages_key": "messages",
+    "json_args": {
+        "model": "gpt-4"
+    }
+  },
+  {
+    "model_type": "dashscope_text_embedding",
+    "config_name": "qwen_emb_config",
+    "model_name": "text-embedding-v2",
+    "api_key": ""
+  },
+  {
+      "model_type": "dashscope_chat",
+      "config_name": "qwen_config",
+      "model_name": "qwen-max",
+      "api_key": ""
+  }
+]

examples/conversation_with_RAG_agents/groupchat_utils.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+""" Group chat utils."""
+import re
+from typing import Sequence
+
+
+def select_next_one(agents: Sequence, rnd: int) -> Sequence:
+    """
+    Select next agent.
+    """
+    return agents[rnd % len(agents)]
+
+
+def filter_agents(string: str, agents: Sequence) -> Sequence:
+    """
+    This function filters the input string for occurrences of the given names
+    prefixed with '@' and returns a list of the found names.
+    """
+    if len(agents) == 0:
+        return []
+
+    # Create a pattern that matches @ followed by any of the candidate names
+    pattern = (
+        r"@(" + "|".join(re.escape(agent.name) for agent in agents) + r")\b"
+    )
+
+    # Find all occurrences of the pattern in the string
+    matches = re.findall(pattern, string)
+
+    # Create a dictionary mapping agent names to agent objects for quick lookup
+    agent_dict = {agent.name: agent for agent in agents}
+
+    # Return the list of matched agent objects preserving the order
+    ordered_agents = [
+        agent_dict[name] for name in matches if name in agent_dict
+    ]
+    return ordered_agents