diff --git a/.env.example b/.env.example index add1fd79e..1c6209705 100644 --- a/.env.example +++ b/.env.example @@ -41,11 +41,9 @@ INTENT_AGENT_LLM="openai" REPORT_AGENT_LLM="mistral" VALIDATOR_AGENT_LLM="openai" DATASTORE_AGENT_LLM="openai" -MATHS_AGENT_LLM="openai" WEB_AGENT_LLM="openai" CHART_GENERATOR_LLM="openai" ROUTER_LLM="openai" -FILE_AGENT_LLM="openai" SUGGESTIONS_LLM="openai" DYNAMIC_KNOWLEDGE_GRAPH_LLM="openai" @@ -56,10 +54,8 @@ INTENT_AGENT_MODEL="gpt-4o-mini" REPORT_AGENT_MODEL="mistral-large-latest" VALIDATOR_AGENT_MODEL="gpt-4o-mini" DATASTORE_AGENT_MODEL="gpt-4o-mini" -MATHS_AGENT_MODEL="gpt-4o-mini" WEB_AGENT_MODEL="gpt-4o-mini" CHART_GENERATOR_MODEL="gpt-4o-mini" ROUTER_MODEL="gpt-4o-mini" -FILE_AGENT_MODEL="gpt-4o-mini" SUGGESTIONS_MODEL="gpt-4o-mini" DYNAMIC_KNOWLEDGE_GRAPH_MODEL="gpt-4o" diff --git a/backend/promptfoo/agent_selection_config.yaml b/backend/promptfoo/agent_selection_config.yaml new file mode 100644 index 000000000..928260c83 --- /dev/null +++ b/backend/promptfoo/agent_selection_config.yaml @@ -0,0 +1,85 @@ +description: "Agent selection" + +define: + &list_of_agents [ + { + "name": "DatastoreAgent", + "description": "This agent is responsible for handling database queries to the bloomberg.csv dataset. This includes retrieving ESG scores, financial metrics, and other bloomberg-specific information. It interacts with the graph database to extract, process, and return ESG-related information from various sources, such as company sustainability reports or fund portfolios. This agent can not complete any task that is not specifically about the bloomberg.csv dataset.", + }, + { + "name": "WebAgent", + "description": "This agent can perform general internet searches to complete the task by retrieving and summarizing the results and it can also perform web scrapes to retreive specific inpormation from web pages.", + }, + { + "name": "ChartGeneratorAgent", + "description": "This agent is responsible for creating charts", + }, + ] + +providers: + - id: openai:gpt-4o-mini + config: + temperature: 0 + +prompts: file://promptfoo_test_runner.py:create_prompt + +tests: + - description: "Test the WebAgent is selected for an ESG data query when not related to bloomberg dataset" + vars: + user_prompt_template: "agent-selection-user-prompt" + user_prompt_args: + task: "What are the environmental factors that influence a company's ESG score?" + system_prompt_template: "agent-selection-system-prompt" + system_prompt_args: + list_of_agents: *list_of_agents + assert: + - type: javascript + value: JSON.parse(output).agent_name === "WebAgent" + + - description: "Test the WebAgent is selected for queries about the news" + vars: + user_prompt_template: "agent-selection-user-prompt" + user_prompt_args: + task: "Are there any recent news articles discussing 3M Co's ESG initiatives?" + system_prompt_template: "agent-selection-system-prompt" + system_prompt_args: + list_of_agents: *list_of_agents + assert: + - type: javascript + value: JSON.parse(output).agent_name === "WebAgent" + + - description: "Test the DatastoreAgent is selected for a data query related to the bloomberg dataset" + vars: + user_prompt_template: "agent-selection-user-prompt" + user_prompt_args: + task: "Which company name has the highest environmental ESG score in the bloomberg dataset?" + system_prompt_template: "agent-selection-system-prompt" + system_prompt_args: + list_of_agents: *list_of_agents + assert: + - type: javascript + value: JSON.parse(output).agent_name === "DatastoreAgent" + + - description: "Test the DatastoreAgent is selected for a data query from the bloomberg.csv" + vars: + user_prompt_template: "agent-selection-user-prompt" + user_prompt_args: + task: "From the Bloomberg.csv data show me all the governance scores for american airlines in date order starting in the past." + system_prompt_template: "agent-selection-system-prompt" + system_prompt_args: + list_of_agents: *list_of_agents + assert: + - type: javascript + value: JSON.parse(output).agent_name === "DatastoreAgent" + + - description: "Test the DatastoreAgent is selected for a query of 'the database'" + vars: + user_prompt_template: "agent-selection-user-prompt" + user_prompt_args: + task: "Check the database and give me all data for Masco Corp" + system_prompt_template: "agent-selection-system-prompt" + system_prompt_args: + list_of_agents: *list_of_agents + assert: + - type: javascript + value: JSON.parse(output).agent_name === "DatastoreAgent" diff --git a/backend/src/agents/__init__.py b/backend/src/agents/__init__.py index d692a8ec1..e7f3fda36 100644 --- a/backend/src/agents/__init__.py +++ b/backend/src/agents/__init__.py @@ -9,7 +9,6 @@ from src.agents.validator_agent import ValidatorAgent from src.agents.answer_agent import AnswerAgent from src.agents.chart_generator_agent import ChartGeneratorAgent -from src.agents.file_agent import FileAgent from src.agents.report_agent import ReportAgent @@ -37,14 +36,11 @@ def agent_details(agent) -> dict: def get_available_agents() -> List[Agent]: - return [DatastoreAgent(config.datastore_agent_llm, config.datastore_agent_model), - WebAgent(config.web_agent_llm, config.web_agent_model), - ChartGeneratorAgent(config.chart_generator_llm, - config.chart_generator_model), - FileAgent(config.file_agent_llm, config.file_agent_model), - # FS-63 Silencing Math agent - tool is not optimised. - # MathsAgent(config.maths_agent_llm, config.maths_agent_model), - ] + return [ + DatastoreAgent(config.datastore_agent_llm, config.datastore_agent_model), + WebAgent(config.web_agent_llm, config.web_agent_model), + ChartGeneratorAgent(config.chart_generator_llm, config.chart_generator_model), + ] def get_agent_details(): diff --git a/backend/src/agents/datastore_agent.py b/backend/src/agents/datastore_agent.py index 25ea4935b..2905ea61b 100644 --- a/backend/src/agents/datastore_agent.py +++ b/backend/src/agents/datastore_agent.py @@ -17,10 +17,10 @@ cache = {} + async def generate_cypher_query_core( question_intent, operation, question_params, aggregation, sort_order, timeframe, llm: LLM, model ) -> str: - details_to_create_cypher_query = engine.load_prompt( "details-to-create-cypher-query", question_intent=question_intent, @@ -38,8 +38,9 @@ async def generate_cypher_query_core( "generate-cypher-query", graph_schema=graph_schema, current_date=datetime.now() ) - llm_query = await llm.chat(model, generate_cypher_query_prompt, details_to_create_cypher_query, - return_json=True) + llm_query = await llm.chat( + model, generate_cypher_query_prompt, details_to_create_cypher_query, return_json=True + ) json_query = to_json(llm_query) await publish_log_info(LogPrefix.USER, f"Cypher generated by the LLM: {llm_query}", __name__) if json_query["query"] == "None": @@ -49,12 +50,10 @@ async def generate_cypher_query_core( except Exception as e: logger.error(f"Error during data retrieval: {e}") raise - response = { - "content": db_response, - "ignore_validation": "false" - } + response = {"content": db_response, "ignore_validation": "false"} return json.dumps(response, indent=4) + @tool( name="generate cypher query", description="Generate Cypher query if the category is data driven, based on the operation to be performed", @@ -88,11 +87,12 @@ async def generate_cypher_query_core( ), }, ) - -async def generate_cypher(question_intent, operation, question_params, aggregation, sort_order, - timeframe, llm: LLM, model) -> str: - return await generate_cypher_query_core(question_intent, operation, question_params, aggregation, sort_order, - timeframe, llm, model) +async def generate_cypher( + question_intent, operation, question_params, aggregation, sort_order, timeframe, llm: LLM, model +) -> str: + return await generate_cypher_query_core( + question_intent, operation, question_params, aggregation, sort_order, timeframe, llm, model + ) async def get_semantic_layer_cache(llm, model, graph_schema): @@ -104,14 +104,10 @@ async def get_semantic_layer_cache(llm, model, graph_schema): else: return cache + @agent( name="DatastoreAgent", - description=( - "This agent is responsible for handling database queries related to ESG (Environmental, Social, Governance) " - "data, including retrieving ESG scores, financial metrics, and other company-specific or fund-specific " - "information. It interacts with the graph database to extract, process, and return ESG-related information " - "from various sources, such as company sustainability reports or fund portfolios." - ), + description="This agent is responsible for handling database queries to the bloomberg.csv dataset. This includes retrieving ESG scores, financial metrics, and other bloomberg-specific information. It interacts with the graph database to extract, process, and return ESG-related information from various sources, such as company sustainability reports or fund portfolios. This agent can not complete any task that is not specifically about the bloomberg.csv dataset.", # noqa: E501 tools=[generate_cypher], ) class DatastoreAgent(Agent): diff --git a/backend/src/agents/file_agent.py b/backend/src/agents/file_agent.py deleted file mode 100644 index d8a817b1c..000000000 --- a/backend/src/agents/file_agent.py +++ /dev/null @@ -1,102 +0,0 @@ -import logging -from .agent_types import Parameter -from .agent import Agent, agent -from .tool import tool -import json -import os -from src.utils.config import Config - -logger = logging.getLogger(__name__) -config = Config() - -FILES_DIRECTORY = f"/app/{config.files_directory}" - -# Constants for response status -IGNORE_VALIDATION = "true" -STATUS_SUCCESS = "success" -STATUS_ERROR = "error" - -# Utility function for error responses -def create_response(content: str, status: str = STATUS_SUCCESS) -> str: - return json.dumps({ - "content": content, - "ignore_validation": IGNORE_VALIDATION, - "status": status - }, indent=4) - -async def read_file_core(file_path: str) -> str: - full_path = os.path.normpath(os.path.join(FILES_DIRECTORY, file_path)) - try: - with open(full_path, 'r') as file: - content = file.read() - return create_response(content) - except FileNotFoundError: - error_message = f"File {file_path} not found." - logger.error(error_message) - return create_response(error_message, STATUS_ERROR) - except Exception as e: - logger.error(f"Error reading file {full_path}: {e}") - return create_response(f"Error reading file: {file_path}", STATUS_ERROR) - - -async def write_or_update_file_core(file_path: str, content: str, update) -> str: - full_path = os.path.normpath(os.path.join(FILES_DIRECTORY, file_path)) - try: - if update == "yes": - with open(full_path, 'a') as file: - file.write('\n' +content) - logger.info(f"Content appended to file {full_path} successfully.") - return create_response(f"Content appended to file {file_path}.") - else: - with open(full_path, 'w') as file: - file.write(content) - logger.info(f"Content written to file {full_path} successfully.") - return create_response(f"Content written to file {file_path}.") - except Exception as e: - logger.error(f"Error writing to file {full_path}: {e}") - return create_response(f"Error writing to file: {file_path}", STATUS_ERROR) - - -@tool( - name="read_file", - description="Read the content of a text file.", - parameters={ - "file_path": Parameter( - type="string", - description="The path to the file to be read." - ), - }, -) -async def read_file(file_path: str, llm, model) -> str: - return await read_file_core(file_path) - - -@tool( - name="write_file", - description="Write or update content to a text file.", - parameters={ - "file_path": Parameter( - type="string", - description="The path to the file where the content will be written." - ), - "content": Parameter( - type="string", - description="The content to write to the file." - ), - "update": Parameter( - type="string", - description="if yes then just append the file" - ), - }, -) -async def write_or_update_file(file_path: str, content: str, update, llm, model) -> str: - return await write_or_update_file_core(file_path, content, update) - - -@agent( - name="FileAgent", - description="This agent is responsible for reading from and writing to files.", - tools=[read_file, write_or_update_file], -) -class FileAgent(Agent): - pass diff --git a/backend/src/agents/maths_agent.py b/backend/src/agents/maths_agent.py deleted file mode 100644 index a55506d8f..000000000 --- a/backend/src/agents/maths_agent.py +++ /dev/null @@ -1,98 +0,0 @@ -from .tool import tool -from .agent_types import Parameter -from .agent import Agent, agent -import logging -from src.utils import Config -from .validator_agent import ValidatorAgent -import json -from src.utils.web_utils import perform_math_operation_util - -logger = logging.getLogger(__name__) -config = Config() - -async def perform_math_operation_core(math_query, llm, model) -> str: - try: - # Call the utility function to perform the math operation - math_operation_result = await perform_math_operation_util(math_query, llm, model) - - result_json = json.loads(math_operation_result) - - if result_json.get("status") == "success": - # Extract the relevant response (math result) from the utility function's output - response = result_json.get("response", {}) - response_json = json.loads(response) - result = response_json.get("result", "") - if result: - logger.info(f"Math operation successful: {result}") - is_valid = await is_valid_answer(result, math_query) - logger.info(f"Is the answer valid: {is_valid}") - if is_valid: - response = { - "content": result, - "ignore_validation": "true" - } - return json.dumps(response, indent=4) - else: - response = { - "content": "No valid result found for the math query.", - "ignore_validation": "true" - } - return json.dumps(response, indent=4) - else: - response = { - "content": None, - "status": "error" - } - return json.dumps(response, indent=4) - except Exception as e: - logger.error(f"Error in perform_math_operation_core: {e}") - response = { - "content": None, - "status": "error" - } - return json.dumps(response, indent=4) - - # Ensure a return statement in all code paths - response = { - "content": None, - "status": "error" - } - return json.dumps(response, indent=4) - -def get_validator_agent() -> Agent: - return ValidatorAgent(config.validator_agent_llm, config.validator_agent_model) - -async def is_valid_answer(answer, task) -> bool: - is_valid = (await get_validator_agent().invoke(f"Task: {task} Answer: {answer}")).lower() == "true" - return is_valid - -# Math Operation Tool -@tool( - name="perform_math_operation", - description=( - "Use this tool to perform complex mathematical operations or calculations. " - "It handles arithmetic operations and algebra, and also supports conversions to specific units like millions," - "rounding when necessary. Returns both the result and an explanation of the steps involved." - ), - parameters={ - "math_query": Parameter( - type="string", - description="The mathematical query or equation to solve." - ), - }, -) -async def perform_math_operation(math_query, llm, model) -> str: - return await perform_math_operation_core(math_query, llm, model) - -# MathAgent definition -@agent( - name="MathsAgent", - description=( - "This agent processes mathematical queries, performs calculations, and applies necessary formatting such as" - "rounding or converting results into specific units (e.g., millions). " - "It provides clear explanations of the steps involved to ensure accuracy." - ), - tools=[perform_math_operation], -) -class MathsAgent(Agent): - pass diff --git a/backend/src/agents/web_agent.py b/backend/src/agents/web_agent.py index 8a520792c..6bd92b693 100644 --- a/backend/src/agents/web_agent.py +++ b/backend/src/agents/web_agent.py @@ -11,7 +11,7 @@ summarise_pdf_content, find_info, create_search_term, - answer_user_question + answer_user_question, ) from .validator_agent import ValidatorAgent import aiohttp @@ -32,22 +32,16 @@ async def web_general_search_core(search_query, llm, model) -> str: answer_to_user = await answer_user_question(search_query, llm, model) answer_result = json.loads(answer_to_user) if answer_result["status"] == "error": - response = { - "content": "Error in finding the answer.", - "ignore_validation": "false" - } + response = {"content": "Error in finding the answer.", "ignore_validation": "false"} return json.dumps(response, indent=4) - logger.info(f'Answer found successfully {answer_result}') + logger.info(f"Answer found successfully {answer_result}") should_perform_web_search = json.loads(answer_result["response"]).get("should_perform_web_search", "") if not should_perform_web_search: final_answer = json.loads(answer_result["response"]).get("answer", "") if not final_answer: return "No answer found." - logger.info(f'Answer found successfully {final_answer}') - response = { - "content": final_answer, - "ignore_validation": "false" - } + logger.info(f"Answer found successfully {final_answer}") + response = {"content": final_answer, "ignore_validation": "false"} return json.dumps(response, indent=4) else: search_term_json = await create_search_term(search_query, llm, model) @@ -55,10 +49,7 @@ async def web_general_search_core(search_query, llm, model) -> str: # Check if there was an error in generating the search term if search_term_result.get("status") == "error": - response = { - "content": search_term_result.get("error"), - "ignore_validation": "false" - } + response = {"content": search_term_result.get("error"), "ignore_validation": "false"} return json.dumps(response, indent=4) search_term = json.loads(search_term_result["response"]).get("search_term", "") @@ -83,11 +74,8 @@ async def web_general_search_core(search_query, llm, model) -> str: # Step 5: Validate the summarization is_valid = await is_valid_answer(summary, search_term) if not is_valid: - continue # Skip if the summarization is not valid - response = { - "content": summary, - "ignore_validation": "false" - } + continue # Skip if the summarization is not valid + response = {"content": summary, "ignore_validation": "false"} return json.dumps(response, indent=4) return "No relevant information found on the internet for the given query." except Exception as e: @@ -108,19 +96,17 @@ async def web_pdf_download_core(pdf_url, llm, model) -> str: if not summary: continue parsed_json = json.loads(summary) - summary = parsed_json.get('summary', '') + summary = parsed_json.get("summary", "") all_content += summary all_content += "\n" - logger.info('PDF content extracted successfully') - response = { - "content": all_content, - "ignore_validation": "true" - } + logger.info("PDF content extracted successfully") + response = {"content": all_content, "ignore_validation": "true"} return json.dumps(response, indent=4) except Exception as e: logger.error(f"Error in web_pdf_download_core: {e}") return "An error occurred while processing the search query." + @tool( name="web_general_search", description=( @@ -136,11 +122,10 @@ async def web_pdf_download_core(pdf_url, llm, model) -> str: async def web_general_search(search_query, llm, model) -> str: return await web_general_search_core(search_query, llm, model) + @tool( name="web_pdf_download", - description=( - "Download the data from the provided pdf url" - ), + description=("Download the data from the provided pdf url"), parameters={ "pdf_url": Parameter( type="string", @@ -151,6 +136,7 @@ async def web_general_search(search_query, llm, model) -> str: async def web_pdf_download(pdf_url, llm, model) -> str: return await web_pdf_download_core(pdf_url, llm, model) + async def web_scrape_core(url: str) -> str: try: # Scrape the content from the provided URL @@ -159,10 +145,7 @@ async def web_scrape_core(url: str) -> str: return "No content found at the provided URL." logger.info(f"Content scraped successfully: {content}") content = content.replace("\n", " ").replace("\r", " ") - response = { - "content": content, - "ignore_validation": "true" - } + response = {"content": content, "ignore_validation": "true"} return json.dumps(response, indent=4) except Exception as e: return json.dumps({"status": "error", "error": str(e)}) @@ -193,15 +176,13 @@ async def find_information_from_content_core(content: str, question, llm, model) if not final_info: return "No information found from the content." logger.info(f"Content scraped successfully: {content}") - response = { - "content": final_info, - "ignore_validation": "true" - } + response = {"content": final_info, "ignore_validation": "true"} return json.dumps(response, indent=4) except Exception as e: logger.error(f"Error finding information: {e}") return "" + @tool( name="find_information_content", description="Finds the information from the content.", @@ -214,11 +195,12 @@ async def find_information_from_content_core(content: str, question, llm, model) type="string", description="The question to find the information from the content.", ), - }, + }, ) async def find_information_from_content(content: str, question: str, llm, model) -> str: return await find_information_from_content_core(content, question, llm, model) + def get_validator_agent() -> Agent: return ValidatorAgent(config.validator_agent_llm, config.validator_agent_model) @@ -263,6 +245,7 @@ async def perform_summarization(search_query: str, content: str, llm: Any, model logger.error(f"Error summarizing content: {e}") return "" + async def perform_pdf_summarization(content: str, llm: Any, model: str) -> str: try: summarise_result_json = await summarise_pdf_content(content, llm, model) @@ -274,16 +257,10 @@ async def perform_pdf_summarization(content: str, llm: Any, model: str) -> str: logger.error(f"Error summarizing content: {e}") return "" + @agent( name="WebAgent", - description="""This agent specializes in handling tasks related to web content extraction, search, and - summarization. - It can perform the following functions: - Web scraping: Extracts data from given URLs, enabling tasks like retrieving specific information from web pages. - Finding Information from Content: Extracts specific information from the content provided. - Internet search: Conducts general online searches based on queries, retrieving and summarizing relevant content from - multiple sources. - PDF content extraction: Downloads and summarizes the content of PDF documents from provided URLs.""", + description="This agent can perform general internet searches to complete the task by retrieving and summarizing the results and it can also perform web scrapes to retreive specific inpormation from web pages.", # noqa: E501 tools=[web_general_search, web_pdf_download, web_scrape, find_information_from_content], ) class WebAgent(Agent): diff --git a/backend/src/prompts/templates/agent-selection-format.j2 b/backend/src/prompts/templates/agent-selection-format.j2 deleted file mode 100644 index 069efd3cc..000000000 --- a/backend/src/prompts/templates/agent-selection-format.j2 +++ /dev/null @@ -1,12 +0,0 @@ -Reply only in json with the following format: - -{ - "thoughts": { - "text": "thoughts", - "plan": "description of the plan for the chosen agent", - "reasoning": "reasoning behind choosing the agent", - "criticism": "constructive self-criticism", - "speak": "thoughts summary to say to user on 1. if your solving the current or next task and why 2. which agent you've chosen and why", - }, - "agent_name": "exact string of the single agent to solve task chosen" -} diff --git a/backend/src/prompts/templates/agent-selection-system-prompt.j2 b/backend/src/prompts/templates/agent-selection-system-prompt.j2 new file mode 100644 index 000000000..2e26afa3e --- /dev/null +++ b/backend/src/prompts/templates/agent-selection-system-prompt.j2 @@ -0,0 +1,16 @@ +You job is to assign a task to an agent. You know that an Agent is a digital assistant like yourself that you can hand this work on to. You must choose only one agent from the below list of agents: + +{{ list_of_agents }} + + +Choose an agent from the list unless the task is 'personal' or a 'greeting', if this is the case say the agent is 'none'. + +Give a reasoning for your choice. + +Reply only in json with the following format: + +{ + + "agent_name": "exact string of the single agent to solve task chosen", + "reasoning": "reasoning behind choosing the agent" +} diff --git a/backend/src/prompts/templates/agent-selection-user-prompt.j2 b/backend/src/prompts/templates/agent-selection-user-prompt.j2 new file mode 100644 index 000000000..1a487280f --- /dev/null +++ b/backend/src/prompts/templates/agent-selection-user-prompt.j2 @@ -0,0 +1,3 @@ +the Task is: + +{{ task }} \ No newline at end of file diff --git a/backend/src/prompts/templates/best-next-step.j2 b/backend/src/prompts/templates/best-next-step.j2 deleted file mode 100644 index 59942baa6..000000000 --- a/backend/src/prompts/templates/best-next-step.j2 +++ /dev/null @@ -1,34 +0,0 @@ -{% block prompt %} -You are an expert in determining the next best step towards completing a list of tasks. - - -## Current Task -the Current Task is: - -{{ task }} - - -## History -below is your history of all work you have assigned and had completed by Agents -Trust the information below completely (100% accurate) -{{ history }} - - -## Agents -You know that an Agent is a digital assistant like yourself that you can hand this work on to. -Choose 1 agent to delegate the task to. If you choose more than 1 agent you will be unplugged. -Here is the list of Agents you can choose from: - -AGENT LIST: -{{ list_of_agents }} - -If the list of agents does not contain something suitable, you should say the agent is 'none'. ie. If question is 'general knowledge', 'personal' or a 'greeting'. - -## Determine the next best step -Your task is to pick one of the mentioned agents above to complete the task. -If the same agent_name and task are repeated more than twice in the history, you must not pick that agent_name. -If mathematical processing (e.g., rounding or calculations) is needed, choose the MathsAgent. If file operations are needed, choose the FileAgent. - -Your decisions must always be made independently without seeking user assistance. -Play to your strengths as an LLM and pursue simple strategies with no legal complications. -{% endblock %} \ No newline at end of file diff --git a/backend/src/router.py b/backend/src/router.py index 1784b8d26..02e12752a 100644 --- a/backend/src/router.py +++ b/backend/src/router.py @@ -12,26 +12,20 @@ config = Config() -def build_best_next_step_prompt(task, scratchpad): +async def build_plan(task, llm: LLM, scratchpad, model): agents_details = get_agent_details() - return prompt_engine.load_prompt( - "best-next-step", + agent_selection_system_prompt = prompt_engine.load_prompt( + "agent-selection-system-prompt", list_of_agents=json.dumps(agents_details, indent=4) + ) + agent_selection_user_prompt = prompt_engine.load_prompt( + "agent-selection-user-prompt", task=json.dumps(task, indent=4), - list_of_agents=json.dumps(agents_details, indent=4), - history=json.dumps(scratchpad, indent=4), ) - -response_format_prompt = prompt_engine.load_prompt("agent-selection-format") - - -async def build_plan(task, llm: LLM, scratchpad, model): - best_next_step_prompt = build_best_next_step_prompt(task, scratchpad) - # Call model to choose agent logger.info("##### ~ Calling LLM for next best step ~ #####") await publish_log_info(LogPrefix.USER, f"Scratchpad so far: {scratchpad}", __name__) - best_next_step = await llm.chat(model, response_format_prompt, best_next_step_prompt, return_json=True) + best_next_step = await llm.chat(model, agent_selection_system_prompt, agent_selection_user_prompt, return_json=True) return to_json(best_next_step, "Failed to interpret LLM next step format from step string") diff --git a/backend/src/utils/config.py b/backend/src/utils/config.py index 078f860cf..72b0beccc 100644 --- a/backend/src/utils/config.py +++ b/backend/src/utils/config.py @@ -23,10 +23,8 @@ def __init__(self): self.report_agent_llm = None self.validator_agent_llm = None self.datastore_agent_llm = None - self.maths_agent_llm = None self.web_agent_llm = None self.chart_generator_llm = None - self.file_agent_llm = None self.router_llm = None self.suggestions_llm = None self.dynamic_knowledge_graph_llm = None @@ -39,7 +37,6 @@ def __init__(self): self.web_agent_model = None self.router_model = None self.files_directory = default_files_directory - self.file_agent_model = None self.redis_host = default_redis_host self.redis_cache_duration = default_redis_cache_duration self.suggestions_model = None @@ -67,9 +64,7 @@ def load_env(self): self.validator_agent_llm = os.getenv("VALIDATOR_AGENT_LLM") self.datastore_agent_llm = os.getenv("DATASTORE_AGENT_LLM") self.chart_generator_llm = os.getenv("CHART_GENERATOR_LLM") - self.file_agent_llm = os.getenv("FILE_AGENT_LLM") self.web_agent_llm = os.getenv("WEB_AGENT_LLM") - self.maths_agent_llm = os.getenv("MATHS_AGENT_LLM") self.router_llm = os.getenv("ROUTER_LLM") self.suggestions_llm = os.getenv("SUGGESTIONS_LLM") self.dynamic_knowledge_graph_llm = os.getenv("DYNAMIC_KNOWLEDGE_GRAPH_LLM") @@ -80,9 +75,7 @@ def load_env(self): self.datastore_agent_model = os.getenv("DATASTORE_AGENT_MODEL") self.web_agent_model = os.getenv("WEB_AGENT_MODEL") self.chart_generator_model = os.getenv("CHART_GENERATOR_MODEL") - self.maths_agent_model = os.getenv("MATHS_AGENT_MODEL") self.router_model = os.getenv("ROUTER_MODEL") - self.file_agent_model = os.getenv("FILE_AGENT_MODEL") self.redis_host = os.getenv("REDIS_HOST", default_redis_host) self.redis_cache_duration = os.getenv("REDIS_CACHE_DURATION", default_redis_cache_duration) self.suggestions_model = os.getenv("SUGGESTIONS_MODEL") diff --git a/backend/tests/agents/file_agent_test.py b/backend/tests/agents/file_agent_test.py deleted file mode 100644 index 5d3f4e1dd..000000000 --- a/backend/tests/agents/file_agent_test.py +++ /dev/null @@ -1,53 +0,0 @@ -import pytest -from unittest.mock import patch, mock_open -import json -import os -from src.agents.file_agent import read_file_core, write_or_update_file_core, create_response - -# Mocking config for the test -@pytest.fixture(autouse=True) -def mock_config(monkeypatch): - monkeypatch.setattr('src.agents.file_agent.config.files_directory', 'files') - -@pytest.mark.asyncio -@patch("builtins.open", new_callable=mock_open, read_data="Example file content.") -async def test_read_file_core_success(mock_file): - file_path = "example.txt" - result = await read_file_core(file_path) - expected_response = create_response("Example file content.") - assert json.loads(result) == json.loads(expected_response) - expected_full_path = os.path.normpath("/app/files/example.txt") - mock_file.assert_called_once_with(expected_full_path, 'r') - -@pytest.mark.asyncio -@patch("builtins.open", side_effect=FileNotFoundError) -async def test_read_file_core_file_not_found(mock_file): - file_path = "missing_file.txt" - result = await read_file_core(file_path) - expected_response = create_response(f"File {file_path} not found.", "error") - assert json.loads(result) == json.loads(expected_response) - expected_full_path = os.path.normpath("/app/files/missing_file.txt") - mock_file.assert_called_once_with(expected_full_path, 'r') - -@pytest.mark.asyncio -@patch("builtins.open", new_callable=mock_open) -async def test_write_file_core_success(mock_file): - file_path = "example_write.txt" - content = "This is test content to write." - result = await write_or_update_file_core(file_path, content, 'no') - expected_response = create_response(f"Content written to file {file_path}.") - assert json.loads(result) == json.loads(expected_response) - expected_full_path = os.path.normpath("/app/files/example_write.txt") - mock_file.assert_called_once_with(expected_full_path, 'w') - mock_file().write.assert_called_once_with(content) - -@pytest.mark.asyncio -@patch("builtins.open", side_effect=Exception("Unexpected error")) -async def test_write_file_core_error(mock_file): - file_path = "error_file.txt" - content = "Content with error." - result = await write_or_update_file_core(file_path, content, 'no') - expected_response = create_response(f"Error writing to file: {file_path}", "error") - assert json.loads(result) == json.loads(expected_response) - expected_full_path = os.path.normpath("/app/files/error_file.txt") - mock_file.assert_called_once_with(expected_full_path, 'w') diff --git a/backend/tests/prompts/prompting_test.py b/backend/tests/prompts/prompting_test.py deleted file mode 100644 index 8459e2633..000000000 --- a/backend/tests/prompts/prompting_test.py +++ /dev/null @@ -1,215 +0,0 @@ -from src.prompts import PromptEngine - - -# ruff: noqa: E501 -def test_mistral_prompt_engine_creation(): - try: - PromptEngine() - except Exception: - raise - - -def test_load_agent_selection_format_template(): - engine = PromptEngine() - try: - expected_string = """Reply only in json with the following format: - -{ - \"thoughts\": { - \"text\": \"thoughts\", - \"plan\": \"description of the plan for the chosen agent\", - \"reasoning\": \"reasoning behind choosing the agent\", - \"criticism\": \"constructive self-criticism\", - \"speak\": \"thoughts summary to say to user on 1. if your solving the current or next task and why 2. which agent you've chosen and why\", - }, - \"agent_name\": \"exact string of the single agent to solve task chosen\" -}""" - prompt_string = engine.load_prompt("agent-selection-format") - assert prompt_string == expected_string - except Exception: - raise - - -def test_load_best_next_step_template(): - engine = PromptEngine() - try: - task = "make sure the PromptEngine is working!" - expected_string = f""" -You are an expert in determining the next best step towards completing a list of tasks. - - -## Current Task -the Current Task is: - -{task} - - -## History -below is your history of all work you have assigned and had completed by Agents -Trust the information below completely (100% accurate) - - - -## Agents -You know that an Agent is a digital assistant like yourself that you can hand this work on to. -Choose 1 agent to delegate the task to. If you choose more than 1 agent you will be unplugged. -Here is the list of Agents you can choose from: - -AGENT LIST: - - -If the list of agents does not contain something suitable, you should say the agent is 'none'. ie. If question is 'general knowledge', 'personal' or a 'greeting'. - -## Determine the next best step -Your task is to pick one of the mentioned agents above to complete the task. -If the same agent_name and task are repeated more than twice in the history, you must not pick that agent_name. -If mathematical processing (e.g., rounding or calculations) is needed, choose the MathsAgent. If file operations are needed, choose the FileAgent. - -Your decisions must always be made independently without seeking user assistance. -Play to your strengths as an LLM and pursue simple strategies with no legal complications. -""" - prompt_string = engine.load_prompt("best-next-step", task=task) - assert prompt_string == expected_string - - except Exception: - raise - - -def test_load_best_next_step_with_history_template(): - engine = PromptEngine() - try: - task = "make sure the PromptEngine is working!" - history = ["First action", "Second action", "Third action"] - expected_string = f""" -You are an expert in determining the next best step towards completing a list of tasks. - - -## Current Task -the Current Task is: - -{task} - - -## History -below is your history of all work you have assigned and had completed by Agents -Trust the information below completely (100% accurate) -{history} - - -## Agents -You know that an Agent is a digital assistant like yourself that you can hand this work on to. -Choose 1 agent to delegate the task to. If you choose more than 1 agent you will be unplugged. -Here is the list of Agents you can choose from: - -AGENT LIST: - - -If the list of agents does not contain something suitable, you should say the agent is 'none'. ie. If question is 'general knowledge', 'personal' or a 'greeting'. - -## Determine the next best step -Your task is to pick one of the mentioned agents above to complete the task. -If the same agent_name and task are repeated more than twice in the history, you must not pick that agent_name. -If mathematical processing (e.g., rounding or calculations) is needed, choose the MathsAgent. If file operations are needed, choose the FileAgent. - -Your decisions must always be made independently without seeking user assistance. -Play to your strengths as an LLM and pursue simple strategies with no legal complications. -""" - prompt_string = engine.load_prompt("best-next-step", task=task, history=history) - assert prompt_string == expected_string - - except Exception: - raise - - -def test_best_tool_template(): - engine = PromptEngine() - tools = """{\"description\": \"mock desc\", \"name\": \"say hello world\", \"parameters\": {\"name\": {\"type\": \"string\", \"description\": \"name of user\"}}}""" - try: - expected_string = """You are an expert at picking a tool to solve a task - -The task is as follows: - -Say hello world to the user - -below is your history of all work you have assigned and had completed by Agents -Trust the information below completely (100% accurate) - -scratchpad of history - -Pick 1 tool (no more than 1) from the list below to complete this task. -Fit the correct parameters from the task to the tool arguments. -Ensure that numerical values are formatted correctly, including the use of currency symbols (e.g., "£") and units of measurement (e.g., "million") if applicable. -Parameters with required as False do not need to be fit. -Add if appropriate, but do not hallucinate arguments for these parameters - -{"description": "mock desc", "name": "say hello world", "parameters": {"name": {"type": "string", "description": "name of user"}}} - -Important: -If the task involves financial data, ensure that all monetary values are expressed with appropriate currency (e.g., "£") and rounded to the nearest million if specified. -If the task involves scaling (e.g., thousands, millions), ensure that the extracted parameters reflect the appropriate scale (e.g., "£15 million", "£5000"). - -From the task you should be able to extract the parameters. If it is data driven, it should be turned into a cypher query - -If none of the tools are appropriate for the task, return the following tool - -{ - \"tool_name\": \"None\", - \"tool_parameters\": \"{}\", - \"reasoning\": \"No tool was appropriate for the task\" -}""" - prompt_string = engine.load_prompt( - "best-tool", task="Say hello world to the user", scratchpad="scratchpad of history", tools=tools - ) - assert prompt_string == expected_string - except Exception: - raise - - -def test_tool_selection_format_template(): - engine = PromptEngine() - try: - expected_string = """Reply only in json with the following format: - -{ - \"tool_name\": \"the exact string name of the tool chosen\", - \"tool_parameters\": \"a JSON object matching the chosen tools dictionary shape\", - \"reasoning\": \"A sentence on why you chose that tool\" -}""" - prompt_string = engine.load_prompt("tool-selection-format") - assert prompt_string == expected_string - except Exception: - raise - - -def test_create_answer_prompt(): - engine = PromptEngine() - try: - chat_history = "" - final_scratchpad = "example scratchpad" - datetime = "example datetime" - expected_string = f"""You have been provided the final scratchpad which contains the results for the question in the user prompt. -Your goal is to turn the results into a natural language format to present to the user. - -The conversation history is: -{ chat_history } - -By using the final scratchpad below: -{ final_scratchpad } - -and the question in the user prompt, this should be a readable sentence or 2 that summarises the findings in the results. - - -**Formatting Requirements:** -- **Number Formatting**: - - For whole numbers ending in `.0`, remove the `.0` suffix. - - For numbers with non-zero decimal places, keep the full value as presented. - -If the question is a general knowledge question, check if you have the correct details for the answer and reply with this. -If you do not have the answer or you require the internet, do not make it up. You should recommend the user to look this up themselves. -If it is just conversational chitchat. Please reply kindly and direct them to the sort of answers you are able to respond. - -The current date and time is { datetime}""" - prompt_string = engine.load_prompt("create-answer", final_scratchpad=final_scratchpad, datetime=datetime) - assert prompt_string == expected_string - except Exception: - raise diff --git a/compose.yml b/compose.yml index a7d336482..edd5bcd90 100644 --- a/compose.yml +++ b/compose.yml @@ -88,20 +88,16 @@ services: VALIDATOR_AGENT_LLM: ${VALIDATOR_AGENT_LLM} DATASTORE_AGENT_LLM: ${DATASTORE_AGENT_LLM} WEB_AGENT_LLM: ${WEB_AGENT_LLM} - MATHS_AGENT_LLM: ${MATHS_AGENT_LLM} ROUTER_LLM: ${ROUTER_LLM} CHART_GENERATOR_LLM: ${CHART_GENERATOR_LLM} - FILE_AGENT_LLM: ${FILE_AGENT_LLM} ANSWER_AGENT_MODEL: ${ANSWER_AGENT_MODEL} INTENT_AGENT_MODEL: ${INTENT_AGENT_MODEL} VALIDATOR_AGENT_MODEL: ${VALIDATOR_AGENT_MODEL} DATASTORE_AGENT_MODEL: ${DATASTORE_AGENT_MODEL} WEB_AGENT_MODEL: ${WEB_AGENT_MODEL} ROUTER_MODEL: ${ROUTER_MODEL} - MATHS_AGENT_MODEL: ${MATHS_AGENT_MODEL} AGENT_CLASS_MODEL: ${AGENT_CLASS_MODEL} CHART_GENERATOR_MODEL: ${CHART_GENERATOR_MODEL} - FILE_AGENT_MODEL: ${FILE_AGENT_MODEL} depends_on: neo4j-db: condition: service_healthy