Fs 104/Fix-Webagent

backend/src/agents/web_agent.py

@@ -86,7 +86,7 @@ async def web_general_search_core(search_query, llm, model) -> str:
                     continue # Skip if the summarization is not valid
                 response = {
                     "content": summary,
-                    "ignore_validation": "false"
+                    "ignore_validation": "true" # This is to ignore the validation of the answer again by the supervisor
                 }
                 return json.dumps(response, indent=4)
             return "No relevant information found on the internet for the given query."

backend/src/prompts/templates/intent-system.j2

@@ -13,4 +13,7 @@ Output your result in the following json format:
     "question": "string of the original question",
     "user_intent": "string of the intent of the user's question",
     "questions": array of singular objective questions or if the question mentions csv, dataset or database an empty array
-}
+}
+
+Guidelines:
+- If the user has asked to check online, then each question in the questions array should also specify that.

backend/src/prompts/templates/validator.j2

@@ -2,7 +2,7 @@ You are an expert validator. You can help with validating the answers to the tas
 
 Your entire purpose is to return a "true" or "false" value to indicate if the answer has fulfilled the task, along with a reasoning to explain your decision.
 
-You will be passed a task and an answer. You need to determine if the answer is correct or not.
+You will be passed a task and an answer. You need to determine if the answer is correct or not, ensuring that the task's specific requirements are addressed.
 
 Output format:
 
@@ -14,10 +14,13 @@ json
 }
 
 **Validation Guidelines:**
-- Be lenient - if the answer looks reasonably accurate, return "true".
-- If multiple entities have the same highest score and this matches the query intent, return "true".
+- The answer must fulfill the specific intent of the task, not just provide related information.
+- Be lenient if the answer is reasonably accurate and fulfills the task's intent, even if it lacks minor details.
+- If specific data (like a list of companies) is requested but missing, return "false."
+- If multiple entities have the same highest score and this matches the query intent, return "true."
 - Spending is negative; ensure any calculations involving spending reflect this if relevant to the task.
 
+
 Example:
 Task: What is 2 + 2?
 Answer: 4
@@ -33,6 +36,20 @@ Answer: 5
     "reasoning": "The answer is incorrect; 2 + 2 equals 4, not 5."
 }
 
+Task: Provide a list of companies with the highest ESG scores in the Technology sector.
+Answer: As of the end of 2023, the Technology sector had the highest weighted-average ESG score among all sectors, according to the MSCI ACWI SRI Index. However, I don't have a specific list of individual companies with the highest scores.
+{
+    "response": "false",
+    "reasoning": "The answer provides general information about ESG scores in the Technology sector but fails to fulfill the task's intent of listing companies with the highest scores."
+}
+
+Task: Provide a list of companies with the highest ESG scores in the Technology sector.
+Answer: Here are the companies with the highest ESG scores in the Technology sector: 1. Apple Inc., 2. Microsoft Corp., 3. Alphabet Inc.
+{
+    "response": "true",
+    "reasoning": "The answer lists companies with the highest ESG scores in the Technology sector, fulfilling the task's intent."
+}
+
 Task: What are Apple's ESG scores?
 Answer: Apple's ESG (Environmental, Social, and Governance) scores are as follows: Environmental Score of 95.0, Social Score of 90.0, Governance Score of 92.0.
 {

backend/src/utils/web_utils.py

@@ -13,7 +13,7 @@
 engine = PromptEngine()
 
 
-async def search_urls(search_query, num_results=10) -> str:
+async def search_urls(search_query, num_results=30) -> str:
     logger.info(f"Searching the web for: {search_query}")
     try:
         https_urls = [str(url) for url in search(search_query, num_results=num_results) if str(url).startswith("https")]