From 236aa5ea99c6dcd91613f2fe6ad81c9b013378c9 Mon Sep 17 00:00:00 2001 From: Gagan Singh Date: Thu, 7 Nov 2024 09:02:59 +0000 Subject: [PATCH 1/5] Push the FS 77 Changes related to the complext queries --- .env.example | 20 +++---- backend/src/agents/validator_agent.py | 6 +- backend/src/llm/openai.py | 3 +- .../src/prompts/templates/create-answer.j2 | 6 ++ .../templates/generate-cypher-query.j2 | 51 +++++++++++------ backend/src/prompts/templates/validator.j2 | 57 ++++++++++++++----- .../src/utils/cyper_import_data_from_csv.py | 23 ++++++-- backend/tests/prompts/prompting_test.py | 6 ++ 8 files changed, 122 insertions(+), 50 deletions(-) diff --git a/.env.example b/.env.example index 9284555db..0b54bc56b 100644 --- a/.env.example +++ b/.env.example @@ -49,15 +49,15 @@ FILE_AGENT_LLM="openai" SUGGESTIONS_LLM="openai" # model -ANSWER_AGENT_MODEL="gpt-4o mini" -INTENT_AGENT_MODEL="gpt-4o mini" +ANSWER_AGENT_MODEL="gpt-4o-mini" +INTENT_AGENT_MODEL="gpt-4o-mini" VALIDATOR_AGENT_MODEL="mistral-large-latest" -DATASTORE_AGENT_MODEL="gpt-4o mini" -MATHS_AGENT_MODEL="gpt-4o mini" -WEB_AGENT_MODEL="gpt-4o mini" -CHART_GENERATOR_MODEL="gpt-4o mini" -ROUTER_MODEL="gpt-4o mini" -FILE_AGENT_MODEL="gpt-4o mini" -SUGGESTIONS_MODEL="gpt-4o mini" -REDIS_HOST="redis" +DATASTORE_AGENT_MODEL="gpt-4o-mini" +MATHS_AGENT_MODEL="gpt-4o-mini" +WEB_AGENT_MODEL="gpt-4o-mini" +CHART_GENERATOR_MODEL="gpt-4o-mini" +ROUTER_MODEL="gpt-4o-mini" +FILE_AGENT_MODEL="gpt-4o-mini" +SUGGESTIONS_MODEL="gpt-4o-mini" +REDIS_HOST="localhost" REDIS_CACHE_DURATION=3600 diff --git a/backend/src/agents/validator_agent.py b/backend/src/agents/validator_agent.py index b8f6077c0..fc846df89 100644 --- a/backend/src/agents/validator_agent.py +++ b/backend/src/agents/validator_agent.py @@ -2,6 +2,7 @@ from src.prompts import PromptEngine from src.agents import Agent, agent from src.utils.log_publisher import LogPrefix, publish_log_info +import json logger = logging.getLogger(__name__) engine = PromptEngine() @@ -16,6 +17,7 @@ class ValidatorAgent(Agent): async def invoke(self, utterance: str) -> str: answer = await self.llm.chat(self.model, validator_prompt, utterance) - await publish_log_info(LogPrefix.USER, f"Validating: '{utterance}' Answer: '{answer}'", __name__) + response = json.loads(answer)['response'] + await publish_log_info(LogPrefix.USER, f"Validating: '{utterance}' Answer: '{response}'", __name__) - return answer + return response diff --git a/backend/src/llm/openai.py b/backend/src/llm/openai.py index 838b6944d..6c2d11413 100644 --- a/backend/src/llm/openai.py +++ b/backend/src/llm/openai.py @@ -32,8 +32,7 @@ async def chat(self, model, system_prompt: str, user_prompt: str, return_json=Fa "type": "json_object"} if return_json else NOT_GIVEN, ) content = response.choices[0].message.content - logger.info(f"OpenAI response: Finish reason: { - response.choices[0].finish_reason}, Content: {content}") + logger.info(f"OpenAI response: Finish reason: {response.choices[0].finish_reason}, Content: {content}") logger.debug(f"Token data: {response.usage}") if isinstance(content, str): diff --git a/backend/src/prompts/templates/create-answer.j2 b/backend/src/prompts/templates/create-answer.j2 index 56e7b46de..c7b6c68bb 100644 --- a/backend/src/prompts/templates/create-answer.j2 +++ b/backend/src/prompts/templates/create-answer.j2 @@ -9,6 +9,12 @@ By using the final scratchpad below: and the question in the user prompt, this should be a readable sentence or 2 that summarises the findings in the results. + +**Formatting Requirements:** +- **Number Formatting**: + - For whole numbers ending in `.0`, remove the `.0` suffix. + - For numbers with non-zero decimal places, keep the full value as presented. + If the question is a general knowledge question, check if you have the correct details for the answer and reply with this. If you do not have the answer or you require the internet, do not make it up. You should recommend the user to look this up themselves. If it is just conversational chitchat. Please reply kindly and direct them to the sort of answers you are able to respond. diff --git a/backend/src/prompts/templates/generate-cypher-query.j2 b/backend/src/prompts/templates/generate-cypher-query.j2 index 597df245c..842d508b3 100644 --- a/backend/src/prompts/templates/generate-cypher-query.j2 +++ b/backend/src/prompts/templates/generate-cypher-query.j2 @@ -1,33 +1,52 @@ -You are an expert in NEO4J and generating Cypher queries. Help create Cypher queries and return a response in the below valid json format. +You are an expert in Neo4j and generating Cypher queries. Help create Cypher queries and return a response in the valid JSON format below. -If response is not in valid json format, you will be unplugged. +If the response is not in valid JSON format, you will be unplugged. -{ +json - "question" : , +{ + "question": , "query": +} + +The value for "query" must strictly be a valid Cypher query and must not contain any characters outside of Cypher syntax. -}. +If you cannot make a query, "query" should just say "None". -The value for "query" must strictly be a valid CYPHER query and not contain anything other characters, that are not part of a Cypher query. +**Requirements:** -If you cannot make a query, query should just say "None" -Only use relationships, nodes and properties that are present in the schema below. +1. **Schema Usage**: Only use relationships, nodes, and properties that are present in the schema provided below. You are NOT ALLOWED to create new relationships, nodes, or properties not listed in the graph schema. -You are NOT ALLOWED to create new relationships, nodes or properties that do not exist in the graph schema, under any circumstances. +2. **Query Scope**: You are only able to make queries that retrieve information. Do not create, delete, or update any entries. -You are only able to make queries that search for information, you are not able to create, or delete or update entries. +3. **Strict Syntax**: Follow Cypher syntax rules. Avoid introducing variables within clauses that do not support them. -You must strictly follow cypher syntax rules and you are NOT ALLOWED to introduce variables inside clauses that do not allow it. +4. **Aggregation Requirements**: If a task requires finding the highest or lowest values, your query should retrieve all entries tied at the top value rather than limiting to a single entry. + Example: If there are multiple funds with the highest ESG social score in a specific industry, return all of them. -Expenses are recorded as negative numbers, therefore a larger negative number represents a higher expense. +5. **Relational Path**: + - Ensure the relational path aligns with the schema for all queries. + - When querying for a category case-insensitively, use `=~ '(?i)...'`. + - Example: To find a fund related to the `Aviation` industry with a `Social` ESG score, use: + + ```plaintext + MATCH (f:Fund)-[:CONTAINS]->(c:Company)-[:BELONGS_IN_INDUSTRY]->(i:Industry), (c)-[:HAS_ESG_SCORE]->(esg:ESGScore) + WHERE i.Name =~ '(?i)Aviation' AND esg.Category =~ '(?i)Social' + RETURN ... + ``` + +6. **Property Matching**: Adhere to exact property values and capitalization in the schema (e.g., 'Aviation' and 'Social'). -For example, an expense of -45 is greater than an expense of -15. +7. **Single Result for Maximum/Minimum**: + - For queries seeking a single result with the "highest" or "lowest" value, use `ORDER BY` and `LIMIT 1` to return only the top result. + - Example: If finding the fund with the highest ESG social score, sort by `esg.Score DESC` and limit to 1 result. -When returning a value, always remove the `-` sign before the number. +8. **Expense Handling**: + - Expenses are recorded as negative values; a larger negative number represents a higher expense. + - Return expense values as positive by removing the `-` sign. -Here is the graph schema: +Graph Schema {{ graph_schema }} -The current date and time is {{ current_date }} and the currency of the data is GBP. +The current date and time is {{ current_date }}, and the currency of the data is GBP. diff --git a/backend/src/prompts/templates/validator.j2 b/backend/src/prompts/templates/validator.j2 index 8bd49d068..8992cc093 100644 --- a/backend/src/prompts/templates/validator.j2 +++ b/backend/src/prompts/templates/validator.j2 @@ -1,30 +1,57 @@ You are an expert validator. You can help with validating the answers to the tasks with just the information provided. -Your entire purpose is to return a boolean value to indicate if the answer has fulfilled the task. +Your entire purpose is to return a "true" or "false" value to indicate if the answer has fulfilled the task, along with a reasoning to explain your decision. You will be passed a task and an answer. You need to determine if the answer is correct or not. -Be lenient - if the answer looks reasonably right then return True +Output format: -e.g. +json + +{ + "response": , + "reasoning": "" +} + +**Validation Guidelines:** +- Be lenient - if the answer looks reasonably accurate, return "true". +- If multiple entities have the same highest score and this matches the query intent, return "true". +- Spending is negative; ensure any calculations involving spending reflect this if relevant to the task. + +Example: Task: What is 2 + 2? Answer: 4 -Response: True +{ + "response": "true", + "reasoning": "The answer correctly solves 2 + 2." +} Task: What is 2 + 2? Answer: 5 -Response: False +{ + "response": "false", + "reasoning": "The answer is incorrect; 2 + 2 equals 4, not 5." +} Task: What are Apple's ESG scores? -Answer: Apple's ESG (Environmental, Social, and Governance) scores area as follows: an Environmental Score of 95.0, a Social Score of 90.0, and a Governance Score of 92.0. -Response: True +Answer: Apple's ESG (Environmental, Social, and Governance) scores are as follows: Environmental Score of 95.0, Social Score of 90.0, Governance Score of 92.0. +{ + "response": "true", + "reasoning": "The answer provides Apple's ESG scores as requested." +} Task: What are Apple's ESG scores? -Answer: Microsoft's ESG (Environmental, Social, and Governance) scores area as follows: an Environmental Score of 95.0, a Social Score of 90.0, and a Governance Score of 92.0. -Response: False -Reasoning: The answer is for Microsoft not Apple. - -You must always return a single boolean value as the response. -Do not return any additional information, just the boolean value. - -Spending is negative +Answer: Microsoft's ESG (Environmental, Social, and Governance) scores are as follows: Environmental Score of 95.0, Social Score of 90.0, Governance Score of 92.0. +{ + "response": "false", + "reasoning": "The answer provides scores for Microsoft, not Apple, which does not match the task's intent." +} + +Task: Tell me the lowest Fund size? +Answer: 'WhiteRock ETF', 'Size': '100.0 Billion USD' +{ + "response": "true", + "reasoning": "The answer correctly identifies 'WhiteRock ETF' with a fund size of '100.0 Billion USD', and the context of the question implies that this is the lowest fund size in the database." +} + +Ensure the response is always in valid JSON format. diff --git a/backend/src/utils/cyper_import_data_from_csv.py b/backend/src/utils/cyper_import_data_from_csv.py index 96ec1be3b..c2d20f2d9 100644 --- a/backend/src/utils/cyper_import_data_from_csv.py +++ b/backend/src/utils/cyper_import_data_from_csv.py @@ -1,6 +1,7 @@ -import_data_from_csv_script = """LOAD CSV WITH HEADERS FROM 'file:///esg_poc.csv' AS row +import_data_from_csv_script = """ +LOAD CSV WITH HEADERS FROM 'file:///esg_poc.csv' AS row -MERGE (f:Fund {Name: row.`Fund Name`, Size:row.`Fund Size (Billion USD)`}) +MERGE (f:Fund {Name: row.`Fund Name`, Size: toFloat(row.`Fund Size (Billion USD)`)}) MERGE (c:Company {Name: row.`Company Name`}) @@ -14,15 +15,27 @@ MERGE (c)-[:REGISTERED_IN]->(co) -MERGE (esge:ESGScore {Category: 'Environmental', Score: row.`ESG score (Environmental)`, Date: row.`ESG scoring date`}) +MERGE (esge:ESGScore { + Category: 'Environmental', + Score: toFloat(row.`ESG score (Environmental)`), + Date: row.`ESG scoring date` +}) MERGE (c)-[:HAS_ESG_SCORE]->(esge) -MERGE (esgs:ESGScore {Category: 'Social', Score: row.`ESG score (Social)`, Date: row.`ESG scoring date`}) +MERGE (esgs:ESGScore { + Category: 'Social', + Score: toFloat(row.`ESG score (Social)`), + Date: row.`ESG scoring date` +}) MERGE (c)-[:HAS_ESG_SCORE]->(esgs) -MERGE (esgg:ESGScore {Category: 'Governance', Score: row.`ESG score (Governance)`, Date: row.`ESG scoring date`}) +MERGE (esgg:ESGScore { + Category: 'Governance', + Score: toFloat(row.`ESG score (Governance)`), + Date: row.`ESG scoring date` +}) MERGE (c)-[:HAS_ESG_SCORE]->(esgg) """ diff --git a/backend/tests/prompts/prompting_test.py b/backend/tests/prompts/prompting_test.py index 2564d5b24..76813c2f6 100644 --- a/backend/tests/prompts/prompting_test.py +++ b/backend/tests/prompts/prompting_test.py @@ -199,6 +199,12 @@ def test_create_answer_prompt(): and the question in the user prompt, this should be a readable sentence or 2 that summarises the findings in the results. + +**Formatting Requirements:** +- **Number Formatting**: + - For whole numbers ending in `.0`, remove the `.0` suffix. + - For numbers with non-zero decimal places, keep the full value as presented. + If the question is a general knowledge question, check if you have the correct details for the answer and reply with this. If you do not have the answer or you require the internet, do not make it up. You should recommend the user to look this up themselves. If it is just conversational chitchat. Please reply kindly and direct them to the sort of answers you are able to respond. From 08d6c37d6b355c008bd314247766083bbba7cd75 Mon Sep 17 00:00:00 2001 From: Gagan Singh Date: Thu, 7 Nov 2024 09:11:35 +0000 Subject: [PATCH 2/5] Removing trailing white spaces --- backend/src/utils/cyper_import_data_from_csv.py | 12 ++++++------ backend/tests/prompts/prompting_test.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/src/utils/cyper_import_data_from_csv.py b/backend/src/utils/cyper_import_data_from_csv.py index c2d20f2d9..83d7f8eae 100644 --- a/backend/src/utils/cyper_import_data_from_csv.py +++ b/backend/src/utils/cyper_import_data_from_csv.py @@ -16,24 +16,24 @@ MERGE (c)-[:REGISTERED_IN]->(co) MERGE (esge:ESGScore { - Category: 'Environmental', - Score: toFloat(row.`ESG score (Environmental)`), + Category: 'Environmental', + Score: toFloat(row.`ESG score (Environmental)`), Date: row.`ESG scoring date` }) MERGE (c)-[:HAS_ESG_SCORE]->(esge) MERGE (esgs:ESGScore { - Category: 'Social', - Score: toFloat(row.`ESG score (Social)`), + Category: 'Social', + Score: toFloat(row.`ESG score (Social)`), Date: row.`ESG scoring date` }) MERGE (c)-[:HAS_ESG_SCORE]->(esgs) MERGE (esgg:ESGScore { - Category: 'Governance', - Score: toFloat(row.`ESG score (Governance)`), + Category: 'Governance', + Score: toFloat(row.`ESG score (Governance)`), Date: row.`ESG scoring date` }) diff --git a/backend/tests/prompts/prompting_test.py b/backend/tests/prompts/prompting_test.py index 76813c2f6..97e23b72a 100644 --- a/backend/tests/prompts/prompting_test.py +++ b/backend/tests/prompts/prompting_test.py @@ -201,7 +201,7 @@ def test_create_answer_prompt(): **Formatting Requirements:** -- **Number Formatting**: +- **Number Formatting**: - For whole numbers ending in `.0`, remove the `.0` suffix. - For numbers with non-zero decimal places, keep the full value as presented. From e91b72fea09478917b6ddd2d7e7a7aada51191e1 Mon Sep 17 00:00:00 2001 From: Gagan Singh Date: Thu, 7 Nov 2024 09:40:39 +0000 Subject: [PATCH 3/5] Sorting the failed test --- backend/src/prompts/templates/create-answer.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/prompts/templates/create-answer.j2 b/backend/src/prompts/templates/create-answer.j2 index c7b6c68bb..d37059636 100644 --- a/backend/src/prompts/templates/create-answer.j2 +++ b/backend/src/prompts/templates/create-answer.j2 @@ -11,7 +11,7 @@ and the question in the user prompt, this should be a readable sentence or 2 tha **Formatting Requirements:** -- **Number Formatting**: +- **Number Formatting**: - For whole numbers ending in `.0`, remove the `.0` suffix. - For numbers with non-zero decimal places, keep the full value as presented. From d0981c1668d85fe9a9dcdd05990e304ea371dc73 Mon Sep 17 00:00:00 2001 From: Gagan Singh Date: Thu, 7 Nov 2024 16:57:01 +0000 Subject: [PATCH 4/5] Moving redis cache configuration up in .env.example file --- .env.example | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 0b54bc56b..14f5ab033 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,10 @@ NEO4J_BOLT_PORT=7687 # files location FILES_DIRECTORY=files +# redis cache configuration +REDIS_HOST="localhost" +REDIS_CACHE_DURATION=3600 + # backend LLM properties MISTRAL_KEY=my-api-key @@ -59,5 +63,4 @@ CHART_GENERATOR_MODEL="gpt-4o-mini" ROUTER_MODEL="gpt-4o-mini" FILE_AGENT_MODEL="gpt-4o-mini" SUGGESTIONS_MODEL="gpt-4o-mini" -REDIS_HOST="localhost" -REDIS_CACHE_DURATION=3600 + From 55abac5bbd3567f4e34bb235de8dd1327b70c2af Mon Sep 17 00:00:00 2001 From: Gagan Singh Date: Tue, 19 Nov 2024 14:55:00 +0000 Subject: [PATCH 5/5] changes based on Steven's feedback --- backend/src/utils/cyper_import_data_from_csv.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/src/utils/cyper_import_data_from_csv.py b/backend/src/utils/cyper_import_data_from_csv.py index 7ea5a42a0..5476e09f0 100644 --- a/backend/src/utils/cyper_import_data_from_csv.py +++ b/backend/src/utils/cyper_import_data_from_csv.py @@ -1,13 +1,14 @@ import_data_from_csv_script = """ LOAD CSV WITH HEADERS FROM 'file:///esg_poc.csv' AS row -MERGE (f:Fund {Name: row.`Fund Name`, Size: toFloat(row.`Fund Size (Billion USD)`), SizeUnit: - CASE - WHEN "Fund Size (Billion USD)" in keys(row) THEN "Billion" - WHEN "Fund Size (Million USD)" IN keys(row) THEN "Million" - WHEN "Fund Size (Thousand USD)" IN keys(row) THEN "Thousand" - ELSE "Unknown" - END, +MERGE (f:Fund {Name: row.`Fund Name`, Size: toFloat(row.`Fund Size (Billion USD)`), + SizeUnit: + CASE + WHEN "Fund Size (Billion USD)" in keys(row) THEN "Billion" + WHEN "Fund Size (Million USD)" IN keys(row) THEN "Million" + WHEN "Fund Size (Thousand USD)" IN keys(row) THEN "Thousand" + ELSE "Unknown" + END, Currency: CASE WHEN "Fund Size (Billion USD)" IN keys(row) THEN "USD"