FS 121: materiality chat agent (#59)

* first pass at materiality chat agent

* Add file upload ids to redis. Add graceful shutdown file cleanup. Add redis-insight with configuration sidecar

* moved LLMFileUploadManager into LLM file

* tidy up

* fixes from rebase

* touch ups for PR

* updating report agent to openai

* rename folder

* make script executable

* clean up Generalist agent a little. fix to materiality agent no files found. tweak to report director messaging

* fix tests and linting

* attempt to fix type checking

* attempt to fix linting and tests again

* fix openai tests

* remove todo comment

* fix report director test

.env.example

@@ -37,7 +37,7 @@ WS_URL=ws://localhost:8250/ws
 # llm
 ANSWER_AGENT_LLM="mistral"
 INTENT_AGENT_LLM="openai"
-REPORT_AGENT_LLM="mistral"
+REPORT_AGENT_LLM="openai"
 MATERIALITY_AGENT_LLM="openai"
 VALIDATOR_AGENT_LLM="openai"
 DATASTORE_AGENT_LLM="openai"
@@ -51,8 +51,8 @@ DYNAMIC_KNOWLEDGE_GRAPH_LLM="openai"
 # model
 ANSWER_AGENT_MODEL="mistral-large-latest"
 INTENT_AGENT_MODEL="gpt-4o-mini"
-REPORT_AGENT_MODEL="mistral-large-latest"
-MATERIALITY_AGENT_MODEL="gpt-4o-mini"
+REPORT_AGENT_MODEL="gpt-4o"
+MATERIALITY_AGENT_MODEL="gpt-4o"
 VALIDATOR_AGENT_MODEL="gpt-4o-mini"
 DATASTORE_AGENT_MODEL="gpt-4o-mini"
 WEB_AGENT_MODEL="gpt-4o-mini"

backend/Dockerfile

@@ -22,4 +22,4 @@ COPY ./src/. ./src
 EXPOSE 8250
 
 # Run our entry file, which will start the server
-CMD ["python", "-m", "src.main", "--host", "0.0.0.0"]
+ENTRYPOINT ["python", "-m", "src.main", "--host", "0.0.0.0"]

backend/promptfoo/materiality_agent_config.yaml

@@ -19,6 +19,9 @@ tests:
         value: JSON.parse(output).files[0] === "Additional-Sector-Guidance-Oil-and-gas.pdf"
       - type: javascript
         value: JSON.parse(output).files[1] === "GRI 11_ Oil and Gas Sector 2021.pdf"
+      - type: javascript
+        value: JSON.parse(output).files.length === 2
+
 
   - description: "test select material documents for BP with focus on nature"
     vars:
@@ -32,6 +35,40 @@ tests:
       - type: javascript
         value: JSON.parse(output).files.length === 1
 
+  - description: "test select material documents for Astra Zeneca with user question"
+    vars:
+      user_prompt: "What activities are part of Astra Zeneca's value chain?"
+      system_prompt_template: "select-material-files-system-prompt"
+      system_prompt_args:
+        catalogue: '{"library":{"TFND":[{"name":"Additional-Sector-Guidance-Biotech-and-Pharma.pdf","sector-label":"Biotechnology and Pharmaceuticals","esg-labels":["Environment","Nature"]},{"name":"Additional-Sector-Guidance-Oil-and-gas.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Nature"]}],"GRI":[{"name":"GRI 11_ Oil and Gas Sector 2021.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Social","Governance"]}]}}'
+    assert:
+      - type: javascript
+        value: JSON.parse(output).files[0] === "Additional-Sector-Guidance-Biotech-and-Pharma.pdf"
+      - type: javascript
+        value: JSON.parse(output).files.length === 1
+
+  - description: "test select material documents for BP with user question and ESG focus"
+    vars:
+      user_prompt: "What social topics are material to BP?"
+      system_prompt_template: "select-material-files-system-prompt"
+      system_prompt_args:
+        catalogue: '{"library":{"TFND":[{"name":"Additional-Sector-Guidance-Biotech-and-Pharma.pdf","sector-label":"Biotechnology and Pharmaceuticals","esg-labels":["Environment","Nature"]},{"name":"Additional-Sector-Guidance-Oil-and-gas.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Nature"]}],"GRI":[{"name":"GRI 11_ Oil and Gas Sector 2021.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Social","Governance"]}]}}'
+    assert:
+      - type: javascript
+        value: JSON.parse(output).files[0] === "GRI 11_ Oil and Gas Sector 2021.pdf"
+      - type: javascript
+        value: JSON.parse(output).files.length === 1
+
+  - description: "test no material documents are selected for NASA "
+    vars:
+      user_prompt: "What topics are material to NASA?"
+      system_prompt_template: "select-material-files-system-prompt"
+      system_prompt_args:
+        catalogue: '{"library":{"TFND":[{"name":"Additional-Sector-Guidance-Biotech-and-Pharma.pdf","sector-label":"Biotechnology and Pharmaceuticals","esg-labels":["Environment","Nature"]},{"name":"Additional-Sector-Guidance-Oil-and-gas.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Nature"]}],"GRI":[{"name":"GRI 11_ Oil and Gas Sector 2021.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Social","Governance"]}]}}'
+    assert:
+      - type: javascript
+        value: JSON.parse(output).files.length === 0
+
   - description: "test list material topics for Astra Zeneca with file"
     vars:
       user_prompt: "What topics are material for AstraZeneca?"
@@ -54,3 +91,123 @@ tests:
         value: JSON.parse(output).material_topics["Biodiversity and Ecosystem Impacts"] === "The potential impact of AstraZeneca's operations on sensitive ecosystems, as well as its reliance on biodiversity for sourcing natural compounds for drug development, highlights the importance of considering biodiversity in the company's sustainability strategy."
       - type: javascript
         value: JSON.parse(output).material_topics["Pollution Prevention"] === "Managing and reducing pollution, particularly non-GHG air pollutants, wastewater discharges, and hazardous waste, is critical for AstraZeneca to mitigate its environmental footprint and comply with environmental regulations."
+
+  - description: "test materiality agent can answer questions about materiality for Astra Zeneca"
+    vars:
+      user_prompt: "What topics are material for AstraZeneca?"
+      system_prompt_template: "answer-materiality-question"
+      file_attachment: "../library/Additional-Sector-Guidance-Biotech-and-Pharma.pdf"
+    assert:
+      - type: javascript
+        value: |
+          var expected = `
+          AstraZeneca, as part of the biotechnology and pharmaceuticals sector, should consider several ESG topics as material based on the provided guidance. These topics include:
+
+          1. **Land/Freshwater/Ocean-use Change:**
+             - Extent of land/freshwater/ocean ecosystem use change by business activity.
+             - Total spatial footprint covering total surface area controlled, disturbed, and rehabilitated.
+
+          2. **Pollution/Pollution Removal:**
+             - Hazardous waste management, both in production and end-of-life treatment, including recycling rates.
+             - Non-compliance incidents with soil and water quality regulations.
+             - Persistent ingredients related to antimicrobial resistance (AMR).
+
+          3. **Resource Use/Replenishment:**
+             - Water withdrawal and consumption from areas of water scarcity.
+             - Quantity and sustainable management of high-risk natural commodities sourced.
+
+          4. **Climate Change:**
+             - Greenhouse Gas (GHG) emissions, especially non-GHG pollutants such as particulate matter, nitrogen oxides, and others.
+
+          5. **State of Nature and Biodiversity:**
+             - Placeholder metrics for ecosystem condition and species extinction risk, indicating monitoring of biodiversity impacts.
+
+          These material topics emphasize the environmental impacts and dependencies AstraZeneca may have, focusing on sustainable management and reduction of negative impacts in their operations and supply chain.
+          `;
+
+          // promptfoo is bad at ignoring whitespace
+          expected_trimmed = expected.replace(/(?:\r\n|\r|\n|\s)/g, '');
+          output = output.replace(/(?:\r\n|\r|\n|\s)/g, '');
+          if (output === expected_trimmed) {
+            return {
+              pass: true,
+              score: 0.5,
+              reason: 'Output matched Expected (whitespace trimmed): ' + expected,
+            };
+          }
+          return {
+            pass: false,
+            score: 0,
+            reason: 'Output: \n' + output + '\n\n did not equal \n\n' + expected_trimmed,
+          };
+
+  - description: "test materiality agent can answer questions about materiality for Astra Zeneca"
+    vars:
+      user_prompt: "What topics are material for AstraZeneca?"
+      system_prompt_template: "answer-materiality-question"
+      file_attachment: "../library/Additional-Sector-Guidance-Biotech-and-Pharma.pdf"
+    assert:
+      - type: llm-rubric
+        value: |
+          Answer matches the following - every word is in the correct order with none missing and no new words added:
+
+          AstraZeneca, as part of the biotechnology and pharmaceuticals sector, should consider several ESG topics as material based on the provided guidance. These topics include:
+
+          1. **Land/Freshwater/Ocean-use Change:**
+             - Extent of land/freshwater/ocean ecosystem use change by business activity.
+             - Total spatial footprint covering total surface area controlled, disturbed, and rehabilitated.
+
+          2. **Pollution/Pollution Removal:**
+             - Hazardous waste management, both in production and end-of-life treatment, including recycling rates.
+             - Non-compliance incidents with soil and water quality regulations.
+             - Persistent ingredients related to antimicrobial resistance (AMR).
+
+          3. **Resource Use/Replenishment:**
+             - Water withdrawal and consumption from areas of water scarcity.
+             - Quantity and sustainable management of high-risk natural commodities sourced.
+
+          4. **Climate Change:**
+             - Greenhouse Gas (GHG) emissions, especially non-GHG pollutants such as particulate matter, nitrogen oxides, and others.
+
+          5. **State of Nature and Biodiversity:**
+             - Placeholder metrics for ecosystem condition and species extinction risk, indicating monitoring of biodiversity impacts.
+
+          These material topics emphasize the environmental impacts and dependencies AstraZeneca may have, focusing on sustainable management and reduction of negative impacts in their operations and supply chain.
+
+  - description: "test materiality agent can answer question about value chain of Biotech & Pharma"
+    vars:
+      user_prompt: "What business activities are typically found in Biotech and Pharma companies?"
+      system_prompt_template: "answer-materiality-question"
+      file_attachment: "../library/Additional-Sector-Guidance-Biotech-and-Pharma.pdf"
+    assert:
+      - type: llm-rubric
+        value: |
+          Answer matches the following - every word is in the correct order with none missing and no new words added:
+
+          Biotechnology and pharmaceuticals companies typically engage in various business activities across their value chains. These activities include:
+
+          1. **Inorganic and Organic Feedstock and Raw Materials**:
+            - Sourcing materials like forestry products, natural substances, agricultural products, fossil fuels, commodity chemicals, metals, minerals, and renewable raw materials.
+
+          2. **Manufacturing, Processing, and Synthesis**:
+            - Producing pharmaceutical products and their packaging.
+
+          3. **Product Design and Research and Development (R&D)**:
+            - Involves pre-clinical trials, clinical trials, and obtaining regulatory approvals.
+            - Prioritizes safe and sustainable by design approaches for product development.
+
+          4. **Distribution**:
+            - Concerned with transporting products to consumers or other locations in the value chain.
+
+          5. **Utilities and Energy Use**:
+            - Employs energy consumption, excess heat management, wastewater management, and energy and water supply management in operations.
+
+          6. **Use of Genetically Modified Organisms (GMOs) and Biofuels**:
+            - Includes utilization of GM cell cultures, GM crops, healthcare products, biodegradable plastics, and other bio-based products.
+
+          7. **End-of-Life Treatment**:
+            - Activities related to recycling, waste management, and final disposal of products.
+
+          Each of these activities has specific dependencies and impacts on nature, such as the requirement for water in manufacturing or the risk of pollutants impacting environmental quality during waste disposal stages.
+
+          These components of the biotech and pharma value chain are critical for organizations when considering ESG materiality, particularly regarding nature-related dependencies, impacts, risks, and opportunities.

backend/src/agents/__init__.py

@@ -50,6 +50,7 @@ def get_available_agents() -> List[ChatAgent]:
         DatastoreAgent(config.datastore_agent_llm, config.datastore_agent_model),
         WebAgent(config.web_agent_llm, config.web_agent_model),
         ChartGeneratorAgent(config.chart_generator_llm, config.chart_generator_model),
+        get_materiality_agent()
     ]
 
 

backend/src/agents/agent.py

@@ -1,7 +1,7 @@
 from abc import ABC
 import json
 import logging
-from typing import List, Type, TypeVar, Optional
+from typing import List, Type, TypeVar
 from src.llm import LLM, get_llm
 from src.utils.log_publisher import LogPrefix, publish_log_info
 
@@ -67,9 +67,7 @@ async def invoke(self, utterance: str) -> str:
 T = TypeVar('T', bound=ChatAgent)
 
 
-def chat_agent(name: str, description: str, tools: Optional[List[Tool]] = None):
-    if not tools:
-        tools = []
+def chat_agent(name: str, description: str, tools: List[Tool]):
 
     def decorator(chat_agent: Type[T]) -> Type[T]:
         chat_agent.name = name

backend/src/agents/generalist_agent.py

@@ -17,39 +17,6 @@
 )
 class GeneralistAgent(ChatAgent):
     async def invoke(self, utterance) -> str:
-        try:
-            answer_to_user = await answer_user_question(utterance, self.llm, self.model)
-            answer_result = json.loads(answer_to_user)
-            final_answer = json.loads(answer_result["response"]).get("answer", "")
-            if not final_answer:
-                response = {"content": "Error in answer format.", "ignore_validation": "false"}
-                return json.dumps(response, indent=4)
-            logger.info(f"Answer found successfully {final_answer}")
-            response = {"content": final_answer, "ignore_validation": "false"}
-            return json.dumps(response, indent=4)
-
-        except Exception as e:
-            logger.error(f"Error in web_general_search_core: {e}")
-            return "An error occurred while processing the search query."
-
-
-async def answer_user_question(search_query, llm, model) -> str:
-    try:
-        summariser_prompt = engine.load_prompt("generalist-answer", question=search_query)
-        response = await llm.chat(model, summariser_prompt, "")
-        return json.dumps(
-            {
-                "status": "success",
-                "response": response,
-                "error": None,
-            }
-        )
-    except Exception as e:
-        logger.error(f"Error during create search term: {e}")
-        return json.dumps(
-            {
-                "status": "error",
-                "response": None,
-                "error": str(e),
-            }
-        )
+        summariser_prompt = engine.load_prompt("generalist-answer", question=utterance)
+        response = await self.llm.chat(self.model, summariser_prompt, "")
+        return json.dumps({"content": response, "ignore_validation": "false"}, indent=4)

backend/src/agents/materiality_agent.py

@@ -3,15 +3,55 @@
 import logging
 
 from src.llm import LLMFile
-from src.agents import Agent
+from src.agents import ChatAgent, chat_agent
 from src.prompts import PromptEngine
 
 engine = PromptEngine()
 logger = logging.getLogger(__name__)
 
 
-class MaterialityAgent(Agent):
-    async def list_material_topics(self, company_name: str) -> dict[str, str]:
+def create_llm_files(filenames: list[str]) -> list[LLMFile]:
+    return [
+        LLMFile(filename=filename, file=Path(f"./library/{filename}"))
+        for filename in filenames
+    ]
+
+
+@chat_agent(
+    name="MaterialityAgent",
+    description="This agent can help answer questions about ESG Materiality, what topics are relevant to a company"
+                "or sector and explain materiality topics in detail. The Materiality Agent can also answer"
+                "questions about typical sector activities, value chain and business relationships.",
+    tools=[]
+)
+class MaterialityAgent(ChatAgent):
+    async def invoke(self, utterance: str) -> str:
+        materiality_files = await self.select_material_files(utterance)
+        if materiality_files:
+            answer = await self.llm.chat_with_file(
+                self.model,
+                system_prompt=engine.load_prompt("answer-materiality-question"),
+                user_prompt=utterance,
+                files=create_llm_files(materiality_files)
+            )
+        else:
+            answer = f"Materiality Agent cannot find suitable reference documents to answer the question: {utterance}"
+        return json.dumps({"content": answer, "ignore_validation": False})
+
+    async def list_material_topics_for_company(self, company_name: str) -> dict[str, str]:
+        materiality_files = await self.select_material_files(company_name)
+        if not materiality_files:
+            logger.info(f"No materiality reference documents could be found for {company_name}")
+            return {}
+        materiality_topics = await self.llm.chat_with_file(
+            self.model,
+            system_prompt=engine.load_prompt("list-material-topics-system-prompt"),
+            user_prompt=f"What topics are material for {company_name}?",
+            files=create_llm_files(materiality_files)
+        )
+        return json.loads(materiality_topics)["material_topics"]
+
+    async def select_material_files(self, utterance) -> list[str]:
         with open('./library/catalogue.json') as file:
             catalogue = json.load(file)
             files_json = await self.llm.chat(
@@ -20,17 +60,7 @@ async def list_material_topics(self, company_name: str) -> dict[str, str]:
                     "select-material-files-system-prompt",
                     catalogue=catalogue
                 ),
-                user_prompt=company_name,
+                user_prompt=utterance,
                 return_json=True
             )
-
-            materiality_topics = await self.llm.chat_with_file(
-                self.model,
-                system_prompt=engine.load_prompt("list-material-topics-system-prompt"),
-                user_prompt=f"What topics are material for {company_name}?",
-                files=[
-                    LLMFile(file_name=file_name, file=Path(f"./library/{file_name}"))
-                    for file_name in json.loads(files_json)["files"]
-                ]
-            )
-            return json.loads(materiality_topics)["material_topics"]
+            return json.loads(files_json)["files"]

backend/src/agents/report_agent.py

@@ -11,10 +11,11 @@
 
 class ReportAgent(Agent):
     async def create_report(self, file: LLMFile, materiality_topics: dict[str, str]) -> str:
+        materiality = materiality_topics if materiality_topics else "No Materiality topics identified."
         return await self.llm.chat_with_file(
             self.model,
             system_prompt=engine.load_prompt("create-report-system-prompt"),
-            user_prompt=engine.load_prompt("create-report-user-prompt", materiality_topics=materiality_topics),
+            user_prompt=engine.load_prompt("create-report-user-prompt", materiality=materiality),
             files=[file],
         )