diff --git a/.env.example b/.env.example index 1c6209705..a3bac4440 100644 --- a/.env.example +++ b/.env.example @@ -39,6 +39,7 @@ WS_URL=ws://localhost:8250/ws ANSWER_AGENT_LLM="mistral" INTENT_AGENT_LLM="openai" REPORT_AGENT_LLM="mistral" +MATERIALITY_AGENT_LLM="openai" VALIDATOR_AGENT_LLM="openai" DATASTORE_AGENT_LLM="openai" WEB_AGENT_LLM="openai" @@ -52,6 +53,7 @@ DYNAMIC_KNOWLEDGE_GRAPH_LLM="openai" ANSWER_AGENT_MODEL="mistral-large-latest" INTENT_AGENT_MODEL="gpt-4o-mini" REPORT_AGENT_MODEL="mistral-large-latest" +MATERIALITY_AGENT_MODEL="gpt-4o-mini" VALIDATOR_AGENT_MODEL="gpt-4o-mini" DATASTORE_AGENT_MODEL="gpt-4o-mini" WEB_AGENT_MODEL="gpt-4o-mini" diff --git a/backend/Dockerfile b/backend/Dockerfile index 74fef9f23..5ff6e0270 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -7,9 +7,12 @@ WORKDIR /backend # Copy just the requirements into the working directory so it gets cached by itself COPY ./requirements.txt ./requirements.txt -# Copy the datasets directory, this should match what local run of application will need +# Copy the datasets directory COPY ./datasets/ ./datasets/ +# Copy the library directory +COPY ./library/ ./library/ + # Install the dependencies from the requirements file RUN pip install --no-cache-dir --upgrade -r /backend/requirements.txt diff --git a/backend/conftest.py b/backend/conftest.py index 3a0e08d61..d9ba26079 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -2,6 +2,7 @@ import pytest import os + @pytest.hookimpl(tryfirst=True) def pytest_configure(config): # Set an environment variable to indicate pytest is running diff --git a/backend/library/Additional-Sector-Guidance-Biotech-and-Pharma.pdf b/backend/library/Additional-Sector-Guidance-Biotech-and-Pharma.pdf new file mode 100644 index 000000000..03378c5f9 Binary files /dev/null and b/backend/library/Additional-Sector-Guidance-Biotech-and-Pharma.pdf differ diff --git a/backend/library/Additional-Sector-Guidance-Oil-and-gas.pdf b/backend/library/Additional-Sector-Guidance-Oil-and-gas.pdf new file mode 100644 index 000000000..3e58699e6 Binary files /dev/null and b/backend/library/Additional-Sector-Guidance-Oil-and-gas.pdf differ diff --git a/backend/library/GRI 11_ Oil and Gas Sector 2021.pdf b/backend/library/GRI 11_ Oil and Gas Sector 2021.pdf new file mode 100644 index 000000000..3e5f2200e Binary files /dev/null and b/backend/library/GRI 11_ Oil and Gas Sector 2021.pdf differ diff --git a/backend/library/catalogue.json b/backend/library/catalogue.json new file mode 100644 index 000000000..1e301596b --- /dev/null +++ b/backend/library/catalogue.json @@ -0,0 +1,23 @@ +{ + "library": { + "TFND": [ + { + "name": "Additional-Sector-Guidance-Biotech-and-Pharma.pdf", + "sector-label": "Biotechnology and Pharmaceuticals", + "esg-labels": ["Environment", "Nature"] + }, + { + "name": "Additional-Sector-Guidance-Oil-and-gas.pdf", + "sector-label": "Oil and Gas", + "esg-labels": ["Environment", "Nature"] + } + ], + "GRI": [ + { + "name": "GRI 11_ Oil and Gas Sector 2021.pdf", + "sector-label": "Oil and Gas", + "esg-labels": ["Environment", "Social", "Governance"] + } + ] + } +} \ No newline at end of file diff --git a/backend/promptfoo/dynamic_knowledge_graph_model_config.yaml b/backend/promptfoo/dynamic_knowledge_graph_config.yaml similarity index 53% rename from backend/promptfoo/dynamic_knowledge_graph_model_config.yaml rename to backend/promptfoo/dynamic_knowledge_graph_config.yaml index 65d941693..38c9ca63e 100644 --- a/backend/promptfoo/dynamic_knowledge_graph_model_config.yaml +++ b/backend/promptfoo/dynamic_knowledge_graph_config.yaml @@ -8,6 +8,23 @@ providers: prompts: file://promptfoo_test_runner.py:create_prompt tests: + - description: "test model prompt references all csv headers in result using valid json format" + vars: + user_prompt_template: "generate-knowledge-graph-cypher-user-prompt" + user_prompt_args: + data_model: '{ "model": { "Company": { "attributes": ["Identifier (RIC)", "Company Name"], "relationships": { "has_report": "Report" }, "type": "main_entity" }, "Industry": { "attributes": ["Industry"], "relationships": {}, "type": "category" }, "Report": { "attributes": ["ESG_score", "BVPS", "Market_cap", "Shares", "Net_income", "RETURN_ON_ASSET", "QUICK_RATIO", "ASSET_GROWTH", "FNCL_LVRG", "PE_RATIO", "Total_assets"], "relationships": { "has_ESG_Environment": "Environment", "has_ESG_Social": "Social", "has_ESG_Governance": "Governance" }, "type": "metrics" }, "Environment": { "attributes": ["Env_score", "Scope_1", "Scope_2", "CO2_emissions", "Energy_use", "Water_use", "Water_recycle", "Toxic_chem_red", "Recycling_Initiatives"], "relationships": {}, "type": "metrics" }, "Social": { "attributes": ["Social_score", "Injury_rate", "Women_Employees", "Human_Rights", "Strikes", "Turnover_empl"], "relationships": {}, "type": "metrics" }, "Governance": { "attributes": ["Gov_score", "Board_Size", "Shareholder_Rights", "Board_gen_div", "Bribery"], "relationships": {}, "type": "metrics" } } }' + input_data: "[['Identifier (RIC)', 'Company Name', 'Date', 'ESG_score', 'Social_score', 'Gov_score', 'Env_score', 'BVPS', 'Market_cap', 'Shares', 'Industry', 'Net_income', 'RETURN_ON_ASSET', 'QUICK_RATIO', 'ASSET_GROWTH', 'FNCL_LVRG', 'PE_RATIO', 'Scope_1', 'Scope_2', 'CO2_emissions', 'Energy_use', 'Water_use', 'Water_recycle', 'Toxic_chem_red', 'Injury_rate', 'Women_Employees', 'Human_Rights', 'Strikes', 'Turnover_empl', 'Board_Size', 'Shareholder_Rights', 'Board_gen_div', 'Bribery', 'Recycling_Initiatives', 'Total_assets'], ['AAL', 'American Airlines Group Inc', '2021', '59.02910721', '64.28601828', '56.39811612', '54.38515247', '-11.39725006', '10198305438', '644015000', 'Airlines', '-1993000000', '-3.1032', '0.7333', '7.1507', '34.6286', '5.8534', '898.9575031', '6.470219367', '41439616', '551629386', '1873777.95', '', '0', '', '41', '1', '1', '5.8', '10', '1', '20', '1', '0', '66467000000'], ['AAL', 'American Airlines Group Inc', '2020', '69.40117811', '68.81448509', '83.94361492', '56.96600148', '-14.19130047', '10626751115', '483888000', 'Airlines', '-8885000000', '-14.5652', '0.4953', '3.3553', '34.6286', '5.8534', '904.2455266', '7.364001706', '40604000', '540190224', '1729932.37', '', '0', '', '41.618', '1', '0', '5.5', '12', '1', '16.66666667', '1', '0', '62008000000'], ['AAL', 'American Airlines Group Inc', '2019', '69.36922798', '72.44505934', '77.10711045', '58.21814638', '-0.266147604', '10574818306', '444269000', 'Airlines', '1686000000', '2.7966', '0.3045', '-0.9657', '34.6286', '5.8534', '916.4750598', '7.648632162', '39388000', '564200000', '1627726.3', '', '0', '41.872', '41.654', '1', '1', '4.242', '13', '1', '15.38461538', '1', '0', '59995000000'], ['AAL', 'American Airlines Group Inc', '2018', '71.63302219', '72.04574143', '83.06962314', '60.68373547', '-0.36403898', '11198012018', '465660000', 'Airlines', '1412000000', '2.4911', '0.3573', '14.7675', '34.6286', '7.0665', '969.3836879', '9.117632405', '39279000', '562000000', '1767786.47', '', '0', '41.942', '42.452', '1', '0', '5.613', '13', '1', '15.38461538', '1', '0', '60580000000'], ['AAL', 'American Airlines Group Inc', '2017', '69.79137149', '73.98311797', '78.63204572', '56.14422918', '-1.594557245', '11334335643', '491692000', 'Airlines', '2105000000', '2.464', '0.4439', '2.9469', '34.6286', '9.8649', '1015.247621', '10.31959014', '42038000', '617300000', '1608799.25', '', '0', '30.7', '42.505', '1', '0', '7.139', '13', '1', '15.38461538', '1', '0', '52785000000'], ['AAL', 'American Airlines Group Inc', '2016', '70.3178445', '78.15023537', '71.82041582', '58.46912521', '6.853060249', '11009755584', '556099000', 'Airlines', '2584000000', '5.3687', '0.5733', '5.9052', '10.5827', '8.1442', '979.2028136', '12.16881594', '42282000', '619000000', '1767786.47', '', '0', '38.95', '42', '1', '0', '', '11', '1', '9.090909091', '1', '0', '51274000000'], ['AAL', 'American Airlines Group Inc', '2015', '51.79390641', '54.30943132', '35.94845361', '62.8321256', '8.430668783', '10802024347', '687355000', 'Airlines', '7610000000', '16.6085', '0.5644', '12.0069', '11.9697', '4.2243', '1581.72232', '', '42300000', '', '', '56781.15', '0', '', '41', '0', '0', '', '12', '1', '8.333333333', '1', '0', '48415000000'], ['AAL', 'American Airlines Group Inc', '2014', '44.30935231', '52.95317347', '34.85299977', '41.3372859', '2.816897482', '11314860839', '734016000', 'Airlines', '2882000000', '6.7413', '0.677', '2.2399', '28.1404', '10.4955', '1074.834037', '18.58780929', '27177000', '389700000', '1935858.67', '98420.66', '0', '', '41', '0', '1', '', '11', '1', '18.18181818', '1', '0', '43225000000'], ['AAL', 'American Airlines Group Inc', '2013', '46.85202135', '51.25924607', '31.05261636', '55.31655844', '-16.74987427', '11600491291', '163046000', 'Airlines', '-1834000000', '-5.5755', '0.7831', '79.8299', '28.1404', '', '1128.904458', '19.30856166', '27533000', '394700000', '2018280.61', '', '0', '', '39', '0', '0', '', '11', '1', '18.18181818', '1', '0', '42278000000'], ['AAL', 'American Airlines Group Inc', '2012', '51.24684551', '70.86964887', '24.12375195', '49.64136041', '-63.778138', '11797714591', '125231000', 'Airlines', '-1876000000', '-7.9226', '0.5391', '-1.4173', '28.1404', '', '', '', '27400000', '469631498', '2022924', '', '1', '', '39', '0', '0', '', '12', '1', '16.66666667', '1', '0', '23510000000'], ['ALK', 'Alaska Air Group Inc', '2021', '50.48254945', '39.63992514', '86.0630969', '32.64819037', '30.39268209', '6474416945', '126775000', 'Airlines', '464000000', '3.4147', '0.9176', '-0.6763', '4.1239', '10.5475', '905.4543902', '2.884637285', '7976125', '107090478', '59089.504', '', '0', '28.803', '', '1', '0', '', '12', '1', '41.66666667', '1', '0', '13951000000'], ['ALK', 'Alaska Air Group Inc', '2020', '54.76478447', '44.82792507', '87.86701992', '37.97117517', '24.20413123', '6625511856', '123450000', 'Airlines', '-1324000000', '-9.7933', '0.8912', '8.1044', '3.6944', '10.5475', '937.9543804', '1.300096805', '7761999', '112275897', '61424.924', '', '0', '27.16', '', '1', '0', '', '12', '1', '41.66666667', '1', '0', '14046000000'], ['ALK', 'Alaska Air Group Inc', '2019', '54.78086974', '47.53378754', '83.23838167', '38.60978203', '35.13169315', '6590256377', '124289000', 'Airlines', '769000000', '6.4338', '0.5761', '19.0707', '2.9578', '10.5475', '949.2740056', '1.254877122', '7503475', '108025619.5', '63348.931', '', '0', '27.506', '', '1', '0', '', '11', '1', '36.36363636', '1', '0', '12993000000'], ['ALK', 'Alaska Air Group Inc', '2018', '61.10430324', '55.14174196', '89.29964586', '43.45236925', '30.43901647', '7068723596', '123975000', 'Airlines', '437000000', '4.0355', '0.5445', '1.5448', '3.0035', '13.6321', '858.5817722', '2.115780591', '5099633', '73414401', '68202.958', '', '0', '32.976', '54', '1', '0', '', '10', '1', '40', '1', '0', '10912000000'], ['ALK', 'Alaska Air Group Inc', '2017', '64.76905583', '67.17896047', '85.4188325', '42.77115416', '28.08190827', '7248778365', '123854000', 'Airlines', '723000000', '9.2718', '0.7305', '7.8699', '3.2402', '11.512', '862.8063594', '1.878885316', '4840510', '69695200', '82806.8', '', '0', '32.853', '54', '0', '0', '', '11', '1', '45.45454545', '1', '0', '10746000000'], ['ALK', 'Alaska Air Group Inc', '2016', '55.66218011', '51.08911554', '87.34169898', '32.98313322', '23.72184498', '7076278341', '124389000', 'Airlines', '797000000', '9.5683', '0.7424', '52.5574', '3.0872', '12.3738', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '9962000000'], ['ALK', 'Alaska Air Group Inc', '2015', '', '', '', '', '18.78120789', '7028431619', '129372000', 'Airlines', '848000000', '13.4667', '0.8532', '7.6847', '2.7752', '12.0625', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '6533000000'], ['ALK', 'Alaska Air Group Inc', '2014', '', '', '', '', '15.70379121', '7281515596', '136801000', 'Airlines', '605000000', '10.1664', '0.8833', '3.8712', '2.8638', '14.7979', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '6064000000'], ['ALK', 'Alaska Air Group Inc', '2013', '', '', '', '', '14.50217997', '7402391525', '141878000', 'Airlines', '508000000', '8.9571', '0.938', '6.049', '3.2878', '13.6386', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '5838000000'], ['ALK', 'Alaska Air Group Inc', '2012', '', '', '', '', '10.04836794', '7414982767', '143568000', 'Airlines', '316000000', '5.922', '0.9207', '6.5415', '4.1125', '9.1566', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '5505000000'], ['ALLE', 'Allegion PLC', '2021', '73.40375216', '83.38290372', '66.44580957', '68.26787654', '8.443826474', '10385605328', '90500000', 'Building Products', '483300000', '15.7833', '1.1331', '-0.5995', '3.8529', '24.4624', '', '', '100618', '408600', '394439.722', '', '1', '1.95', '36', '1', '0', '', '8', '1', '12.5', '1', '0', '3051000000'], ['ALLE', 'Allegion PLC', '2020', '59.52521192', '62.27435928', '67.24358974', '41.10701352', '8.985915493', '10355606978', '92800000', 'Building Products', '314500000', '10.4131', '1.5383', '3.4443', '3.8043', '22.7661', '', '', '', '', '', '', '1', '', '', '1', '0', '', '7', '1', '28.57142857', '1', '0', '3069400000'], ['ALLE', 'Allegion PLC', '2019', '59.57897254', '59.97902967', '67.4', '45.07902049', '8.091880342', '10113855566', '94300000', 'Building Products', '402100000', '13.9094', '1.3513', '5.5868', '4.1021', '25.4909', '8.628692755', '28.83442545', '102338', '462330', '', '', '1', '', '', '1', '0', '', '7', '1', '28.57142857', '1', '0', '2967200000'], ['ALLE', 'Allegion PLC', '2018', '47.47726304', '58.84089693', '45.47619048', '31.17791234', '6.852631579', '10104150218', '95700000', 'Building Products', '413500000', '16.2513', '1.1688', '10.5507', '5.0847', '17.1241', '', '', '', '', '', '', '1', '', '', '1', '0', '', '7', '1', '28.57142857', '1', '0', '2810200000'], ['ALLE', 'Allegion PLC', '2017', '37.20505888', '43.26741466', '46.33333333', '10.51693405', '4.222923239', '10155323874', '96000000', 'Building Products', '329000000', '11.4127', '1.6554', '13.1085', '9.3016', '21.1839', '', '', '', '', '', '', '1', '', '', '1', '0', '', '6', '1', '16.66666667', '1', '0', '2542000000'], ['ALLE', 'Allegion PLC', '2016', '33.00965314', '46.7541872', '36.17886179', '3.431372549', '1.182672234', '9855340371', '96900000', 'Building Products', '231200000', '10.1587', '1.3324', '-0.6894', '32.4723', '19.1713', '8.827909676', '28.74474155', '77704', '', '', '', '0', '', '', '1', '0', '', '6', '1', '16.66666667', '1', '0', '2247400000'], ['ALLE', 'Allegion PLC', '2015', '28.54361028', '30.94209162', '38.50877193', '6.77244582', '0.266944734', '9666527226', '96900000', 'Building Products', '154700000', '7.1934', '1.1253', '12.2576', '205.7163', '21.7584', '', '', '', '', '', '', '0', '', '', '0', '0', '', '6', '1', '16.66666667', '1', '0', '2263000000'], ['ALLE', 'Allegion PLC', '2014', '41.9183995', '32.57938447', '73.05555556', '3.267973856', '-0.049947971', '10015037471', '97200000', 'Building Products', '183700000', '8.724', '0.9782', '0.7648', '3.1199', '21.3734', '', '', '', '', '', '', '0', '', '', '0', '0', '', '5', '1', '20', '1', '0', '2015900000'], ['ALLE', 'Allegion PLC', '2013', '', '', '', '', '-0.688541667', '10069740345', '96100000', 'Building Products', '48400000', '1.6213', '0.9937', '0.8469', '3.1199', '26.9942', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2000600000'], ['ALLE', 'Allegion PLC', '2012', '', '', '', '', '13.99166667', '10110326349', '96000000', 'Building Products', '230000000', '10.9502', '1.5827', '-2.5734', '1.4581', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '1983800000'], ['AME', 'AMETEK Inc', '2021', '48.43311116', '50.37985133', '64.36906602', '33.0261549', '29.75421186', '30364402028', '232813000', 'Electrical Equipment', '990053000', '8.8971', '0.7522', '14.8753', '1.7358', '34.7739', '', '', '', '', '', '', '0', '', '', '0', '0', '', '8', '1', '37.5', '1', '0', '11898187000'], ['AME', 'AMETEK Inc', '2020', '46.57431694', '50.92936263', '58.75348611', '31.66671862', '25.93042038', '30350499974', '231150000', 'Electrical Equipment', '872439000', '8.6371', '1.6826', '5.2102', '1.8258', '27.7144', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '10357483000'], ['AME', 'AMETEK Inc', '2019', '42.26573632', '40.45126171', '65.38635236', '24.98103781', '22.46010915', '29516376759', '229395000', 'Electrical Equipment', '861297000', '9.3079', '0.798', '13.6485', '1.9778', '26.7282', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '9844559000'], ['AME', 'AMETEK Inc', '2018', '26.17277106', '29.15026235', '26.42022008', '22.71337861', '18.37738007', '30072458902', '232712000', 'Electrical Equipment', '766133000', '9.4534', '0.8634', '11.111', '1.9902', '20.5798', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '8662288000'], ['AME', 'AMETEK Inc', '2017', '30.66502908', '29.55969049', '44.8033748', '20.09075908', '17.49402986', '30090994974', '231845000', 'Electrical Equipment', '589870000', '9.1493', '1.1544', '9.7933', '2.0451', '27.7205', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '7796064000'], ['AME', 'AMETEK Inc', '2016', '33.20225163', '26.6121386', '56.66255398', '20.85230835', '14.00090716', '29567350956', '233730000', 'Electrical Equipment', '512158000', '7.4435', '1.4166', '6.6095', '2.1135', '21.136', '', '', '', '', '', '', '0', '', '', '0', '0', '', '8', '1', '25', '1', '0', '7100674000'], ['AME', 'AMETEK Inc', '2015', '32.44116267', '29.91329851', '53.04810997', '18.03063224', '13.56625512', '28909320420', '241586000', 'Electrical Equipment', '590859000', '9.0336', '0.9612', '3.7298', '2.0143', '20.998', '', '', '', '', '', '', '0', '', '', '0', '0', '', '7', '1', '14.28571429', '1', '0', '6660450000'], ['AME', 'AMETEK Inc', '2014', '29.54343011', '33.91813565', '37.77892207', '17.90074928', '13.22890745', '29527961804', '247102000', 'Electrical Equipment', '584460000', '9.5043', '1.0288', '9.239', '1.929', '21.6921', '', '', '', '', '', '', '0', '', '', '0', '0', '', '7', '1', '14.28571429', '1', '0', '6420963000'], ['AME', 'AMETEK Inc', '2013', '25.66447145', '33.68505357', '27.27989037', '15.55506041', '12.85743394', '29708688500', '246065000', 'Electrical Equipment', '516999000', '9.3423', '0.9512', '13.2532', '1.9516', '25.4523', '', '', '', '', '', '', '0', '', '', '0', '0', '', '7', '1', '14.28571429', '1', '0', '5877902000'], ['AME', 'AMETEK Inc', '2012', '12.24247583', '22.79737658', '10.33164129', '2.302631579', '10.49699808', '29731858590', '243986000', 'Electrical Equipment', '459132000', '9.6562', '0.7567', '20.1544', '2.0727', '19.984', '', '', '', '', '', '', '0', '', '', '0', '0', '', '8', '1', '12.5', '1', '0', '5190056000'], ['AOS', 'A O Smith Corp', '2021', '54.00154295', '46.2908336', '58.07782524', '59.02930495', '11.45792209', '10784338541', '161319900', 'Building Products', '487100000', '14.6825', '1.1314', '9.925', '1.8028', '29.695', '22.00053463', '24.36361814', '138754', '2030842.8', '1370806.738', '', '0', '', '42', '1', '0', '', '10', '1', '20', '1', '0', '3474400000'], ['AOS', 'A O Smith Corp', '2020', '52.04805145', '41.10365709', '56.74584021', '60.09099171', '11.44241472', '10897395090', '162604150', 'Building Products', '344900000', '11.0924', '1.4381', '3.3584', '1.7691', '25.3944', '20.7663352', '24.32416324', '143744', '1902160.8', '964340.768', '', '0', '', '', '1', '0', '', '10', '1', '20', '1', '0', '3160700000'], ['AOS', 'A O Smith Corp', '2019', '41.56413871', '32.65950433', '58.37558372', '37.28373876', '10.07431591', '10462441424', '166710900', 'Building Products', '370000000', '12.0728', '1.4885', '-0.4395', '1.8114', '21.4595', '22.10197884', '27.59135049', '148916', '1894870.8', '1048918.184', '', '0', '', '', '1', '0', '', '11', '1', '18.18181818', '1', '0', '3058000000'], ['AOS', 'A O Smith Corp', '2018', '40.35379016', '34.81925394', '54.62287941', '34.50987574', '10.06510664', '10768636243', '172194040', 'Building Products', '444200000', '14.1715', '1.6456', '-3.9376', '1.8647', '16.1191', '', '', '', '', '', '', '0', '', '', '0', '0', '', '10', '1', '20', '1', '0', '3071500000'], ['AOS', 'A O Smith Corp', '2017', '39.21154111', '31.54732662', '60.62491062', '29.74093049', '9.526481348', '10895824860', '174605190', 'Building Products', '378300000', '9.7398', '1.7797', '10.5984', '1.9266', '35.9134', '', '', '', '', '', '', '0', '', '', '0', '0', '', '10', '1', '10', '1', '0', '3197400000'], ['AOS', 'A O Smith Corp', '2016', '26.94238012', '27.99118757', '50.88011516', '5.848348348', '8.673096881', '10641447626', '176825280', 'Building Products', '326500000', '11.8293', '1.6631', '9.9574', '1.8664', '25.4379', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2891000000'], ['AOS', 'A O Smith Corp', '2015', '', '', '', '', '8.120039896', '10660290384', '179009180', 'Building Products', '282900000', '10.9982', '1.7554', '4.5283', '1.822', '24.5036', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2629200000'], ['AOS', 'A O Smith Corp', '2014', '', '', '', '', '7.648944406', '10917808078', '181973960', 'Building Products', '207800000', '8.4699', '1.6809', '5.1767', '1.8106', '24.7412', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2515300000'], ['AOS', 'A O Smith Corp', '2013', '', '', '', '', '7.211933413', '11180036461', '185575340', 'Building Products', '169700000', '7.2672', '1.5987', '4.9456', '1.8512', '27.8981', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2391500000']]" + system_prompt_template: "generate-knowledge-graph-cypher-system-prompt" + assert: + - type: is-json + value: + required: ["cypher_query"] + type: object + - type: contains-all + value: + - "COALESCE" + - "WITH $data AS data UNWIND data.all_data[1..] AS row WITH data.all_data[0] AS headers, row" + - description: "test dynamic knowledge graph cypher prompt creates suitable data model" vars: user_prompt: "[['Identifier (RIC)', 'Company Name', 'Date', 'ESG_score', 'Social_score', 'Gov_score', 'Env_score', 'BVPS', 'Market_cap', 'Shares', 'Industry', 'Net_income', 'RETURN_ON_ASSET', 'QUICK_RATIO', 'ASSET_GROWTH', 'FNCL_LVRG', 'PE_RATIO', 'Scope_1', 'Scope_2', 'CO2_emissions', 'Energy_use', 'Water_use', 'Water_recycle', 'Toxic_chem_red', 'Injury_rate', 'Women_Employees', 'Human_Rights', 'Strikes', 'Turnover_empl', 'Board_Size', 'Shareholder_Rights', 'Board_gen_div', 'Bribery', 'Recycling_Initiatives', 'Total_assets'], ['AAL', 'American Airlines Group Inc', '2021', '59.02910721', '64.28601828', '56.39811612', '54.38515247', '-11.39725006', '10198305438', '644015000', 'Airlines', '-1993000000', '-3.1032', '0.7333', '7.1507', '34.6286', '5.8534', '898.9575031', '6.470219367', '41439616', '551629386', '1873777.95', '', '0', '', '41', '1', '1', '5.8', '10', '1', '20', '1', '0', '66467000000'], ['AAL', 'American Airlines Group Inc', '2020', '69.40117811', '68.81448509', '83.94361492', '56.96600148', '-14.19130047', '10626751115', '483888000', 'Airlines', '-8885000000', '-14.5652', '0.4953', '3.3553', '34.6286', '5.8534', '904.2455266', '7.364001706', '40604000', '540190224', '1729932.37', '', '0', '', '41.618', '1', '0', '5.5', '12', '1', '16.66666667', '1', '0', '62008000000'], ['AAL', 'American Airlines Group Inc', '2019', '69.36922798', '72.44505934', '77.10711045', '58.21814638', '-0.266147604', '10574818306', '444269000', 'Airlines', '1686000000', '2.7966', '0.3045', '-0.9657', '34.6286', '5.8534', '916.4750598', '7.648632162', '39388000', '564200000', '1627726.3', '', '0', '41.872', '41.654', '1', '1', '4.242', '13', '1', '15.38461538', '1', '0', '59995000000'], ['AAL', 'American Airlines Group Inc', '2018', '71.63302219', '72.04574143', '83.06962314', '60.68373547', '-0.36403898', '11198012018', '465660000', 'Airlines', '1412000000', '2.4911', '0.3573', '14.7675', '34.6286', '7.0665', '969.3836879', '9.117632405', '39279000', '562000000', '1767786.47', '', '0', '41.942', '42.452', '1', '0', '5.613', '13', '1', '15.38461538', '1', '0', '60580000000'], ['AAL', 'American Airlines Group Inc', '2017', '69.79137149', '73.98311797', '78.63204572', '56.14422918', '-1.594557245', '11334335643', '491692000', 'Airlines', '2105000000', '2.464', '0.4439', '2.9469', '34.6286', '9.8649', '1015.247621', '10.31959014', '42038000', '617300000', '1608799.25', '', '0', '30.7', '42.505', '1', '0', '7.139', '13', '1', '15.38461538', '1', '0', '52785000000'], ['AAL', 'American Airlines Group Inc', '2016', '70.3178445', '78.15023537', '71.82041582', '58.46912521', '6.853060249', '11009755584', '556099000', 'Airlines', '2584000000', '5.3687', '0.5733', '5.9052', '10.5827', '8.1442', '979.2028136', '12.16881594', '42282000', '619000000', '1767786.47', '', '0', '38.95', '42', '1', '0', '', '11', '1', '9.090909091', '1', '0', '51274000000'], ['AAL', 'American Airlines Group Inc', '2015', '51.79390641', '54.30943132', '35.94845361', '62.8321256', '8.430668783', '10802024347', '687355000', 'Airlines', '7610000000', '16.6085', '0.5644', '12.0069', '11.9697', '4.2243', '1581.72232', '', '42300000', '', '', '56781.15', '0', '', '41', '0', '0', '', '12', '1', '8.333333333', '1', '0', '48415000000'], ['AAL', 'American Airlines Group Inc', '2014', '44.30935231', '52.95317347', '34.85299977', '41.3372859', '2.816897482', '11314860839', '734016000', 'Airlines', '2882000000', '6.7413', '0.677', '2.2399', '28.1404', '10.4955', '1074.834037', '18.58780929', '27177000', '389700000', '1935858.67', '98420.66', '0', '', '41', '0', '1', '', '11', '1', '18.18181818', '1', '0', '43225000000'], ['AAL', 'American Airlines Group Inc', '2013', '46.85202135', '51.25924607', '31.05261636', '55.31655844', '-16.74987427', '11600491291', '163046000', 'Airlines', '-1834000000', '-5.5755', '0.7831', '79.8299', '28.1404', '', '1128.904458', '19.30856166', '27533000', '394700000', '2018280.61', '', '0', '', '39', '0', '0', '', '11', '1', '18.18181818', '1', '0', '42278000000'], ['AAL', 'American Airlines Group Inc', '2012', '51.24684551', '70.86964887', '24.12375195', '49.64136041', '-63.778138', '11797714591', '125231000', 'Airlines', '-1876000000', '-7.9226', '0.5391', '-1.4173', '28.1404', '', '', '', '27400000', '469631498', '2022924', '', '1', '', '39', '0', '0', '', '12', '1', '16.66666667', '1', '0', '23510000000'], ['ALK', 'Alaska Air Group Inc', '2021', '50.48254945', '39.63992514', '86.0630969', '32.64819037', '30.39268209', '6474416945', '126775000', 'Airlines', '464000000', '3.4147', '0.9176', '-0.6763', '4.1239', '10.5475', '905.4543902', '2.884637285', '7976125', '107090478', '59089.504', '', '0', '28.803', '', '1', '0', '', '12', '1', '41.66666667', '1', '0', '13951000000'], ['ALK', 'Alaska Air Group Inc', '2020', '54.76478447', '44.82792507', '87.86701992', '37.97117517', '24.20413123', '6625511856', '123450000', 'Airlines', '-1324000000', '-9.7933', '0.8912', '8.1044', '3.6944', '10.5475', '937.9543804', '1.300096805', '7761999', '112275897', '61424.924', '', '0', '27.16', '', '1', '0', '', '12', '1', '41.66666667', '1', '0', '14046000000'], ['ALK', 'Alaska Air Group Inc', '2019', '54.78086974', '47.53378754', '83.23838167', '38.60978203', '35.13169315', '6590256377', '124289000', 'Airlines', '769000000', '6.4338', '0.5761', '19.0707', '2.9578', '10.5475', '949.2740056', '1.254877122', '7503475', '108025619.5', '63348.931', '', '0', '27.506', '', '1', '0', '', '11', '1', '36.36363636', '1', '0', '12993000000'], ['ALK', 'Alaska Air Group Inc', '2018', '61.10430324', '55.14174196', '89.29964586', '43.45236925', '30.43901647', '7068723596', '123975000', 'Airlines', '437000000', '4.0355', '0.5445', '1.5448', '3.0035', '13.6321', '858.5817722', '2.115780591', '5099633', '73414401', '68202.958', '', '0', '32.976', '54', '1', '0', '', '10', '1', '40', '1', '0', '10912000000'], ['ALK', 'Alaska Air Group Inc', '2017', '64.76905583', '67.17896047', '85.4188325', '42.77115416', '28.08190827', '7248778365', '123854000', 'Airlines', '723000000', '9.2718', '0.7305', '7.8699', '3.2402', '11.512', '862.8063594', '1.878885316', '4840510', '69695200', '82806.8', '', '0', '32.853', '54', '0', '0', '', '11', '1', '45.45454545', '1', '0', '10746000000'], ['ALK', 'Alaska Air Group Inc', '2016', '55.66218011', '51.08911554', '87.34169898', '32.98313322', '23.72184498', '7076278341', '124389000', 'Airlines', '797000000', '9.5683', '0.7424', '52.5574', '3.0872', '12.3738', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '9962000000'], ['ALK', 'Alaska Air Group Inc', '2015', '', '', '', '', '18.78120789', '7028431619', '129372000', 'Airlines', '848000000', '13.4667', '0.8532', '7.6847', '2.7752', '12.0625', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '6533000000'], ['ALK', 'Alaska Air Group Inc', '2014', '', '', '', '', '15.70379121', '7281515596', '136801000', 'Airlines', '605000000', '10.1664', '0.8833', '3.8712', '2.8638', '14.7979', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '6064000000'], ['ALK', 'Alaska Air Group Inc', '2013', '', '', '', '', '14.50217997', '7402391525', '141878000', 'Airlines', '508000000', '8.9571', '0.938', '6.049', '3.2878', '13.6386', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '5838000000'], ['ALK', 'Alaska Air Group Inc', '2012', '', '', '', '', '10.04836794', '7414982767', '143568000', 'Airlines', '316000000', '5.922', '0.9207', '6.5415', '4.1125', '9.1566', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '5505000000'], ['ALLE', 'Allegion PLC', '2021', '73.40375216', '83.38290372', '66.44580957', '68.26787654', '8.443826474', '10385605328', '90500000', 'Building Products', '483300000', '15.7833', '1.1331', '-0.5995', '3.8529', '24.4624', '', '', '100618', '408600', '394439.722', '', '1', '1.95', '36', '1', '0', '', '8', '1', '12.5', '1', '0', '3051000000'], ['ALLE', 'Allegion PLC', '2020', '59.52521192', '62.27435928', '67.24358974', '41.10701352', '8.985915493', '10355606978', '92800000', 'Building Products', '314500000', '10.4131', '1.5383', '3.4443', '3.8043', '22.7661', '', '', '', '', '', '', '1', '', '', '1', '0', '', '7', '1', '28.57142857', '1', '0', '3069400000'], ['ALLE', 'Allegion PLC', '2019', '59.57897254', '59.97902967', '67.4', '45.07902049', '8.091880342', '10113855566', '94300000', 'Building Products', '402100000', '13.9094', '1.3513', '5.5868', '4.1021', '25.4909', '8.628692755', '28.83442545', '102338', '462330', '', '', '1', '', '', '1', '0', '', '7', '1', '28.57142857', '1', '0', '2967200000'], ['ALLE', 'Allegion PLC', '2018', '47.47726304', '58.84089693', '45.47619048', '31.17791234', '6.852631579', '10104150218', '95700000', 'Building Products', '413500000', '16.2513', '1.1688', '10.5507', '5.0847', '17.1241', '', '', '', '', '', '', '1', '', '', '1', '0', '', '7', '1', '28.57142857', '1', '0', '2810200000'], ['ALLE', 'Allegion PLC', '2017', '37.20505888', '43.26741466', '46.33333333', '10.51693405', '4.222923239', '10155323874', '96000000', 'Building Products', '329000000', '11.4127', '1.6554', '13.1085', '9.3016', '21.1839', '', '', '', '', '', '', '1', '', '', '1', '0', '', '6', '1', '16.66666667', '1', '0', '2542000000'], ['ALLE', 'Allegion PLC', '2016', '33.00965314', '46.7541872', '36.17886179', '3.431372549', '1.182672234', '9855340371', '96900000', 'Building Products', '231200000', '10.1587', '1.3324', '-0.6894', '32.4723', '19.1713', '8.827909676', '28.74474155', '77704', '', '', '', '0', '', '', '1', '0', '', '6', '1', '16.66666667', '1', '0', '2247400000'], ['ALLE', 'Allegion PLC', '2015', '28.54361028', '30.94209162', '38.50877193', '6.77244582', '0.266944734', '9666527226', '96900000', 'Building Products', '154700000', '7.1934', '1.1253', '12.2576', '205.7163', '21.7584', '', '', '', '', '', '', '0', '', '', '0', '0', '', '6', '1', '16.66666667', '1', '0', '2263000000'], ['ALLE', 'Allegion PLC', '2014', '41.9183995', '32.57938447', '73.05555556', '3.267973856', '-0.049947971', '10015037471', '97200000', 'Building Products', '183700000', '8.724', '0.9782', '0.7648', '3.1199', '21.3734', '', '', '', '', '', '', '0', '', '', '0', '0', '', '5', '1', '20', '1', '0', '2015900000'], ['ALLE', 'Allegion PLC', '2013', '', '', '', '', '-0.688541667', '10069740345', '96100000', 'Building Products', '48400000', '1.6213', '0.9937', '0.8469', '3.1199', '26.9942', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2000600000'], ['ALLE', 'Allegion PLC', '2012', '', '', '', '', '13.99166667', '10110326349', '96000000', 'Building Products', '230000000', '10.9502', '1.5827', '-2.5734', '1.4581', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '1983800000'], ['AME', 'AMETEK Inc', '2021', '48.43311116', '50.37985133', '64.36906602', '33.0261549', '29.75421186', '30364402028', '232813000', 'Electrical Equipment', '990053000', '8.8971', '0.7522', '14.8753', '1.7358', '34.7739', '', '', '', '', '', '', '0', '', '', '0', '0', '', '8', '1', '37.5', '1', '0', '11898187000'], ['AME', 'AMETEK Inc', '2020', '46.57431694', '50.92936263', '58.75348611', '31.66671862', '25.93042038', '30350499974', '231150000', 'Electrical Equipment', '872439000', '8.6371', '1.6826', '5.2102', '1.8258', '27.7144', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '10357483000'], ['AME', 'AMETEK Inc', '2019', '42.26573632', '40.45126171', '65.38635236', '24.98103781', '22.46010915', '29516376759', '229395000', 'Electrical Equipment', '861297000', '9.3079', '0.798', '13.6485', '1.9778', '26.7282', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '9844559000'], ['AME', 'AMETEK Inc', '2018', '26.17277106', '29.15026235', '26.42022008', '22.71337861', '18.37738007', '30072458902', '232712000', 'Electrical Equipment', '766133000', '9.4534', '0.8634', '11.111', '1.9902', '20.5798', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '8662288000'], ['AME', 'AMETEK Inc', '2017', '30.66502908', '29.55969049', '44.8033748', '20.09075908', '17.49402986', '30090994974', '231845000', 'Electrical Equipment', '589870000', '9.1493', '1.1544', '9.7933', '2.0451', '27.7205', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '7796064000'], ['AME', 'AMETEK Inc', '2016', '33.20225163', '26.6121386', '56.66255398', '20.85230835', '14.00090716', '29567350956', '233730000', 'Electrical Equipment', '512158000', '7.4435', '1.4166', '6.6095', '2.1135', '21.136', '', '', '', '', '', '', '0', '', '', '0', '0', '', '8', '1', '25', '1', '0', '7100674000'], ['AME', 'AMETEK Inc', '2015', '32.44116267', '29.91329851', '53.04810997', '18.03063224', '13.56625512', '28909320420', '241586000', 'Electrical Equipment', '590859000', '9.0336', '0.9612', '3.7298', '2.0143', '20.998', '', '', '', '', '', '', '0', '', '', '0', '0', '', '7', '1', '14.28571429', '1', '0', '6660450000'], ['AME', 'AMETEK Inc', '2014', '29.54343011', '33.91813565', '37.77892207', '17.90074928', '13.22890745', '29527961804', '247102000', 'Electrical Equipment', '584460000', '9.5043', '1.0288', '9.239', '1.929', '21.6921', '', '', '', '', '', '', '0', '', '', '0', '0', '', '7', '1', '14.28571429', '1', '0', '6420963000'], ['AME', 'AMETEK Inc', '2013', '25.66447145', '33.68505357', '27.27989037', '15.55506041', '12.85743394', '29708688500', '246065000', 'Electrical Equipment', '516999000', '9.3423', '0.9512', '13.2532', '1.9516', '25.4523', '', '', '', '', '', '', '0', '', '', '0', '0', '', '7', '1', '14.28571429', '1', '0', '5877902000'], ['AME', 'AMETEK Inc', '2012', '12.24247583', '22.79737658', '10.33164129', '2.302631579', '10.49699808', '29731858590', '243986000', 'Electrical Equipment', '459132000', '9.6562', '0.7567', '20.1544', '2.0727', '19.984', '', '', '', '', '', '', '0', '', '', '0', '0', '', '8', '1', '12.5', '1', '0', '5190056000'], ['AOS', 'A O Smith Corp', '2021', '54.00154295', '46.2908336', '58.07782524', '59.02930495', '11.45792209', '10784338541', '161319900', 'Building Products', '487100000', '14.6825', '1.1314', '9.925', '1.8028', '29.695', '22.00053463', '24.36361814', '138754', '2030842.8', '1370806.738', '', '0', '', '42', '1', '0', '', '10', '1', '20', '1', '0', '3474400000'], ['AOS', 'A O Smith Corp', '2020', '52.04805145', '41.10365709', '56.74584021', '60.09099171', '11.44241472', '10897395090', '162604150', 'Building Products', '344900000', '11.0924', '1.4381', '3.3584', '1.7691', '25.3944', '20.7663352', '24.32416324', '143744', '1902160.8', '964340.768', '', '0', '', '', '1', '0', '', '10', '1', '20', '1', '0', '3160700000'], ['AOS', 'A O Smith Corp', '2019', '41.56413871', '32.65950433', '58.37558372', '37.28373876', '10.07431591', '10462441424', '166710900', 'Building Products', '370000000', '12.0728', '1.4885', '-0.4395', '1.8114', '21.4595', '22.10197884', '27.59135049', '148916', '1894870.8', '1048918.184', '', '0', '', '', '1', '0', '', '11', '1', '18.18181818', '1', '0', '3058000000'], ['AOS', 'A O Smith Corp', '2018', '40.35379016', '34.81925394', '54.62287941', '34.50987574', '10.06510664', '10768636243', '172194040', 'Building Products', '444200000', '14.1715', '1.6456', '-3.9376', '1.8647', '16.1191', '', '', '', '', '', '', '0', '', '', '0', '0', '', '10', '1', '20', '1', '0', '3071500000'], ['AOS', 'A O Smith Corp', '2017', '39.21154111', '31.54732662', '60.62491062', '29.74093049', '9.526481348', '10895824860', '174605190', 'Building Products', '378300000', '9.7398', '1.7797', '10.5984', '1.9266', '35.9134', '', '', '', '', '', '', '0', '', '', '0', '0', '', '10', '1', '10', '1', '0', '3197400000'], ['AOS', 'A O Smith Corp', '2016', '26.94238012', '27.99118757', '50.88011516', '5.848348348', '8.673096881', '10641447626', '176825280', 'Building Products', '326500000', '11.8293', '1.6631', '9.9574', '1.8664', '25.4379', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2891000000'], ['AOS', 'A O Smith Corp', '2015', '', '', '', '', '8.120039896', '10660290384', '179009180', 'Building Products', '282900000', '10.9982', '1.7554', '4.5283', '1.822', '24.5036', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2629200000'], ['AOS', 'A O Smith Corp', '2014', '', '', '', '', '7.648944406', '10917808078', '181973960', 'Building Products', '207800000', '8.4699', '1.6809', '5.1767', '1.8106', '24.7412', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2515300000'], ['AOS', 'A O Smith Corp', '2013', '', '', '', '', '7.211933413', '11180036461', '185575340', 'Building Products', '169700000', '7.2672', '1.5987', '4.9456', '1.8512', '27.8981', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2391500000']]" @@ -32,8 +49,8 @@ tests: Company: type: object required: - - attributes - - relationships + - attributes + - relationships properties: attributes: type: array @@ -50,9 +67,9 @@ tests: attributes: type: object required: - - Date + - date properties: - Date: + date: type: string type: type: string @@ -160,4 +177,4 @@ tests: - "Board_gen_div" - "Bribery" - "Recycling_Initiatives" - - "Total_assets" \ No newline at end of file + - "Total_assets" diff --git a/backend/promptfoo/dynamic_knowledge_graph_cypher_config.yaml b/backend/promptfoo/dynamic_knowledge_graph_cypher_config.yaml deleted file mode 100644 index fc820d19f..000000000 --- a/backend/promptfoo/dynamic_knowledge_graph_cypher_config.yaml +++ /dev/null @@ -1,26 +0,0 @@ -description: "Test Dynamic Knowledge Graph Prompts" - -providers: - - id: openai:gpt-4o - config: - temperature: 0 - -prompts: file://promptfoo_test_runner.py:create_prompt - -tests: - - description: "test model prompt references all csv headers in result using valid json format" - vars: - user_prompt_template: "generate-knowledge-graph-cypher-user-prompt" - user_prompt_args: - data_model: '{ "model": { "Company": { "attributes": ["Identifier (RIC)", "Company Name"], "relationships": { "has_report": "Report" }, "type": "main_entity" }, "Industry": { "attributes": ["Industry"], "relationships": {}, "type": "category" }, "Report": { "attributes": ["ESG_score", "BVPS", "Market_cap", "Shares", "Net_income", "RETURN_ON_ASSET", "QUICK_RATIO", "ASSET_GROWTH", "FNCL_LVRG", "PE_RATIO", "Total_assets"], "relationships": { "has_ESG_Environment": "Environment", "has_ESG_Social": "Social", "has_ESG_Governance": "Governance" }, "type": "metrics" }, "Environment": { "attributes": ["Env_score", "Scope_1", "Scope_2", "CO2_emissions", "Energy_use", "Water_use", "Water_recycle", "Toxic_chem_red", "Recycling_Initiatives"], "relationships": {}, "type": "metrics" }, "Social": { "attributes": ["Social_score", "Injury_rate", "Women_Employees", "Human_Rights", "Strikes", "Turnover_empl"], "relationships": {}, "type": "metrics" }, "Governance": { "attributes": ["Gov_score", "Board_Size", "Shareholder_Rights", "Board_gen_div", "Bribery"], "relationships": {}, "type": "metrics" } } }' - input_data: "[['Identifier (RIC)', 'Company Name', 'Date', 'ESG_score', 'Social_score', 'Gov_score', 'Env_score', 'BVPS', 'Market_cap', 'Shares', 'Industry', 'Net_income', 'RETURN_ON_ASSET', 'QUICK_RATIO', 'ASSET_GROWTH', 'FNCL_LVRG', 'PE_RATIO', 'Scope_1', 'Scope_2', 'CO2_emissions', 'Energy_use', 'Water_use', 'Water_recycle', 'Toxic_chem_red', 'Injury_rate', 'Women_Employees', 'Human_Rights', 'Strikes', 'Turnover_empl', 'Board_Size', 'Shareholder_Rights', 'Board_gen_div', 'Bribery', 'Recycling_Initiatives', 'Total_assets'], ['AAL', 'American Airlines Group Inc', '2021', '59.02910721', '64.28601828', '56.39811612', '54.38515247', '-11.39725006', '10198305438', '644015000', 'Airlines', '-1993000000', '-3.1032', '0.7333', '7.1507', '34.6286', '5.8534', '898.9575031', '6.470219367', '41439616', '551629386', '1873777.95', '', '0', '', '41', '1', '1', '5.8', '10', '1', '20', '1', '0', '66467000000'], ['AAL', 'American Airlines Group Inc', '2020', '69.40117811', '68.81448509', '83.94361492', '56.96600148', '-14.19130047', '10626751115', '483888000', 'Airlines', '-8885000000', '-14.5652', '0.4953', '3.3553', '34.6286', '5.8534', '904.2455266', '7.364001706', '40604000', '540190224', '1729932.37', '', '0', '', '41.618', '1', '0', '5.5', '12', '1', '16.66666667', '1', '0', '62008000000'], ['AAL', 'American Airlines Group Inc', '2019', '69.36922798', '72.44505934', '77.10711045', '58.21814638', '-0.266147604', '10574818306', '444269000', 'Airlines', '1686000000', '2.7966', '0.3045', '-0.9657', '34.6286', '5.8534', '916.4750598', '7.648632162', '39388000', '564200000', '1627726.3', '', '0', '41.872', '41.654', '1', '1', '4.242', '13', '1', '15.38461538', '1', '0', '59995000000'], ['AAL', 'American Airlines Group Inc', '2018', '71.63302219', '72.04574143', '83.06962314', '60.68373547', '-0.36403898', '11198012018', '465660000', 'Airlines', '1412000000', '2.4911', '0.3573', '14.7675', '34.6286', '7.0665', '969.3836879', '9.117632405', '39279000', '562000000', '1767786.47', '', '0', '41.942', '42.452', '1', '0', '5.613', '13', '1', '15.38461538', '1', '0', '60580000000'], ['AAL', 'American Airlines Group Inc', '2017', '69.79137149', '73.98311797', '78.63204572', '56.14422918', '-1.594557245', '11334335643', '491692000', 'Airlines', '2105000000', '2.464', '0.4439', '2.9469', '34.6286', '9.8649', '1015.247621', '10.31959014', '42038000', '617300000', '1608799.25', '', '0', '30.7', '42.505', '1', '0', '7.139', '13', '1', '15.38461538', '1', '0', '52785000000'], ['AAL', 'American Airlines Group Inc', '2016', '70.3178445', '78.15023537', '71.82041582', '58.46912521', '6.853060249', '11009755584', '556099000', 'Airlines', '2584000000', '5.3687', '0.5733', '5.9052', '10.5827', '8.1442', '979.2028136', '12.16881594', '42282000', '619000000', '1767786.47', '', '0', '38.95', '42', '1', '0', '', '11', '1', '9.090909091', '1', '0', '51274000000'], ['AAL', 'American Airlines Group Inc', '2015', '51.79390641', '54.30943132', '35.94845361', '62.8321256', '8.430668783', '10802024347', '687355000', 'Airlines', '7610000000', '16.6085', '0.5644', '12.0069', '11.9697', '4.2243', '1581.72232', '', '42300000', '', '', '56781.15', '0', '', '41', '0', '0', '', '12', '1', '8.333333333', '1', '0', '48415000000'], ['AAL', 'American Airlines Group Inc', '2014', '44.30935231', '52.95317347', '34.85299977', '41.3372859', '2.816897482', '11314860839', '734016000', 'Airlines', '2882000000', '6.7413', '0.677', '2.2399', '28.1404', '10.4955', '1074.834037', '18.58780929', '27177000', '389700000', '1935858.67', '98420.66', '0', '', '41', '0', '1', '', '11', '1', '18.18181818', '1', '0', '43225000000'], ['AAL', 'American Airlines Group Inc', '2013', '46.85202135', '51.25924607', '31.05261636', '55.31655844', '-16.74987427', '11600491291', '163046000', 'Airlines', '-1834000000', '-5.5755', '0.7831', '79.8299', '28.1404', '', '1128.904458', '19.30856166', '27533000', '394700000', '2018280.61', '', '0', '', '39', '0', '0', '', '11', '1', '18.18181818', '1', '0', '42278000000'], ['AAL', 'American Airlines Group Inc', '2012', '51.24684551', '70.86964887', '24.12375195', '49.64136041', '-63.778138', '11797714591', '125231000', 'Airlines', '-1876000000', '-7.9226', '0.5391', '-1.4173', '28.1404', '', '', '', '27400000', '469631498', '2022924', '', '1', '', '39', '0', '0', '', '12', '1', '16.66666667', '1', '0', '23510000000'], ['ALK', 'Alaska Air Group Inc', '2021', '50.48254945', '39.63992514', '86.0630969', '32.64819037', '30.39268209', '6474416945', '126775000', 'Airlines', '464000000', '3.4147', '0.9176', '-0.6763', '4.1239', '10.5475', '905.4543902', '2.884637285', '7976125', '107090478', '59089.504', '', '0', '28.803', '', '1', '0', '', '12', '1', '41.66666667', '1', '0', '13951000000'], ['ALK', 'Alaska Air Group Inc', '2020', '54.76478447', '44.82792507', '87.86701992', '37.97117517', '24.20413123', '6625511856', '123450000', 'Airlines', '-1324000000', '-9.7933', '0.8912', '8.1044', '3.6944', '10.5475', '937.9543804', '1.300096805', '7761999', '112275897', '61424.924', '', '0', '27.16', '', '1', '0', '', '12', '1', '41.66666667', '1', '0', '14046000000'], ['ALK', 'Alaska Air Group Inc', '2019', '54.78086974', '47.53378754', '83.23838167', '38.60978203', '35.13169315', '6590256377', '124289000', 'Airlines', '769000000', '6.4338', '0.5761', '19.0707', '2.9578', '10.5475', '949.2740056', '1.254877122', '7503475', '108025619.5', '63348.931', '', '0', '27.506', '', '1', '0', '', '11', '1', '36.36363636', '1', '0', '12993000000'], ['ALK', 'Alaska Air Group Inc', '2018', '61.10430324', '55.14174196', '89.29964586', '43.45236925', '30.43901647', '7068723596', '123975000', 'Airlines', '437000000', '4.0355', '0.5445', '1.5448', '3.0035', '13.6321', '858.5817722', '2.115780591', '5099633', '73414401', '68202.958', '', '0', '32.976', '54', '1', '0', '', '10', '1', '40', '1', '0', '10912000000'], ['ALK', 'Alaska Air Group Inc', '2017', '64.76905583', '67.17896047', '85.4188325', '42.77115416', '28.08190827', '7248778365', '123854000', 'Airlines', '723000000', '9.2718', '0.7305', '7.8699', '3.2402', '11.512', '862.8063594', '1.878885316', '4840510', '69695200', '82806.8', '', '0', '32.853', '54', '0', '0', '', '11', '1', '45.45454545', '1', '0', '10746000000'], ['ALK', 'Alaska Air Group Inc', '2016', '55.66218011', '51.08911554', '87.34169898', '32.98313322', '23.72184498', '7076278341', '124389000', 'Airlines', '797000000', '9.5683', '0.7424', '52.5574', '3.0872', '12.3738', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '9962000000'], ['ALK', 'Alaska Air Group Inc', '2015', '', '', '', '', '18.78120789', '7028431619', '129372000', 'Airlines', '848000000', '13.4667', '0.8532', '7.6847', '2.7752', '12.0625', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '6533000000'], ['ALK', 'Alaska Air Group Inc', '2014', '', '', '', '', '15.70379121', '7281515596', '136801000', 'Airlines', '605000000', '10.1664', '0.8833', '3.8712', '2.8638', '14.7979', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '6064000000'], ['ALK', 'Alaska Air Group Inc', '2013', '', '', '', '', '14.50217997', '7402391525', '141878000', 'Airlines', '508000000', '8.9571', '0.938', '6.049', '3.2878', '13.6386', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '5838000000'], ['ALK', 'Alaska Air Group Inc', '2012', '', '', '', '', '10.04836794', '7414982767', '143568000', 'Airlines', '316000000', '5.922', '0.9207', '6.5415', '4.1125', '9.1566', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '5505000000'], ['ALLE', 'Allegion PLC', '2021', '73.40375216', '83.38290372', '66.44580957', '68.26787654', '8.443826474', '10385605328', '90500000', 'Building Products', '483300000', '15.7833', '1.1331', '-0.5995', '3.8529', '24.4624', '', '', '100618', '408600', '394439.722', '', '1', '1.95', '36', '1', '0', '', '8', '1', '12.5', '1', '0', '3051000000'], ['ALLE', 'Allegion PLC', '2020', '59.52521192', '62.27435928', '67.24358974', '41.10701352', '8.985915493', '10355606978', '92800000', 'Building Products', '314500000', '10.4131', '1.5383', '3.4443', '3.8043', '22.7661', '', '', '', '', '', '', '1', '', '', '1', '0', '', '7', '1', '28.57142857', '1', '0', '3069400000'], ['ALLE', 'Allegion PLC', '2019', '59.57897254', '59.97902967', '67.4', '45.07902049', '8.091880342', '10113855566', '94300000', 'Building Products', '402100000', '13.9094', '1.3513', '5.5868', '4.1021', '25.4909', '8.628692755', '28.83442545', '102338', '462330', '', '', '1', '', '', '1', '0', '', '7', '1', '28.57142857', '1', '0', '2967200000'], ['ALLE', 'Allegion PLC', '2018', '47.47726304', '58.84089693', '45.47619048', '31.17791234', '6.852631579', '10104150218', '95700000', 'Building Products', '413500000', '16.2513', '1.1688', '10.5507', '5.0847', '17.1241', '', '', '', '', '', '', '1', '', '', '1', '0', '', '7', '1', '28.57142857', '1', '0', '2810200000'], ['ALLE', 'Allegion PLC', '2017', '37.20505888', '43.26741466', '46.33333333', '10.51693405', '4.222923239', '10155323874', '96000000', 'Building Products', '329000000', '11.4127', '1.6554', '13.1085', '9.3016', '21.1839', '', '', '', '', '', '', '1', '', '', '1', '0', '', '6', '1', '16.66666667', '1', '0', '2542000000'], ['ALLE', 'Allegion PLC', '2016', '33.00965314', '46.7541872', '36.17886179', '3.431372549', '1.182672234', '9855340371', '96900000', 'Building Products', '231200000', '10.1587', '1.3324', '-0.6894', '32.4723', '19.1713', '8.827909676', '28.74474155', '77704', '', '', '', '0', '', '', '1', '0', '', '6', '1', '16.66666667', '1', '0', '2247400000'], ['ALLE', 'Allegion PLC', '2015', '28.54361028', '30.94209162', '38.50877193', '6.77244582', '0.266944734', '9666527226', '96900000', 'Building Products', '154700000', '7.1934', '1.1253', '12.2576', '205.7163', '21.7584', '', '', '', '', '', '', '0', '', '', '0', '0', '', '6', '1', '16.66666667', '1', '0', '2263000000'], ['ALLE', 'Allegion PLC', '2014', '41.9183995', '32.57938447', '73.05555556', '3.267973856', '-0.049947971', '10015037471', '97200000', 'Building Products', '183700000', '8.724', '0.9782', '0.7648', '3.1199', '21.3734', '', '', '', '', '', '', '0', '', '', '0', '0', '', '5', '1', '20', '1', '0', '2015900000'], ['ALLE', 'Allegion PLC', '2013', '', '', '', '', '-0.688541667', '10069740345', '96100000', 'Building Products', '48400000', '1.6213', '0.9937', '0.8469', '3.1199', '26.9942', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2000600000'], ['ALLE', 'Allegion PLC', '2012', '', '', '', '', '13.99166667', '10110326349', '96000000', 'Building Products', '230000000', '10.9502', '1.5827', '-2.5734', '1.4581', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '1983800000'], ['AME', 'AMETEK Inc', '2021', '48.43311116', '50.37985133', '64.36906602', '33.0261549', '29.75421186', '30364402028', '232813000', 'Electrical Equipment', '990053000', '8.8971', '0.7522', '14.8753', '1.7358', '34.7739', '', '', '', '', '', '', '0', '', '', '0', '0', '', '8', '1', '37.5', '1', '0', '11898187000'], ['AME', 'AMETEK Inc', '2020', '46.57431694', '50.92936263', '58.75348611', '31.66671862', '25.93042038', '30350499974', '231150000', 'Electrical Equipment', '872439000', '8.6371', '1.6826', '5.2102', '1.8258', '27.7144', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '10357483000'], ['AME', 'AMETEK Inc', '2019', '42.26573632', '40.45126171', '65.38635236', '24.98103781', '22.46010915', '29516376759', '229395000', 'Electrical Equipment', '861297000', '9.3079', '0.798', '13.6485', '1.9778', '26.7282', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '9844559000'], ['AME', 'AMETEK Inc', '2018', '26.17277106', '29.15026235', '26.42022008', '22.71337861', '18.37738007', '30072458902', '232712000', 'Electrical Equipment', '766133000', '9.4534', '0.8634', '11.111', '1.9902', '20.5798', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '8662288000'], ['AME', 'AMETEK Inc', '2017', '30.66502908', '29.55969049', '44.8033748', '20.09075908', '17.49402986', '30090994974', '231845000', 'Electrical Equipment', '589870000', '9.1493', '1.1544', '9.7933', '2.0451', '27.7205', '', '', '', '', '', '', '0', '', '', '0', '0', '', '9', '1', '33.33333333', '1', '0', '7796064000'], ['AME', 'AMETEK Inc', '2016', '33.20225163', '26.6121386', '56.66255398', '20.85230835', '14.00090716', '29567350956', '233730000', 'Electrical Equipment', '512158000', '7.4435', '1.4166', '6.6095', '2.1135', '21.136', '', '', '', '', '', '', '0', '', '', '0', '0', '', '8', '1', '25', '1', '0', '7100674000'], ['AME', 'AMETEK Inc', '2015', '32.44116267', '29.91329851', '53.04810997', '18.03063224', '13.56625512', '28909320420', '241586000', 'Electrical Equipment', '590859000', '9.0336', '0.9612', '3.7298', '2.0143', '20.998', '', '', '', '', '', '', '0', '', '', '0', '0', '', '7', '1', '14.28571429', '1', '0', '6660450000'], ['AME', 'AMETEK Inc', '2014', '29.54343011', '33.91813565', '37.77892207', '17.90074928', '13.22890745', '29527961804', '247102000', 'Electrical Equipment', '584460000', '9.5043', '1.0288', '9.239', '1.929', '21.6921', '', '', '', '', '', '', '0', '', '', '0', '0', '', '7', '1', '14.28571429', '1', '0', '6420963000'], ['AME', 'AMETEK Inc', '2013', '25.66447145', '33.68505357', '27.27989037', '15.55506041', '12.85743394', '29708688500', '246065000', 'Electrical Equipment', '516999000', '9.3423', '0.9512', '13.2532', '1.9516', '25.4523', '', '', '', '', '', '', '0', '', '', '0', '0', '', '7', '1', '14.28571429', '1', '0', '5877902000'], ['AME', 'AMETEK Inc', '2012', '12.24247583', '22.79737658', '10.33164129', '2.302631579', '10.49699808', '29731858590', '243986000', 'Electrical Equipment', '459132000', '9.6562', '0.7567', '20.1544', '2.0727', '19.984', '', '', '', '', '', '', '0', '', '', '0', '0', '', '8', '1', '12.5', '1', '0', '5190056000'], ['AOS', 'A O Smith Corp', '2021', '54.00154295', '46.2908336', '58.07782524', '59.02930495', '11.45792209', '10784338541', '161319900', 'Building Products', '487100000', '14.6825', '1.1314', '9.925', '1.8028', '29.695', '22.00053463', '24.36361814', '138754', '2030842.8', '1370806.738', '', '0', '', '42', '1', '0', '', '10', '1', '20', '1', '0', '3474400000'], ['AOS', 'A O Smith Corp', '2020', '52.04805145', '41.10365709', '56.74584021', '60.09099171', '11.44241472', '10897395090', '162604150', 'Building Products', '344900000', '11.0924', '1.4381', '3.3584', '1.7691', '25.3944', '20.7663352', '24.32416324', '143744', '1902160.8', '964340.768', '', '0', '', '', '1', '0', '', '10', '1', '20', '1', '0', '3160700000'], ['AOS', 'A O Smith Corp', '2019', '41.56413871', '32.65950433', '58.37558372', '37.28373876', '10.07431591', '10462441424', '166710900', 'Building Products', '370000000', '12.0728', '1.4885', '-0.4395', '1.8114', '21.4595', '22.10197884', '27.59135049', '148916', '1894870.8', '1048918.184', '', '0', '', '', '1', '0', '', '11', '1', '18.18181818', '1', '0', '3058000000'], ['AOS', 'A O Smith Corp', '2018', '40.35379016', '34.81925394', '54.62287941', '34.50987574', '10.06510664', '10768636243', '172194040', 'Building Products', '444200000', '14.1715', '1.6456', '-3.9376', '1.8647', '16.1191', '', '', '', '', '', '', '0', '', '', '0', '0', '', '10', '1', '20', '1', '0', '3071500000'], ['AOS', 'A O Smith Corp', '2017', '39.21154111', '31.54732662', '60.62491062', '29.74093049', '9.526481348', '10895824860', '174605190', 'Building Products', '378300000', '9.7398', '1.7797', '10.5984', '1.9266', '35.9134', '', '', '', '', '', '', '0', '', '', '0', '0', '', '10', '1', '10', '1', '0', '3197400000'], ['AOS', 'A O Smith Corp', '2016', '26.94238012', '27.99118757', '50.88011516', '5.848348348', '8.673096881', '10641447626', '176825280', 'Building Products', '326500000', '11.8293', '1.6631', '9.9574', '1.8664', '25.4379', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2891000000'], ['AOS', 'A O Smith Corp', '2015', '', '', '', '', '8.120039896', '10660290384', '179009180', 'Building Products', '282900000', '10.9982', '1.7554', '4.5283', '1.822', '24.5036', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2629200000'], ['AOS', 'A O Smith Corp', '2014', '', '', '', '', '7.648944406', '10917808078', '181973960', 'Building Products', '207800000', '8.4699', '1.6809', '5.1767', '1.8106', '24.7412', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2515300000'], ['AOS', 'A O Smith Corp', '2013', '', '', '', '', '7.211933413', '11180036461', '185575340', 'Building Products', '169700000', '7.2672', '1.5987', '4.9456', '1.8512', '27.8981', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '2391500000']]" - system_prompt_template: "generate-knowledge-graph-cypher-system-prompt" - assert: - - type: is-json - value: - required: ["cypher_query"] - type: object - - type: contains-all - value: - - "COALESCE" - - "WITH $data AS data UNWIND data.all_data[1..] AS row WITH data.all_data[0] AS headers, row" diff --git a/backend/promptfoo/materiality_agent_config.yaml b/backend/promptfoo/materiality_agent_config.yaml new file mode 100644 index 000000000..0a38bc7c6 --- /dev/null +++ b/backend/promptfoo/materiality_agent_config.yaml @@ -0,0 +1,56 @@ +description: "Test Materiality Agent Prompts" + +providers: + - id: openai:gpt-4o + config: + temperature: 0 + +prompts: file://promptfoo_test_runner.py:create_prompt + +tests: + - description: "test select material documents for BP" + vars: + user_prompt: "BP" + system_prompt_template: "select-material-files-system-prompt" + system_prompt_args: + catalogue: '{"library":{"TFND":[{"name":"Additional-Sector-Guidance-Biotech-and-Pharma.pdf","sector-label":"Biotechnology and Pharmaceuticals","esg-labels":["Environment","Nature"]},{"name":"Additional-Sector-Guidance-Oil-and-gas.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Nature"]}],"GRI":[{"name":"GRI 11_ Oil and Gas Sector 2021.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Social","Governance"]}]}}' + assert: + - type: javascript + value: JSON.parse(output).files[0] === "Additional-Sector-Guidance-Oil-and-gas.pdf" + - type: javascript + value: JSON.parse(output).files[1] === "GRI 11_ Oil and Gas Sector 2021.pdf" + + - description: "test select material documents for BP with focus on nature" + vars: + user_prompt: "BP with focus on Nature materiality topics" + system_prompt_template: "select-material-files-system-prompt" + system_prompt_args: + catalogue: '{"library":{"TFND":[{"name":"Additional-Sector-Guidance-Biotech-and-Pharma.pdf","sector-label":"Biotechnology and Pharmaceuticals","esg-labels":["Environment","Nature"]},{"name":"Additional-Sector-Guidance-Oil-and-gas.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Nature"]}],"GRI":[{"name":"GRI 11_ Oil and Gas Sector 2021.pdf","sector-label":"Oil and Gas","esg-labels":["Environment","Social","Governance"]}]}}' + assert: + - type: javascript + value: JSON.parse(output).files[0] === "Additional-Sector-Guidance-Oil-and-gas.pdf" + - type: javascript + value: JSON.parse(output).files.length === 1 + + - description: "test list material topics for Astra Zeneca with file" + vars: + user_prompt: "What topics are material for AstraZeneca?" + system_prompt_template: "list-material-topics-system-prompt" + file_attachment: "../library/Additional-Sector-Guidance-Biotech-and-Pharma.pdf" + assert: + - type: is-json + value: + required: ["material_topics"] + type: object + - type: javascript + value: JSON.parse(output).material_topics["Environmental Stewardship"] === "AstraZeneca, like other companies in the biotechnology and pharmaceuticals sector, has significant dependencies and impacts on natural ecosystems. The company relies on biomass provisioning, genetic material for drug development, and water resources for manufacturing. The management of these dependencies and mitigating environmental impacts such as water and soil pollution is crucial for sustainable operations." + - type: javascript + value: JSON.parse(output).material_topics["Climate Change and GHG Emissions"] === "As part of a sector that is intensive in resource and energy use, managing greenhouse gas emissions and transitioning to sustainable energy sources is essential for AstraZeneca to address climate change risks and opportunities, comply with global regulatory standards and meet the expectations of stakeholders." + - type: javascript + value: JSON.parse(output).material_topics["Product Stewardship and Safety"] === "Given the nature of pharmaceuticals, AstraZeneca must ensure the safe production, handling, and disposal of products, preventing environmental contamination, and addressing the issue of pharmaceuticals in the environment, including environmentally persistent pharmaceutical pollutants (EPPPs)." + - type: javascript + value: JSON.parse(output).material_topics["Supply Chain Management"] === "AstraZeneca sources various inorganic and organic feedstock and raw materials that may pose environmental risks if not managed sustainably. Effective supply chain management, including traceability and engagement with suppliers on nature-related impacts, is essential to minimize dependencies and risks." + - type: javascript + value: JSON.parse(output).material_topics["Biodiversity and Ecosystem Impacts"] === "The potential impact of AstraZeneca's operations on sensitive ecosystems, as well as its reliance on biodiversity for sourcing natural compounds for drug development, highlights the importance of considering biodiversity in the company's sustainability strategy." + - type: javascript + value: JSON.parse(output).material_topics["Pollution Prevention"] === "Managing and reducing pollution, particularly non-GHG air pollutants, wastewater discharges, and hazardous waste, is critical for AstraZeneca to mitigate its environmental footprint and comply with environmental regulations." diff --git a/backend/promptfoo/promptfoo_test_runner.py b/backend/promptfoo/promptfoo_test_runner.py index 96331bec7..9920101ff 100644 --- a/backend/promptfoo/promptfoo_test_runner.py +++ b/backend/promptfoo/promptfoo_test_runner.py @@ -1,10 +1,19 @@ import sys +from pypdf import PdfReader sys.path.append("../") from src.prompts.prompting import PromptEngine # noqa: E402 engine = PromptEngine() +def read_pdf_file_for_promptfoo(file_path: str) -> str: + pdf_file = PdfReader(file_path) + content = "\n".join([ + page.extract_text() for page in pdf_file.pages + ]) + return content + + def create_prompt(context): config = context["vars"] @@ -20,4 +29,7 @@ def create_prompt(context): else: raise Exception("Must provide either user_prompt or user_prompt_template") + if "file_attachment" in config: + user_prompt = f"{user_prompt}\n\nAttached file: {read_pdf_file_for_promptfoo(config["file_attachment"])}" + return [{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}] diff --git a/backend/promptfoo/create_report_config.yaml b/backend/promptfoo/report_agent_config.yaml similarity index 50% rename from backend/promptfoo/create_report_config.yaml rename to backend/promptfoo/report_agent_config.yaml index 33e5e96e0..458e86b4c 100644 --- a/backend/promptfoo/create_report_config.yaml +++ b/backend/promptfoo/report_agent_config.yaml @@ -1,4 +1,4 @@ -description: "Test Report Prompt" +description: "Test Report Agent Prompts" providers: - id: mistral:mistral-large-latest @@ -14,6 +14,116 @@ tests: system_prompt_template: "create-report-system-prompt" user_prompt_args: document_text: "Published September 2024 Carbon Reduction Plan +Supplier name: Amazon Web Services EU SARL (UK Branch) (“AWS UK”) +Publication date: September 30, 2024 +Commitment to Achieving Net Zero +AWS UK, as part of Amazon.com, Inc. (“Amazon”), is committed to achieving net -zero +emissions by 2040. In 2019, Amazon co -founded The Climate Pledge, a public commitment +to innovate, use our scale for good and go faster to address the urgency of the climate crisis +to reach net -zero carbon across the entire organization by 2040. Since committing to the +Pledge, we’ve changed how we conduct our business and the running of our operations, and +we’ve increased funding and implementation of new technologies and services that +decarbonize and help preserve the natural world, alon gside the ambitious goals outlined in +The Climate Pledge. We’re fully committed to our goals and our work to build a better planet. +Baseline Emissions Footprint +Base Year emissions are a record of the greenhouse gases that have been produced in the +past an d are the reference point against which emissions reduction can be measured. +Baseline Year: 2020 +Additional Details relating to the Baseline Emissions calculations: +AWS UK utilized January 1, 2020 to December 31, 2020 as the baseline year for emissions +reporting under this Carbon Reduction Plan. Our plan includes emissions data from relevant +affiliate companies helping to provide AWS UK’s services to our customers. We ’ve included both +location -based and market -based method Scope 2 emissions in the following tables. AWS UK +benefits from contractual arrangements entered into by our affiliate(s) for renewable electricity +and/or renewable attributes that are reflected in t he market -based data set. More information +about our corporate carbon footprint and methodology can be found on our website . +Our baseline year does not include Scope 1 emissions. In 2022 we updated our methodology +and Scope 1 emissions are now included in total emissions for AWS UK + + Published September 2024 Baseline year emissions: +EMISSIONS TOTAL (tCO 2e) +Scope 1 0 +Scope 2 61,346 – Location -based method +2,813 – Market -based method +Scope 3 (Included +Sources) 3,770 +Total Emissions 65,116 – Location -based method +6,583 – Market -based method +Current Emissions Reporting +Reporting Year: 202 3 (January 1, 202 3 to December 31, 202 3) +EMISSIONS TOTAL (tCO 2e) +Scope 1 2,23 3 +Scope 2 126,755 – Location -based method +0 – Market -based method +Scope 3 (Included +Sources) 13,188 +Total Emissions 142,17 6 – Location -based method +15,42 1 – Market -based method + + Published September 2024 Emissions Reduction Targets +In 2019, we set an ambitious goal to match 100% of the electricity we use with renewable +energy by 2030. This goal includes all data centres , logistics facilities, physical stores, and +corporate offices, as well as on -site charg ing points and our financially integrated subsidiaries. +We are proud to have achieved this goal in 2023, seven years early, with 100% of the electricity +consum ed by Amazon matched with renewable energy sources. +Amazon continue s to be transparent and share our progress to reach net -zero carbon in our +annual Sustainability Report , which also includes details on how we measure carbon . +Carbon Reduction Projects +Completed Carbon Reduction Initiatives +Amazon continues to take actions across our operations to drive carbon reduction around the +world, including in the UK. As of January 202 4, Amazon’s renewable energy portfolio includes +243 wind and solar farms and 2 70 rooftop solar projects, totalling 513 projects and 28 +gigawatts of renewable energy capacity. This includes several utility -scale renewable energy +projects located within the UK: +•In 2019, Amazon announced our first power purchase agreement in the UK, located in +Kintyre Peninsula, Scotland. The “Amazon Wind Farm Scotland – Beinn an Tuirc 3” +began o perating in 2021, providing 50 megawatts (MW) of new renewable capacity to +the electricity grid with expected generation of 168,000 megawatt hours (MWh) of +clean energy annually. That’s enough to power 46,000 UK homes every year. +•In December 2020, Amazon a nnounced a two -phase renewable energy project located +in South Lanarkshire, Scotland, the Kennoxhead wind farm. Kennoxhead will be the +largest single -site onshore wind project in the UK, enabled through corporate +procurement. Once fully operational, Kenno xhead will produce 129 MW of renewable +capacity and is expected to generate 439,000 MWh of clean energy annually. Phase 1 +(60 MW) began operating in 2022, and Phase 2 (69 MW) will begin operations in 2024 . +•In 2022, Amazon announced its first project in Nor thern Ireland, a 16 MW onshore +windfarm in Co Antrim. +•In 2022, Amazon also announced a new 473 MW offshore wind farm, Moray West, +located off the coast of Scotland . Amazon expects completion of Moray West in 2024. +This is Amazon’s largest project in Scotland and the largest corporate renewable +energy deal announced by any company in the UK to date. +•In 2023, Amazon announced a new 47 MW solar farm, Warl ey located in Essex. +This project is expected to be operational in 2024. + + Published September 2024 Declaration and Sign Off +This Carbon Reduction Plan has been completed in accordance with PPN 06/21 and +associated guidance and reporting standard for Carbon Reduction Plans. +Emiss ions have been reported and recorded in accordance with the published reporting +standard for Carbon Reduction Plans and the GHG Reporting Protocol corporate standard1 +and uses the appropri ate Government emission conversion factors for greenhouse gas +company reporting2. +Scope 1 and Scope 2 emissions have been reported in accordance with S ECR requirements, +and the required subset of Scope 3 emissions have been reported in accordance with the +published reporting standard for Carbon Reduction Plans and the Corporate Value Chain +(Scope 3) Standard3. +This Carbon Reduction Plan has been reviewed and signed off by the board of directors (or +equivalent management body)." + assert: + - type: contains-all + value: + - "# Basic" + - "# ESG" + - "# Environmental" + - "# Social" + - "# Governance" + - "# Conclusion" + + - description: "Test getting company name from file" + vars: + user_prompt_template: "find-company-name-from-file-user-prompt" + system_prompt_template: "find-company-name-from-file-system-prompt" + user_prompt_args: + file_content: "Published September 2024 Carbon Reduction Plan Supplier name: Amazon Web Services EU SARL (UK Branch) (“AWS UK”) Publication date: September 30, 2024 Commitment to Achieving Net Zero @@ -109,11 +219,9 @@ published reporting standard for Carbon Reduction Plans and the Corporate Value This Carbon Reduction Plan has been reviewed and signed off by the board of directors (or equivalent management body)." assert: - - type: contains-all + - type: is-json value: - - "# Basic" - - "# ESG" - - "# Environmental" - - "# Social" - - "# Governance" - - "# Conclusion" \ No newline at end of file + required: ["company_name"] + type: object + - type: javascript + value: JSON.parse(output).company_name === "Amazon" diff --git a/backend/src/agents/__init__.py b/backend/src/agents/__init__.py index e7f3fda36..3a9731107 100644 --- a/backend/src/agents/__init__.py +++ b/backend/src/agents/__init__.py @@ -1,7 +1,7 @@ from typing import List from src.utils import Config -from src.agents.agent import Agent, agent +from src.agents.agent import Agent, ChatAgent, chat_agent from src.agents.datastore_agent import DatastoreAgent from src.agents.web_agent import WebAgent from src.agents.intent_agent import IntentAgent @@ -10,32 +10,37 @@ from src.agents.answer_agent import AnswerAgent from src.agents.chart_generator_agent import ChartGeneratorAgent from src.agents.report_agent import ReportAgent +from src.agents.materiality_agent import MaterialityAgent config = Config() -def get_validator_agent() -> Agent: +def get_validator_agent() -> ChatAgent: return ValidatorAgent(config.validator_agent_llm, config.validator_agent_model) -def get_intent_agent() -> Agent: +def get_intent_agent() -> ChatAgent: return IntentAgent(config.intent_agent_llm, config.intent_agent_model) -def get_answer_agent() -> Agent: +def get_answer_agent() -> ChatAgent: return AnswerAgent(config.answer_agent_llm, config.answer_agent_model) -def get_report_agent() -> Agent: +def get_report_agent() -> ReportAgent: return ReportAgent(config.report_agent_llm, config.report_agent_model) -def agent_details(agent) -> dict: +def get_materiality_agent() -> MaterialityAgent: + return MaterialityAgent(config.materiality_agent_llm, config.materiality_agent_model) + + +def agent_details(agent: ChatAgent) -> dict: return {"name": agent.name, "description": agent.description} -def get_available_agents() -> List[Agent]: +def get_available_agents() -> List[ChatAgent]: return [ DatastoreAgent(config.datastore_agent_llm, config.datastore_agent_model), WebAgent(config.web_agent_llm, config.web_agent_model), @@ -49,8 +54,9 @@ def get_agent_details(): __all__ = [ - "agent", "Agent", + "ChatAgent", + "chat_agent", "agent_details", "get_agent_details", "get_answer_agent", @@ -58,6 +64,7 @@ def get_agent_details(): "get_available_agents", "get_validator_agent", "get_report_agent", + "get_materiality_agent", "Parameter", "tool", ] diff --git a/backend/src/agents/agent.py b/backend/src/agents/agent.py index 32224b55d..a431efa2f 100644 --- a/backend/src/agents/agent.py +++ b/backend/src/agents/agent.py @@ -1,7 +1,7 @@ from abc import ABC import json import logging -from typing import List, Type +from typing import List, Type, TypeVar, Optional from src.llm import LLM, get_llm from src.utils.log_publisher import LogPrefix, publish_log_info @@ -18,9 +18,6 @@ class Agent(ABC): - name: str - description: str - tools: List[Tool] llm: LLM model: str @@ -30,6 +27,12 @@ def __init__(self, llm_name: str | None, model: str | None): raise ValueError("LLM Model Not Provided") self.model = model + +class ChatAgent(Agent): + name: str + description: str + tools: List[Tool] + async def __get_action(self, utterance: str) -> Action_and_args: tool_descriptions = create_all_tools_str(self.tools) @@ -52,7 +55,7 @@ async def __get_action(self, utterance: str) -> Action_and_args: validate_args(chosen_tool_parameters, chosen_tool) except Exception: raise Exception(f"Unable to extract chosen tool and parameters from {response}") - return (chosen_tool.action, chosen_tool_parameters) + return chosen_tool.action, chosen_tool_parameters async def invoke(self, utterance: str) -> str: (action, args) = await self.__get_action(utterance) @@ -61,11 +64,17 @@ async def invoke(self, utterance: str) -> str: return result_of_action -def agent(name: str, description: str, tools: List[Tool]): - def decorator(agent: Type[Agent]): - agent.name = name - agent.description = description - agent.tools = tools - return agent +T = TypeVar('T', bound=ChatAgent) + + +def chat_agent(name: str, description: str, tools: Optional[List[Tool]] = None): + if not tools: + tools = [] + + def decorator(chat_agent: Type[T]) -> Type[T]: + chat_agent.name = name + chat_agent.description = description + chat_agent.tools = tools + return chat_agent return decorator diff --git a/backend/src/agents/answer_agent.py b/backend/src/agents/answer_agent.py index 06a77458a..82aa51d47 100644 --- a/backend/src/agents/answer_agent.py +++ b/backend/src/agents/answer_agent.py @@ -1,18 +1,18 @@ from datetime import datetime from src.utils import get_scratchpad from src.prompts import PromptEngine -from src.agents import Agent, agent +from src.agents import ChatAgent, chat_agent from src.session import get_session_chat engine = PromptEngine() -@agent( +@chat_agent( name="AnswerAgent", description="This agent is responsible for generating an answer for the user, based on results in the scratchpad", tools=[], ) -class AnswerAgent(Agent): +class AnswerAgent(ChatAgent): async def invoke(self, utterance: str) -> str: final_scratchpad = get_scratchpad() create_answer = engine.load_prompt( diff --git a/backend/src/agents/chart_generator_agent.py b/backend/src/agents/chart_generator_agent.py index 8479bc128..df6a4346b 100644 --- a/backend/src/agents/chart_generator_agent.py +++ b/backend/src/agents/chart_generator_agent.py @@ -1,6 +1,6 @@ import logging from src.prompts import PromptEngine -from .agent import Agent, agent +from .agent import ChatAgent, chat_agent from .tool import tool from src.llm.llm import LLM from .agent_types import Parameter @@ -91,10 +91,10 @@ async def generate_code_chart(question_intent, data_provided, question_params, l return await generate_chart(question_intent, data_provided, question_params, llm, model) -@agent( +@chat_agent( name="ChartGeneratorAgent", description="This agent is responsible for creating charts", tools=[generate_code_chart], ) -class ChartGeneratorAgent(Agent): +class ChartGeneratorAgent(ChatAgent): pass diff --git a/backend/src/agents/datastore_agent.py b/backend/src/agents/datastore_agent.py index 2905ea61b..fc5a83981 100644 --- a/backend/src/agents/datastore_agent.py +++ b/backend/src/agents/datastore_agent.py @@ -7,7 +7,7 @@ from src.utils import to_json from .agent_types import Parameter from src.utils.log_publisher import LogPrefix, publish_log_info -from .agent import Agent, agent +from .agent import ChatAgent, chat_agent from .tool import tool from src.utils.semantic_layer_builder import get_semantic_layer @@ -31,7 +31,7 @@ async def generate_cypher_query_core( timeframe=timeframe, ) try: - graph_schema = await get_semantic_layer_cache(llm, model, cache) + graph_schema = await get_semantic_layer_cache(llm, model) graph_schema = json.dumps(graph_schema, separators=(",", ":")) generate_cypher_query_prompt = engine.load_prompt( @@ -95,7 +95,7 @@ async def generate_cypher( ) -async def get_semantic_layer_cache(llm, model, graph_schema): +async def get_semantic_layer_cache(llm, model): global cache if not cache: graph_schema = await get_semantic_layer(llm, model) @@ -105,10 +105,10 @@ async def get_semantic_layer_cache(llm, model, graph_schema): return cache -@agent( +@chat_agent( name="DatastoreAgent", description="This agent is responsible for handling database queries to the bloomberg.csv dataset. This includes retrieving ESG scores, financial metrics, and other bloomberg-specific information. It interacts with the graph database to extract, process, and return ESG-related information from various sources, such as company sustainability reports or fund portfolios. This agent can not complete any task that is not specifically about the bloomberg.csv dataset.", # noqa: E501 tools=[generate_cypher], ) -class DatastoreAgent(Agent): +class DatastoreAgent(ChatAgent): pass diff --git a/backend/src/agents/intent_agent.py b/backend/src/agents/intent_agent.py index 8233b8683..cc986b665 100644 --- a/backend/src/agents/intent_agent.py +++ b/backend/src/agents/intent_agent.py @@ -1,5 +1,5 @@ from src.prompts import PromptEngine -from src.agents import Agent, agent +from src.agents import ChatAgent, chat_agent from src.session import get_session_chat import logging from src.utils.config import Config @@ -12,12 +12,12 @@ logger = logging.getLogger(__name__) -@agent( +@chat_agent( name="IntentAgent", description="This agent is responsible for determining the intent of the user's utterance", tools=[], ) -class IntentAgent(Agent): +class IntentAgent(ChatAgent): async def invoke(self, utterance: str) -> str: session_chat = get_session_chat() user_prompt = engine.load_prompt( diff --git a/backend/src/agents/materiality_agent.py b/backend/src/agents/materiality_agent.py new file mode 100644 index 000000000..d63479559 --- /dev/null +++ b/backend/src/agents/materiality_agent.py @@ -0,0 +1,36 @@ +import json +from pathlib import Path +import logging + +from src.llm import LLMFile +from src.agents import Agent +from src.prompts import PromptEngine + +engine = PromptEngine() +logger = logging.getLogger(__name__) + + +class MaterialityAgent(Agent): + async def list_material_topics(self, company_name: str) -> dict[str, str]: + with open('./library/catalogue.json') as file: + catalogue = json.load(file) + files_json = await self.llm.chat( + self.model, + system_prompt=engine.load_prompt( + "select-material-files-system-prompt", + catalogue=catalogue + ), + user_prompt=company_name, + return_json=True + ) + + materiality_topics = await self.llm.chat_with_file( + self.model, + system_prompt=engine.load_prompt("list-material-topics-system-prompt"), + user_prompt=f"What topics are material for {company_name}?", + files=[ + LLMFile(file_name=file_name, file=Path(f"./library/{file_name}")) + for file_name in json.loads(files_json)["files"] + ] + ) + return json.loads(materiality_topics)["material_topics"] diff --git a/backend/src/agents/report_agent.py b/backend/src/agents/report_agent.py index dd2c0db00..a2f911d2c 100644 --- a/backend/src/agents/report_agent.py +++ b/backend/src/agents/report_agent.py @@ -1,20 +1,33 @@ -from src.agents import Agent, agent +import json +import logging + +from src.agents import Agent from src.prompts import PromptEngine +logger = logging.getLogger(__name__) engine = PromptEngine() -@agent( - name="ReportAgent", - description="This agent is responsible for generating an ESG focused report on a narrative document", - tools=[], -) class ReportAgent(Agent): - async def invoke(self, utterance: str) -> str: + async def create_report(self, file_content: str, materiality_topics: dict[str, str]) -> str: user_prompt = engine.load_prompt( "create-report-user-prompt", - document_text=utterance) + document_text=file_content, + materiality_topics=materiality_topics + ) system_prompt = engine.load_prompt("create-report-system-prompt") return await self.llm.chat(self.model, system_prompt=system_prompt, user_prompt=user_prompt) + + async def get_company_name(self, file_content: str) -> str: + response = await self.llm.chat( + self.model, + system_prompt=engine.load_prompt("find-company-name-from-file-system-prompt"), + user_prompt=engine.load_prompt( + "find-company-name-from-file-user-prompt", + file_content=file_content + ), + return_json=True + ) + return json.loads(response)["company_name"] diff --git a/backend/src/agents/validator_agent.py b/backend/src/agents/validator_agent.py index fc846df89..925e51b25 100644 --- a/backend/src/agents/validator_agent.py +++ b/backend/src/agents/validator_agent.py @@ -1,6 +1,6 @@ import logging from src.prompts import PromptEngine -from src.agents import Agent, agent +from src.agents import ChatAgent, chat_agent from src.utils.log_publisher import LogPrefix, publish_log_info import json @@ -9,12 +9,12 @@ validator_prompt = engine.load_prompt("validator") -@agent( +@chat_agent( name="ValidatorAgent", description="This agent is responsible for validating the answers to the tasks", tools=[], ) -class ValidatorAgent(Agent): +class ValidatorAgent(ChatAgent): async def invoke(self, utterance: str) -> str: answer = await self.llm.chat(self.model, validator_prompt, utterance) response = json.loads(answer)['response'] diff --git a/backend/src/agents/web_agent.py b/backend/src/agents/web_agent.py index 3d324c059..d4b6005a8 100644 --- a/backend/src/agents/web_agent.py +++ b/backend/src/agents/web_agent.py @@ -1,7 +1,7 @@ import logging from src.prompts import PromptEngine from .agent_types import Parameter -from .agent import Agent, agent +from .agent import ChatAgent, chat_agent from .tool import tool from src.utils import Config from src.utils.web_utils import ( @@ -207,7 +207,7 @@ async def find_information_from_content(content: str, question: str, llm, model) return await find_information_from_content_core(content, question, llm, model) -def get_validator_agent() -> Agent: +def get_validator_agent() -> ChatAgent: return ValidatorAgent(config.validator_agent_llm, config.validator_agent_model) @@ -264,10 +264,10 @@ async def perform_pdf_summarization(content: str, llm: Any, model: str) -> str: return "" -@agent( +@chat_agent( name="WebAgent", description="This agent can perform general internet searches to complete the task by retrieving and summarizing the results and it can also perform web scrapes to retreive specific inpormation from web pages.", # noqa: E501 tools=[web_general_search, web_pdf_download, web_scrape, find_information_from_content], ) -class WebAgent(Agent): +class WebAgent(ChatAgent): pass diff --git a/backend/src/directors/report_director.py b/backend/src/directors/report_director.py index 80b021414..f3f4a18c5 100644 --- a/backend/src/directors/report_director.py +++ b/backend/src/directors/report_director.py @@ -1,22 +1,41 @@ from fastapi import UploadFile from src.session.file_uploads import FileUploadReport, store_report -from src.utils.scratchpad import clear_scratchpad, update_scratchpad from src.utils.file_utils import handle_file_upload -from src.agents import get_report_agent +from src.agents import get_report_agent, get_materiality_agent + async def report_on_file_upload(upload: UploadFile) -> FileUploadReport: file = handle_file_upload(upload) - update_scratchpad(result=file["content"]) + report_agent = get_report_agent() + + company_name = await report_agent.get_company_name(file["content"]) - report = await get_report_agent().invoke(file["content"]) + topics = await get_materiality_agent().list_material_topics(company_name) - clear_scratchpad() + report = await get_report_agent().create_report(file["content"], topics) - report_upload = FileUploadReport(filename=file["filename"], id=file["uploadId"], report=report) + report_upload = FileUploadReport( + filename=file["filename"], + id=file["uploadId"], + report=report, + answer=create_report_chat_message(file["filename"], company_name, topics) + ) store_report(report_upload) return report_upload + + +def create_report_chat_message(file_name: str, company_name: str, topics: dict[str, str]) -> str: + topics_with_markdown = [ + f"{key}\n{value}" for key, value in topics.items() + ] + return f"""Your report for {file_name} is ready to view. + +The following materiality topics were identified for {company_name} which the report focuses on: + +{"\n\n".join(topics_with_markdown)} +""" diff --git a/backend/src/llm/__init__.py b/backend/src/llm/__init__.py index 2c575cf7a..6d97c839d 100644 --- a/backend/src/llm/__init__.py +++ b/backend/src/llm/__init__.py @@ -1,9 +1,7 @@ -from .llm import LLM +from .llm import LLM, LLMFile from .factory import get_llm from .mistral import Mistral from .count_calls import count_calls -from .mock import MockLLM from .openai import OpenAI -from .openai_client import OpenAIClient -__all__ = ["count_calls", "get_llm", "LLM", "Mistral", "MockLLM", "OpenAI", "OpenAIClient"] +__all__ = ["count_calls", "get_llm", "LLM", "LLMFile", "Mistral", "OpenAI"] diff --git a/backend/src/llm/llm.py b/backend/src/llm/llm.py index 3221592bf..64a13e659 100644 --- a/backend/src/llm/llm.py +++ b/backend/src/llm/llm.py @@ -1,6 +1,17 @@ from abc import ABC, ABCMeta, abstractmethod +from os import PathLike from typing import Any, Coroutine from .count_calls import count_calls +from dataclasses import dataclass + + +count_calls_of_functions = ["chat", "chat_with_file"] + + +@dataclass +class LLMFile(ABC): + file_name: str + file: PathLike[str] | bytes class LLMMeta(ABCMeta): @@ -12,8 +23,9 @@ def __init__(cls, name, bases, namespace): cls.instances[name.lower()] = cls() def __new__(cls, name, bases, attrs): - if "chat" in attrs: - attrs["chat"] = count_calls(attrs["chat"]) + for function in count_calls_of_functions: + if function in attrs: + attrs[function] = count_calls(attrs[function]) return super().__new__(cls, name, bases, attrs) @@ -24,5 +36,21 @@ def get_instances(cls): return cls.instances @abstractmethod - def chat(self, model: str, system_prompt: str, user_prompt: str, return_json=False) -> Coroutine[Any, Any, str]: + def chat( + self, + model: str, + system_prompt: str, + user_prompt: str, + return_json: bool = False + ) -> Coroutine[Any, Any, str]: + pass + + @abstractmethod + def chat_with_file( + self, + model: str, + system_prompt: str, + user_prompt: str, + files: list[LLMFile] + ) -> Coroutine: pass diff --git a/backend/src/llm/mistral.py b/backend/src/llm/mistral.py index 18974b4ef..bf303fdf8 100644 --- a/backend/src/llm/mistral.py +++ b/backend/src/llm/mistral.py @@ -1,7 +1,9 @@ +from typing import Coroutine + from mistralai import Mistral as MistralApi, UserMessage, SystemMessage import logging from src.utils import Config -from .llm import LLM +from .llm import LLM, LLMFile logger = logging.getLogger(__name__) config = Config() @@ -32,3 +34,12 @@ async def chat(self, model, system_prompt: str, user_prompt: str, return_json=Fa logger.debug('{0} response : "{1}"'.format(model, content)) return content + + def chat_with_file( + self, + model: str, + system_prompt: str, + user_prompt: str, + files: list[LLMFile] + ) -> Coroutine: + raise Exception("Mistral does not support chat_with_file") diff --git a/backend/src/llm/mock.py b/backend/src/llm/mock.py deleted file mode 100644 index 5fd02dc0f..000000000 --- a/backend/src/llm/mock.py +++ /dev/null @@ -1,6 +0,0 @@ -from .llm import LLM - - -class MockLLM(LLM): - async def chat(self, model: str, system_prompt: str, user_prompt: str, return_json=False) -> str: - return "mocked response" diff --git a/backend/src/llm/openai.py b/backend/src/llm/openai.py index 6c2d11413..8b42f6181 100644 --- a/backend/src/llm/openai.py +++ b/backend/src/llm/openai.py @@ -1,17 +1,22 @@ -# src/llm/openai_llm.py import logging -from .openai_client import OpenAIClient + from src.utils import Config -from .llm import LLM +from src.llm import LLM, LLMFile from openai import NOT_GIVEN, AsyncOpenAI +from openai.types.beta.threads import Text, TextContentBlock logger = logging.getLogger(__name__) config = Config() +def remove_citations(message: Text): + value = message.value + for annotation in message.annotations: + value = value.replace(annotation.text, "") + return value + + class OpenAI(LLM): - def __init__(self): - self.client = OpenAIClient() async def chat(self, model, system_prompt: str, user_prompt: str, return_json=False) -> str: logger.debug( @@ -19,8 +24,8 @@ async def chat(self, model, system_prompt: str, user_prompt: str, return_json=Fa str([system_prompt, user_prompt]) ) ) - client = AsyncOpenAI(api_key=config.openai_key) try: + client = AsyncOpenAI(api_key=config.openai_key) response = await client.chat.completions.create( model=model, messages=[ @@ -28,19 +33,71 @@ async def chat(self, model, system_prompt: str, user_prompt: str, return_json=Fa {"role": "user", "content": user_prompt}, ], temperature=0, - response_format={ - "type": "json_object"} if return_json else NOT_GIVEN, + response_format={"type": "json_object"} if return_json else NOT_GIVEN ) content = response.choices[0].message.content logger.info(f"OpenAI response: Finish reason: {response.choices[0].finish_reason}, Content: {content}") logger.debug(f"Token data: {response.usage}") - if isinstance(content, str): - return content - elif isinstance(content, list): - return " ".join(content) - else: - return "Unexpected content format" + if not content: + logger.error("Call to Mistral API failed: message content is None") + return "An error occurred while processing the request." + + return content except Exception as e: logger.error(f"Error calling OpenAI model: {e}") return "An error occurred while processing the request." + + async def chat_with_file( + self, + model: str, + system_prompt: str, + user_prompt: str, + files: list[LLMFile] + ) -> str: + client = AsyncOpenAI(api_key=config.openai_key) + file_ids = await self.__upload_files(files) + + file_assistant = await client.beta.assistants.create( + name="ESG Analyst", + instructions=system_prompt, + model=model, + tools=[{"type": "file_search"}], + ) + + thread = await client.beta.threads.create( + messages=[ + { + "role": "user", + "content": user_prompt, + "attachments": [ + {"file_id": file_id, "tools": [{"type": "file_search"}]} + for file_id in file_ids + ], + } + ] + ) + + run = await client.beta.threads.runs.create_and_poll( + thread_id=thread.id, assistant_id=file_assistant.id + ) + + messages = await client.beta.threads.messages.list(thread_id=thread.id, run_id=run.id) + + if isinstance(messages.data[0].content[0], TextContentBlock): + message = remove_citations(messages.data[0].content[0].text) + else: + message = messages.data[0].content[0].to_json() + + logger.info(f"OpenAI response: {message}") + return message + + async def __upload_files(self, files: list[LLMFile]) -> list[str]: + client = AsyncOpenAI(api_key=config.openai_key) + + file_ids = [] + for file in files: + logger.info(f"Uploading file '{file.file_name}' to OpenAI") + file = await client.files.create(file=file.file, purpose="assistants") + file_ids.append(file.id) + return file_ids diff --git a/backend/src/llm/openai_client.py b/backend/src/llm/openai_client.py deleted file mode 100644 index d0cd481eb..000000000 --- a/backend/src/llm/openai_client.py +++ /dev/null @@ -1,26 +0,0 @@ -# src/llm/openai_client.py -import openai -from src.utils import Config -import logging - -config = Config() -logger = logging.getLogger(__name__) - - -class OpenAIClient: - def __init__(self): - self.api_key = config.openai_key - openai.api_key = self.api_key - - def chat(self, model, messages, temperature=0, max_tokens=150): - try: - response = openai.ChatCompletion.create( # type: ignore - model=model, - messages=messages, - ) - content = response["choices"][0]["message"]["content"] - logger.debug(f'{model} response: "{content}"') - return content - except Exception as e: - logger.error(f"Error calling OpenAI model: {e}") - return "An error occurred while processing the request." diff --git a/backend/src/prompts/templates/create-report-user-prompt.j2 b/backend/src/prompts/templates/create-report-user-prompt.j2 index 8b5d3ee44..9d7c8af6b 100644 --- a/backend/src/prompts/templates/create-report-user-prompt.j2 +++ b/backend/src/prompts/templates/create-report-user-prompt.j2 @@ -1,3 +1,7 @@ +Using the following information about ESG Materiality: + +{{ materiality_topics }} + Generate an ESG report using the following document: {{ document_text }} \ No newline at end of file diff --git a/backend/src/prompts/templates/find-company-name-from-file-system-prompt.j2 b/backend/src/prompts/templates/find-company-name-from-file-system-prompt.j2 new file mode 100644 index 000000000..e4c4da7b2 --- /dev/null +++ b/backend/src/prompts/templates/find-company-name-from-file-system-prompt.j2 @@ -0,0 +1,9 @@ +You will receive a file containing information about a company. Your task is to identify the name of the company mentioned in the file. + +Output only the company name as they are commonly known. +Do not include additional text, explanations. + +Output Requirements: +Output must be in JSON format with no additional markdown or formatting as shown below: + +{ "company_name": "COMPANY_NAME" } diff --git a/backend/src/prompts/templates/find-company-name-from-file-user-prompt.j2 b/backend/src/prompts/templates/find-company-name-from-file-user-prompt.j2 new file mode 100644 index 000000000..0b2edfce7 --- /dev/null +++ b/backend/src/prompts/templates/find-company-name-from-file-user-prompt.j2 @@ -0,0 +1,3 @@ +What company is this file about? + +{{ file_content }} diff --git a/backend/src/prompts/templates/list-material-topics-system-prompt.j2 b/backend/src/prompts/templates/list-material-topics-system-prompt.j2 new file mode 100644 index 000000000..2fd7d2f07 --- /dev/null +++ b/backend/src/prompts/templates/list-material-topics-system-prompt.j2 @@ -0,0 +1,19 @@ +You are an ESG specialist conducting materiality assessments using attached reference files and company-specific information. + +Core Responsibility: +- Determine which ESG topics from provided files are relevant to the company given by the user +- Explain why the chosen ESG topics are relevant to the company + +Assessment Method: +- Carefully analyze attached files for materiality frameworks +- Apply materiality guidance from documents to company context +- Evaluate topics through lens of company's unique attributes + +Output Requirements: +Your output must be strict JSON format with not additional markdown or formatting. If you fail to do this you will be unplugged. +{ "material_topics": { "topic_1": "explanation of topic_1 relevance to company", "topic_2": "explanation of topic_2 relevance to company", "topic_n": "explanation of topic_n relevance to company" }} + +Key Principles: +- Use reference documents as primary assessment framework +- Provide context-specific materiality determination +- Output must be in JSON format with no additional markdown or formatting diff --git a/backend/src/prompts/templates/select-material-files-system-prompt.j2 b/backend/src/prompts/templates/select-material-files-system-prompt.j2 new file mode 100644 index 000000000..0359386cb --- /dev/null +++ b/backend/src/prompts/templates/select-material-files-system-prompt.j2 @@ -0,0 +1,23 @@ +You are an advanced ESG (Environmental, Social, and Governance) specialist AI assistant with a specialized PDF library containing the following documents: + +{{ catalogue }} + +Input: +- Accept company name as primary input +- Optional ESG focus can be provided + +Recommendation Guidelines: +- Carefully match company to sector labels +- Consider potential sector matches beyond exact wording +- If no specific ESG focus provided, recommend all relevant sector PDFs +- If ESG focus specified, filter recommendations accordingly +- Return results in strict JSON format +- If no match found, return an empty JSON list + +Output Format: +{ "files": [ "filename.pdf" ]} + +Response Requirements: +- Always provide a JSON response, do not use any markdown or new line characters +- Include only file names in recommendations +- Be precise in sector and ESG label matching \ No newline at end of file diff --git a/backend/src/router.py b/backend/src/router.py index 02e12752a..96276b760 100644 --- a/backend/src/router.py +++ b/backend/src/router.py @@ -4,7 +4,7 @@ from src.utils import to_json, Config from src.utils.log_publisher import publish_log_info, LogPrefix from src.prompts import PromptEngine -from src.agents import Agent, get_available_agents, get_agent_details +from src.agents import ChatAgent, get_available_agents, get_agent_details from src.llm import get_llm logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ def find_agent_from_name(name): return (agent for agent in agents if agent.name == name) -async def get_agent_for_task(task, scratchpad) -> Agent | None: +async def get_agent_for_task(task, scratchpad) -> ChatAgent | None: llm = get_llm(config.router_llm) model = config.router_model plan = await build_plan(task, llm, scratchpad, model) diff --git a/backend/src/session/file_uploads.py b/backend/src/session/file_uploads.py index 9e6dcb4dd..6b0d7108c 100644 --- a/backend/src/session/file_uploads.py +++ b/backend/src/session/file_uploads.py @@ -1,5 +1,5 @@ import json -from typing import TypedDict +from typing import TypedDict, Optional import logging import redis @@ -27,14 +27,17 @@ class FileUploadMeta(TypedDict): class FileUpload(TypedDict): uploadId: str content: str - filename: str | None - contentType: str | None - size: int | None + filename: str + contentType: Optional[str] + size: Optional[int] + class FileUploadReport(TypedDict): id: str - filename: str | None - report: str | None + answer: str + filename: Optional[str] + report: Optional[str] + def get_session_file_uploads_meta() -> list[FileUploadMeta] | None: return get_session(UPLOADS_META_SESSION_KEY, []) @@ -52,7 +55,7 @@ def get_session_file_upload(upload_id) -> FileUpload | None: return _get_key(UPLOADS_KEY_PREFIX + upload_id) -def update_session_file_uploads(file_upload:FileUpload): +def update_session_file_uploads(file_upload: FileUpload): file_uploads_meta_session = get_session(UPLOADS_META_SESSION_KEY, []) if not file_uploads_meta_session: # initialise the session object @@ -80,7 +83,7 @@ def clear_session_file_uploads(): set_session(UPLOADS_META_SESSION_KEY, []) -def store_report(report:FileUploadReport): +def store_report(report: FileUploadReport): redis_client.set(REPORT_KEY_PREFIX + report["id"], json.dumps(report)) diff --git a/backend/src/session/redis_session_middleware.py b/backend/src/session/redis_session_middleware.py index 3e3f1e99c..1b6a16065 100644 --- a/backend/src/session/redis_session_middleware.py +++ b/backend/src/session/redis_session_middleware.py @@ -19,6 +19,7 @@ request_context = contextvars.ContextVar(REQUEST_CONTEXT_KEY) redis_client = redis.Redis(host=config.redis_host, port=6379, decode_responses=True) + class RedisSessionMiddleware(BaseHTTPMiddleware): async def dispatch(self, request: Request, call_next): request_context.set(request) @@ -56,11 +57,13 @@ def set_session(key: str, value): request: Request = request_context.get() request.state.session[key] = value + def reset_session(): logger.info("Reset chat session") request: Request = request_context.get() request.state.session = {} + def get_redis_session(request: Request): session_id = request.cookies.get(SESSION_COOKIE_NAME) logger.info(f"Attempting to get session for session_id: {session_id}") diff --git a/backend/src/utils/config.py b/backend/src/utils/config.py index 72b0beccc..2b8e20010 100644 --- a/backend/src/utils/config.py +++ b/backend/src/utils/config.py @@ -21,6 +21,7 @@ def __init__(self): self.answer_agent_llm = None self.intent_agent_llm = None self.report_agent_llm = None + self.materiality_agent_llm = None self.validator_agent_llm = None self.datastore_agent_llm = None self.web_agent_llm = None @@ -32,6 +33,7 @@ def __init__(self): self.intent_agent_model = None self.answer_agent_model = None self.report_agent_model = None + self.materiality_agent_model = None self.datastore_agent_model = None self.chart_generator_model = None self.web_agent_model = None @@ -61,6 +63,7 @@ def load_env(self): self.answer_agent_llm = os.getenv("ANSWER_AGENT_LLM") self.intent_agent_llm = os.getenv("INTENT_AGENT_LLM") self.report_agent_llm = os.getenv("REPORT_AGENT_LLM") + self.materiality_agent_llm = os.getenv("MATERIALITY_AGENT_LLM") self.validator_agent_llm = os.getenv("VALIDATOR_AGENT_LLM") self.datastore_agent_llm = os.getenv("DATASTORE_AGENT_LLM") self.chart_generator_llm = os.getenv("CHART_GENERATOR_LLM") @@ -71,6 +74,7 @@ def load_env(self): self.answer_agent_model = os.getenv("ANSWER_AGENT_MODEL") self.intent_agent_model = os.getenv("INTENT_AGENT_MODEL") self.report_agent_model = os.getenv("REPORT_AGENT_MODEL") + self.materiality_agent_model = os.getenv("MATERIALITY_AGENT_MODEL") self.validator_agent_model = os.getenv("VALIDATOR_AGENT_MODEL") self.datastore_agent_model = os.getenv("DATASTORE_AGENT_MODEL") self.web_agent_model = os.getenv("WEB_AGENT_MODEL") diff --git a/backend/src/utils/dynamic_knowledge_graph.py b/backend/src/utils/dynamic_knowledge_graph.py index 26c71f7cb..edcf5b555 100644 --- a/backend/src/utils/dynamic_knowledge_graph.py +++ b/backend/src/utils/dynamic_knowledge_graph.py @@ -17,27 +17,24 @@ async def generate_dynamic_knowledge_graph(csv_data: list[list[str]]) -> dict[st reduced_data_set = csv_data[slice(50)] - model_system_prompt = engine.load_prompt("generate-knowledge-graph-model") - model_response = await llm.chat( llm_model, # type: ignore[reportArgumentType] - model_system_prompt, - user_prompt=str(reduced_data_set) + system_prompt=engine.load_prompt("generate-knowledge-graph-model"), + user_prompt=str(reduced_data_set), + return_json=True ) data_model = json.loads(model_response)["model"] - system_prompt = engine.load_prompt("generate-knowledge-graph-cypher-system-prompt") - user_prompt = engine.load_prompt( - "generate-knowledge-graph-cypher-user-prompt", - input_data=reduced_data_set, - data_model=data_model - ) - query_response = await llm.chat( llm_model, # type: ignore[reportArgumentType] - system_prompt, - user_prompt=user_prompt + system_prompt=engine.load_prompt("generate-knowledge-graph-cypher-system-prompt"), + user_prompt=engine.load_prompt( + "generate-knowledge-graph-cypher-user-prompt", + input_data=reduced_data_set, + data_model=data_model + ), + return_json=True ) query = json.loads(query_response)["cypher_query"] diff --git a/backend/src/utils/file_utils.py b/backend/src/utils/file_utils.py index e4fcaf2d5..9931b60fa 100644 --- a/backend/src/utils/file_utils.py +++ b/backend/src/utils/file_utils.py @@ -17,6 +17,9 @@ def handle_file_upload(file: UploadFile) -> FileUpload: if (file.size or 0) > MAX_FILE_SIZE: raise HTTPException(status_code=413, detail=f"File upload must be less than {MAX_FILE_SIZE} bytes") + if not file.filename: + raise HTTPException(status_code=400, detail="Filename missing from file upload") + if "application/pdf" == file.content_type: start_time = time.time() pdf_file = PdfReader(file.file) diff --git a/backend/tests/BDD/step_defs/test_prompts.py b/backend/tests/BDD/step_defs/test_prompts.py index 1d48a4026..35b3572cf 100644 --- a/backend/tests/BDD/step_defs/test_prompts.py +++ b/backend/tests/BDD/step_defs/test_prompts.py @@ -35,8 +35,8 @@ def get_response(context): @then(parsers.parse("the response to this '{prompt}' should match the '{expected_response}'")) -def check_response_includes_expected_response(context, prompt, expected_response): - response = send_prompt(prompt) +async def check_response_includes_expected_response(context, prompt, expected_response): + response = await send_prompt(prompt) actual_response = response.json()["answer"] # Allow `expected_response` to be a list of possible valid responses @@ -83,7 +83,7 @@ def check_response_includes_expected_response(context, prompt, expected_response @then(parsers.parse("the response to this '{prompt}' should give a confident answer")) -def check_bot_response_confidence(prompt): - response = send_prompt(prompt) +async def check_bot_response_confidence(prompt): + response = await send_prompt(prompt) result = check_response_confidence(prompt, response.json()["answer"]) assert result["score"] == 1, "The bot response is not confident enough. \nReasoning: " + result["reasoning"] diff --git a/backend/tests/BDD/test_utilities.py b/backend/tests/BDD/test_utilities.py index 5ae61eee6..32f908ae1 100644 --- a/backend/tests/BDD/test_utilities.py +++ b/backend/tests/BDD/test_utilities.py @@ -19,7 +19,7 @@ def app_healthcheck(): return healthcheck_response -def send_prompt(prompt: str): +async def send_prompt(prompt: str): start_response = client.get(START_ENDPOINT_URL.format(utterance=prompt)) return start_response diff --git a/backend/tests/agents/__init__.py b/backend/tests/agents/__init__.py index 783ca2b82..cdf0d5f20 100644 --- a/backend/tests/agents/__init__.py +++ b/backend/tests/agents/__init__.py @@ -1,9 +1,11 @@ -from src.agents import Agent, agent, tool, Parameter +from src.agents import ChatAgent, chat_agent, tool, Parameter +from tests.llm.mock_llm import MockLLM name_a = "Mock Tool A" name_b = "Mock Tool B" description = "A test tool" param_description = "A string" +MockLLM() # initialise MockLLM so future calls to get_llm will return this object @tool( @@ -37,9 +39,9 @@ async def mock_tool_b(input: str, llm, model): mock_tools = [mock_tool_a, mock_tool_b] -@agent(name=mock_agent_name, description=mock_agent_description, tools=mock_tools) -class MockAgent(Agent): +@chat_agent(name=mock_agent_name, description=mock_agent_description, tools=mock_tools) +class MockChatAgent(ChatAgent): pass -__all__ = ["MockAgent", "mock_agent_description", "mock_agent_name", "mock_tools", "mock_tool_a", "mock_tool_b"] +__all__ = ["MockChatAgent", "mock_agent_description", "mock_agent_name", "mock_tools", "mock_tool_a", "mock_tool_b"] diff --git a/backend/tests/agents/agent_test.py b/backend/tests/agents/agent_test.py index 6cfe16880..c2a2b0f3d 100644 --- a/backend/tests/agents/agent_test.py +++ b/backend/tests/agents/agent_test.py @@ -1,57 +1,55 @@ -from pytest import raises -import pytest -from src.llm.factory import get_llm -from tests.agents import MockAgent, mock_agent_description, mock_agent_name, mock_tools - - -def test_agent_metadata_description(): - assert MockAgent.description == mock_agent_description - - -def test_agent_metadata_name(): - assert MockAgent.name == mock_agent_name - - -def test_agent_metadata_tools(): - assert MockAgent.tools == mock_tools - - -mock_model = "mockmodel" -mock_llm = get_llm("mockllm") -mock_agent_instance = MockAgent("mockllm", mock_model) - - -@pytest.mark.asyncio -async def test_agent_invoke_uses_tool(mocker): - mock_response = """{"tool_name": "Mock Tool A", "tool_parameters": { "input": "value for input" }, "reasoning": "Mock reasoning" }""" # noqa: E501 - mock_llm.chat = mocker.AsyncMock(return_value=mock_response) - - response = await mock_agent_instance.invoke("Mock task to solve") - - assert response == "value for input" - - -@pytest.mark.asyncio -async def test_agent_invoke_with_no_tool(mocker): - mock_response = """{"tool_name": "Undefined Tool", "tool_parameters": {}, "reasoning": "Mock reasoning"}""" - mock_llm.chat = mocker.AsyncMock(return_value=mock_response) - - with raises(Exception) as error: - await mock_agent_instance.invoke("Mock task to solve") - - expected = "Unable to extract chosen tool and parameters from {'tool_name': 'Undefined Tool', 'tool_parameters': {}, 'reasoning': 'Mock reasoning'}" # noqa: E501 - assert str(error.value) == expected - - -@pytest.mark.asyncio -async def test_agent_invoke_no_appropriate_tool_for_task(mocker): - mock_response = ( - """{"tool_name": "None", "tool_parameters": {}, "reasoning": "No tool was appropriate for the task"}""" - ) - mock_llm.chat = mocker.AsyncMock(return_value=mock_response) - - with raises(Exception) as error: - await mock_agent_instance.invoke("Mock task to solve") - - expected = "Unable to extract chosen tool and parameters from {'tool_name': 'None', 'tool_parameters': {}, 'reasoning': 'No tool was appropriate for the task'}" # noqa: E501 - assert str(error.value) == expected # noqa: E501 +import json + +import pytest +from pytest import raises +from tests.agents import MockChatAgent +from src.llm.factory import get_llm + + +mock_model = "mockmodel" +mock_llm = get_llm("mockllm") +mock_agent_instance = MockChatAgent("mockllm", mock_model) + + +def mock_response(tool_name: str, tool_parameters: dict[str,str]) -> str: + return json.dumps({"tool_name": tool_name, "tool_parameters": tool_parameters, "reasoning": "Mock Reasoning"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mocked_response, expect_success, expected", [ + ( + mock_response("Mock Tool A", {"input": "string for tool to output"}), + True, + "string for tool to output" + ), + ( + mock_response("Undefined Tool", {}), + False, + "Unable to extract chosen tool and parameters from {'tool_name': 'None', 'tool_parameters': {}," + " 'reasoning': 'No tool was appropriate for the task'}" + ), + ( + mock_response("None", {}), + False, + "Unable to extract chosen tool and parameters from {'tool_name': 'None', 'tool_parameters': {}," + " 'reasoning': 'No tool was appropriate for the task'}" + ) + ], + ids=[ + "When appropriate tool selected, Test Chat Agent invoke func will call tool with parameters", + "When 'Undefined Tool' selected, Test Chat Agent invoke func will explain no tool was appropriate for task", + "When 'None' selected, Test Chat Agent invoke func will explain no tool was appropriate for task" + ] +) +async def test_chat_agent_invoke_uses_tool(mocker, mocked_response: str, expect_success: bool, expected: str): + mock_llm.chat = mocker.AsyncMock(return_value=str(mocked_response)) + + if expect_success: + response = await mock_agent_instance.invoke("Mock task to solve") + + assert response == expected + else: + with raises(Exception) as error: + await mock_agent_instance.invoke("Mock task to solve") + + assert str(error.value) == expected diff --git a/backend/tests/agents/materiality_agent_test.py b/backend/tests/agents/materiality_agent_test.py new file mode 100644 index 000000000..3c5bbe1bc --- /dev/null +++ b/backend/tests/agents/materiality_agent_test.py @@ -0,0 +1,33 @@ +from unittest.mock import patch, mock_open + +import pytest +import json + +from src.agents.materiality_agent import MaterialityAgent +from src.llm.factory import get_llm + +mock_model = "mockmodel" +mock_llm = get_llm("mockllm") +mock_selected_files = {"files": ["file1.pdf", "file2.pdf"]} + +mock_materiality_topics = {"material_topics": {"topic1": "topic1 description", "topic2": "topic2 description"}} + +mock_catalogue = { + "library": { + "Standard1": [{"name": "name"}], + "Standard2": [{"name2": "name2"}], + } +} + + +@pytest.mark.asyncio +async def test_invoke_calls_llm(mocker): + agent = MaterialityAgent(llm_name="mockllm", model=mock_model) + + with patch("builtins.open", mock_open(read_data=json.dumps(mock_catalogue))): + mock_llm.chat = mocker.AsyncMock(return_value=json.dumps(mock_selected_files)) + mock_llm.chat_with_file = mocker.AsyncMock(return_value=json.dumps(mock_materiality_topics)) + + response = await agent.list_material_topics("AstraZeneca") + + assert response == mock_materiality_topics["material_topics"] diff --git a/backend/tests/agents/report_agent_test.py b/backend/tests/agents/report_agent_test.py index a8e1ca366..0d9b8c20c 100644 --- a/backend/tests/agents/report_agent_test.py +++ b/backend/tests/agents/report_agent_test.py @@ -6,6 +6,7 @@ mock_model = "mockmodel" mock_llm = get_llm("mockllm") + @pytest.mark.asyncio async def test_invoke_calls_llm(mocker): report_agent = ReportAgent(llm_name="mockllm", model=mock_model) @@ -13,7 +14,6 @@ async def test_invoke_calls_llm(mocker): mock_llm.chat = mocker.AsyncMock(return_value=mock_response) - response = await report_agent.invoke("Test Document") + response = await report_agent.create_report("Test Document", materiality_topics={"abc": "123"}) assert response == mock_response - diff --git a/backend/tests/api/app_test.py b/backend/tests/api/app_test.py index 68420c3ca..2fb72e745 100644 --- a/backend/tests/api/app_test.py +++ b/backend/tests/api/app_test.py @@ -49,6 +49,7 @@ def test_chat_response_failure(mocker): assert response.status_code == 500 assert response.json() == chat_fail_response + def test_chat_delete(mocker): mock_reset_session = mocker.patch("src.api.app.reset_session") mock_clear_files = mocker.patch("src.api.app.clear_session_file_uploads") @@ -60,6 +61,7 @@ def test_chat_delete(mocker): assert response.status_code == 204 + def test_chat_message_success(mocker): message = ChatResponse(id="1", question="Question", answer="Answer", reasoning="Reasoning", dataset="dataset") mock_get_chat_message = mocker.patch("src.api.app.get_chat_message", return_value=message) @@ -70,6 +72,7 @@ def test_chat_message_success(mocker): assert response.status_code == 200 assert response.json() == message + def test_chat_message_not_found(mocker): mock_get_chat_message = mocker.patch("src.api.app.get_chat_message", return_value=None) @@ -78,15 +81,17 @@ def test_chat_message_not_found(mocker): mock_get_chat_message.assert_called_with("123") assert response.status_code == 404 + def test_report_response_success(mocker): - mock_reponse = FileUploadReport(filename="filename", id="1", report="some report md") - mock_report = mocker.patch("src.api.app.report_on_file_upload", return_value=mock_reponse) + mock_response = FileUploadReport(filename="filename", id="1", report="some report md", answer="chat message") + mock_report = mocker.patch("src.api.app.report_on_file_upload", return_value=mock_response) response = client.post("/report", files={"file": ("filename", "test data".encode("utf-8"), "text/plain")}) mock_report.assert_called_once() assert response.status_code == 200 - assert response.json() == {'filename': 'filename', 'id': '1', 'report': 'some report md'} + assert response.json() == {'filename': 'filename', 'id': '1', 'report': 'some report md', 'answer': 'chat message'} + @pytest.mark.asyncio async def test_lifespan_populates_db(mocker) -> None: @@ -95,8 +100,9 @@ async def test_lifespan_populates_db(mocker) -> None: with client: mock_dataset_upload.assert_called_once_with() + def test_get_report_success(mocker): - report = FileUploadReport(id="12", filename="test.pdf", report="test report") + report = FileUploadReport(id="12", filename="test.pdf", report="test report", answer='chat message') mock_get_report = mocker.patch("src.api.app.get_report", return_value=report) response = client.get("/report/12") @@ -106,6 +112,7 @@ def test_get_report_success(mocker): assert response.headers.get('Content-Disposition') == 'attachment; filename="report.md"' assert response.headers.get('Content-Type') == 'text/markdown; charset=utf-8' + def test_get_report_not_found(mocker): mock_get_report = mocker.patch("src.api.app.get_report", return_value=None) diff --git a/backend/tests/directors/report_director_test.py b/backend/tests/directors/report_director_test.py index b52d6ed7a..7539f8b02 100644 --- a/backend/tests/directors/report_director_test.py +++ b/backend/tests/directors/report_director_test.py @@ -3,29 +3,46 @@ from fastapi.datastructures import Headers import pytest -from src.session.file_uploads import FileUpload, FileUploadReport +from src.session.file_uploads import FileUpload from src.directors.report_director import report_on_file_upload +mock_topics = {"topic1": "topic1 description", "topic2": "topic2 description"} +mock_report = "#Report on upload as markdown" +expected_answer = ('Your report for test.txt is ready to view.\n\nThe following materiality topics were identified for ' + 'CompanyABC which the report focuses on:\n\ntopic1\ntopic1 description\n\ntopic2\ntopic2 ' + 'description\n') + + @pytest.mark.asyncio async def test_report_on_file_upload(mocker): file_upload = FileUpload(uploadId="1", filename="test.txt", content="test", contentType="text/plain", size=4) - mock_agent = mocker.AsyncMock() - mock_agent.invoke.return_value = "#Report on upload as markdown" - mocker.patch("src.directors.report_director.get_report_agent", return_value=mock_agent) + mock_report_agent = mocker.AsyncMock() + mock_report_agent.get_company_name.return_value = "CompanyABC" + mock_report_agent.create_report.return_value = mock_report + mocker.patch("src.directors.report_director.get_report_agent", return_value=mock_report_agent) mock_handle_file_upload = mocker.patch("src.directors.report_director.handle_file_upload", return_value=file_upload) mock_store_report = mocker.patch("src.directors.report_director.store_report", return_value=file_upload) - headers = Headers({"content-type": "text/plain"}) - file = BytesIO(b"test content") - request_upload_file = UploadFile(file=file, size=12, headers=headers, filename="test.txt") + mock_materiality_agent = mocker.AsyncMock() + mock_materiality_agent.list_material_topics.return_value = mock_topics + mocker.patch("src.directors.report_director.get_materiality_agent", return_value=mock_materiality_agent) + + request_upload_file = UploadFile( + file=BytesIO(b"test"), + size=12, + headers=Headers({"content-type": "text/plain"}), + filename="test.txt" + ) response = await report_on_file_upload(request_upload_file) - report_upload = FileUploadReport(filename=file_upload["filename"], - id=file_upload["uploadId"], - report="#Report on upload as markdown") + expected_response = {"filename": "test.txt", "id": "1", "report": mock_report, "answer": expected_answer} + + mock_report_agent.get_company_name.assert_called_once_with("test") mock_handle_file_upload.assert_called_once_with(request_upload_file) - mock_store_report.assert_called_once_with(report_upload) + mock_store_report.assert_called_once_with(expected_response) + + mock_materiality_agent.list_material_topics.assert_called_once_with("CompanyABC") - assert response == {"filename": "test.txt", "id": "1", "report": "#Report on upload as markdown"} + assert response == expected_response diff --git a/backend/tests/llm/llm_test.py b/backend/tests/llm/llm_test.py index 605b353ab..d358cedca 100644 --- a/backend/tests/llm/llm_test.py +++ b/backend/tests/llm/llm_test.py @@ -1,6 +1,6 @@ import pytest from src.llm.count_calls import Counter -from src.llm import MockLLM +from tests.llm.mock_llm import MockLLM model = MockLLM() diff --git a/backend/tests/llm/mock_llm.py b/backend/tests/llm/mock_llm.py new file mode 100644 index 000000000..d5ed21af1 --- /dev/null +++ b/backend/tests/llm/mock_llm.py @@ -0,0 +1,15 @@ +from src.llm import LLM, LLMFile + + +class MockLLM(LLM): + async def chat(self, model: str, system_prompt: str, user_prompt: str, return_json=False) -> str: + return "mocked response" + + async def chat_with_file( + self, + model: str, + system_prompt: str, + user_prompt: str, + files: list[LLMFile] + ) -> str: + return "mocked response" diff --git a/backend/tests/llm/openai_test.py b/backend/tests/llm/openai_test.py index b02d50938..b9afedccc 100644 --- a/backend/tests/llm/openai_test.py +++ b/backend/tests/llm/openai_test.py @@ -1,67 +1,69 @@ -# tests/test_openai_llm.py import pytest -from unittest.mock import MagicMock, patch -from src.llm.openai_client import OpenAIClient -from src.utils import Config +from dataclasses import dataclass +from pathlib import Path -mock_config = MagicMock(spec=Config) -mock_config.openai_model = "gpt-3.5-turbo" -system_prompt = "system_prompt" -user_prompt = "user_prompt" -content_response = "Hello there" -openapi_response = "Hello! How can I assist you today?" +from unittest.mock import patch, AsyncMock +from openai.types.beta.threads import Text, FileCitationAnnotation, TextContentBlock +from openai.types.beta.threads.file_citation_annotation import FileCitation +from src.llm import LLMFile +from src.llm.openai import OpenAI -def create_mock_chat_response(content): - return {"choices": [{"message": {"role": "system", "content": content}}]} +@dataclass +class MockResponse: + id: str -@patch("src.llm.openai_client.openai.ChatCompletion.create") -def test_chat_content_string_returns_string(mock_create): - mock_create.return_value = create_mock_chat_response(content_response) - client = OpenAIClient() - response = client.chat( - model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ], - ) - assert response == content_response +@dataclass +class MockMessage: + content: list[TextContentBlock] -@patch("src.llm.openai_client.openai.ChatCompletion.create") -def test_chat_content_list_returns_string(mock_create): - content_list = ["Hello", "there"] - mock_create.return_value = create_mock_chat_response(content_list) - client = OpenAIClient() - response = client.chat( - model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ], - ) +class MockListResponse: + data = [MockMessage(content=[TextContentBlock( + text=Text( + annotations=[ + FileCitationAnnotation( + file_citation=FileCitation(file_id="123"), + text="【7†source】", + end_index=1, + start_index=2, + type="file_citation" + ), + FileCitationAnnotation( + file_citation=FileCitation(file_id="123"), + text="【1:9†source】", + end_index=1, + start_index=2, + type="file_citation" + ) + ], + value="Response with quote【7†source】【1:9†source】" + ), + type="text" + )])] - assert " ".join(response) == content_response +mock_message_list = {"data"} -@patch("src.llm.openai_client.openai.ChatCompletion.create") -def test_chat_handles_exception(mock_create): - mock_create.side_effect = Exception("API error") - client = OpenAIClient() - response = client.chat( - model="gpt-3.5-turbo", - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ], - ) +@pytest.mark.asyncio +@patch("src.llm.openai.AsyncOpenAI") +async def test_chat_with_file_removes_citations(mock_async_openai): + mock_instance = mock_async_openai.return_value - assert response == "An error occurred while processing the request." + mock_instance.files.create = AsyncMock(return_value=MockResponse(id="file-id")) + mock_instance.beta.assistants.create = AsyncMock(return_value=MockResponse(id="assistant-id")) + mock_instance.beta.threads.create = AsyncMock(return_value=MockResponse(id="thread-id")) + mock_instance.beta.threads.runs.create_and_poll = AsyncMock(return_value=MockResponse(id="run-id")) + mock_instance.beta.threads.messages.list = AsyncMock(return_value=MockListResponse) - -if __name__ == "__main__": - pytest.main() + client = OpenAI() + response = await client .chat_with_file( + model="", + user_prompt="", + system_prompt="", + files=[LLMFile("file_name", Path("file/path"))] + ) + assert response == "Response with quote" diff --git a/backend/tests/router_test.py b/backend/tests/router_test.py index 22a491c2a..863a5c1a8 100644 --- a/backend/tests/router_test.py +++ b/backend/tests/router_test.py @@ -1,15 +1,15 @@ import json import pytest -from src.llm import MockLLM from src.agents import agent_details -from tests.agents import MockAgent, mock_agent_name +from tests.llm.mock_llm import MockLLM +from tests.agents import MockChatAgent, mock_agent_name from src.router import get_agent_for_task mock_model = "mockmodel" mock_llm = MockLLM() -mock_agent = MockAgent("mockllm", mock_model) +mock_agent = MockChatAgent("mockllm", mock_model) mock_agents = [mock_agent] task = {"summary": "task1"} scratchpad = [] diff --git a/backend/tests/session/test_file_uploads.py b/backend/tests/session/test_file_uploads.py index 63df80d26..dac086e2f 100644 --- a/backend/tests/session/test_file_uploads.py +++ b/backend/tests/session/test_file_uploads.py @@ -91,16 +91,17 @@ def test_clear_session_file_uploads_meta(mocker, mock_redis, mock_request_contex def test_store_report(mocker, mock_redis): mocker.patch("src.session.file_uploads.redis_client", mock_redis) - report = FileUploadReport(filename="test.txt", id="12", report="test report") + report = FileUploadReport(filename="test.txt", id="12", report="test report", answer="chat message") store_report(report) mock_redis.set.assert_called_with("report_12", json.dumps(report)) + def test_get_report(mocker, mock_redis): mocker.patch("src.session.file_uploads.redis_client", mock_redis) - report = FileUploadReport(filename="test.txt", id="12", report="test report") + report = FileUploadReport(filename="test.txt", id="12", report="test report", answer="chat message") mock_redis.get.return_value = json.dumps(report) value = get_report("12") diff --git a/backend/tests/supervisors/supervisor_test.py b/backend/tests/supervisors/supervisor_test.py index 31c0a4147..780939edb 100644 --- a/backend/tests/supervisors/supervisor_test.py +++ b/backend/tests/supervisors/supervisor_test.py @@ -1,5 +1,5 @@ import pytest -from tests.agents import MockAgent +from tests.agents import MockChatAgent import json from src.supervisors import solve_all, solve_task, no_questions_response, unsolvable_response, no_agent_response @@ -25,7 +25,7 @@ ], } -agent = MockAgent("mockllm", mock_model) +chat_agent = MockChatAgent("mockllm", mock_model) @pytest.mark.asyncio @@ -57,14 +57,14 @@ async def test_solve_task_first_attempt_solves(mocker): "content": "the answer is 42", "ignore_validation": "false" }) - agent.invoke = mocker.AsyncMock(return_value=mock_answer) - mocker.patch("src.supervisors.supervisor.get_agent_for_task", return_value=agent) + chat_agent.invoke = mocker.AsyncMock(return_value=mock_answer) + mocker.patch("src.supervisors.supervisor.get_agent_for_task", return_value=chat_agent) mocker.patch("src.supervisors.supervisor.is_valid_answer", return_value=True) answer = await solve_task(task, scratchpad) mock_answer_json = json.loads(mock_answer) # Ensure that the result is returned directly without validation - assert answer == (agent.name, mock_answer_json.get('content', ''), "success") + assert answer == (chat_agent.name, mock_answer_json.get('content', ''), "success") @pytest.mark.asyncio @@ -74,8 +74,8 @@ async def test_solve_task_ignore_validation(mocker): "content": "the answer is 42", "ignore_validation": "true" }) - agent.invoke = mocker.AsyncMock(return_value=mock_answer) - mocker.patch("src.supervisors.supervisor.get_agent_for_task", return_value=agent) + chat_agent.invoke = mocker.AsyncMock(return_value=mock_answer) + mocker.patch("src.supervisors.supervisor.get_agent_for_task", return_value=chat_agent) mock_is_valid_answer = mocker.patch("src.supervisors.supervisor.is_valid_answer") # Run the solve_task function @@ -83,13 +83,13 @@ async def test_solve_task_ignore_validation(mocker): mock_answer_json = json.loads(mock_answer) # Ensure that the result is returned directly without validation - assert answer == (agent.name, mock_answer_json.get('content', ''), "success") + assert answer == (chat_agent.name, mock_answer_json.get('content', ''), "success") mock_is_valid_answer.assert_not_called() # Validation should not be called @pytest.mark.asyncio async def test_solve_task_unsolvable(mocker): - agent.invoke = mocker.MagicMock(return_value=mock_answer) - mocker.patch("src.supervisors.supervisor.get_agent_for_task", return_value=agent) + chat_agent.invoke = mocker.MagicMock(return_value=mock_answer) + mocker.patch("src.supervisors.supervisor.get_agent_for_task", return_value=chat_agent) mocker.patch("src.supervisors.supervisor.is_valid_answer", return_value=False) with pytest.raises(Exception) as error: diff --git a/backend/tests/utils/file_utils_test.py b/backend/tests/utils/file_utils_test.py index 307dbccac..e7188597a 100644 --- a/backend/tests/utils/file_utils_test.py +++ b/backend/tests/utils/file_utils_test.py @@ -7,8 +7,8 @@ from src.utils.file_utils import handle_file_upload -def test_handle_file_upload_size(): +def test_handle_file_upload_size(): with pytest.raises(HTTPException) as err: handle_file_upload(UploadFile(file=BinaryIO(), size=15*1024*1024)) @@ -17,16 +17,24 @@ def test_handle_file_upload_size(): def test_handle_file_upload_unsupported_type(): - headers = Headers({"content-type": "text/html"}) with pytest.raises(HTTPException) as err: - handle_file_upload(UploadFile(file=BinaryIO(), size=15*1024, headers=headers)) + handle_file_upload(UploadFile(file=BinaryIO(), size=15*1024, headers=headers, filename="test.txt")) assert err.value.status_code == 400 assert err.value.detail == 'File upload must be supported type (text/plain or application/pdf)' -def test_handle_file_upload_text(mocker): +def test_handle_file_upload_missing_file_name(): + headers = Headers({"content-type": "text/html"}) + with pytest.raises(HTTPException) as err: + handle_file_upload(UploadFile(file=BytesIO(b"test content"), size=12, headers=headers)) + + assert err.value.status_code == 400 + assert err.value.detail == 'Filename missing from file upload' + + +def test_handle_file_upload_text(mocker): mock = mocker.patch("src.utils.file_utils.update_session_file_uploads", MagicMock()) headers = Headers({"content-type": "text/plain"}) @@ -37,7 +45,6 @@ def test_handle_file_upload_text(mocker): def test_handle_file_upload_pdf(mocker): - mock = mocker.patch("src.utils.file_utils.update_session_file_uploads", MagicMock()) pdf_mock = mocker.patch("src.utils.file_utils.PdfReader", MagicMock()) diff --git a/frontend/src/components/input.tsx b/frontend/src/components/input.tsx index 40510c337..812ab4593 100644 --- a/frontend/src/components/input.tsx +++ b/frontend/src/components/input.tsx @@ -72,12 +72,12 @@ export const Input = ({ setUploadInProgress(true); try { - const { filename, report, id } = await uploadFileToServer(file); + const { filename, report, id, answer } = await uploadFileToServer(file); setUploadedFile(file); appendMessage( { id, - answer: `Your ESG report for ${filename} is ready to view.`, + answer, }, Role.Bot, report, diff --git a/frontend/src/server.ts b/frontend/src/server.ts index 9c53a3b8d..55a510d35 100644 --- a/frontend/src/server.ts +++ b/frontend/src/server.ts @@ -81,7 +81,12 @@ export const resetChat = async (): Promise => { export const uploadFileToServer = async ( file: File, -): Promise<{ filename: string; id: string; report: string }> => { +): Promise<{ + filename: string; + id: string; + report: string; + answer: string; +}> => { const formData = new FormData(); formData.append('file', file);