From 20b1ad09bbb795b14bc3cbae3a5d14b5dd9e2e7c Mon Sep 17 00:00:00 2001 From: Emma Pearce Date: Wed, 30 Oct 2024 16:32:38 +0000 Subject: [PATCH 1/5] FS-60: Add promptfoo test config for suggestions prompt --- backend/src/prompts/README.md | 10 ++++++ .../generate_message_suggestions_config.yaml | 31 +++++++++++++++++++ ...get_generate_message_suggestions_prompt.py | 11 +++++++ backend/src/prompts/promptfooconfig.yaml | 4 +++ 4 files changed, 56 insertions(+) create mode 100644 backend/src/prompts/README.md create mode 100644 backend/src/prompts/generate_message_suggestions_config.yaml create mode 100644 backend/src/prompts/get_generate_message_suggestions_prompt.py create mode 100644 backend/src/prompts/promptfooconfig.yaml diff --git a/backend/src/prompts/README.md b/backend/src/prompts/README.md new file mode 100644 index 00000000..53c18ae1 --- /dev/null +++ b/backend/src/prompts/README.md @@ -0,0 +1,10 @@ +To get started, set your OPENAI_API_KEY environment variable, or other required keys for the providers you selected. + +install promptfoo by running `npx install promptfoo` + +Next, edit promptfooconfig.yaml. + +Then run `promptfoo eval` to run promptfooconfig.yaml +ro run a specific file for example `generate_message_suggestions_config.yaml` run `promptfoo eval -c generate_message_suggestions_config.yaml` + +Afterwards, you can view the results by running `promptfoo view` diff --git a/backend/src/prompts/generate_message_suggestions_config.yaml b/backend/src/prompts/generate_message_suggestions_config.yaml new file mode 100644 index 00000000..87933278 --- /dev/null +++ b/backend/src/prompts/generate_message_suggestions_config.yaml @@ -0,0 +1,31 @@ +description: "Generate Message Suggestions" + +providers: + - id: openai:chat # openai:chat - defaults to gpt-4o-mini + config: + temperature: 0 + +prompts: file://get_generate_message_suggestions_prompt.py:get_prompt + +tests: + - description: "test the output has the correct format and content when there is no chat history " + vars: + chatHistory: [] + assert: + - type: javascript + value: JSON.parse(output).suggestions.length === 5 + - type: contains + value: ESG + + - description: "test the output has content containing coca-cola when the chat history contains a previous question about coca-cola" + vars: + chatHistory: + [ + "User: Can you find recent news articles discussing the ESG initiatives of Coca-Cola?", + "System: In 2023, Coca-Cola HBC has strengthened its commitment to Environmental, Social, and Governance (ESG) initiatives by embedding sustainability into its operations. The company aims for a net zero carbon footprint and net positive biodiversity by 2040, and it has been recognized as the world's most sustainable beverage company by the Dow Jones Sustainability Indices for the seventh consecutive year. Key efforts include collaborating with suppliers to improve sustainability practices, reducing carbon emissions, and promoting responsible sourcing. Additionally, Coca-Cola HBC has expanded its sustainability strategy to Egypt, reflecting its global approach to these initiatives.", + ] + assert: + - type: contains + value: Coca-Cola + - type: llm-rubric + value: the suggestions are all related to the topic of sustainability and ESG (Environment, Social, Governance) diff --git a/backend/src/prompts/get_generate_message_suggestions_prompt.py b/backend/src/prompts/get_generate_message_suggestions_prompt.py new file mode 100644 index 00000000..1f7a178c --- /dev/null +++ b/backend/src/prompts/get_generate_message_suggestions_prompt.py @@ -0,0 +1,11 @@ +from prompting import PromptEngine + +engine = PromptEngine() + + +def get_prompt(context): + chat_history = context["vars"]["chatHistory"] + + system_prompt = engine.load_prompt("generate_message_suggestions", chat_history=chat_history) + + return [{"role": "system", "content": system_prompt}, {"role": "user", "content": "Give me 5 suggestions."}] diff --git a/backend/src/prompts/promptfooconfig.yaml b/backend/src/prompts/promptfooconfig.yaml new file mode 100644 index 00000000..b2dfaee0 --- /dev/null +++ b/backend/src/prompts/promptfooconfig.yaml @@ -0,0 +1,4 @@ +providers: + id: openai:chat # openai:chat - defaults to gpt-4o-mini + config: + temperature: 0 From 2b4414f7bc17578e963b275f64a7cfcb619fec68 Mon Sep 17 00:00:00 2001 From: evpearce <30893538+evpearce@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:59:58 +0000 Subject: [PATCH 2/5] Update backend/src/prompts/README.md Co-authored-by: Charlie Leopard --- backend/src/prompts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/prompts/README.md b/backend/src/prompts/README.md index 53c18ae1..0c5e9d4a 100644 --- a/backend/src/prompts/README.md +++ b/backend/src/prompts/README.md @@ -5,6 +5,6 @@ install promptfoo by running `npx install promptfoo` Next, edit promptfooconfig.yaml. Then run `promptfoo eval` to run promptfooconfig.yaml -ro run a specific file for example `generate_message_suggestions_config.yaml` run `promptfoo eval -c generate_message_suggestions_config.yaml` +To run a specific file for example `generate_message_suggestions_config.yaml` run `promptfoo eval -c generate_message_suggestions_config.yaml` Afterwards, you can view the results by running `promptfoo view` From cbb764459f90e5b86210562d801e90ac9f3e9260 Mon Sep 17 00:00:00 2001 From: "Ivan Mladjenovic (He/Him)" Date: Fri, 1 Nov 2024 11:26:44 +0000 Subject: [PATCH 3/5] Move promptfoo to its own folder in /backend. Load OpenAI key from env file. Simplify test structure so less utility .py classes are needed --- backend/promtfoo/README.md | 21 +++++++++++++++++++ .../generate_message_suggestions_config.yaml | 2 +- .../prompt_foo_runner.py} | 11 ++++++++-- .../prompts => promtfoo}/promptfooconfig.yaml | 0 backend/requirements.txt | 7 ++++--- backend/src/prompts/README.md | 10 --------- 6 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 backend/promtfoo/README.md rename backend/{src/prompts => promtfoo}/generate_message_suggestions_config.yaml (95%) rename backend/{src/prompts/get_generate_message_suggestions_prompt.py => promtfoo/prompt_foo_runner.py} (50%) rename backend/{src/prompts => promtfoo}/promptfooconfig.yaml (100%) delete mode 100644 backend/src/prompts/README.md diff --git a/backend/promtfoo/README.md b/backend/promtfoo/README.md new file mode 100644 index 00000000..cf3b4fda --- /dev/null +++ b/backend/promtfoo/README.md @@ -0,0 +1,21 @@ +# Promptfoo + +Promptfoo is a CLI and library for evaluating and red-teaming LLM apps. + +See https://www.promptfoo.dev/docs/intro/ + +## Setup + +### Install Promptfoo +Install promptfoo by running `npx install promptfoo` + +### Activate Python venv +Promptfoo must be run in a python virtual environment as python is used to load the jinja prompt templates. +See [Running Locally](../README.md) + +## Run Promptfoo +Promptfoo configuration (e.g. LLM model) can be set in `promptfooconfig.yaml` + +* Use `promptfoo eval` to run all promptfoo tests. +* Use `promptfoo eval -c generate_message_suggestions_config.yaml` to run a specific test suite. +* Use `promptfoo view` to view the results in browser. diff --git a/backend/src/prompts/generate_message_suggestions_config.yaml b/backend/promtfoo/generate_message_suggestions_config.yaml similarity index 95% rename from backend/src/prompts/generate_message_suggestions_config.yaml rename to backend/promtfoo/generate_message_suggestions_config.yaml index 87933278..cfb16d52 100644 --- a/backend/src/prompts/generate_message_suggestions_config.yaml +++ b/backend/promtfoo/generate_message_suggestions_config.yaml @@ -5,7 +5,7 @@ providers: config: temperature: 0 -prompts: file://get_generate_message_suggestions_prompt.py:get_prompt +prompts: file://prompt_foo_runner.py:generate_message_suggestions tests: - description: "test the output has the correct format and content when there is no chat history " diff --git a/backend/src/prompts/get_generate_message_suggestions_prompt.py b/backend/promtfoo/prompt_foo_runner.py similarity index 50% rename from backend/src/prompts/get_generate_message_suggestions_prompt.py rename to backend/promtfoo/prompt_foo_runner.py index 1f7a178c..1763c366 100644 --- a/backend/src/prompts/get_generate_message_suggestions_prompt.py +++ b/backend/promtfoo/prompt_foo_runner.py @@ -1,9 +1,16 @@ -from prompting import PromptEngine +import sys +import os +sys.path.append("../") +from dotenv import load_dotenv, find_dotenv # noqa: E402 +from src.prompts.prompting import PromptEngine # noqa: E402 +load_dotenv(find_dotenv()) + +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") engine = PromptEngine() -def get_prompt(context): +def generate_message_suggestions(context): chat_history = context["vars"]["chatHistory"] system_prompt = engine.load_prompt("generate_message_suggestions", chat_history=chat_history) diff --git a/backend/src/prompts/promptfooconfig.yaml b/backend/promtfoo/promptfooconfig.yaml similarity index 100% rename from backend/src/prompts/promptfooconfig.yaml rename to backend/promtfoo/promptfooconfig.yaml diff --git a/backend/requirements.txt b/backend/requirements.txt index cd69944d..0ff63e12 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,9 +5,6 @@ pycodestyle==2.11.1 python-dotenv==1.0.1 neo4j==5.18.0 ruff==0.3.5 -pytest==8.1.1 -pytest-mock==3.14.0 -pytest-asyncio==0.23.7 jinja2==3.1.3 websockets==12.0 azure-core==1.30.1 @@ -26,3 +23,7 @@ pypdf==4.3.1 hiredis==3.0.0 redis==5.0.8 +# tests +pytest==8.1.1 +pytest-mock==3.14.0 +pytest-asyncio==0.23.7 diff --git a/backend/src/prompts/README.md b/backend/src/prompts/README.md deleted file mode 100644 index 53c18ae1..00000000 --- a/backend/src/prompts/README.md +++ /dev/null @@ -1,10 +0,0 @@ -To get started, set your OPENAI_API_KEY environment variable, or other required keys for the providers you selected. - -install promptfoo by running `npx install promptfoo` - -Next, edit promptfooconfig.yaml. - -Then run `promptfoo eval` to run promptfooconfig.yaml -ro run a specific file for example `generate_message_suggestions_config.yaml` run `promptfoo eval -c generate_message_suggestions_config.yaml` - -Afterwards, you can view the results by running `promptfoo view` From fcc51ec857714853f77b8f37eafa75fa040cecff Mon Sep 17 00:00:00 2001 From: "Ivan Mladjenovic (He/Him)" Date: Fri, 1 Nov 2024 11:29:16 +0000 Subject: [PATCH 4/5] update readme --- backend/promtfoo/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/promtfoo/README.md b/backend/promtfoo/README.md index cf3b4fda..ca1809ed 100644 --- a/backend/promtfoo/README.md +++ b/backend/promtfoo/README.md @@ -11,7 +11,7 @@ Install promptfoo by running `npx install promptfoo` ### Activate Python venv Promptfoo must be run in a python virtual environment as python is used to load the jinja prompt templates. -See [Running Locally](../README.md) +To set up a virtual environment, see [Running Locally](../README.md) ## Run Promptfoo Promptfoo configuration (e.g. LLM model) can be set in `promptfooconfig.yaml` From 0765378d6db68a0c57201ec799886994bbc84ddf Mon Sep 17 00:00:00 2001 From: "Ivan Mladjenovic (He/Him)" Date: Tue, 5 Nov 2024 16:31:24 +0000 Subject: [PATCH 5/5] rename folder --- backend/{promtfoo => promptfoo}/README.md | 0 .../generate_message_suggestions_config.yaml | 0 backend/{promtfoo => promptfoo}/prompt_foo_runner.py | 0 backend/{promtfoo => promptfoo}/promptfooconfig.yaml | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename backend/{promtfoo => promptfoo}/README.md (100%) rename backend/{promtfoo => promptfoo}/generate_message_suggestions_config.yaml (100%) rename backend/{promtfoo => promptfoo}/prompt_foo_runner.py (100%) rename backend/{promtfoo => promptfoo}/promptfooconfig.yaml (100%) diff --git a/backend/promtfoo/README.md b/backend/promptfoo/README.md similarity index 100% rename from backend/promtfoo/README.md rename to backend/promptfoo/README.md diff --git a/backend/promtfoo/generate_message_suggestions_config.yaml b/backend/promptfoo/generate_message_suggestions_config.yaml similarity index 100% rename from backend/promtfoo/generate_message_suggestions_config.yaml rename to backend/promptfoo/generate_message_suggestions_config.yaml diff --git a/backend/promtfoo/prompt_foo_runner.py b/backend/promptfoo/prompt_foo_runner.py similarity index 100% rename from backend/promtfoo/prompt_foo_runner.py rename to backend/promptfoo/prompt_foo_runner.py diff --git a/backend/promtfoo/promptfooconfig.yaml b/backend/promptfoo/promptfooconfig.yaml similarity index 100% rename from backend/promtfoo/promptfooconfig.yaml rename to backend/promptfoo/promptfooconfig.yaml