From 8523246a42f58522e84cc4732ee9f8822c6a09c9 Mon Sep 17 00:00:00 2001 From: steven krawczyk Date: Mon, 17 Jul 2023 13:43:52 -0700 Subject: [PATCH] Fix notebooks --- examples/notebooks/AutoEval.ipynb | 5 +- examples/notebooks/HuggingFaceHub.ipynb | 14 +- examples/notebooks/HumanFeedback.ipynb | 5 +- examples/notebooks/ModelComparison.ipynb | 245 +------------- examples/notebooks/SemanticSimilarity.ipynb | 318 +++++------------- .../harness/prompt_template_harness.py | 3 +- prompttools/harness/system_prompt_harness.py | 3 +- 7 files changed, 106 insertions(+), 487 deletions(-) diff --git a/examples/notebooks/AutoEval.ipynb b/examples/notebooks/AutoEval.ipynb index 8a034e74..fba1333a 100644 --- a/examples/notebooks/AutoEval.ipynb +++ b/examples/notebooks/AutoEval.ipynb @@ -260,7 +260,8 @@ "outputs": [], "source": [ "from typing import Dict\n", - "from prompttools.harness import PromptTemplateExperimentationHarness" + "from prompttools.harness import PromptTemplateExperimentationHarness\n", + "from prompttools.experiment import OpenAICompletionExperiment" ] }, { @@ -308,7 +309,7 @@ "metadata": {}, "outputs": [], "source": [ - "harness = PromptTemplateExperimentationHarness(\"text-davinci-003\", prompt_templates, user_inputs)" + "harness = PromptTemplateExperimentationHarness(OpenAICompletionExperiment, \"text-davinci-003\", prompt_templates, user_inputs)" ] }, { diff --git a/examples/notebooks/HuggingFaceHub.ipynb b/examples/notebooks/HuggingFaceHub.ipynb index 5031f4b3..0bbf2ce6 100644 --- a/examples/notebooks/HuggingFaceHub.ipynb +++ b/examples/notebooks/HuggingFaceHub.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "id": "0a13ddc8", "metadata": {}, @@ -9,6 +10,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "623f0cfe", "metadata": {}, @@ -27,6 +29,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "2eac35f8", "metadata": {}, @@ -35,6 +38,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "5edba05a", "metadata": {}, @@ -63,10 +67,11 @@ "outputs": [], "source": [ "from typing import Dict, List, Tuple\n", - "from prompttools.experiment.huggingface_hub_experiment import HuggingFaceHubExperiment" + "from prompttools.experiment import HuggingFaceHubExperiment" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "622dea9a", "metadata": {}, @@ -75,6 +80,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "3babfe5a", "metadata": {}, @@ -101,6 +107,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "f3fa5450", "metadata": {}, @@ -119,6 +126,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "266c13eb", "metadata": {}, @@ -127,6 +135,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "bebb8023", "metadata": {}, @@ -166,6 +175,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "974d6065", "metadata": {}, @@ -182,7 +192,7 @@ }, "outputs": [], "source": [ - "experiment.evaluate(\"similar_to_expected\", measure_similarity)\n" + "experiment.evaluate(\"similar_to_expected\", measure_similarity)" ] }, { diff --git a/examples/notebooks/HumanFeedback.ipynb b/examples/notebooks/HumanFeedback.ipynb index c4e7a5f4..10fdb4fd 100644 --- a/examples/notebooks/HumanFeedback.ipynb +++ b/examples/notebooks/HumanFeedback.ipynb @@ -259,7 +259,8 @@ "metadata": {}, "outputs": [], "source": [ - "from prompttools.harness import PromptTemplateExperimentationHarness" + "from prompttools.harness import PromptTemplateExperimentationHarness\n", + "from prompttools.experiment import OpenAICompletionExperiment" ] }, { @@ -307,7 +308,7 @@ "metadata": {}, "outputs": [], "source": [ - "harness = PromptTemplateExperimentationHarness(\"text-davinci-003\", prompt_templates, user_inputs)" + "harness = PromptTemplateExperimentationHarness(OpenAICompletionExperiment, \"text-davinci-003\", prompt_templates, user_inputs)" ] }, { diff --git a/examples/notebooks/ModelComparison.ipynb b/examples/notebooks/ModelComparison.ipynb index 1502b24b..155e2a41 100644 --- a/examples/notebooks/ModelComparison.ipynb +++ b/examples/notebooks/ModelComparison.ipynb @@ -20,194 +20,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "4175323f", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: prompttools in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (0.0.5)\n", - "Requirement already satisfied: openai in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/openai-0.27.8-py3.10.egg (0.27.8)\n", - "Requirement already satisfied: jinja2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/Jinja2-3.1.2-py3.10.egg (3.1.2)\n", - "Requirement already satisfied: pandas in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/pandas-2.0.3-py3.10-macosx-10.9-x86_64.egg (from prompttools) (2.0.3)\n", - "Requirement already satisfied: ipywidgets in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/ipywidgets-8.0.7-py3.10.egg (from prompttools) (8.0.7)\n", - "Requirement already satisfied: hegel in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/hegel-0.0.3-py3.10.egg (from prompttools) (0.0.3)\n", - "Requirement already satisfied: jupyterlab in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/jupyterlab-4.0.2-py3.10.egg (from prompttools) (4.0.2)\n", - "Requirement already satisfied: sentence-transformers in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/sentence_transformers-2.2.2-py3.10.egg (from prompttools) (2.2.2)\n", - "Requirement already satisfied: tabulate in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/tabulate-0.9.0-py3.10.egg (from prompttools) (0.9.0)\n", - "Requirement already satisfied: chromadb in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/chromadb-0.3.26-py3.10.egg (from prompttools) (0.3.26)\n", - "Requirement already satisfied: tenacity in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/tenacity-8.2.2-py3.10.egg (from prompttools) (8.2.2)\n", - "Requirement already satisfied: requests>=2.20 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from openai) (2.28.1)\n", - "Requirement already satisfied: tqdm in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from openai) (4.65.0)\n", - "Requirement already satisfied: aiohttp in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from openai) (3.8.4)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jinja2) (2.1.3)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.20->openai) (1.26.15)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.20->openai) (2022.12.7)\n", - "Requirement already satisfied: idna<4,>=2.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.20->openai) (3.4)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.20->openai) (2.0.4)\n", - "Requirement already satisfied: attrs>=17.3.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (23.1.0)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (1.3.3)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (1.3.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (1.9.2)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (4.0.2)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (6.0.4)\n", - "Requirement already satisfied: pydantic>=1.9 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (2.0.2)\n", - "Requirement already satisfied: hnswlib>=0.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.7.0)\n", - "Requirement already satisfied: clickhouse-connect>=0.5.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.6.6)\n", - "Requirement already satisfied: duckdb>=0.7.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.8.1)\n", - "Requirement already satisfied: fastapi>=0.85.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.100.0)\n", - "Requirement already satisfied: uvicorn[standard]>=0.18.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.22.0)\n", - "Requirement already satisfied: numpy>=1.21.6 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (1.25.0)\n", - "Requirement already satisfied: posthog>=2.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (3.0.1)\n", - "Requirement already satisfied: typing-extensions>=4.5.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (4.7.1)\n", - "Requirement already satisfied: pulsar-client>=3.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (3.2.0)\n", - "Requirement already satisfied: onnxruntime>=1.14.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (1.15.1)\n", - "Requirement already satisfied: tokenizers>=0.13.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.13.3)\n", - "Requirement already satisfied: overrides>=7.3.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (7.3.1)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pandas->prompttools) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pandas->prompttools) (2023.3)\n", - "Requirement already satisfied: tzdata>=2022.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pandas->prompttools) (2023.3)\n", - "Requirement already satisfied: python-dotenv in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/python_dotenv-1.0.0-py3.10.egg (from hegel->prompttools) (1.0.0)\n", - "Requirement already satisfied: ipykernel>=4.5.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (6.24.0)\n", - "Requirement already satisfied: ipython>=6.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (8.14.0)\n", - "Requirement already satisfied: traitlets>=4.3.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (5.9.0)\n", - "Requirement already satisfied: widgetsnbextension~=4.0.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (4.0.8)\n", - "Requirement already satisfied: jupyterlab-widgets~=3.0.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (3.0.8)\n", - "Requirement already satisfied: async-lru>=1.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.0.3)\n", - "Requirement already satisfied: jupyter-core in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (5.3.1)\n", - "Requirement already satisfied: jupyter-lsp>=2.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.2.0)\n", - "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.7.0)\n", - "Requirement already satisfied: jupyterlab-server<3,>=2.19.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.23.0)\n", - "Requirement already satisfied: notebook-shim>=0.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (0.2.3)\n", - "Requirement already satisfied: packaging in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (23.0)\n", - "Requirement already satisfied: tomli in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.0.1)\n", - "Requirement already satisfied: tornado>=6.2.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (6.3.2)\n", - "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sentence-transformers->prompttools) (4.30.2)\n", - "Requirement already satisfied: torch>=1.6.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sentence-transformers->prompttools) (2.0.1)\n", - "Requirement already satisfied: torchvision in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sentence-transformers->prompttools) (0.15.2)\n", - "Requirement already satisfied: scikit-learn in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/scikit_learn-1.3.0-py3.10-macosx-10.9-x86_64.egg (from sentence-transformers->prompttools) (1.3.0)\n", - "Requirement already satisfied: scipy in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/scipy-1.11.1-py3.10-macosx-10.9-x86_64.egg (from sentence-transformers->prompttools) (1.11.1)\n", - "Requirement already satisfied: nltk in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/nltk-3.8.1-py3.10.egg (from sentence-transformers->prompttools) (3.8.1)\n", - "Requirement already satisfied: sentencepiece in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/sentencepiece-0.1.99-py3.10-macosx-10.9-x86_64.egg (from sentence-transformers->prompttools) (0.1.99)\n", - "Requirement already satisfied: huggingface-hub>=0.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/huggingface_hub-0.16.4-py3.10.egg (from sentence-transformers->prompttools) (0.16.4)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: zstandard in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (0.19.0)\n", - "Requirement already satisfied: importlib-metadata in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (6.8.0)\n", - "Requirement already satisfied: lz4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (4.3.2)\n", - "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from fastapi>=0.85.1->chromadb->prompttools) (0.27.0)\n", - "Requirement already satisfied: filelock in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers->prompttools) (3.12.2)\n", - "Requirement already satisfied: fsspec in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers->prompttools) (2023.6.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers->prompttools) (6.0)\n", - "Requirement already satisfied: appnope in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (0.1.3)\n", - "Requirement already satisfied: jupyter-client>=6.1.12 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (8.3.0)\n", - "Requirement already satisfied: debugpy>=1.6.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (1.6.7)\n", - "Requirement already satisfied: nest-asyncio in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (1.5.6)\n", - "Requirement already satisfied: pyzmq>=20 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (25.1.0)\n", - "Requirement already satisfied: psutil in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (5.9.5)\n", - "Requirement already satisfied: matplotlib-inline>=0.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (0.1.6)\n", - "Requirement already satisfied: comm>=0.1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (0.1.3)\n", - "Requirement already satisfied: pexpect>4.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (4.8.0)\n", - "Requirement already satisfied: decorator in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (5.1.1)\n", - "Requirement already satisfied: stack-data in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.6.2)\n", - "Requirement already satisfied: pygments>=2.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (2.15.1)\n", - "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (3.0.39)\n", - "Requirement already satisfied: backcall in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.2.0)\n", - "Requirement already satisfied: jedi>=0.16 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.18.2)\n", - "Requirement already satisfied: pickleshare in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.7.5)\n", - "Requirement already satisfied: platformdirs>=2.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-core->jupyterlab->prompttools) (3.8.1)\n", - "Requirement already satisfied: prometheus-client in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.17.0)\n", - "Requirement already satisfied: nbconvert>=6.4.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (7.6.0)\n", - "Requirement already satisfied: terminado>=0.8.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.17.1)\n", - "Requirement already satisfied: websocket-client in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.6.1)\n", - "Requirement already satisfied: anyio>=3.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (3.7.1)\n", - "Requirement already satisfied: nbformat>=5.3.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (5.9.0)\n", - "Requirement already satisfied: jupyter-events>=0.6.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.6.3)\n", - "Requirement already satisfied: jupyter-server-terminals in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.4.4)\n", - "Requirement already satisfied: send2trash in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.8.2)\n", - "Requirement already satisfied: argon2-cffi in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (21.3.0)\n", - "Requirement already satisfied: json5>=0.9.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (0.9.14)\n", - "Requirement already satisfied: babel>=2.10 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (2.12.1)\n", - "Requirement already satisfied: jsonschema>=4.17.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (4.18.0)\n", - "Requirement already satisfied: flatbuffers in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (23.5.26)\n", - "Requirement already satisfied: sympy in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (1.12)\n", - "Requirement already satisfied: protobuf in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (4.23.4)\n", - "Requirement already satisfied: coloredlogs in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (15.0.1)\n", - "Requirement already satisfied: backoff>=1.10.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->prompttools) (2.2.1)\n", - "Requirement already satisfied: six>=1.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->prompttools) (1.16.0)\n", - "Requirement already satisfied: monotonic>=1.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->prompttools) (1.6)\n", - "Requirement already satisfied: pydantic-core==2.1.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pydantic>=1.9->chromadb->prompttools) (2.1.2)\n", - "Requirement already satisfied: annotated-types>=0.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pydantic>=1.9->chromadb->prompttools) (0.5.0)\n", - "Requirement already satisfied: networkx in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from torch>=1.6.0->sentence-transformers->prompttools) (3.1)\n", - "Requirement already satisfied: regex!=2019.12.17 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->prompttools) (2023.6.3)\n", - "Requirement already satisfied: safetensors>=0.3.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->prompttools) (0.3.1)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: click>=7.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (8.1.4)\n", - "Requirement already satisfied: h11>=0.8 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.14.0)\n", - "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.17.0)\n", - "Requirement already satisfied: watchfiles>=0.13 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.19.0)\n", - "Requirement already satisfied: httptools>=0.5.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.6.0)\n", - "Requirement already satisfied: websockets>=10.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (11.0.3)\n", - "Requirement already satisfied: joblib in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nltk->sentence-transformers->prompttools) (1.3.1)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from scikit-learn->sentence-transformers->prompttools) (3.1.0)\n", - "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from torchvision->sentence-transformers->prompttools) (10.0.0)\n", - "Requirement already satisfied: sniffio>=1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.3.0)\n", - "Requirement already satisfied: exceptiongroup in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.1.2)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets->prompttools) (0.8.3)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (2023.6.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (0.29.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (0.8.8)\n", - "Requirement already satisfied: rfc3986-validator>=0.1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.1.1)\n", - "Requirement already satisfied: rfc3339-validator in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.1.4)\n", - "Requirement already satisfied: python-json-logger>=2.0.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.0.7)\n", - "Requirement already satisfied: pandocfilters>=1.4.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.5.0)\n", - "Requirement already satisfied: nbclient>=0.5.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.8.0)\n", - "Requirement already satisfied: jupyterlab-pygments in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.2.2)\n", - "Requirement already satisfied: beautifulsoup4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (4.12.2)\n", - "Requirement already satisfied: defusedxml in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.7.1)\n", - "Requirement already satisfied: tinycss2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.2.1)\n", - "Requirement already satisfied: bleach!=5.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (6.0.0)\n", - "Requirement already satisfied: mistune<4,>=2.0.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (3.0.1)\n", - "Requirement already satisfied: fastjsonschema in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbformat>=5.3.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.17.1)\n", - "Requirement already satisfied: ptyprocess>=0.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets->prompttools) (0.7.0)\n", - "Requirement already satisfied: wcwidth in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets->prompttools) (0.2.6)\n", - "Requirement already satisfied: argon2-cffi-bindings in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (21.2.0)\n", - "Requirement already satisfied: humanfriendly>=9.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb->prompttools) (10.0)\n", - "Requirement already satisfied: zipp>=0.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from importlib-metadata->clickhouse-connect>=0.5.7->chromadb->prompttools) (3.15.0)\n", - "Requirement already satisfied: pure-eval in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets->prompttools) (0.2.2)\n", - "Requirement already satisfied: asttokens>=2.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets->prompttools) (2.2.1)\n", - "Requirement already satisfied: executing>=1.2.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets->prompttools) (1.2.0)\n", - "Requirement already satisfied: mpmath>=0.19 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sympy->onnxruntime>=1.14.1->chromadb->prompttools) (1.3.0)\n", - "Requirement already satisfied: webencodings in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.5.1)\n", - "Requirement already satisfied: fqdn in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.5.1)\n", - "Requirement already satisfied: webcolors>=1.11 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.13)\n", - "Requirement already satisfied: jsonpointer>1.13 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (2.1)\n", - "Requirement already satisfied: uri-template in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.3.0)\n", - "Requirement already satisfied: isoduration in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (20.11.0)\n", - "Requirement already satisfied: cffi>=1.0.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.15.1)\n", - "Requirement already satisfied: soupsieve>1.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.4.1)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: pycparser in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.21)\n", - "Requirement already satisfied: arrow>=0.15.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from isoduration->jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.2.3)\n" - ] - } - ], + "outputs": [], "source": [ "!pip install --quiet --force-reinstall prompttools" ] @@ -259,7 +75,8 @@ "metadata": {}, "outputs": [], "source": [ - "from prompttools.harness import ChatModelComparisonHarness" + "from prompttools.harness import ChatModelComparisonHarness\n", + "from prompttools.experiment import OpenAIChatExperiment" ] }, { @@ -309,7 +126,7 @@ "metadata": {}, "outputs": [], "source": [ - "harness = ChatModelComparisonHarness([\"gpt-3.5-turbo\", \"gpt-3.5-turbo-0613\"], chat_histories)" + "harness = ChatModelComparisonHarness(OpenAIChatExperiment, [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-0613\"], chat_histories)" ] }, { @@ -359,7 +176,7 @@ " 0\n", " [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who won the world series in 2020?'}]\n", " [George Washington]\n", - " 0.000004\n", + " 0.000003\n", " gpt-3.5-turbo\n", " \n", " \n", @@ -379,7 +196,7 @@ "1 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who won the world series in 2020?'}] \n", "\n", " response(s) latency model \n", - "0 [George Washington] 0.000004 gpt-3.5-turbo \n", + "0 [George Washington] 0.000003 gpt-3.5-turbo \n", "1 [George Washington] 0.000002 gpt-3.5-turbo-0613 " ] }, @@ -430,7 +247,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b07a6a0a1c0c436d9d530634bca0482c", + "model_id": "29487dd5fca7453d99a1d159ba434136", "version_major": 2, "version_minor": 0 }, @@ -440,52 +257,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
gpt-3.5-turbofeedback
0[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who won the world series in 2020?'}]0
\n", - "
" - ], - "text/plain": [ - " gpt-3.5-turbo \n", - "0 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who won the world series in 2020?'}] \\\n", - "\n", - " feedback \n", - "0 0 " - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ diff --git a/examples/notebooks/SemanticSimilarity.ipynb b/examples/notebooks/SemanticSimilarity.ipynb index 42047d1e..49bd3882 100644 --- a/examples/notebooks/SemanticSimilarity.ipynb +++ b/examples/notebooks/SemanticSimilarity.ipynb @@ -20,194 +20,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "f5e5b81b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: prompttools in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (0.0.5)\n", - "Requirement already satisfied: sentence-transformers in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/sentence_transformers-2.2.2-py3.10.egg (from prompttools) (2.2.2)\n", - "Requirement already satisfied: openai in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/openai-0.27.8-py3.10.egg (from prompttools) (0.27.8)\n", - "Requirement already satisfied: hegel in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/hegel-0.0.3-py3.10.egg (from prompttools) (0.0.3)\n", - "Requirement already satisfied: tenacity in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/tenacity-8.2.2-py3.10.egg (from prompttools) (8.2.2)\n", - "Requirement already satisfied: tabulate in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/tabulate-0.9.0-py3.10.egg (from prompttools) (0.9.0)\n", - "Requirement already satisfied: jinja2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/Jinja2-3.1.2-py3.10.egg (from prompttools) (3.1.2)\n", - "Requirement already satisfied: ipywidgets in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/ipywidgets-8.0.7-py3.10.egg (from prompttools) (8.0.7)\n", - "Requirement already satisfied: jupyterlab in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/jupyterlab-4.0.2-py3.10.egg (from prompttools) (4.0.2)\n", - "Requirement already satisfied: pandas in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/pandas-2.0.3-py3.10-macosx-10.9-x86_64.egg (from prompttools) (2.0.3)\n", - "Requirement already satisfied: chromadb in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/chromadb-0.3.26-py3.10.egg (from prompttools) (0.3.26)\n", - "Requirement already satisfied: requests>=2.28 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (2.28.1)\n", - "Requirement already satisfied: pydantic>=1.9 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (2.0.2)\n", - "Requirement already satisfied: hnswlib>=0.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.7.0)\n", - "Requirement already satisfied: clickhouse-connect>=0.5.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.6.6)\n", - "Requirement already satisfied: duckdb>=0.7.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.8.1)\n", - "Requirement already satisfied: fastapi>=0.85.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.100.0)\n", - "Requirement already satisfied: uvicorn[standard]>=0.18.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.22.0)\n", - "Requirement already satisfied: numpy>=1.21.6 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (1.25.0)\n", - "Requirement already satisfied: posthog>=2.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (3.0.1)\n", - "Requirement already satisfied: typing-extensions>=4.5.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (4.7.1)\n", - "Requirement already satisfied: pulsar-client>=3.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (3.2.0)\n", - "Requirement already satisfied: onnxruntime>=1.14.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (1.15.1)\n", - "Requirement already satisfied: tokenizers>=0.13.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.13.3)\n", - "Requirement already satisfied: tqdm>=4.65.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (4.65.0)\n", - "Requirement already satisfied: overrides>=7.3.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (7.3.1)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pandas->prompttools) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pandas->prompttools) (2023.3)\n", - "Requirement already satisfied: tzdata>=2022.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pandas->prompttools) (2023.3)\n", - "Requirement already satisfied: python-dotenv in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/python_dotenv-1.0.0-py3.10.egg (from hegel->prompttools) (1.0.0)\n", - "Requirement already satisfied: ipykernel>=4.5.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (6.24.0)\n", - "Requirement already satisfied: ipython>=6.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (8.14.0)\n", - "Requirement already satisfied: traitlets>=4.3.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (5.9.0)\n", - "Requirement already satisfied: widgetsnbextension~=4.0.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (4.0.8)\n", - "Requirement already satisfied: jupyterlab-widgets~=3.0.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (3.0.8)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jinja2->prompttools) (2.1.3)\n", - "Requirement already satisfied: async-lru>=1.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.0.3)\n", - "Requirement already satisfied: jupyter-core in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (5.3.1)\n", - "Requirement already satisfied: jupyter-lsp>=2.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.2.0)\n", - "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.7.0)\n", - "Requirement already satisfied: jupyterlab-server<3,>=2.19.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.23.0)\n", - "Requirement already satisfied: notebook-shim>=0.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (0.2.3)\n", - "Requirement already satisfied: packaging in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (23.0)\n", - "Requirement already satisfied: tomli in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.0.1)\n", - "Requirement already satisfied: tornado>=6.2.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (6.3.2)\n", - "Requirement already satisfied: aiohttp in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from openai->prompttools) (3.8.4)\n", - "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sentence-transformers->prompttools) (4.30.2)\n", - "Requirement already satisfied: torch>=1.6.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sentence-transformers->prompttools) (2.0.1)\n", - "Requirement already satisfied: torchvision in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sentence-transformers->prompttools) (0.15.2)\n", - "Requirement already satisfied: scikit-learn in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/scikit_learn-1.3.0-py3.10-macosx-10.9-x86_64.egg (from sentence-transformers->prompttools) (1.3.0)\n", - "Requirement already satisfied: scipy in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/scipy-1.11.1-py3.10-macosx-10.9-x86_64.egg (from sentence-transformers->prompttools) (1.11.1)\n", - "Requirement already satisfied: nltk in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/nltk-3.8.1-py3.10.egg (from sentence-transformers->prompttools) (3.8.1)\n", - "Requirement already satisfied: sentencepiece in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/sentencepiece-0.1.99-py3.10-macosx-10.9-x86_64.egg (from sentence-transformers->prompttools) (0.1.99)\n", - "Requirement already satisfied: huggingface-hub>=0.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/huggingface_hub-0.16.4-py3.10.egg (from sentence-transformers->prompttools) (0.16.4)\n", - "Requirement already satisfied: lz4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (4.3.2)\n", - "Requirement already satisfied: certifi in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (2022.12.7)\n", - "Requirement already satisfied: zstandard in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (0.19.0)\n", - "Requirement already satisfied: importlib-metadata in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (6.8.0)\n", - "Requirement already satisfied: urllib3>=1.26 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (1.26.15)\n", - "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from fastapi>=0.85.1->chromadb->prompttools) (0.27.0)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: filelock in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers->prompttools) (3.12.2)\n", - "Requirement already satisfied: fsspec in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers->prompttools) (2023.6.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers->prompttools) (6.0)\n", - "Requirement already satisfied: comm>=0.1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (0.1.3)\n", - "Requirement already satisfied: pyzmq>=20 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (25.1.0)\n", - "Requirement already satisfied: appnope in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (0.1.3)\n", - "Requirement already satisfied: nest-asyncio in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (1.5.6)\n", - "Requirement already satisfied: debugpy>=1.6.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (1.6.7)\n", - "Requirement already satisfied: psutil in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (5.9.5)\n", - "Requirement already satisfied: jupyter-client>=6.1.12 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (8.3.0)\n", - "Requirement already satisfied: matplotlib-inline>=0.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (0.1.6)\n", - "Requirement already satisfied: pexpect>4.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (4.8.0)\n", - "Requirement already satisfied: jedi>=0.16 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.18.2)\n", - "Requirement already satisfied: decorator in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (5.1.1)\n", - "Requirement already satisfied: stack-data in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.6.2)\n", - "Requirement already satisfied: pygments>=2.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (2.15.1)\n", - "Requirement already satisfied: pickleshare in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.7.5)\n", - "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (3.0.39)\n", - "Requirement already satisfied: backcall in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.2.0)\n", - "Requirement already satisfied: platformdirs>=2.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-core->jupyterlab->prompttools) (3.8.1)\n", - "Requirement already satisfied: websocket-client in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.6.1)\n", - "Requirement already satisfied: prometheus-client in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.17.0)\n", - "Requirement already satisfied: argon2-cffi in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (21.3.0)\n", - "Requirement already satisfied: nbconvert>=6.4.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (7.6.0)\n", - "Requirement already satisfied: send2trash in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.8.2)\n", - "Requirement already satisfied: jupyter-server-terminals in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.4.4)\n", - "Requirement already satisfied: nbformat>=5.3.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (5.9.0)\n", - "Requirement already satisfied: anyio>=3.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (3.7.1)\n", - "Requirement already satisfied: terminado>=0.8.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.17.1)\n", - "Requirement already satisfied: jupyter-events>=0.6.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.6.3)\n", - "Requirement already satisfied: jsonschema>=4.17.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (4.18.0)\n", - "Requirement already satisfied: babel>=2.10 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (2.12.1)\n", - "Requirement already satisfied: json5>=0.9.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (0.9.14)\n", - "Requirement already satisfied: flatbuffers in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (23.5.26)\n", - "Requirement already satisfied: coloredlogs in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (15.0.1)\n", - "Requirement already satisfied: protobuf in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (4.23.4)\n", - "Requirement already satisfied: sympy in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (1.12)\n", - "Requirement already satisfied: six>=1.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->prompttools) (1.16.0)\n", - "Requirement already satisfied: backoff>=1.10.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->prompttools) (2.2.1)\n", - "Requirement already satisfied: monotonic>=1.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->prompttools) (1.6)\n", - "Requirement already satisfied: annotated-types>=0.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pydantic>=1.9->chromadb->prompttools) (0.5.0)\n", - "Requirement already satisfied: pydantic-core==2.1.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pydantic>=1.9->chromadb->prompttools) (2.1.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.28->chromadb->prompttools) (3.4)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.28->chromadb->prompttools) (2.0.4)\n", - "Requirement already satisfied: networkx in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from torch>=1.6.0->sentence-transformers->prompttools) (3.1)\n", - "Requirement already satisfied: safetensors>=0.3.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->prompttools) (0.3.1)\n", - "Requirement already satisfied: regex!=2019.12.17 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->prompttools) (2023.6.3)\n", - "Requirement already satisfied: click>=7.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (8.1.4)\n", - "Requirement already satisfied: h11>=0.8 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.14.0)\n", - "Requirement already satisfied: httptools>=0.5.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.6.0)\n", - "Requirement already satisfied: websockets>=10.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (11.0.3)\n", - "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.17.0)\n", - "Requirement already satisfied: watchfiles>=0.13 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.19.0)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai->prompttools) (1.9.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai->prompttools) (1.3.3)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai->prompttools) (4.0.2)\n", - "Requirement already satisfied: attrs>=17.3.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai->prompttools) (23.1.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai->prompttools) (6.0.4)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai->prompttools) (1.3.1)\n", - "Requirement already satisfied: joblib in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nltk->sentence-transformers->prompttools) (1.3.1)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from scikit-learn->sentence-transformers->prompttools) (3.1.0)\n", - "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from torchvision->sentence-transformers->prompttools) (10.0.0)\n", - "Requirement already satisfied: sniffio>=1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.3.0)\n", - "Requirement already satisfied: exceptiongroup in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.1.2)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets->prompttools) (0.8.3)\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (0.29.1)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (2023.6.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (0.8.8)\n", - "Requirement already satisfied: rfc3339-validator in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.1.4)\n", - "Requirement already satisfied: rfc3986-validator>=0.1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.1.1)\n", - "Requirement already satisfied: python-json-logger>=2.0.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.0.7)\n", - "Requirement already satisfied: defusedxml in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.7.1)\n", - "Requirement already satisfied: mistune<4,>=2.0.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (3.0.1)\n", - "Requirement already satisfied: jupyterlab-pygments in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.2.2)\n", - "Requirement already satisfied: pandocfilters>=1.4.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.5.0)\n", - "Requirement already satisfied: tinycss2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.2.1)\n", - "Requirement already satisfied: nbclient>=0.5.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.8.0)\n", - "Requirement already satisfied: bleach!=5.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (6.0.0)\n", - "Requirement already satisfied: beautifulsoup4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (4.12.2)\n", - "Requirement already satisfied: fastjsonschema in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbformat>=5.3.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.17.1)\n", - "Requirement already satisfied: ptyprocess>=0.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets->prompttools) (0.7.0)\n", - "Requirement already satisfied: wcwidth in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets->prompttools) (0.2.6)\n", - "Requirement already satisfied: argon2-cffi-bindings in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (21.2.0)\n", - "Requirement already satisfied: humanfriendly>=9.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb->prompttools) (10.0)\n", - "Requirement already satisfied: zipp>=0.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from importlib-metadata->clickhouse-connect>=0.5.7->chromadb->prompttools) (3.15.0)\n", - "Requirement already satisfied: executing>=1.2.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets->prompttools) (1.2.0)\n", - "Requirement already satisfied: asttokens>=2.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets->prompttools) (2.2.1)\n", - "Requirement already satisfied: pure-eval in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets->prompttools) (0.2.2)\n", - "Requirement already satisfied: mpmath>=0.19 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sympy->onnxruntime>=1.14.1->chromadb->prompttools) (1.3.0)\n", - "Requirement already satisfied: webencodings in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.5.1)\n", - "Requirement already satisfied: fqdn in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.5.1)\n", - "Requirement already satisfied: uri-template in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.3.0)\n", - "Requirement already satisfied: jsonpointer>1.13 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (2.1)\n", - "Requirement already satisfied: isoduration in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (20.11.0)\n", - "Requirement already satisfied: webcolors>=1.11 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.13)\n", - "Requirement already satisfied: cffi>=1.0.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.15.1)\n", - "Requirement already satisfied: soupsieve>1.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.4.1)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: pycparser in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.21)\n", - "Requirement already satisfied: arrow>=0.15.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from isoduration->jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.2.3)\n" - ] - } - ], + "outputs": [], "source": [ "!pip install --quiet --force-reinstall prompttools" ] @@ -232,13 +48,13 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 10, "id": "ed4e635e", "metadata": {}, "outputs": [], "source": [ "import os\n", - "os.environ['DEBUG'] = \"\" # Set to \"1\" if you want to use debug mode.\n", + "os.environ['DEBUG'] = \"1\" # Set to \"1\" if you want to use debug mode.\n", "os.environ['HEGELAI_API_KEY'] = \"\" # Optional, it will be needed to use with `HegelScribe` to persist/visualize your experiments\n", "os.environ['OPENAI_API_KEY'] = \"\"" ] @@ -254,13 +70,14 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 11, "id": "f0841dd5", "metadata": {}, "outputs": [], "source": [ "from typing import Dict, Tuple\n", - "from prompttools.harness import PromptTemplateExperimentationHarness" + "from prompttools.harness import PromptTemplateExperimentationHarness\n", + "from prompttools.experiment import OpenAICompletionExperiment" ] }, { @@ -283,7 +100,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 12, "id": "dbab6b3e", "metadata": {}, "outputs": [], @@ -303,12 +120,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 13, "id": "8b3086e6", "metadata": {}, "outputs": [], "source": [ - "harness = PromptTemplateExperimentationHarness(\"text-davinci-003\", \n", + "harness = PromptTemplateExperimentationHarness(OpenAICompletionExperiment,\n", + " \"text-davinci-003\", \n", " prompt_templates, \n", " user_inputs,\n", " model_arguments={\"temperature\": 0.9})" @@ -325,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 14, "id": "84304957", "metadata": {}, "outputs": [ @@ -353,32 +171,37 @@ " messages\n", " response(s)\n", " latency\n", + " prompt\n", " \n", " \n", " \n", " \n", " 0\n", " Answer the following question: Who was the first president?\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", - " 0.000005\n", + " [\\n\\nGeorge Washington]\n", + " 0.000004\n", + " Answer the following question: Who was the first president?\n", " \n", " \n", " 1\n", " Answer the following question: Who was the first president of India?\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", + " [\\n\\nGeorge Washington]\n", " 0.000002\n", + " Answer the following question: Who was the first president of India?\n", " \n", " \n", " 2\n", " Respond the following query: Who was the first president?\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", + " [\\n\\nGeorge Washington]\n", " 0.000002\n", + " Respond the following query: Who was the first president?\n", " \n", " \n", " 3\n", " Respond the following query: Who was the first president of India?\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", + " [\\n\\nGeorge Washington]\n", " 0.000002\n", + " Respond the following query: Who was the first president of India?\n", " \n", " \n", "\n", @@ -391,11 +214,17 @@ "2 Respond the following query: Who was the first president? \n", "3 Respond the following query: Who was the first president of India? \n", "\n", - " response(s) latency \n", - "0 [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] 0.000005 \n", - "1 [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] 0.000002 \n", - "2 [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] 0.000002 \n", - "3 [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] 0.000002 " + " response(s) latency \n", + "0 [\\n\\nGeorge Washington] 0.000004 \\\n", + "1 [\\n\\nGeorge Washington] 0.000002 \n", + "2 [\\n\\nGeorge Washington] 0.000002 \n", + "3 [\\n\\nGeorge Washington] 0.000002 \n", + "\n", + " prompt \n", + "0 Answer the following question: Who was the first president? \n", + "1 Answer the following question: Who was the first president of India? \n", + "2 Respond the following query: Who was the first president? \n", + "3 Respond the following query: Who was the first president of India? " ] }, "metadata": {}, @@ -418,7 +247,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 15, "id": "fc2f1bde", "metadata": { "scrolled": true @@ -457,28 +286,28 @@ " \n", " \n", " {'input': 'Who was the first president of India?'}\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", + " [\\n\\nGeorge Washington]\n", + " [\\n\\nGeorge Washington]\n", " \n", " \n", " {'input': 'Who was the first president?'}\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", + " [\\n\\nGeorge Washington]\n", + " [\\n\\nGeorge Washington]\n", " \n", " \n", "\n", "" ], "text/plain": [ - "prompt_template Answer the following question: {{input}} \n", - "user_input \n", - "{'input': 'Who was the first president of India?'} [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] \\\n", - "{'input': 'Who was the first president?'} [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] \n", + "prompt_template Answer the following question: {{input}} \n", + "user_input \n", + "{'input': 'Who was the first president of India?'} [\\n\\nGeorge Washington] \\\n", + "{'input': 'Who was the first president?'} [\\n\\nGeorge Washington] \n", "\n", - "prompt_template Respond the following query: {{input}} \n", - "user_input \n", - "{'input': 'Who was the first president of India?'} [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] \n", - "{'input': 'Who was the first president?'} [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] " + "prompt_template Respond the following query: {{input}} \n", + "user_input \n", + "{'input': 'Who was the first president of India?'} [\\n\\nGeorge Washington] \n", + "{'input': 'Who was the first president?'} [\\n\\nGeorge Washington] " ] }, "metadata": {}, @@ -509,7 +338,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 16, "id": "63de3014", "metadata": {}, "outputs": [], @@ -550,7 +379,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 17, "id": "233f25a6", "metadata": {}, "outputs": [ @@ -558,10 +387,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2023-07-05 15:08:51,242] WARNING in Collection: No embedding_function provided, using default embedding function: DefaultEmbeddingFunction https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n", - "[2023-07-05 15:08:51,469] WARNING in Collection: No embedding_function provided, using default embedding function: DefaultEmbeddingFunction https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n", - "[2023-07-05 15:08:51,723] WARNING in Collection: No embedding_function provided, using default embedding function: DefaultEmbeddingFunction https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n", - "[2023-07-05 15:08:51,968] WARNING in Collection: No embedding_function provided, using default embedding function: DefaultEmbeddingFunction https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n" + "WARNING:chromadb.api.models.Collection:No embedding_function provided, using default embedding function: DefaultEmbeddingFunction https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n", + "WARNING:chromadb.api.models.Collection:No embedding_function provided, using default embedding function: DefaultEmbeddingFunction https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n", + "WARNING:chromadb.api.models.Collection:No embedding_function provided, using default embedding function: DefaultEmbeddingFunction https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n", + "WARNING:chromadb.api.models.Collection:No embedding_function provided, using default embedding function: DefaultEmbeddingFunction https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n" ] }, { @@ -589,36 +418,41 @@ " response(s)\n", " latency\n", " similar_to_expected\n", + " prompt\n", " \n", " \n", " \n", " \n", " 0\n", " Answer the following question: Who was the first president?\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", - " 0.000005\n", - " 0.965657\n", + " [\\n\\nGeorge Washington]\n", + " 0.000004\n", + " 0.000000\n", + " Answer the following question: Who was the first president?\n", " \n", " \n", " 1\n", " Answer the following question: Who was the first president of India?\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", + " [\\n\\nGeorge Washington]\n", " 0.000002\n", - " 1.021784\n", + " 0.626794\n", + " Answer the following question: Who was the first president of India?\n", " \n", " \n", " 2\n", " Respond the following query: Who was the first president?\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", + " [\\n\\nGeorge Washington]\n", " 0.000002\n", - " 0.965657\n", + " 0.000000\n", + " Respond the following query: Who was the first president?\n", " \n", " \n", " 3\n", " Respond the following query: Who was the first president of India?\n", - " [\\n\\nThe Los Angeles Dodgers won the World Series in 2020]\n", + " [\\n\\nGeorge Washington]\n", " 0.000002\n", - " 1.021784\n", + " 0.626794\n", + " Respond the following query: Who was the first president of India?\n", " \n", " \n", "\n", @@ -631,17 +465,17 @@ "2 Respond the following query: Who was the first president? \n", "3 Respond the following query: Who was the first president of India? \n", "\n", - " response(s) latency \n", - "0 [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] 0.000005 \\\n", - "1 [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] 0.000002 \n", - "2 [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] 0.000002 \n", - "3 [\\n\\nThe Los Angeles Dodgers won the World Series in 2020] 0.000002 \n", + " response(s) latency similar_to_expected \n", + "0 [\\n\\nGeorge Washington] 0.000004 0.000000 \\\n", + "1 [\\n\\nGeorge Washington] 0.000002 0.626794 \n", + "2 [\\n\\nGeorge Washington] 0.000002 0.000000 \n", + "3 [\\n\\nGeorge Washington] 0.000002 0.626794 \n", "\n", - " similar_to_expected \n", - "0 0.965657 \n", - "1 1.021784 \n", - "2 0.965657 \n", - "3 1.021784 " + " prompt \n", + "0 Answer the following question: Who was the first president? \n", + "1 Answer the following question: Who was the first president of India? \n", + "2 Respond the following query: Who was the first president? \n", + "3 Respond the following query: Who was the first president of India? " ] }, "metadata": {}, diff --git a/prompttools/harness/prompt_template_harness.py b/prompttools/harness/prompt_template_harness.py index ca9fea00..94ffc6b9 100644 --- a/prompttools/harness/prompt_template_harness.py +++ b/prompttools/harness/prompt_template_harness.py @@ -26,13 +26,14 @@ class PromptTemplateExperimentationHarness(ExperimentationHarness): def __init__( self, + experiment_classname, model_name: str, prompt_templates: List[str], user_inputs: List[Dict[str, str]], model_arguments: Optional[Dict[str, object]] = None, ): self.environment = jinja2.Environment() - self.experiment_classname = model_arguments["experiment_classname"] + self.experiment_classname = experiment_classname self.model_name = model_name self.prompt_templates = prompt_templates self.user_inputs = user_inputs diff --git a/prompttools/harness/system_prompt_harness.py b/prompttools/harness/system_prompt_harness.py index a8ba1e07..edc9074e 100644 --- a/prompttools/harness/system_prompt_harness.py +++ b/prompttools/harness/system_prompt_harness.py @@ -24,12 +24,13 @@ class SystemPromptExperimentationHarness(ExperimentationHarness): def __init__( self, + experiment_classname, model_name: str, system_prompts: List[str], human_messages: List[str], model_arguments: Optional[Dict[str, object]] = None, ): - self.experiment_classname = model_arguments["experiment_classname"] + self.experiment_classname = experiment_classname, self.model_name = model_name self.system_prompts = system_prompts self.human_messages = human_messages