Fix notebooks

hegelai · Jul 17, 2023 · 8523246 · 8523246
1 parent b34cbf1
commit 8523246
Show file tree

Hide file tree

Showing 7 changed files with 106 additions and 487 deletions.
diff --git a/examples/notebooks/AutoEval.ipynb b/examples/notebooks/AutoEval.ipynb
@@ -260,7 +260,8 @@
  "outputs": [],
  "source": [
  "from typing import Dict\n",
- "from prompttools.harness import PromptTemplateExperimentationHarness"
+ "from prompttools.harness import PromptTemplateExperimentationHarness\n",
+ "from prompttools.experiment import OpenAICompletionExperiment"
  ]
  },
  {
@@ -308,7 +309,7 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "harness = PromptTemplateExperimentationHarness(\"text-davinci-003\", prompt_templates, user_inputs)"
+ "harness = PromptTemplateExperimentationHarness(OpenAICompletionExperiment, \"text-davinci-003\", prompt_templates, user_inputs)"
  ]
  },
  {

diff --git a/examples/notebooks/HuggingFaceHub.ipynb b/examples/notebooks/HuggingFaceHub.ipynb
@@ -1,6 +1,7 @@
 {
  "cells": [
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "0a13ddc8",
  "metadata": {},
@@ -9,6 +10,7 @@
  ]
  },
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "623f0cfe",
  "metadata": {},
@@ -27,6 +29,7 @@
  ]
  },
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "2eac35f8",
  "metadata": {},
@@ -35,6 +38,7 @@
  ]
  },
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "5edba05a",
  "metadata": {},
@@ -63,10 +67,11 @@
  "outputs": [],
  "source": [
  "from typing import Dict, List, Tuple\n",
- "from prompttools.experiment.huggingface_hub_experiment import HuggingFaceHubExperiment"
+ "from prompttools.experiment import HuggingFaceHubExperiment"
  ]
  },
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "622dea9a",
  "metadata": {},
@@ -75,6 +80,7 @@
  ]
  },
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "3babfe5a",
  "metadata": {},
@@ -101,6 +107,7 @@
  ]
  },
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "f3fa5450",
  "metadata": {},
@@ -119,6 +126,7 @@
  ]
  },
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "266c13eb",
  "metadata": {},
@@ -127,6 +135,7 @@
  ]
  },
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "bebb8023",
  "metadata": {},
@@ -166,6 +175,7 @@
  ]
  },
  {
+ "attachments": {},
  "cell_type": "markdown",
  "id": "974d6065",
  "metadata": {},
@@ -182,7 +192,7 @@
  },
  "outputs": [],
  "source": [
- "experiment.evaluate(\"similar_to_expected\", measure_similarity)\n"
+ "experiment.evaluate(\"similar_to_expected\", measure_similarity)"
  ]
  },
  {

diff --git a/examples/notebooks/HumanFeedback.ipynb b/examples/notebooks/HumanFeedback.ipynb
@@ -259,7 +259,8 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "from prompttools.harness import PromptTemplateExperimentationHarness"
+ "from prompttools.harness import PromptTemplateExperimentationHarness\n",
+ "from prompttools.experiment import OpenAICompletionExperiment"
  ]
  },
  {
@@ -307,7 +308,7 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "harness = PromptTemplateExperimentationHarness(\"text-davinci-003\", prompt_templates, user_inputs)"
+ "harness = PromptTemplateExperimentationHarness(OpenAICompletionExperiment, \"text-davinci-003\", prompt_templates, user_inputs)"
  ]
  },
  {

diff --git a/examples/notebooks/ModelComparison.ipynb b/examples/notebooks/ModelComparison.ipynb
diff --git a/examples/notebooks/SemanticSimilarity.ipynb b/examples/notebooks/SemanticSimilarity.ipynb
diff --git a/prompttools/harness/prompt_template_harness.py b/prompttools/harness/prompt_template_harness.py
@@ -26,13 +26,14 @@ class PromptTemplateExperimentationHarness(ExperimentationHarness):
 
  def __init__(
  self,
+ experiment_classname,
  model_name: str,
  prompt_templates: List[str],
  user_inputs: List[Dict[str, str]],
  model_arguments: Optional[Dict[str, object]] = None,
  ):
  self.environment = jinja2.Environment()
- self.experiment_classname = model_arguments["experiment_classname"]
+ self.experiment_classname = experiment_classname
  self.model_name = model_name
  self.prompt_templates = prompt_templates
  self.user_inputs = user_inputs

diff --git a/prompttools/harness/system_prompt_harness.py b/prompttools/harness/system_prompt_harness.py
@@ -24,12 +24,13 @@ class SystemPromptExperimentationHarness(ExperimentationHarness):
 
  def __init__(
  self,
+ experiment_classname,
  model_name: str,
  system_prompts: List[str],
  human_messages: List[str],
  model_arguments: Optional[Dict[str, object]] = None,
  ):
- self.experiment_classname = model_arguments["experiment_classname"]
+ self.experiment_classname = experiment_classname,
  self.model_name = model_name
  self.system_prompts = system_prompts
  self.human_messages = human_messages