dspy-rag-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports and Global Declarations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import os\n",
    "import random\n",
    "import re\n",
    "from typing import List, NamedTuple\n",
    "\n",
    "import dspy\n",
    "from dspy.teleprompt import BootstrapFewShotWithRandomSearch\n",
    "from dspy.retrieve.weaviate_rm import WeaviateRM\n",
    "\n",
    "from wcs_client_adapter import COLLECTION_TEXT_KEY, WCS_COLLECTION_NAME, WcsClientAdapter\n",
    "from indexers import NaiveWcsIndexer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Index Paper for Retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_uri = \"https://arxiv.org/html/2312.10997v5\"\n",
    "indexer = NaiveWcsIndexer(doc_uri) # TODO: this calling syntax doesn't make it clear what side effects the constructor has"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Configure Language Model and Retrieval Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_lm = dspy.OpenAI(model=\"gpt-3.5-turbo\")\n",
    "\n",
    "wcs_client = WcsClientAdapter.get_wcs_client()\n",
    "wcs_rm = WeaviateRM(WCS_COLLECTION_NAME, weaviate_client=wcs_client, weaviate_collection_text_key=COLLECTION_TEXT_KEY)\n",
    "dspy.settings.configure(lm=default_lm, rm=wcs_rm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Questions Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "answerable_questions_path = \"./data/answerable-questions.csv\"\n",
    "unanswerable_questions_path = \"./data/unanswerable-questions.csv\"\n",
    "\n",
    "def load_questions_from_csv(file_path: str) -> List[str]:\n",
    "    questions = []\n",
    "    with open(file_path, mode='r', newline='', encoding='utf-8') as file:\n",
    "        reader = csv.reader(file)\n",
    "        for row in reader:\n",
    "            if row:\n",
    "                questions.append(row[0])\n",
    "    return questions\n",
    "\n",
    "class DataSplits(NamedTuple):\n",
    "    train: List\n",
    "    dev: List\n",
    "    test: List\n",
    "\n",
    "def split_data(data: List, train_size: float, dev_size: float, test_size: float) -> DataSplits:\n",
    "    if train_size + dev_size + test_size != 1:\n",
    "        raise ValueError(\"The sum of train_size, dev_size, and test_size must be 1.\")\n",
    "\n",
    "    random.shuffle(data)  \n",
    "    \n",
    "    train_end = int(train_size * len(data))\n",
    "    dev_end = train_end + int(dev_size * len(data))\n",
    "    \n",
    "    train_set = data[:train_end]\n",
    "    dev_set = data[train_end:dev_end]\n",
    "    test_set = data[dev_end:]\n",
    "    \n",
    "    return DataSplits(train=train_set, dev=dev_set, test=test_set)\n",
    "\n",
    "answerable_questions = load_questions_from_csv(answerable_questions_path)\n",
    "unanswerable_questions = load_questions_from_csv(unanswerable_questions_path)\n",
    "all_questions = answerable_questions + unanswerable_questions\n",
    "all_qs_as_dspy_examples = trainset = [dspy.Example(question=question).with_inputs(\"question\") for question in all_questions]\n",
    "\n",
    "splits = split_data(all_qs_as_dspy_examples, 0.7, 0.15, 0.15)\n",
    "trainset = splits.train\n",
    "devset = splits.dev\n",
    "testset = splits.test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build Signatures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "metricLM = dspy.OpenAI(model='gpt-4-turbo', max_tokens=1000, model_type='chat')\n",
    "\n",
    "class GenerateAnswer(dspy.Signature):\n",
    "    context = dspy.InputField(desc=\"may contain relevant facts\")\n",
    "    question = dspy.InputField()\n",
    "    answer = dspy.OutputField(desc=\"between 1 and 4 sentences\")\n",
    "    \n",
    "class Assess(dspy.Signature):\n",
    "    \"\"\"Assess the quality of an answer to a question.\"\"\"\n",
    "    \n",
    "    context = dspy.InputField(desc=\"The context for answering the question.\")\n",
    "    assessed_question = dspy.InputField(desc=\"The evaluation criterion.\")\n",
    "    assessed_answer = dspy.InputField(desc=\"The answer to the question.\")\n",
    "    assessment_answer = dspy.OutputField(desc=\"A rating between 1 and 5. Only output the rating and nothing else.\")\n",
    "\n",
    "def structured_llm_feedback(gold, pred, trace=None):\n",
    "    predicted_answer = pred.answer\n",
    "    question = gold.question\n",
    "    \n",
    "    print(f\"Test Question: {question}\")\n",
    "    print(f\"Predicted Answer: {predicted_answer}\")\n",
    "    \n",
    "    detail = \"Is the assessed answer detailed?\"\n",
    "    faithful = \"Is the assessed text grounded in the context? Say no if it includes significant facts not in the context.\"\n",
    "    overall = f\"Please rate how well this answer answers the question, `{question}` based on the context.\\n `{predicted_answer}`\"\n",
    "    \n",
    "    with dspy.context(lm=metricLM):\n",
    "        context = dspy.Retrieve(k=5)(question).passages\n",
    "        detail = dspy.ChainOfThought(Assess)(context=\"N/A\", assessed_question=detail, assessed_answer=predicted_answer)\n",
    "        faithful = dspy.ChainOfThought(Assess)(context=context, assessed_question=faithful, assessed_answer=predicted_answer)\n",
    "        overall = dspy.ChainOfThought(Assess)(context=context, assessed_question=overall, assessed_answer=predicted_answer)\n",
    "    \n",
    "    print(f\"Faithful: {faithful.assessment_answer}\")\n",
    "    print(f\"Detail: {detail.assessment_answer}\")\n",
    "    print(f\"Overall: {overall.assessment_answer}\")\n",
    "    \n",
    "    total = float(detail.assessment_answer) + float(faithful.assessment_answer)*2 + float(overall.assessment_answer)\n",
    "    \n",
    "    return total / 5.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build Rag Pipeline as DSPy Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RAG(dspy.Module):\n",
    "    def __init__(self, num_passages=3):\n",
    "        super().__init__()\n",
    "\n",
    "        self.retrieve = dspy.Retrieve(k=num_passages)\n",
    "        self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
    "    \n",
    "    def forward(self, question):\n",
    "        context = self.retrieve(question).passages\n",
    "        prediction = self.generate_answer(context=context, question=question)\n",
    "        return dspy.Prediction(context=context, answer=prediction.answer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimize Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'save_optimized_rag' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 40\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m contextlib\u001b[38;5;241m.\u001b[39mredirect_stdout(stdout_file), contextlib\u001b[38;5;241m.\u001b[39mredirect_stderr(stderr_file):\n\u001b[1;32m     39\u001b[0m         compiled_rag \u001b[38;5;241m=\u001b[39m teleprompter\u001b[38;5;241m.\u001b[39mcompile(RAG(), trainset\u001b[38;5;241m=\u001b[39mtrainset)\n\u001b[0;32m---> 40\u001b[0m \u001b[43msave_optimized_rag\u001b[49m(compiled_rag, teleprompter)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'save_optimized_rag' is not defined"
     ]
    }
   ],
   "source": [
    "def save_compiled_rag(compiled_rag, teleprompter):\n",
    "    teleprompter_name = teleprompter.__class__.__name__\n",
    "    directory = 'dspy-optimized-programs'\n",
    "    os.makedirs(directory, exist_ok=True)\n",
    "    \n",
    "    highest_number = 0\n",
    "    pattern = re.compile(rf'^rag-{re.escape(teleprompter_name)}-(\\d+)\\.dspy\\.txt$')\n",
    "    \n",
    "    for filename in os.listdir(directory):\n",
    "        match = pattern.match(filename)\n",
    "        if match:\n",
    "            current_number = int(match.group(1))\n",
    "            if current_number > highest_number:\n",
    "                highest_number = current_number\n",
    "    \n",
    "    next_number = highest_number + 1\n",
    "    \n",
    "    file_path = os.path.join(directory, f'rag-{teleprompter_name}-{next_number}.dspy.txt')\n",
    "    \n",
    "    compiled_rag.save(file_path)\n",
    "    print(f\"Saved optimized program to {file_path}\")\n",
    "\n",
    "teleprompter = BootstrapFewShotWithRandomSearch(\n",
    "    metric=structured_llm_feedback,\n",
    "    max_bootstrapped_demos=4,\n",
    "    max_labeled_demos=4,\n",
    "    max_rounds=1,\n",
    "    num_candidate_programs=2,\n",
    "    num_threads=2\n",
    "    ) \n",
    "\n",
    "import contextlib\n",
    "import sys\n",
    "\n",
    "# Follow output of this cell using tail -f due to large output size\n",
    "stdout_file_path = 'dspy-optimized-programs/compile_stdout.txt'\n",
    "stderr_file_path = 'dspy-optimized-programs/compile_stderr.txt'\n",
    "with open(stdout_file_path, 'w') as stdout_file, open(stderr_file_path, 'w') as stderr_file:\n",
    "    with contextlib.redirect_stdout(stdout_file), contextlib.redirect_stderr(stderr_file):\n",
    "        compiled_rag = teleprompter.compile(RAG(), trainset=trainset)\n",
    "save_compiled_rag(compiled_rag, teleprompter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Execute Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: What methodologies are used for evaluating RAG systems?\n",
      "Predicted Answer: The methodologies used for evaluating RAG systems include assessing retrieval quality through metrics like Hit Rate, MRR, and NDCG, and evaluating generation quality based on metrics such as answer faithfulness, answer relevance, and counterfactual robustness. These evaluations aim to measure the effectiveness of the context sourced by the retriever component and the generator's capacity to synthesize coherent answers.\n",
      "Retrieved Contexts (truncated): ['Target Historically, RAG models assessments have centered on their execution in specific downstream tasks. These evaluations employ established metrics suitable to the tasks at hand. For instance, que...', 'charting its evolution and anticipated future paths, with a focus on the integration of RAG within LLMs. This paper considers both technical paradigms and research methods, summarizing three main rese...', 'information from multiple documents to address complex questions. Counterfactual Robustnesstests the model’s ability to recognize and disregard known inaccuracies within documents, even when instructe...']\n"
     ]
    }
   ],
   "source": [
    "my_question = \"What methodologies are used for evaluating RAG systems?\"\n",
    "\n",
    "pred = uncompiled_rag(my_question)\n",
    "\n",
    "print(f\"Question: {my_question}\")\n",
    "print(f\"Predicted Answer: {pred.answer}\")\n",
    "print(f\"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Given the fields `context`, `question`, produce the fields `answer`.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Context: may contain relevant facts\n",
      "\n",
      "Question: ${question}\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: between 1 and 4 sentences\n",
      "\n",
      "---\n",
      "\n",
      "Context:\n",
      "[1] «Target Historically, RAG models assessments have centered on their execution in specific downstream tasks. These evaluations employ established metrics suitable to the tasks at hand. For instance, question answering evaluations might rely on EM and F1 scores[45,72,59,7], whereas fact-checking tasks often hinge on Accuracy as the primary metric[4,42,14]. BLEU and ROUGE metrics are also commonly used to evaluate answer quality[26,78,52,32]. Tools like RALLE, designed for the automatic evaluation of RAG applications, similarly base their assessments on these task-specific metrics[160]. Despite this, there is a notable paucity of research dedicated to evaluating the distinct characteristics of RAG models.The main evaluation objectives include: Retrieval Quality. Evaluating the retrieval quality is crucial for determining the effectiveness of the context sourced by the retriever component. Standard metrics from the domains of search engines, recommendation systems, and information retrieval systems are employed to measure the performance of the RAG retrieval module. Metrics such as Hit Rate, MRR, and NDCG are commonly utilized for this purpose[161,162]. Generation Quality. The assessment of generation quality centers on the generator’s capacity to synthesize coherent»\n",
      "[2] «charting its evolution and anticipated future paths, with a focus on the integration of RAG within LLMs. This paper considers both technical paradigms and research methods, summarizing three main research paradigms from over 100 RAG studies, and analyzing key technologies in the core stages of “Retrieval,” “Generation,” and “Augmentation.” On the other hand, current research tends to focus more on methods, lacking analysis and summarization of how to evaluate RAG. This paper comprehensively reviews the downstream tasks, datasets, benchmarks, and evaluation methods applicable to RAG. Overall, this paper sets out to meticulously compile and categorize the foundational technical concepts, historical progression, and the spectrum of RAG methodologies and applications that have emerged post-LLMs. It is designed to equip readers and professionals with a detailed and structured understanding of both large models and RAG. It aims to illuminate the evolution of retrieval augmentation techniques, assess the strengths and weaknesses of various approaches in their respective contexts, and speculate on upcoming trends and innovations. Our contributions are as follows: In this survey, we present a thorough and»\n",
      "[3] «information from multiple documents to address complex questions. Counterfactual Robustnesstests the model’s ability to recognize and disregard known inaccuracies within documents, even when instructed about potential misinformation. Context relevance and noise robustness are important for evaluating the quality of retrieval, while answer faithfulness, answer relevance, negative rejection, information integration, and counterfactual robustness are important for evaluating the quality of generation. The specific metrics for each evaluation aspect are summarized in TableIII. It is essential to recognize that these metrics, derived from related work, are traditional measures and do not yet represent a mature or standardized approach for quantifying RAG evaluation aspects. Custom metrics tailored to the nuances of RAG models, though not included here, have also been developed in some evaluation studies. ContextRelevance Faithfulness AnswerRelevance NoiseRobustness NegativeRejection InformationIntegration CounterfactualRobustness Accuracy ✓ ✓ ✓ ✓ ✓ ✓ ✓ EM ✓ Recall ✓ Precision ✓ ✓ R-Rate ✓ Cosine Similarity ✓ Hit Rate ✓ MRR ✓ NDCG ✓ BLEU ✓ ✓ ✓ ROUGE/ROUGE-L ✓ ✓ ✓ VI-DEvaluation Benchmarks and Tools A series of benchmark tests and tools»\n",
      "\n",
      "Question: What methodologies are used for evaluating RAG systems?\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We need to consider the different aspects of RAG systems that need to be evaluated, such as retrieval quality and generation quality. By understanding the specific metrics and benchmarks mentioned in the context, we can determine the methodologies used for evaluating RAG systems.\n",
      "\n",
      "Answer: The methodologies used for evaluating RAG systems include assessing retrieval quality through metrics like Hit Rate, MRR, and NDCG, and evaluating generation quality based on metrics such as answer faithfulness, answer relevance, and counterfactual robustness. These evaluations aim to measure the effectiveness of the context sourced by the retriever component and the generator's capacity to synthesize coherent answers.\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\n\\n\\nGiven the fields `context`, `question`, produce the fields `answer`.\\n\\n---\\n\\nFollow the following format.\\n\\nContext: may contain relevant facts\\n\\nQuestion: ${question}\\n\\nReasoning: Let's think step by step in order to ${produce the answer}. We ...\\n\\nAnswer: between 1 and 4 sentences\\n\\n---\\n\\nContext:\\n[1] «Target Historically, RAG models assessments have centered on their execution in specific downstream tasks. These evaluations employ established metrics suitable to the tasks at hand. For instance, question answering evaluations might rely on EM and F1 scores[45,72,59,7], whereas fact-checking tasks often hinge on Accuracy as the primary metric[4,42,14]. BLEU and ROUGE metrics are also commonly used to evaluate answer quality[26,78,52,32]. Tools like RALLE, designed for the automatic evaluation of RAG applications, similarly base their assessments on these task-specific metrics[160]. Despite this, there is a notable paucity of research dedicated to evaluating the distinct characteristics of RAG models.The main evaluation objectives include: Retrieval Quality. Evaluating the retrieval quality is crucial for determining the effectiveness of the context sourced by the retriever component. Standard metrics from the domains of search engines, recommendation systems, and information retrieval systems are employed to measure the performance of the RAG retrieval module. Metrics such as Hit Rate, MRR, and NDCG are commonly utilized for this purpose[161,162]. Generation Quality. The assessment of generation quality centers on the generator’s capacity to synthesize coherent»\\n[2] «charting its evolution and anticipated future paths, with a focus on the integration of RAG within LLMs. This paper considers both technical paradigms and research methods, summarizing three main research paradigms from over 100 RAG studies, and analyzing key technologies in the core stages of “Retrieval,” “Generation,” and “Augmentation.” On the other hand, current research tends to focus more on methods, lacking analysis and summarization of how to evaluate RAG. This paper comprehensively reviews the downstream tasks, datasets, benchmarks, and evaluation methods applicable to RAG. Overall, this paper sets out to meticulously compile and categorize the foundational technical concepts, historical progression, and the spectrum of RAG methodologies and applications that have emerged post-LLMs. It is designed to equip readers and professionals with a detailed and structured understanding of both large models and RAG. It aims to illuminate the evolution of retrieval augmentation techniques, assess the strengths and weaknesses of various approaches in their respective contexts, and speculate on upcoming trends and innovations. Our contributions are as follows: In this survey, we present a thorough and»\\n[3] «information from multiple documents to address complex questions. Counterfactual Robustnesstests the model’s ability to recognize and disregard known inaccuracies within documents, even when instructed about potential misinformation. Context relevance and noise robustness are important for evaluating the quality of retrieval, while answer faithfulness, answer relevance, negative rejection, information integration, and counterfactual robustness are important for evaluating the quality of generation. The specific metrics for each evaluation aspect are summarized in TableIII. It is essential to recognize that these metrics, derived from related work, are traditional measures and do not yet represent a mature or standardized approach for quantifying RAG evaluation aspects. Custom metrics tailored to the nuances of RAG models, though not included here, have also been developed in some evaluation studies. ContextRelevance Faithfulness AnswerRelevance NoiseRobustness NegativeRejection InformationIntegration CounterfactualRobustness Accuracy ✓ ✓ ✓ ✓ ✓ ✓ ✓ EM ✓ Recall ✓ Precision ✓ ✓ R-Rate ✓ Cosine Similarity ✓ Hit Rate ✓ MRR ✓ NDCG ✓ BLEU ✓ ✓ ✓ ROUGE/ROUGE-L ✓ ✓ ✓ VI-DEvaluation Benchmarks and Tools A series of benchmark tests and tools»\\n\\nQuestion: What methodologies are used for evaluating RAG systems?\\n\\nReasoning: Let's think step by step in order to\\x1b[32m produce the answer. We need to consider the different aspects of RAG systems that need to be evaluated, such as retrieval quality and generation quality. By understanding the specific metrics and benchmarks mentioned in the context, we can determine the methodologies used for evaluating RAG systems.\\n\\nAnswer: The methodologies used for evaluating RAG systems include assessing retrieval quality through metrics like Hit Rate, MRR, and NDCG, and evaluating generation quality based on metrics such as answer faithfulness, answer relevance, and counterfactual robustness. These evaluations aim to measure the effectiveness of the context sourced by the retriever component and the generator's capacity to synthesize coherent answers.\\x1b[0m\\n\\n\\n\""
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_lm.inspect_history(n=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: What methodologies are used for evaluating RAG systems?\n",
      "Predicted Answer: The methodologies used for evaluating RAG systems include assessing retrieval quality through metrics like Hit Rate, MRR, and NDCG, and evaluating generation quality based on metrics such as answer faithfulness, answer relevance, and counterfactual robustness. These evaluations aim to measure the effectiveness of the context sourced by the retriever component and the generator's capacity to synthesize coherent answers.\n",
      "Retrieved Contexts (truncated): ['Target Historically, RAG models assessments have centered on their execution in specific downstream tasks. These evaluations employ established metrics suitable to the tasks at hand. For instance, que...', 'charting its evolution and anticipated future paths, with a focus on the integration of RAG within LLMs. This paper considers both technical paradigms and research methods, summarizing three main rese...', 'information from multiple documents to address complex questions. Counterfactual Robustnesstests the model’s ability to recognize and disregard known inaccuracies within documents, even when instructe...']\n"
     ]
    }
   ],
   "source": [
    "my_question = \"What methodologies are used for evaluating RAG systems?\"\n",
    "\n",
    "pred = compiled_rag(my_question)\n",
    "\n",
    "print(f\"Question: {my_question}\")\n",
    "print(f\"Predicted Answer: {pred.answer}\")\n",
    "print(f\"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Given the fields `context`, `question`, produce the fields `answer`.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Context: may contain relevant facts\n",
      "\n",
      "Question: ${question}\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: between 1 and 4 sentences\n",
      "\n",
      "---\n",
      "\n",
      "Context:\n",
      "[1] «Target Historically, RAG models assessments have centered on their execution in specific downstream tasks. These evaluations employ established metrics suitable to the tasks at hand. For instance, question answering evaluations might rely on EM and F1 scores[45,72,59,7], whereas fact-checking tasks often hinge on Accuracy as the primary metric[4,42,14]. BLEU and ROUGE metrics are also commonly used to evaluate answer quality[26,78,52,32]. Tools like RALLE, designed for the automatic evaluation of RAG applications, similarly base their assessments on these task-specific metrics[160]. Despite this, there is a notable paucity of research dedicated to evaluating the distinct characteristics of RAG models.The main evaluation objectives include: Retrieval Quality. Evaluating the retrieval quality is crucial for determining the effectiveness of the context sourced by the retriever component. Standard metrics from the domains of search engines, recommendation systems, and information retrieval systems are employed to measure the performance of the RAG retrieval module. Metrics such as Hit Rate, MRR, and NDCG are commonly utilized for this purpose[161,162]. Generation Quality. The assessment of generation quality centers on the generator’s capacity to synthesize coherent»\n",
      "[2] «charting its evolution and anticipated future paths, with a focus on the integration of RAG within LLMs. This paper considers both technical paradigms and research methods, summarizing three main research paradigms from over 100 RAG studies, and analyzing key technologies in the core stages of “Retrieval,” “Generation,” and “Augmentation.” On the other hand, current research tends to focus more on methods, lacking analysis and summarization of how to evaluate RAG. This paper comprehensively reviews the downstream tasks, datasets, benchmarks, and evaluation methods applicable to RAG. Overall, this paper sets out to meticulously compile and categorize the foundational technical concepts, historical progression, and the spectrum of RAG methodologies and applications that have emerged post-LLMs. It is designed to equip readers and professionals with a detailed and structured understanding of both large models and RAG. It aims to illuminate the evolution of retrieval augmentation techniques, assess the strengths and weaknesses of various approaches in their respective contexts, and speculate on upcoming trends and innovations. Our contributions are as follows: In this survey, we present a thorough and»\n",
      "[3] «information from multiple documents to address complex questions. Counterfactual Robustnesstests the model’s ability to recognize and disregard known inaccuracies within documents, even when instructed about potential misinformation. Context relevance and noise robustness are important for evaluating the quality of retrieval, while answer faithfulness, answer relevance, negative rejection, information integration, and counterfactual robustness are important for evaluating the quality of generation. The specific metrics for each evaluation aspect are summarized in TableIII. It is essential to recognize that these metrics, derived from related work, are traditional measures and do not yet represent a mature or standardized approach for quantifying RAG evaluation aspects. Custom metrics tailored to the nuances of RAG models, though not included here, have also been developed in some evaluation studies. ContextRelevance Faithfulness AnswerRelevance NoiseRobustness NegativeRejection InformationIntegration CounterfactualRobustness Accuracy ✓ ✓ ✓ ✓ ✓ ✓ ✓ EM ✓ Recall ✓ Precision ✓ ✓ R-Rate ✓ Cosine Similarity ✓ Hit Rate ✓ MRR ✓ NDCG ✓ BLEU ✓ ✓ ✓ ROUGE/ROUGE-L ✓ ✓ ✓ VI-DEvaluation Benchmarks and Tools A series of benchmark tests and tools»\n",
      "\n",
      "Question: What methodologies are used for evaluating RAG systems?\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We need to consider the different aspects of RAG systems that need to be evaluated, such as retrieval quality and generation quality. By understanding the specific metrics and benchmarks mentioned in the context, we can determine the methodologies used for evaluating RAG systems.\n",
      "\n",
      "Answer: The methodologies used for evaluating RAG systems include assessing retrieval quality through metrics like Hit Rate, MRR, and NDCG, and evaluating generation quality based on metrics such as answer faithfulness, answer relevance, and counterfactual robustness. These evaluations aim to measure the effectiveness of the context sourced by the retriever component and the generator's capacity to synthesize coherent answers.\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\n\\n\\nGiven the fields `context`, `question`, produce the fields `answer`.\\n\\n---\\n\\nFollow the following format.\\n\\nContext: may contain relevant facts\\n\\nQuestion: ${question}\\n\\nReasoning: Let's think step by step in order to ${produce the answer}. We ...\\n\\nAnswer: between 1 and 4 sentences\\n\\n---\\n\\nContext:\\n[1] «Target Historically, RAG models assessments have centered on their execution in specific downstream tasks. These evaluations employ established metrics suitable to the tasks at hand. For instance, question answering evaluations might rely on EM and F1 scores[45,72,59,7], whereas fact-checking tasks often hinge on Accuracy as the primary metric[4,42,14]. BLEU and ROUGE metrics are also commonly used to evaluate answer quality[26,78,52,32]. Tools like RALLE, designed for the automatic evaluation of RAG applications, similarly base their assessments on these task-specific metrics[160]. Despite this, there is a notable paucity of research dedicated to evaluating the distinct characteristics of RAG models.The main evaluation objectives include: Retrieval Quality. Evaluating the retrieval quality is crucial for determining the effectiveness of the context sourced by the retriever component. Standard metrics from the domains of search engines, recommendation systems, and information retrieval systems are employed to measure the performance of the RAG retrieval module. Metrics such as Hit Rate, MRR, and NDCG are commonly utilized for this purpose[161,162]. Generation Quality. The assessment of generation quality centers on the generator’s capacity to synthesize coherent»\\n[2] «charting its evolution and anticipated future paths, with a focus on the integration of RAG within LLMs. This paper considers both technical paradigms and research methods, summarizing three main research paradigms from over 100 RAG studies, and analyzing key technologies in the core stages of “Retrieval,” “Generation,” and “Augmentation.” On the other hand, current research tends to focus more on methods, lacking analysis and summarization of how to evaluate RAG. This paper comprehensively reviews the downstream tasks, datasets, benchmarks, and evaluation methods applicable to RAG. Overall, this paper sets out to meticulously compile and categorize the foundational technical concepts, historical progression, and the spectrum of RAG methodologies and applications that have emerged post-LLMs. It is designed to equip readers and professionals with a detailed and structured understanding of both large models and RAG. It aims to illuminate the evolution of retrieval augmentation techniques, assess the strengths and weaknesses of various approaches in their respective contexts, and speculate on upcoming trends and innovations. Our contributions are as follows: In this survey, we present a thorough and»\\n[3] «information from multiple documents to address complex questions. Counterfactual Robustnesstests the model’s ability to recognize and disregard known inaccuracies within documents, even when instructed about potential misinformation. Context relevance and noise robustness are important for evaluating the quality of retrieval, while answer faithfulness, answer relevance, negative rejection, information integration, and counterfactual robustness are important for evaluating the quality of generation. The specific metrics for each evaluation aspect are summarized in TableIII. It is essential to recognize that these metrics, derived from related work, are traditional measures and do not yet represent a mature or standardized approach for quantifying RAG evaluation aspects. Custom metrics tailored to the nuances of RAG models, though not included here, have also been developed in some evaluation studies. ContextRelevance Faithfulness AnswerRelevance NoiseRobustness NegativeRejection InformationIntegration CounterfactualRobustness Accuracy ✓ ✓ ✓ ✓ ✓ ✓ ✓ EM ✓ Recall ✓ Precision ✓ ✓ R-Rate ✓ Cosine Similarity ✓ Hit Rate ✓ MRR ✓ NDCG ✓ BLEU ✓ ✓ ✓ ROUGE/ROUGE-L ✓ ✓ ✓ VI-DEvaluation Benchmarks and Tools A series of benchmark tests and tools»\\n\\nQuestion: What methodologies are used for evaluating RAG systems?\\n\\nReasoning: Let's think step by step in order to\\x1b[32m produce the answer. We need to consider the different aspects of RAG systems that need to be evaluated, such as retrieval quality and generation quality. By understanding the specific metrics and benchmarks mentioned in the context, we can determine the methodologies used for evaluating RAG systems.\\n\\nAnswer: The methodologies used for evaluating RAG systems include assessing retrieval quality through metrics like Hit Rate, MRR, and NDCG, and evaluating generation quality based on metrics such as answer faithfulness, answer relevance, and counterfactual robustness. These evaluations aim to measure the effectiveness of the context sourced by the retriever component and the generator's capacity to synthesize coherent answers.\\x1b[0m\\n\\n\\n\""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_lm.inspect_history(n=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate Complete RAG Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dspy.evaluate.evaluate import Evaluate\n",
    "\n",
    "evaluate_with_llm_feedback = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5)\n",
    "metric = structured_llm_feedback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/15 [00:00<?, ?it/s]Test Question: What role does RAG play in the development of virtual assistants and conversational agents?\n",
      "Predicted Answer: RAG plays a crucial role in the development of virtual assistants and conversational agents by providing chunked retrieval and on-demand input to improve operational efficiency. Additionally, RAG-based generation helps quickly locate original references for LLMs, enabling users to verify generated answers. The observable retrieval and reasoning process of RAG contrasts with the black box nature of generation solely relying on long context, making it an essential tool for developing advanced AI technologies.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 4.0 / 1  (400.0):   7%|▋         | 1/15 [00:17<04:06, 17.62s/it]Test Question: How is the integration of domain-specific information handled in RAG systems?\n",
      "Predicted Answer: The integration of domain-specific information in Retrieval-Augmented Generation (RAG) systems is achieved by incorporating knowledge from external databases. This enhances the accuracy and credibility of the generation, particularly for knowledge-intensive tasks. RAG synergistically merges the intrinsic knowledge of Large Language Models (LLMs) with the vast repositories of external databases, allowing for continuous knowledge updates and integration of domain\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 7.8 / 2  (390.0):  13%|█▎        | 2/15 [00:18<01:42,  7.89s/it]Test Question: How does RAG contribute to real-world applications such as chatbots?\n",
      "Predicted Answer: RAG contributes to real-world applications such as chatbots by enhancing the accuracy and reliability of large language models through the retrieval of relevant information from external knowledge bases. This technology reduces the generation of factually incorrect content, making chatbots more effective in handling queries beyond their training data and providing current information to users.\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 11.6 / 3  (386.7):  20%|██        | 3/15 [00:19<00:56,  4.74s/it]Test Question: How does RAG deal with the retrieval of contradictory information from different sources?\n",
      "Predicted Answer: RAG may encounter difficulties in handling contradictory information from different sources during retrieval due to precision and recall challenges. This can lead to the selection of misaligned or irrelevant chunks, impacting the quality and reliability of the responses generated. Additionally, issues like hallucination, irrelevance, toxicity, or bias in the outputs may further complicate the integration of contradictory information in RAG's responses.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 15.6 / 4  (390.0):  27%|██▋       | 4/15 [00:20<00:35,  3.27s/it]Test Question: How is external knowledge integrated into the generation process in RAG systems?\n",
      "Predicted Answer: External knowledge is integrated into the generation process in RAG systems through methods like Flare and Self-RAG, which refine the framework by determining optimal moments and content for retrieval. Models like Graph-Toolformer and WebGPT also employ active judgment to autonomously use search engines during text generation, enhancing the capabilities of LLMs in generating accurate responses.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 19.6 / 5  (392.0):  33%|███▎      | 5/15 [00:21<00:24,  2.44s/it]Test Question: How can RAG contribute to the development of autonomous systems like self-driving cars or smart homes?\n",
      "Predicted Answer: RAG models can contribute to the development of autonomous systems like self-driving cars or smart homes by enhancing their ability to actively retrieve relevant information and make informed decisions. By refining the retrieval framework and enabling LLMs to determine optimal moments and content for retrieval, RAG models can improve the efficiency and relevance of information sourced, ultimately enhancing the autonomous judgment capabilities of these systems. Additionally, RAG's integration with other AI methodologies like fine-tuning and reinforcement learning further expands its capabilities, making it a valuable tool for developing advanced autonomous systems.\n",
      "Faithful: 2\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 22.400000000000002 / 6  (373.3):  40%|████      | 6/15 [00:22<00:17,  1.93s/it]Test Question: How is the trade-off between retrieval accuracy and generation quality managed?\n",
      "Predicted Answer: The trade-off between retrieval accuracy and generation quality in RAG models can be managed by developing specialized strategies to integrate retrieval with language generation models. Research findings have shown that including irrelevant documents can actually increase accuracy, highlighting the need for further exploration into the robustness of RAG models. Additionally, combining RAG with fine-tuning is emerging as a leading strategy to optimize the integration of retrieval accuracy and generation quality.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 26.400000000000002 / 7  (377.1):  47%|████▋     | 7/15 [00:23<00:12,  1.61s/it]Test Question: How are changes in external databases handled to ensure the RAG system remains up-to-date?\n",
      "Predicted Answer: Changes in external databases are handled by continuously updating the retrieval process in the RAG system to ensure that the most current information is being accessed. This allows the system to remain up-to-date by dynamically retrieving relevant document chunks from external knowledge bases through semantic similarity calculations. Additionally, advancements in RAG technology focus on improving retrieval efficiency and document recall in large knowledge bases to ensure the system remains aligned with the most recent information available.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 30.400000000000002 / 8  (380.0):  53%|█████▎    | 8/15 [00:24<00:09,  1.42s/it]Test Question: What are the best practices for curating and maintaining the external knowledge bases used in RAG systems?\n",
      "Predicted Answer: The best practices for curating and maintaining external knowledge bases in RAG systems include regular updates to ensure accuracy, verification of sources to maintain credibility, and organization of information for easy retrieval. Additionally, implementing quality control measures and establishing clear guidelines for adding new information can help optimize the performance of RAG systems.\n",
      "Faithful: 3\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 33.400000000000006 / 9  (371.1):  60%|██████    | 9/15 [00:25<00:07,  1.26s/it]Test Question: What strategies are suggested for overcoming limitations in naive RAG models?\n",
      "Predicted Answer: The strategies suggested for overcoming limitations in naive RAG models include refining indexing techniques through a sliding window approach, fine-grained segmentation, and the incorporation of metadata in Advanced RAG. Additionally, optimizing the indexing structure and the original query in the pre-retrieval process, selecting essential information, emphasizing critical sections, and shortening the context to be processed are recommended strategies.\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 37.2 / 10  (372.0):  67%|██████▋   | 10/15 [00:26<00:05,  1.15s/it]            Test Question: How does query optimization enhance the performance of RAG systems?\n",
      "Predicted Answer: Query optimization enhances the performance of RAG systems by making the user's original question clearer and more suitable for the retrieval task. Common methods include query rewriting, query transformation, query expansion, and other techniques. By refining the indexing structure and enhancing data granularity, query optimization ensures that the content being indexed is of high quality, ultimately leading to improved retrieval results in RAG systems.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 41.2 / 11  (374.5):  73%|███████▎  | 11/15 [00:27<00:04,  1.08s/it]Test Question: How does RAG handle ambiguous queries where multiple interpretations are possible?\n",
      "Predicted Answer: RAG addresses ambiguous queries by expanding a single query into multiple queries, enriching the context and providing further nuances to ensure optimal relevance in the generated answers. Additionally, techniques such as recursive retrieval and multi-hop retrieval are utilized to refine search results and extract interconnected information from graph-structured data sources. This adaptive approach significantly improves the quality and relevance of retrieved information, catering to a wide array of tasks and queries with enhanced precision and flexibility.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 45.2 / 12  (376.7):  80%|████████  | 12/15 [00:28<00:03,  1.05s/it]Test Question: How does embedding optimization improve retrieval results in RAG?\n",
      "Predicted Answer: Embedding optimization techniques like HyDE focus on embedding similarities between generated answers and real documents, enhancing retrieval relevance in RAG systems. By adjusting module arrangement and interaction, such as in the Demonstrate-Search-Predict framework, RAG systems can dynamically use module outputs to bolster functionality and improve retrieval results. This sophisticated understanding of enhancing module synergy ultimately leads to more effective retrieval in RAG systems.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 49.2 / 13  (378.5):  87%|████████▋ | 13/15 [00:29<00:02,  1.06s/it]Test Question: How does RAG ensure the privacy and security of the data it retrieves and generates?\n",
      "Predicted Answer: RAG addresses privacy and security concerns by focusing on enhancing retrieval efficiency, improving document recall in large knowledge bases, and preventing inadvertent disclosure of document sources or metadata by LLMs. The development of the RAG ecosystem is greatly impacted by the progression of its technical stack, with key tools like LangChain and LLamaIndex playing a crucial role in ensuring data security. Traditional software and cloud service providers are also expanding their offerings to include RAG-centric services, further emphasizing the\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 53.0 / 14  (378.6):  93%|█████████▎| 14/15 [00:30<00:01,  1.01s/it]Test Question: How do different RAG implementations impact the latency of response generation in real-time systems?\n",
      "Predicted Answer: Different RAG implementations impact the latency of response generation in real-time systems differently. RAG offers real-time knowledge updates but comes with higher latency due to ethical considerations regarding data retrieval. In contrast, Fine-tuning (FT) may be more static and require retraining for updates, potentially affecting response generation latency.\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 56.8 / 15  (378.7): 100%|██████████| 15/15 [00:31<00:00,  2.08s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_de5a6 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_de5a6 td {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_de5a6_row0_col0, #T_de5a6_row0_col1, #T_de5a6_row0_col2, #T_de5a6_row0_col3, #T_de5a6_row1_col0, #T_de5a6_row1_col1, #T_de5a6_row1_col2, #T_de5a6_row1_col3, #T_de5a6_row2_col0, #T_de5a6_row2_col1, #T_de5a6_row2_col2, #T_de5a6_row2_col3, #T_de5a6_row3_col0, #T_de5a6_row3_col1, #T_de5a6_row3_col2, #T_de5a6_row3_col3, #T_de5a6_row4_col0, #T_de5a6_row4_col1, #T_de5a6_row4_col2, #T_de5a6_row4_col3 {\n",
       "  text-align: left;\n",
       "  white-space: pre-wrap;\n",
       "  word-wrap: break-word;\n",
       "  max-width: 400px;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_de5a6\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_de5a6_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
       "      <th id=\"T_de5a6_level0_col1\" class=\"col_heading level0 col1\" >context</th>\n",
       "      <th id=\"T_de5a6_level0_col2\" class=\"col_heading level0 col2\" >answer</th>\n",
       "      <th id=\"T_de5a6_level0_col3\" class=\"col_heading level0 col3\" >structured_llm_feedback</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_de5a6_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_de5a6_row0_col0\" class=\"data row0 col0\" >What role does RAG play in the development of virtual assistants and conversational agents?</td>\n",
       "      <td id=\"T_de5a6_row0_col1\" class=\"data row0 col1\" >['discussions on whether RAG is still necessary when LLMs are not constrained by context. In fact, RAG still plays an irreplaceable role. On one hand,...</td>\n",
       "      <td id=\"T_de5a6_row0_col2\" class=\"data row0 col2\" >RAG plays a crucial role in the development of virtual assistants and conversational agents by providing chunked retrieval and on-demand input to improve operational efficiency....</td>\n",
       "      <td id=\"T_de5a6_row0_col3\" class=\"data row0 col3\" >✔️ [4.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_de5a6_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_de5a6_row1_col0\" class=\"data row1 col0\" >How is the integration of domain-specific information handled in RAG systems?</td>\n",
       "      <td id=\"T_de5a6_row1_col1\" class=\"data row1 col1\" >['University College of Design and Innovation, Tongji University Abstract Large Language Models (LLMs) showcase impressive capabilities but encounter challenges like hallucination, outdated knowledge, and non-transparent,...</td>\n",
       "      <td id=\"T_de5a6_row1_col2\" class=\"data row1 col2\" >The integration of domain-specific information in Retrieval-Augmented Generation (RAG) systems is achieved by incorporating knowledge from external databases. This enhances the accuracy and credibility of...</td>\n",
       "      <td id=\"T_de5a6_row1_col3\" class=\"data row1 col3\" >✔️ [3.8]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_de5a6_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_de5a6_row2_col0\" class=\"data row2 col0\" >How does RAG contribute to real-world applications such as chatbots?</td>\n",
       "      <td id=\"T_de5a6_row2_col1\" class=\"data row2 col1\" >['paper introduces up-to-date evaluation framework and benchmark. At the end, this article delineates the challenges currently faced and points out prospective avenues for research and...</td>\n",
       "      <td id=\"T_de5a6_row2_col2\" class=\"data row2 col2\" >RAG contributes to real-world applications such as chatbots by enhancing the accuracy and reliability of large language models through the retrieval of relevant information from...</td>\n",
       "      <td id=\"T_de5a6_row2_col3\" class=\"data row2 col3\" >✔️ [3.8]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_de5a6_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_de5a6_row3_col0\" class=\"data row3 col0\" >How does RAG deal with the retrieval of contradictory information from different sources?</td>\n",
       "      <td id=\"T_de5a6_row3_col1\" class=\"data row3 col1\" >['or contradictory information during retrieval can detrimentally affect RAG’s output quality. This situation is figuratively referred to as “Misinformation can be worse than no information...</td>\n",
       "      <td id=\"T_de5a6_row3_col2\" class=\"data row3 col2\" >RAG may encounter difficulties in handling contradictory information from different sources during retrieval due to precision and recall challenges. This can lead to the selection...</td>\n",
       "      <td id=\"T_de5a6_row3_col3\" class=\"data row3 col3\" >✔️ [4.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_de5a6_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_de5a6_row4_col0\" class=\"data row4 col0\" >How is external knowledge integrated into the generation process in RAG systems?</td>\n",
       "      <td id=\"T_de5a6_row4_col1\" class=\"data row4 col1\" >['University College of Design and Innovation, Tongji University Abstract Large Language Models (LLMs) showcase impressive capabilities but encounter challenges like hallucination, outdated knowledge, and non-transparent,...</td>\n",
       "      <td id=\"T_de5a6_row4_col2\" class=\"data row4 col2\" >External knowledge is integrated into the generation process in RAG systems through methods like Flare and Self-RAG, which refine the framework by determining optimal moments...</td>\n",
       "      <td id=\"T_de5a6_row4_col3\" class=\"data row4 col3\" >✔️ [4.0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x14f34f710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <div style='\n",
       "                    text-align: center;\n",
       "                    font-size: 16px;\n",
       "                    font-weight: bold;\n",
       "                    color: #555;\n",
       "                    margin: 10px 0;'>\n",
       "                    ... 10 more rows not displayed ...\n",
       "                </div>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "378.67"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uncompiled_rag = RAG()\n",
    "evaluate_with_llm_feedback(uncompiled_rag, metric=metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/15 [00:00<?, ?it/s]Test Question: What role does RAG play in the development of virtual assistants and conversational agents?\n",
      "Predicted Answer: RAG plays a crucial role in the development of virtual assistants and conversational agents by providing chunked retrieval and on-demand input to improve operational efficiency. Additionally, RAG-based generation helps quickly locate original references for LLMs, enabling users to verify generated answers. The observable retrieval and reasoning process of RAG contrasts with the black box nature of generation solely relying on long context, making it an essential tool for developing advanced AI technologies.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 4.0 / 1  (400.0):   7%|▋         | 1/15 [00:00<00:13,  1.04it/s]Test Question: How is the integration of domain-specific information handled in RAG systems?\n",
      "Predicted Answer: The integration of domain-specific information in Retrieval-Augmented Generation (RAG) systems is achieved by incorporating knowledge from external databases. This enhances the accuracy and credibility of the generation, particularly for knowledge-intensive tasks. RAG synergistically merges the intrinsic knowledge of Large Language Models (LLMs) with the vast repositories of external databases, allowing for continuous knowledge updates and integration of domain\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 7.8 / 2  (390.0):  13%|█▎        | 2/15 [00:01<00:12,  1.02it/s]Test Question: How does RAG contribute to real-world applications such as chatbots?\n",
      "Predicted Answer: RAG contributes to real-world applications such as chatbots by enhancing the accuracy and reliability of large language models through the retrieval of relevant information from external knowledge bases. This technology reduces the generation of factually incorrect content, making chatbots more effective in handling queries beyond their training data and providing current information to users.\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 11.6 / 3  (386.7):  20%|██        | 3/15 [00:03<00:12,  1.01s/it]Test Question: How does RAG deal with the retrieval of contradictory information from different sources?\n",
      "Predicted Answer: RAG may encounter difficulties in handling contradictory information from different sources during retrieval due to precision and recall challenges. This can lead to the selection of misaligned or irrelevant chunks, impacting the quality and reliability of the responses generated. Additionally, issues like hallucination, irrelevance, toxicity, or bias in the outputs may further complicate the integration of contradictory information in RAG's responses.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 15.6 / 4  (390.0):  27%|██▋       | 4/15 [00:03<00:10,  1.01it/s]Test Question: How is external knowledge integrated into the generation process in RAG systems?\n",
      "Predicted Answer: External knowledge is integrated into the generation process in RAG systems through methods like Flare and Self-RAG, which refine the framework by determining optimal moments and content for retrieval. Models like Graph-Toolformer and WebGPT also employ active judgment to autonomously use search engines during text generation, enhancing the capabilities of LLMs in generating accurate responses.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 19.6 / 5  (392.0):  33%|███▎      | 5/15 [00:04<00:09,  1.02it/s]Test Question: How can RAG contribute to the development of autonomous systems like self-driving cars or smart homes?\n",
      "Predicted Answer: RAG models can contribute to the development of autonomous systems like self-driving cars or smart homes by enhancing their ability to actively retrieve relevant information and make informed decisions. By refining the retrieval framework and enabling LLMs to determine optimal moments and content for retrieval, RAG models can improve the efficiency and relevance of information sourced, ultimately enhancing the autonomous judgment capabilities of these systems. Additionally, RAG's integration with other AI methodologies like fine-tuning and reinforcement learning further expands its capabilities, making it a valuable tool for developing advanced autonomous systems.\n",
      "Faithful: 2\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 22.400000000000002 / 6  (373.3):  40%|████      | 6/15 [00:05<00:08,  1.04it/s]Test Question: How is the trade-off between retrieval accuracy and generation quality managed?\n",
      "Predicted Answer: The trade-off between retrieval accuracy and generation quality in RAG models can be managed by developing specialized strategies to integrate retrieval with language generation models. Research findings have shown that including irrelevant documents can actually increase accuracy, highlighting the need for further exploration into the robustness of RAG models. Additionally, combining RAG with fine-tuning is emerging as a leading strategy to optimize the integration of retrieval accuracy and generation quality.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 26.400000000000002 / 7  (377.1):  47%|████▋     | 7/15 [00:06<00:07,  1.05it/s]Test Question: How are changes in external databases handled to ensure the RAG system remains up-to-date?\n",
      "Predicted Answer: Changes in external databases are handled by continuously updating the retrieval process in the RAG system to ensure that the most current information is being accessed. This allows the system to remain up-to-date by dynamically retrieving relevant document chunks from external knowledge bases through semantic similarity calculations. Additionally, advancements in RAG technology focus on improving retrieval efficiency and document recall in large knowledge bases to ensure the system remains aligned with the most recent information available.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 30.400000000000002 / 8  (380.0):  53%|█████▎    | 8/15 [00:09<00:10,  1.43s/it]Test Question: What are the best practices for curating and maintaining the external knowledge bases used in RAG systems?\n",
      "Predicted Answer: The best practices for curating and maintaining external knowledge bases in RAG systems include regular updates to ensure accuracy, verification of sources to maintain credibility, and organization of information for easy retrieval. Additionally, implementing quality control measures and establishing clear guidelines for adding new information can help optimize the performance of RAG systems.\n",
      "Faithful: 3\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 33.400000000000006 / 9  (371.1):  60%|██████    | 9/15 [00:10<00:07,  1.28s/it]Test Question: What strategies are suggested for overcoming limitations in naive RAG models?\n",
      "Predicted Answer: The strategies suggested for overcoming limitations in naive RAG models include refining indexing techniques through a sliding window approach, fine-grained segmentation, and the incorporation of metadata in Advanced RAG. Additionally, optimizing the indexing structure and the original query in the pre-retrieval process, selecting essential information, emphasizing critical sections, and shortening the context to be processed are recommended strategies.\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 37.2 / 10  (372.0):  67%|██████▋   | 10/15 [00:11<00:05,  1.17s/it]            Test Question: How does query optimization enhance the performance of RAG systems?\n",
      "Predicted Answer: Query optimization enhances the performance of RAG systems by making the user's original question clearer and more suitable for the retrieval task. Common methods include query rewriting, query transformation, query expansion, and other techniques. By refining the query, RAG systems can retrieve more relevant and accurate information, ultimately improving their overall performance.\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 41.0 / 11  (372.7):  73%|███████▎  | 11/15 [00:29<00:25,  6.36s/it]Test Question: How does RAG handle ambiguous queries where multiple interpretations are possible?\n",
      "Predicted Answer: RAG addresses ambiguous queries by expanding a single query into multiple queries, enriching the context and providing further nuances to ensure optimal relevance in the generated answers. Additionally, techniques such as recursive retrieval and multi-hop retrieval are utilized to refine search results and extract interconnected information from graph-structured data sources. This adaptive approach significantly improves the quality and relevance of retrieved information, catering to a wide array of tasks and queries with enhanced precision and flexibility.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 45.0 / 12  (375.0):  80%|████████  | 12/15 [00:30<00:14,  4.71s/it]Test Question: How does embedding optimization improve retrieval results in RAG?\n",
      "Predicted Answer: Embedding optimization techniques like HyDE focus on embedding similarities between generated answers and real documents, enhancing retrieval relevance in RAG systems. By adjusting module arrangement and interaction, such as in the Demonstrate-Search-Predict framework, RAG systems can dynamically use module outputs to bolster functionality and improve retrieval results. This sophisticated understanding of enhancing module synergy ultimately leads to more effective retrieval in RAG systems.\n",
      "Faithful: 5\n",
      "Detail: 5\n",
      "Overall: 5\n",
      "Average Metric: 49.0 / 13  (376.9):  87%|████████▋ | 13/15 [00:31<00:07,  3.57s/it]Test Question: How does RAG ensure the privacy and security of the data it retrieves and generates?\n",
      "Predicted Answer: RAG addresses privacy and security concerns by focusing on enhancing retrieval efficiency, improving document recall in large knowledge bases, and preventing inadvertent disclosure of document sources or metadata by LLMs. The development of the RAG ecosystem is greatly impacted by the progression of its technical stack, with key tools like LangChain and LLamaIndex playing a crucial role in ensuring data security. Traditional software and cloud service providers are also expanding their offerings to include RAG-centric services, further emphasizing the\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 52.8 / 14  (377.1):  93%|█████████▎| 14/15 [00:32<00:02,  2.76s/it]Test Question: How do different RAG implementations impact the latency of response generation in real-time systems?\n",
      "Predicted Answer: Different RAG implementations impact the latency of response generation in real-time systems differently. RAG offers real-time knowledge updates but comes with higher latency due to ethical considerations regarding data retrieval. In contrast, Fine-tuning (FT) may be more static and require retraining for updates, potentially affecting response generation latency.\n",
      "Faithful: 5\n",
      "Detail: 4\n",
      "Overall: 5\n",
      "Average Metric: 56.599999999999994 / 15  (377.3): 100%|██████████| 15/15 [00:33<00:00,  2.21s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_bc2ab th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_bc2ab td {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_bc2ab_row0_col0, #T_bc2ab_row0_col1, #T_bc2ab_row0_col2, #T_bc2ab_row0_col3, #T_bc2ab_row1_col0, #T_bc2ab_row1_col1, #T_bc2ab_row1_col2, #T_bc2ab_row1_col3, #T_bc2ab_row2_col0, #T_bc2ab_row2_col1, #T_bc2ab_row2_col2, #T_bc2ab_row2_col3, #T_bc2ab_row3_col0, #T_bc2ab_row3_col1, #T_bc2ab_row3_col2, #T_bc2ab_row3_col3, #T_bc2ab_row4_col0, #T_bc2ab_row4_col1, #T_bc2ab_row4_col2, #T_bc2ab_row4_col3 {\n",
       "  text-align: left;\n",
       "  white-space: pre-wrap;\n",
       "  word-wrap: break-word;\n",
       "  max-width: 400px;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_bc2ab\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_bc2ab_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
       "      <th id=\"T_bc2ab_level0_col1\" class=\"col_heading level0 col1\" >context</th>\n",
       "      <th id=\"T_bc2ab_level0_col2\" class=\"col_heading level0 col2\" >answer</th>\n",
       "      <th id=\"T_bc2ab_level0_col3\" class=\"col_heading level0 col3\" >structured_llm_feedback</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_bc2ab_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_bc2ab_row0_col0\" class=\"data row0 col0\" >What role does RAG play in the development of virtual assistants and conversational agents?</td>\n",
       "      <td id=\"T_bc2ab_row0_col1\" class=\"data row0 col1\" >['discussions on whether RAG is still necessary when LLMs are not constrained by context. In fact, RAG still plays an irreplaceable role. On one hand,...</td>\n",
       "      <td id=\"T_bc2ab_row0_col2\" class=\"data row0 col2\" >RAG plays a crucial role in the development of virtual assistants and conversational agents by providing chunked retrieval and on-demand input to improve operational efficiency....</td>\n",
       "      <td id=\"T_bc2ab_row0_col3\" class=\"data row0 col3\" >✔️ [4.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_bc2ab_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_bc2ab_row1_col0\" class=\"data row1 col0\" >How is the integration of domain-specific information handled in RAG systems?</td>\n",
       "      <td id=\"T_bc2ab_row1_col1\" class=\"data row1 col1\" >['University College of Design and Innovation, Tongji University Abstract Large Language Models (LLMs) showcase impressive capabilities but encounter challenges like hallucination, outdated knowledge, and non-transparent,...</td>\n",
       "      <td id=\"T_bc2ab_row1_col2\" class=\"data row1 col2\" >The integration of domain-specific information in Retrieval-Augmented Generation (RAG) systems is achieved by incorporating knowledge from external databases. This enhances the accuracy and credibility of...</td>\n",
       "      <td id=\"T_bc2ab_row1_col3\" class=\"data row1 col3\" >✔️ [3.8]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_bc2ab_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_bc2ab_row2_col0\" class=\"data row2 col0\" >How does RAG contribute to real-world applications such as chatbots?</td>\n",
       "      <td id=\"T_bc2ab_row2_col1\" class=\"data row2 col1\" >['paper introduces up-to-date evaluation framework and benchmark. At the end, this article delineates the challenges currently faced and points out prospective avenues for research and...</td>\n",
       "      <td id=\"T_bc2ab_row2_col2\" class=\"data row2 col2\" >RAG contributes to real-world applications such as chatbots by enhancing the accuracy and reliability of large language models through the retrieval of relevant information from...</td>\n",
       "      <td id=\"T_bc2ab_row2_col3\" class=\"data row2 col3\" >✔️ [3.8]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_bc2ab_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_bc2ab_row3_col0\" class=\"data row3 col0\" >How does RAG deal with the retrieval of contradictory information from different sources?</td>\n",
       "      <td id=\"T_bc2ab_row3_col1\" class=\"data row3 col1\" >['or contradictory information during retrieval can detrimentally affect RAG’s output quality. This situation is figuratively referred to as “Misinformation can be worse than no information...</td>\n",
       "      <td id=\"T_bc2ab_row3_col2\" class=\"data row3 col2\" >RAG may encounter difficulties in handling contradictory information from different sources during retrieval due to precision and recall challenges. This can lead to the selection...</td>\n",
       "      <td id=\"T_bc2ab_row3_col3\" class=\"data row3 col3\" >✔️ [4.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_bc2ab_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_bc2ab_row4_col0\" class=\"data row4 col0\" >How is external knowledge integrated into the generation process in RAG systems?</td>\n",
       "      <td id=\"T_bc2ab_row4_col1\" class=\"data row4 col1\" >['University College of Design and Innovation, Tongji University Abstract Large Language Models (LLMs) showcase impressive capabilities but encounter challenges like hallucination, outdated knowledge, and non-transparent,...</td>\n",
       "      <td id=\"T_bc2ab_row4_col2\" class=\"data row4 col2\" >External knowledge is integrated into the generation process in RAG systems through methods like Flare and Self-RAG, which refine the framework by determining optimal moments...</td>\n",
       "      <td id=\"T_bc2ab_row4_col3\" class=\"data row4 col3\" >✔️ [4.0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x14f06d0a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <div style='\n",
       "                    text-align: center;\n",
       "                    font-size: 16px;\n",
       "                    font-weight: bold;\n",
       "                    color: #555;\n",
       "                    margin: 10px 0;'>\n",
       "                    ... 10 more rows not displayed ...\n",
       "                </div>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "377.33"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_with_llm_feedback(compiled_rag, metric=metric)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}