cross_encoder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dotenv import load_dotenv, find_dotenv\n",
    "from langchain_community.document_loaders import TextLoader, DirectoryLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_community.vectorstores import Chroma\n",
    "load_dotenv(find_dotenv())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading the txt files from source directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = DirectoryLoader('./source', glob=\"./*.txt\", loader_cls=TextLoader)\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Chunks using RecursiveCharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=100,\n",
    "    chunk_overlap=10,\n",
    "    length_function=len\n",
    ")\n",
    "new_docs = text_splitter.split_documents(documents=docs)\n",
    "doc_strings = [doc.page_content for doc in new_docs]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  BGE Embddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "\n",
    "model_name = \"BAAI/bge-base-en-v1.5\"\n",
    "model_kwargs = {'device': 'cpu'}\n",
    "encode_kwargs = {'normalize_embeddings': True} # set True to compute cosine similarity\n",
    "embeddings = HuggingFaceBgeEmbeddings(\n",
    "    model_name=model_name,\n",
    "    model_kwargs=model_kwargs,\n",
    "    encode_kwargs=encode_kwargs,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Vectors from the Doc String using Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = embeddings.embed_documents(doc_strings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Retriever using Vector DB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "db = Chroma.from_documents(new_docs, embeddings)\n",
    "retriever = db.as_retriever(search_kwargs={\"k\": 4})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### UMAP for Visulation of Vector Space along with Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! pip install umap-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aritra.sen/Aritra/coding/my_llm_app/.venv/lib/python3.10/site-packages/umap/umap_.py:1943: UserWarning: n_jobs value -1 overridden to 1 by setting random_state. Use no seed for parallelism.\n",
      "  warn(f\"n_jobs value {self.n_jobs} overridden to 1 by setting random_state. Use no seed for parallelism.\")\n"
     ]
    }
   ],
   "source": [
    "import umap.umap_ as umap\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "umap_transformer = umap.UMAP(random_state=0, transform_seed=0).fit(vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[12.246204 ,  3.7773688]], dtype=float32)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "umap_transformer.transform([vectors[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embed_docs(vectors,umap_transformer):\n",
    "    \"\"\"Function to transform the vectors to 2 dimnesional vectors using UMAP\"\"\"\n",
    "    return np.array([umap_transformer.transform([vector])[0] for vector in vectors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Base UPMAP vectors for all the chunks\n",
    "base_umap_vectors = embed_docs(vectors=vectors,umap_transformer=umap_transformer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8.223255252838134, 13.836990261077881, -4.899656009674072, 4.535670948028565)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGbCAYAAABZBpPkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAY20lEQVR4nO3df5DVZd038M8CpYkJOwIBA4usKDdgI6s92VC05Ui0ajpk2jSloM3UQtnwR8Y6Nak5pVDO+NyMxB+NmGN3Yyhr/tEa5n1Lj0N1+8j65IykMqDrrzJwUUFLhe/zRw/7ePYXu8s5e77fc71eM/xxrnPO91xnz+h5n+v6XNdVl2VZFgBAssZUuwMAQHUJAwCQOGEAABInDABA4oQBAEicMAAAiRMGACBxwgAAJE4YAIDECQNQAKecckpceOGFFX+dZ599Nurq6uKOO+446mNXrFgRp5xySklbXV1dXH/99RXpG1A5wgAcgzvuuCPq6uoG/PfHP/6x2l0EOKpx1e4A1IIf/OAHMXv27D7tc+bMqUJvquett96KceP8bwWKxn+1UAYtLS3xkY98pNrdqLrjjz++2l0ARsA0AVTYkXn4n/zkJ3HbbbdFY2NjnHDCCfGZz3wmnn/++ciyLG688caYMWNGfOADH4iLL744Xn311X6vtXXr1li4cGEcf/zxMX/+/NiyZUufx+zfvz9Wr14dM2fOjOOOOy7mzJkTa9eujcOHD/d53IoVK2LChAkxceLEWL58eezfv7/f173vvvvijDPOiOOPPz7OOOOMaG9v7/dxvWsGrr/++qirq4tdu3bFihUrYuLEiTFhwoS48sor48033yx57ltvvRXf+ta3YtKkSfHBD34wLrroonjxxRf7XPONN96I1atXxymnnBLHHXdcTJkyJZYsWRI7duzot0/A0RkZgDJ47bXXYu/evSVtdXV1cfLJJ/fc/sUvfhFvv/12XH311fHqq6/GunXr4rLLLotzzz03Hn744VizZk3s2rUr1q9fH9/+9rfj9ttvL7neM888E1/84hejtbU1li9fHps2bYpLL700HnjggViyZElERLz55pvR3NwcL774Ynz961+PhoaG2L59e1x77bXx8ssvx6233hoREVmWxcUXXxyPPPJItLa2xrx586K9vT2WL1/e571t3bo1Lrnkkpg/f37cdNNNsW/fvrjyyitjxowZQ/77XHbZZTF79uy46aabYseOHfGzn/0spkyZEmvXru15zIoVK+JXv/pVXH755fGxj30stm3bFhdccEGfa7W2tsY999wT3/zmN2P+/Pmxb9++eOSRR2Lnzp1x1llnDblPwHtkwIht2rQpi4h+/x133HFZlmXZnj17sojIJk+enO3fv7/nuddee20WEdmZZ56ZvfPOOz3tX/rSl7L3v//92T/+8Y+etlmzZmURkd177709ba+99lo2bdq0rKmpqaftxhtvzMaPH589/fTTJf1sa2vLxo4dm3V1dWVZlmX33XdfFhHZunXreh7z7rvvZosXL84iItu0aVNP+8KFC7Np06aV9H3r1q1ZRGSzZs0qeZ2IyK677rqe29ddd10WEdlVV11V8rhly5ZlJ598cs/txx57LIuIbPXq1SWPW7FiRZ9rTpgwIfvGN76RAeVjmgDK4LbbbosHH3yw5F9HR0fJYy699NKYMGFCz+1zzjknIiK+8pWvlBTdnXPOOfH222/Hiy++WPL86dOnx7Jly3pun3TSSXHFFVdEZ2dn/PWvf42IiM2bN8fixYujvr4+9u7d2/PvvPPOi0OHDsXvf//7iIj4zW9+E+PGjYuVK1f2XG/s2LFx9dVXl7zmyy+/HI8//ngsX768pO9LliyJ+fPnD/nv09raWnJ78eLFsW/fvnj99dcjIuKBBx6IiIhVq1aVPK53fyIiJk6cGH/605/ipZdeGvLrA4MzTQBl8NGPfvSoBYQNDQ0lt498uc6cObPf9u7u7pL2OXPmRF1dXUnb6aefHhH/qkuYOnVqPPPMM/HnP/85Jk+e3G8fXnnllYiIeO6552LatGlx4oknltw/d+7cktvPPfdcREScdtppfa41d+7cIc/T937v9fX1EfGv93jSSSfFc889F2PGjOmzIqO/1Rjr1q2L5cuXx8yZM+Pss8+O888/P6644opobGwcUl+AvoQBGCVjx44dVnuWZcN+jcOHD8eSJUviO9/5Tr/3HwkPo62c7/Gyyy6LxYsXR3t7e2zdujV+/OMfx9q1a2PLli3R0tJyrF2FJAkDUBC7du2KLMtKRgeefvrpiIienQBPPfXUOHDgQJx33nmDXmvWrFnx0EMPxYEDB0pGB5566qk+j4v4V/Fib70feyxmzZoVhw8fjj179pSMQuzatavfx0+bNi1WrVoVq1atildeeSXOOuus+OEPfygMwAipGYCCeOmll0qW9L3++utx5513xsKFC2Pq1KkR8a9fzX/4wx/it7/9bZ/n79+/P959992IiDj//PPj3XffjZ/+9Kc99x86dCjWr19f8pxp06bFwoUL4+c//3m89tprPe0PPvhgPPnkk2V7b0uXLo2IiA0bNpS09+7PoUOHSvoRETFlypSYPn16/POf/yxbfyA1RgagDDo6OuIvf/lLn/ZFixbFmDHlydynn356fPWrX41HH300PvShD8Xtt98ef/vb32LTpk09j7nmmmvi/vvvjwsvvDBWrFgRZ599dhw8eDCeeOKJuOeee+LZZ5+NSZMmxec+97n4+Mc/Hm1tbfHss8/27FnQ+4s2IuKmm26KCy64ID7xiU/EVVddFa+++mqsX78+FixYEAcOHCjLezv77LPjkksuiVtvvTX27dvXs7TwyMjHkdGQN954I2bMmBFf+MIX4swzz4wTTzwxfve738Wjjz4at9xyS1n6AikSBqAMvv/97/fbvmnTpvjUpz5Vltc47bTTYv369XHNNdfEU089FbNnz467776751d1RMQJJ5wQ27Ztix/96EexefPmuPPOO+Okk06K008/PW644Yae4sQxY8bE/fffH6tXr4677ror6urq4qKLLopbbrklmpqaSl73s5/9bGzevDm+973vxbXXXhunnnpqbNq0KX7961/Hww8/XJb3FhFx5513xtSpU+OXv/xltLe3x3nnnRd33313zJ07t2dnwxNOOCFWrVoVW7dujS1btsThw4djzpw5sWHDhpKVEcDw1GUjqeABGAWPP/54NDU1xV133RVf/vKXq90dqFlqBoBceOutt/q03XrrrTFmzJj45Cc/WYUeQTpMEwC5sG7dunjsscfi05/+dIwbNy46Ojqio6Mjvva1r/XZiwEoL9MEQC48+OCDccMNN8STTz4ZBw4ciIaGhrj88svju9/9rmORocKEAQBInJoBAEicMAAAiRMGACBxwgAAJE4YAIDECQMAkDhhAAASJwwAQOKEAQBInDAAAIkTBgAgccIAACROGACAxAkDAJA4YQAAEicMAEDixlW7A0Dt6+zqjj17D8bsSeOjqaG+2t0BehEGgIq6uWNnbNy2u+d2a3NjtLXMq2KPgN5MEwAV09nVXRIEIiI2btsdnV3dVeoR0B9hAKiYPXsPDqsdqA5hAKiY2ZPGD6sdqA41A8Axe2+BYESUFAu2NjeWTBWsbG5URAg5U5dlWVbtTgDF1btA8L2OFAtaTQD5JgwAI9bZ1R3LNmwf9DHtqxYJAJBzagaAERtKIaBiQcg/YQAYsaEUAioWhPwTBoARO1IgOBDFglAMagaAQQ2l+G+w1QRA/gkDwIBsJQxpME0A9KsoWwl3dnXHlh0v5K5fUCQ2HQL6NdhWwnkZ/jdyAeVhZADoV963Ei7KyAUUgTAA9Ku/lQJ5Wh3gECQoH9MEkKihrBJoa5kXSxdMzeXqgLyPXECRWE0ACaqVufbe72Nlc2OsOYb34QwFUiUMQGIGOk+gqGcIlOsLvFYCEoyEmgFITK3NtTc11Mfnz5pxTEFAMSKpEwYgEUfW479z6HC/96c8115rAQmGSwEh1LjOru7494eeif966u89bQtnTojHn3+t53aeVglUg2JEUqdmAGpY73nw91p7yYfjfWPHKJb7f8pdjAhFYmQAalR/8+Dv9b6xY+LzZ80YxR7l23CWUVp1QK0RBqBG/ftDzwx6vyHwvpoa6o/65W7VAbVIASHUoM6u7pIagd5SrxEYKasOqFVGBqAGDVQFP2/aB+NHyz4sCIxQEQ5vgpEwMgA1aKApAEHg2Fh1QK0SBqAG5f2QoaIa7O96ZB8HUwYUkaWFUMNUvVdG77+rokKKThgAOAa1dtYDaTJNAHAMbGVMLRAGAI6BokJqgTAAcAwUa1IL1AxADVAoWH0+A4pMGICCU8kOHCvTBFBgtscFykEYgAJTyQ6UgzAABaaSHSgHYQAKTCU7UA4KCKEGqGQHjoUwAACJM00AAIkTBgAgccIAACROGACAxAkDAJC4cdXuADB0lhAClSAMQEE4kAioFNMEUAAOJKp9nV3dsWXHCz5TqsLIABTAYAcSmS4oPqM+VJuRASgABxLVLqM+5IEwAAXQ1FAfy5qml7Q5kKg2OIaaPBAGoABu7tgZ7Z0v9dxe1jQ91hhGrglGfcgDYQByrr9h5PbOlwwj1wjHUJMHCggh5xQP1r62lnmxdMFUe0hQNcIA5Jxh5DQ0NdQLAVSNaQLIOcPIQKXVZVmWVbsTwNHZihioFGEAABJnmgAAEicMAEDihAEASJwwAACJEwYAIHHCAAAkzg6EkHP2FwAqTRiAHLu5Y2fJIUWtzY3R5rRCoMxME0BO9Xda4cZtu51WCJSdMAA5NdhphQDlJAxATjmtEBgtwgDklNMKgdHioCLIOasJgEoTBgAgcaYJACBxwgAAJE4YAIDECQMAkDjbEQPkiNUjVIMwAJATzqKgWkwTAOSAsyioJmEAIAecRUE1CQMAOeAsCqpJGADIAWdRUE22IwbIEasJqAZhAAASZ2khFIhfjbXF50leCANQENag1xafJ3migBAKwBr02uLzJG+EASgAa9Bri8+TvBEGoACsQa8tPk/yRhiAArAGvbb4PMkbSwuhQFSf1xafJ3khDABA4iwtBBglRgLIK2EAYBT03lfg3H+bHFefe5pQQC6YJgCosM6u7li2YXu/9wkF5IHVBFAQnV3dsWXHCzamKaDB9g/4z7/8PZZt2B43d+wcxR5BKdMEUAC2ri22oewfsHHb7li6YKoRAqrCyADknK1ri6+/fQX6YwdCqkUYgJyzdW1taGuZF+2rFsWn504e8DF2IKRahAHIuXcOHR5WO/nV1FAfm678aL+hwA6EVJOaAci5943tP7MP1E7+HQkF9h0gL4QByDmH2tSupoZ6IYBc8NMCcs6hNkCl2XQICsKQMlApwgAAJM40AQAkThgAgMQJAwCQOGEAABInDABA4oQBAEicMAAAiRMGACBxwgAAJE4YAIDECQMAkDhhAAASJwwAQOKEAQBInDAAAIkTBgAgccIAACROGACAxAkDAJA4YQAAEicMAEDihAEASJwwAACJEwYAIHHCAAAkbly1OwBQazq7umPP3oMxe9L4aGqor3Z34KiEAYAyurljZ2zctrvndmtzY7S1zKtij+DoTBMAlElnV3dJEIiI2Lhtd3R2dVepRzA0wgDAEHV2dceWHS8M+OW+Z+/BYbVDXpgmABiCoQz/z540vt/nDtQOeWFkAOAohjr839RQH63NjSVtK5sbFRGSe0YGAI5isOH/3l/0bS3zYumCqVYTUCjCAMBRDHf4v6mhXgigUEwTAByF4X9qXV2WZVm1OwFQBDYTolYJAwCQONMEAJA4BYRQEIaogUoRBqAAem94c+6/TY6rzz1NKKgwAYxUqBmAnOvs6o5lG7b3e59DcCrHgUOkRM0A5Nxg+9o7BKcyHDhEaoQByLmj7WvvEJzyc+AQqREGIOf62/DmvRyCU34OHCI1wgAUQFvLvGhftSg+PXdySbtd8CrDjoOkRgEhFIwK99Hjb00qhAEASJxpAgBInDAAAIkTBgAgccIAACROGACAxAkDAJA4YQAAEucIY6hRNswBhkoYgBrk+F1gOEwTQI1x/C4wXMIA1BjH7wLDJQxAjXH8LjBcwgDUGMfvAsPl1EKoUdVcTWAlAxSLMACUlZUMUDymCYCyKedKhs6u7tiy4wWrIGAU2GcAKJvBVjIMZ7rA6AKMLiMDQNmUYyWDfRJg9AkDQNmUYyWDfRJg9JkmAMqqrWVeLF0wdcSrCeyTAKPPagIgd3rXDKxsbow1agagYoQBIJfsVQCjRxgAgMQpIASAxAkDAJA4YQAAEicMAEDihAEASJwwAACJEwYAIHHCAAAkThgAgMQJAwCQOGEAABInDABA4oQBAEicMAAAiRMGACBxwgAAJE4YAIDECQMAkDhhAAASJwwAQOKEAQBInDAAAIkTBgAgccIAACROGACAxAkDAJA4YQAAEicMAEDihAEASJwwAACJEwYAIHHCAAAkThgAgMQJAwCQOGEAABInDABA4oQBAEicMAAAiRMGACBxwgAAJE4YAIDECQMAkDhhAAASJwwAQOKEAQBInDAAAIkTBgAgccIAACROGACAxAkDAJA4YQAAEicMAEDihAEASNy4ancAiqKzqzv27D0YsyeNj6aG+mp3B6BshAEYgps7dsbGbbt7brc2N0Zby7yyXFvIAKqtLsuyrNqdgDzr7OqOZRu292lvX7XomL+8KxkyAIZKzQAcxZ69B/tt3/y/n4/Oru4RX7ezq7skCEREbNy2+5iuCTASwgAcxTuHDvfb/h///Xws27A9bu7YOaLrDhQyBmoHqBRhAAZxc8fOWHPvE4M+ZqS/5mdPGj+sdoBKEQZgAP0N4w9kJL/mmxrqo7W5saRtZXOjIkJg1FlNAAMYzhf8SH/Nt7XMi6ULplpNAFSVMAADGOgLflnT9GjvfKnn9rH+mm9qqBcCgKqytBAG0Xvp38rmxljTMs/eAEBNEQbgKHzxA7VOGACAxFlNAACJEwYAIHHCAAAkThgAgMTZZ4CaZzUAwOCEAWqaI4IBjs40ATXLEcEAQyMMULMcEQwwNMIANet/PfP3ftsdEQxQShigJnV2dZccJnTE55umKyIE6EUYoCYNNBXwidMmj3JPAPJPGKAmDTQVYIoAoC9hgJrU1FAfrc2NJW0rmxtNEVRQZ1d3bNnxgtUaUEBOLaSm2XBodNjPAYpNGACOSWdXdyzbsL1Pe/uqRQIYFIRpAuCY2M8Bik8YAI6JYk0oPmGAwlGoli+KNaH41AxQKArV8kuxJhSXMEBhKFQDqAzTBBSGQjWAyhAGKAyFagCVIQxQGArVACpDzQCFo1ANoLyEAQBInGkCAEicMAAAiRtX7Q7AUKgTAKgcYYDcs+sgQGWZJiDXOru6S4JARMTGbbudS1AQzpGAYjAyQK4Ntuug6YJ8M6IDxWFkgFyz62AxGdGBYhEGyDW7DhaTcySgWEwTkHttLfNi6YKpVhMUiBEdKBY7EJI7lhHWht41AyubG2ONmgHIJWGAXFF0VlsEOygGYYDc6OzqjmUbtvdpb1+1yBcJQAUpICQ3FJ3VFnsMQHEoICQ3FJ3VDtM9UCxGBsgNywhrgz0GoHiMDJArlhEWn10joXiEAXKnqaHel0aBme6B4jFNAJSV6R4oHksLgYqwxwAUhzAAAIkzTQAAiRMGACBxwgAAJE4YAIDECQMAkDibDjHqLDkDyBdhgFHlABuA/DFNwKhxgA1APgkDjJrBDrABoHqEAUaNA2wA8kkYYNQ4wAYgn5xNwKizmgAgX4QBAEicaQIASJwwAACJEwYAIHHCAAAkThgAgMQJAwCQOGEAABInDABA4oQBAEicMAAAiRMGACBxwgAAJE4YAIDEjat2B4D8ctw0pEEYAPp1c8fO2Lhtd8/t1ubGaGuZV8UeAZVimgDoo7OruyQIRERs3LY7Oru6q9QjoJKEAaCPPXsPDqsdKDZhAOhj9qTxw2oHik0YAPpoaqiP1ubGkraVzY2KCKFG1WVZllW7E0B1DbRqwGoCSIMwAImzagAwTQAJs2oAiBAGIGlWDQARwgAkzaoBIEIYgKRZNQBEKCAEwqoBSJ0wAACJM00AAIkTBgAgccIAACROGACAxAkDAJC4cdXuADA0lv8BlSIMQAE4TAioJPsMQE4dGQl459DhWHPvE33ub1+1yAgBUBZGBiCHeo8E9GfP3oPCAFAWwgDJ6uzqjoefeiUiIj41d0rJF2s55+eHe63+jhXuj8OEgHIRBkhS71/e//OhXT3z8OWcnx/JtYZyfLDDhIByEgZIzkC/vDdu2x2zJ43vc9/Gbbtj6YKpw/7y7e91hnKtgX7xr73kw/G+sWOsJgDKzj4DJGewX97/5/n9w37OcF/naNca6FjhL/6Phvj8WTMEAaDsjAyQnMHm2s+cOTH+47+fH9Zzhvs6Q7lWW8u8WLpgqn0FgFFhZIDk9PfLO+L///ru71f5SL6MB/qFP9RrNTXUGwkARoV9BkhWXlcTAIw2YQAAEmeaAAASJwwAQOKEAQBInDAAAIkTBgAgccIAACROGACAxAkDAJA4YQAAEicMAEDihAEASJwwAACJEwYAIHHCAAAkThgAgMQJAwCQOGEAABInDABA4oQBAEicMAAAiRMGACBxwgAAJO7/AlnqQnetm3TzAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(base_umap_vectors[:, 0], base_umap_vectors[:, 1], s=10)\n",
    "plt.gca().set_aspect('equal', 'datalim')\n",
    "plt.title('Embeddings')\n",
    "plt.axis('off')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function to Visualize the embeddings with the Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_query_with_base_embeddings(query, embeddings, retriever, umap_transformer, base_umap_vectors):\n",
    "    q_embedding = embeddings.embed_query(query)\n",
    "\n",
    "    docs = retriever.get_relevant_documents(query)\n",
    "    page_contents = [doc.page_content for doc in docs]\n",
    "    retrieved_documents = embeddings.embed_documents(page_contents)\n",
    "\n",
    "    query_embeddings = embed_docs([q_embedding], umap_transformer)\n",
    "    retrieved_embeddings = embed_docs(retrieved_documents, umap_transformer)\n",
    "\n",
    "    plt.figure()\n",
    "    plt.scatter(base_umap_vectors[:, 0], base_umap_vectors[:, 1], s=10, color='gray')\n",
    "    plt.scatter(query_embeddings[:, 0], query_embeddings[:, 1], s=150, marker='X', color='r')\n",
    "    plt.scatter(retrieved_embeddings[:, 0], retrieved_embeddings[:, 1], s=50, facecolors='none', edgecolors='g')\n",
    "    plt.gca().set_aspect('equal', 'datalim')\n",
    "    plt.title(f'{query}')\n",
    "    plt.axis('off')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGbCAYAAABZBpPkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnjUlEQVR4nO3de3TT5eHH8U9aSgsUKLQF1AClxUmHChNU1q7UuQbYQKYTOqebMItDUZnzurkdAZljXnB439ivBzsn4gGFHcEBZSqWdXiZWkGcUrn4iziByk2xXNrn90d/jU2TtmmbNt/keb/O4WiefPPNk3zb5pPn6jLGGAEAAGvFRboCAAAgsggDAABYjjAAAIDlCAMAAFiOMAAAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlCAMx5IILLtAFF1zgu71r1y65XC498cQTEatTOM2dO1cul0v79++PdFUQIfU/A07y8ssvy+Vy6eWXX271Y534ejpK/d+j+++/P9JVQRCEgQhzuVwh/WvLHxqEz7Zt2zR37lzt2rUrpOOjNbg88cQTYftwuuCCC3TmmWeG5VydZenSpVq0aFGkq9GksrIyFRYW6rTTTlPXrl3Vu3dvnX/++brrrrv06aefRrp6kqQXXnhBc+fOjXQ10EpdIl0B2z355JN+t//yl7+otLQ0oDw7O7szq4VGtm3bpnnz5umCCy5QRkZGpKuDDrJ06VJt3bpVN954Y8iPGTt2rL788kt17dq14yom6c4779T8+fOVmZmp6dOnKzMzU9XV1fr3v/+thQsXqqSkRB9++GGH1iEUL7zwgh599FECQZQhDETYj3/8Y7/bmzdvVmlpaUA5AGeprq5W165dFRcXp6SkpA59rmeeeUbz589XYWGhnnzyyYDg8Yc//EF/+MMfmj2HMUbV1dXq1q1bR1YVUYpugihQW1urRYsWafjw4UpKSlL//v01c+ZMHThwICznr28a3rRpk2bPnq309HSlpKRo5syZOn78uA4ePKgrr7xSffr0UZ8+fXTbbbep8WaX7anjO++84/umk5SUpAEDBuiqq65SVVVV0OP379+vwsJC9erVS6mpqfr5z3+u6upqv2NOnjyp+fPnKysrS4mJicrIyNAdd9yhY8eO+R3ncrmCfoPJyMjQ9OnTfe/P1KlTJUnf/va329x1U1ZWpqlTp2rQoEFKTEzUwIED9Ytf/EJffvml33HTp09XcnKyduzYofHjx6tHjx469dRTdddddwW87/fff79ycnKUmpqqbt26adSoUVqxYkXAc7tcLl1//fVatWqVzjzzTCUmJmr48OFau3Zti/V+4403NH78eKWlpalbt24aMmSIrrrqqla99rbUY9OmTTr33HOVlJSkrKws/elPfwo4prlxMY2v7ZEjR3TjjTcqIyNDiYmJ6tevnzwej958801Jdd0aa9as0e7du33XuL4VqH5cwLJly/Sb3/xGp512mrp3767Dhw8HHTMQ6rUO1Z133qm0tDQVFxcHbYHo3bt3wM9xRkaGJk2apHXr1mn06NHq1q2b7z3csWOHpk6dqr59+6p79+4aM2aM1qxZ43usMUZpaWm66aabfGW1tbVKSUlRfHy8Dh486Cu/55571KVLF33++eeaPn26Hn30UUn+XaCNLV682Pe7ee655+r1119v0/uC8KFlIArMnDlTTzzxhH76059q9uzZ2rlzpx555BG99dZb+uc//6mEhISwPM8NN9ygAQMGaN68edq8ebMWL16slJQUlZeXa9CgQfrd736nF154Qffdd5/OPPNMXXnllWGpY2lpqXbs2KGf/vSnGjBggN59910tXrxY7777rjZv3hzwx6SwsFAZGRlasGCBNm/erIceekgHDhzQX/7yF98xM2bMUElJiaZMmaKbb75Zr776qhYsWKD33ntPK1eubNX7MnbsWM2ePVsPPfSQ7rjjDl+XTWu7bpYvX66jR4/q2muvVWpqql577TU9/PDD8nq9Wr58ud+xNTU1mjBhgsaMGaN7771Xa9eu1Zw5c3Ty5EndddddvuMefPBBTZ48WVdccYWOHz+uZcuWaerUqVq9erUmTpzod85Nmzbpueee06xZs9SzZ0899NBDuvTSS/XRRx8pNTU1aJ337t2rcePGKT09Xb/85S+VkpKiXbt26bnnnmvVa29tPbZs2eJ73rlz5+rkyZOaM2eO+vfv3+bnveaaa7RixQpdf/31+vrXv66qqipt2rRJ7733ns455xz9+te/1qFDh+T1en3fspOTk/3OMX/+fHXt2lW33HKLjh071mTXQGuudUs++OADffDBB5oxY0ZAfVry/vvv60c/+pFmzpypq6++WmeccYY+/fRT5eTk6OjRo5o9e7ZSU1NVUlKiyZMna8WKFbrkkkvkcrmUm5urV155xXeud955R4cOHVJcXJz++c9/+n6+ysrK9I1vfEPJycmaOXOm9uzZE7Srs97SpUt15MgRzZw5Uy6XS/fee69+8IMfaMeOHWH7W4Y2MHCU6667zjS8LGVlZUaSeeqpp/yOW7t2bUB5fn6+yc/P993euXOnkWSWLFnS7HMuWbLESDLjx483tbW1vvJvfvObxuVymWuuucZXdvLkSeN2u/2epzV1DObo0aMBZU8//bSRZF555RVf2Zw5c4wkM3nyZL9jZ82aZSSZiooKY4wxb7/9tpFkZsyY4XfcLbfcYiSZF1980VcmycyZMyfg+QcPHmymTZvmu718+XIjybz00kvNvpbGdd23b1+zr3PBggXG5XKZ3bt3+8qmTZtmJJkbbrjBV1ZbW2smTpxounbt2uw5jx8/bs4880xz4YUX+pVLMl27djWVlZW+soqKCiPJPPzww02+jpUrVxpJ5vXXXw/hVfvLz883w4cPb1M9Lr74YpOUlOT3vmzbts3Ex8f7/X409zPe+Nr27t3bXHfddc3WeeLEiWbw4MEB5S+99JKRZDIzMwPe8/r7Gv5shHqt639OmvO3v/3NSDKLFi3yK6+trTX79u3z+3fixAnf/YMHDzaSzNq1a/0ed+ONNxpJpqyszFd25MgRM2TIEJORkWFqamqMMcbcd999Jj4+3hw+fNgYY8xDDz1kBg8ebM477zxz++23G2OMqampMSkpKeYXv/iF71yN/4bVq79Wqamp5rPPPgt4fc8//3yz7wM6Ft0EDrd8+XL17t1bHo9H+/fv9/0bNWqUkpOT9dJLL4XtuYqKivy+hZ9//vkyxqioqMhXFh8fr9GjR2vHjh1hq2PDPszq6mrt379fY8aMkSRfE25D1113nd/tG264QVLdwKWG/23YxClJN998syT5NYd2poav84svvtD+/fuVk5MjY4zeeuutgOOvv/563//XN68fP35cGzZsCHrOAwcO6NChQ8rLywv6vhUUFCgrK8t3++yzz1avXr38rmVjKSkpkqTVq1frxIkTob3QFrRUj5qaGq1bt04XX3yxBg0a5DsuOztb48ePb/PzpqSk6NVXX9WePXvafI5p06aF1Ofe2mvdnMOHD0sKbKU4dOiQ0tPT/f69/fbbfscMGTIk4D174YUXdN555+lb3/qWryw5OVk/+9nPtGvXLm3btk2SlJeXp5qaGpWXl0uqawHIy8tTXl6eysrKJElbt27VwYMHlZeXF/Lr+eEPf6g+ffr4btc/trmfQ3Q8woDDbd++XYcOHVK/fv0CfvE///xz7d27N2zP1fAPr1TXDylJAwcODChvOBagvXX87LPP9POf/1z9+/dXt27dlJ6eriFDhkiq+4PX2Omnn+53OysrS3Fxcb5pf7t371ZcXJyGDh3qd9yAAQOUkpKi3bt3N1ufjvLRRx9p+vTp6tu3r5KTk5Wenq78/HxJga8zLi5OmZmZfmVf+9rXJMlveuPq1as1ZswYJSUlqW/fvkpPT9fjjz8e9H1rfH0lqU+fPs2O68jPz9ell16qefPmKS0tTd///ve1ZMmSgLEXrdFSPfbt26cvv/wy4DpL0hlnnNHm57333nu1detWDRw4UOedd57mzp3b6g+g+p/LlrTmWrekZ8+ekqTPP//crzw5OVmlpaUqLS3VrbfeGnJ9d+/eHfR9rO/2qv/9OOecc9S9e3ffB399GBg7dqzeeOMNVVdX++5rGCxa0vj61weDcI2BQtswZsDhamtr1a9fPz311FNB709PTw/bc8XHx4dcbhoMZGtvHQsLC1VeXq5bb71VI0eOVHJysmprazVhwgTV1ta2WO+m5sW3Z758TU1Nmx/b1Pk8Ho8+++wz3X777Ro2bJh69Oihjz/+WNOnTw/pdTZWVlamyZMna+zYsXrsscd0yimnKCEhQUuWLNHSpUsDjm/q+ppGgxIbcrlcWrFihTZv3qznn39e69at01VXXaWFCxdq8+bNre7Dbms9mqtfMMGuX2FhofLy8rRy5UqtX79e9913n+655x4999xz+u53vxvS84XSKhDuaz1s2DBJdd/CG+rSpYsKCgokSV6vt831bUpCQoLOP/98vfLKK6qsrNR///tf5eXlqX///jpx4oReffVVlZWVadiwYa36OxTO64/wIQw4XFZWljZs2KDc3FzHTglqTx0PHDigf/zjH5o3b57uvPNOX/n27dubfMz27dv9vvFUVlaqtrbWN/J78ODBqq2t1fbt2/0G+X366ac6ePCgBg8e7Cvr06eP38hoSTp+/Lg++eQTv7L2LsSzZcsWffDBByopKfEbeFlaWhr0+NraWu3YscPXGiDVDSST5Hudzz77rJKSkrRu3TolJib6jluyZEm76hrMmDFjNGbMGN19991aunSprrjiCi1btkwzZswI+3Olp6erW7duQX8G3n//fb/b9d8qG1/Dplp/TjnlFM2aNUuzZs3S3r17dc455+juu+/2hYFwLLjU2mvdkjPOOEOnn366Vq1apUWLFqlHjx7tqt/gwYMD3kdJ+s9//uO7v15eXp7uuecebdiwQWlpaRo2bJhcLpeGDx+usrIylZWVadKkSX7nsWVFxVhDN4HDFRYWqqamRvPnzw+47+TJkwF/BCOhPXWs/5bQ+FtBc6vA1U9dqvfwww9Lku8P+ve+972g53jggQckyW+UfVZWlt+Iaalu2lPjb5b1f4Db+n4He53GGD344INNPuaRRx7xO/aRRx5RQkKCvvOd7/jO6XK5/Oq6a9curVq1qk11DObAgQMB12bkyJGS1K6ugubEx8dr/PjxWrVqlT766CNf+Xvvvad169b5HdurVy+lpaUFXMPHHnvM73ZNTU1A83y/fv106qmn+r2OHj16tLoZP1j9pdZd65bMnTtX+/fv19VXXx107EZrvlV/73vf02uvvaZ//etfvrIvvvhCixcvVkZGhr7+9a/7yvPy8nTs2DEtWrRI3/rWt3wf9Hl5eXryySe1Z8+egPEC7f1dQWTQMuBw+fn5mjlzphYsWKC3335b48aNU0JCgrZv367ly5frwQcf1JQpU6K2jr169dLYsWN177336sSJEzrttNO0fv167dy5s8nn27lzpyZPnqwJEyboX//6l/7617/q8ssv14gRIyRJI0aM0LRp07R48WIdPHhQ+fn5eu2111RSUqKLL75Y3/72t33nmjFjhq655hpdeuml8ng8qqio0Lp165SWlub3nCNHjlR8fLzuueceHTp0SImJibrwwgvVr1+/kN6jYcOGKSsrS7fccos+/vhj9erVS88++2yT/aRJSUlau3atpk2bpvPPP19///vftWbNGt1xxx2+JtmJEyfqgQce0IQJE3T55Zdr7969evTRRzV06FC98847IdWrJSUlJXrsscd0ySWXKCsrS0eOHNGf//xn9erVyxe6OsK8efO0du1a5eXladasWTp58qQefvhhDR8+POC1zZgxQ7///e81Y8YMjR49Wq+88oqvFaXekSNH5Ha7NWXKFI0YMULJycnasGGDXn/9dS1cuNB33KhRo/TMM8/opptu0rnnnqvk5GRddNFFrap7a691KC6//HJt3bpVCxYs0GuvvabLLrtMQ4YM0RdffKGtW7fq6aefVs+ePf0G5jXll7/8pZ5++ml997vf1ezZs9W3b1+VlJRo586devbZZxUX99V3xG9+85vq0qWL3n//ff3sZz/zlY8dO1aPP/64JAWEgVGjRkmSZs+erfHjxys+Pl6XXXZZm187OkkEZjCgGU1Ny1m8eLEZNWqU6datm+nZs6c566yzzG233Wb27NnjO6a9UwsbTx8LNj3OmLqpbz169GhTHYPxer3mkksuMSkpKaZ3795m6tSpZs+ePQFTw+rrs23bNjNlyhTTs2dP06dPH3P99debL7/80u+cJ06cMPPmzTNDhgwxCQkJZuDAgeZXv/qVqa6u9juupqbG3H777SYtLc10797djB8/3lRWVgZMLTTGmD//+c8mMzPTN72tuWmGd955p5HkN4Vq27ZtpqCgwCQnJ5u0tDRz9dVX+6bVNbxG9e/vhx9+aMaNG2e6d+9u+vfvb+bMmeOb9lWvuLjYnH766SYxMdEMGzbMLFmyJOh0NUlBp9UFe50Nvfnmm+ZHP/qRGTRokElMTDT9+vUzkyZNMm+88UaTj6nX1NTCUOuxceNGM2rUKNO1a1eTmZlp/vjHPwZ9bUePHjVFRUWmd+/epmfPnqawsNDs3bvX7+fn2LFj5tZbbzUjRowwPXv2ND169DAjRowwjz32mN+5Pv/8c3P55ZeblJQUI8k3zbB++uDy5csD6h5samGo1zqUqYUNvfzyy2bKlCnmlFNOMQkJCaZXr15m9OjRZs6cOeaTTz4JeE8nTpwY9DwffvihmTJliklJSTFJSUnmvPPOM6tXrw567LnnnmskmVdffdVX5vV6jSQzcODAgONPnjxpbrjhBpOenm5cLpfv9dX/PbrvvvsCHtP4dx2dz2UMozaAcLvpppv04IMPqrq6utULqUyfPl0rVqwIGD0OAB2FMQNAB3j99dc1dOhQVlQDEBUIA0AYLVmyRD/5yU+0adMmTZs2LdLVAYCQMIAQCKOioiINGDBAt912m26//fZIVwcAQsKYAQAALEc3AQAAliMMAABgOcIAAACWIwwAAGA5wgAAAJYjDAAAYDnCAAAAliMMAABgOcIAAACWIwwAAGA5wgAAAJYjDAAAYDnCAAAAliMMAABgOcIAAACWIwwAAGC5LpGuAIDY5/V6VVVVpdTUVLnd7khXB0AjhAEAHaq0tFTl5eW+2zk5OfJ4PBGsEYDG6CYA0GG8Xq9fEJCk8vJyeb3eCNUIQDCEAQAdpqqqqlXlACKDMACgw6SmpraqHEBkMGYAQLs1HCAoyW+wYE5Ojl9XQW5uLoMIAYdxGWNMpCsBIHo1HiDYUP1gQWYTAM5GywCANgs2QLCh8vJyZWdny+12EwIAB2PMAIA2C2UgIIMFAecjDABos1AGAjJYEHA+wgCANqsfINgUBgsC0YEBhACaFcrgv+ZmEwBwPsIAgCaxlDBgB7oJAAQVLUsJe71eVVRUOK5eQDRhaiGAoJpbStgpzf+0XADhQcsAgKCcvpRwtLRcANGAMAAgqGAzBZw0O4BNkIDwoZsAsFQoswQ8Ho+ys7MdOTvA6S0XQDRhNgFgoVjpa2/8OnJzc1VQUNDm87GHAmxFGAAs4/V6VVxcHFBeVFQUlR+A4foAj5WABLQFYwYAy8RaX7vb7daIESPaFQQYjAjbEQYAS9TPx6+pqQl6v8197bEWkIDWYgAhEOO8Xq82btyoyspKX9mpp56qPXv2+G47aZZAJDAYEbZjzAAQwxr3gzd00UUXKT4+nsFy/y/cgxGBaELLABCjgvWDNxQfH68RI0Z0Yo2crTXTKJl1gFhDGABi1MaNG5u9nybwQG63u8UPd2YdIBYxgBCIQV6v12+MQGO2jxFoK2YdIFbRMgDEoKZGwffv31+TJk0iCLRRNGzeBLQFLQNADGqqC4Ag0D7MOkCsIgwAMcjpmwxFq+be1/p1HOgyQDRiaiEQwxj13jEav68MKkS0IwwAQDvE2l4PsBPdBADQDixljFhAGACAdmBQIWIBYQAA2oHBmogFjBkAYgADBSOPa4BoRhgAohwj2QG0F90EQBRjeVwA4UAYAKIYI9kBhANhAIhijGQHEA6EASCKMZIdQDgwgBCIAYxkB9AehAEAACxHNwEAAJYjDAAAYDnCAAAAliMMAABgOcIAAACW6xLpCgAIHVMIAXQEwgAQJdiQCEBHoZsAiAJsSBT7vF6vKioquKaICFoGgCjQ3IZEdBdEP1p9EGm0DABRgA2JYhetPnACwgAQBdxut8466yy/MjYkig1sQw0nIAwAUaC0tFRbtmzx3T7rrLNUUFAQwRohXGj1gRMQBgCHC9aMvGXLFpqRYwTbUMMJGEAIOByDB2Ofx+NRdnY2a0ggYggDgMPRjGwHt9tNCEDE0E0AOBzNyAA6mssYYyJdCQAtYyliAB2FMAAAgOXoJgAAwHKEAQAALEcYAADAcoQBAAAsRxgAAMByhAEAACzHCoSAw7G+AICORhgAHKy0tNRvk6KcnBx5PJ4I1ghALKKbAHCoYLsVlpeXs1shgLAjDAAO1dxuhQAQToQBwKHYrRBAZyEMAA7FboUAOgsbFQEOx2wCAB2NMAAAgOXoJgAAwHKEAQAALEcYAADAcoQBAAAsx3LEAOAgzB5BJBAGAMAh2IsCkUI3AQA4AHtRIJIIAwDgAOxFgUgiDACAA7AXBSKJMAAADsBeFIgkliMGAAdhNgEigTAAAIDlmFoIRBG+NcYWriecgjAARAnmoMcWriechAGEQBRgDnps4XrCaQgDQBRgDnps4XrCaQgDQBRgDnps4XrCaQgDQBRgDnps4XrCaZhaCEQRRp/HFq4nnIIwAACA5ZhaCACdhJYAOBVhAAA6QeN1BYYOHar8/HxCARyBbgIA6GBer1fFxcVB7yMUwAloGQCiBE3M0au59QMqKytVWVnJCoSIKMIAEAVYuja6hbJ+QHl5ubKzswl6iAjWGQAcjqVro1+wdQWCYQVCRAphAHA4lq6NDR6PR0VFRRo6dGiTx7ACISKFbgLA4WpqalpVDudyu9264oor5PV6tXHjRlVWVvruYwVCRBJhAHC4+Pj4VpXD+RqGAgaFwgkIA4DDsalN7HK73YQAOAJjBgCHY1MbAB2NRYeAKEGTMoCOQhgAAMBydBMAAGA5wgAAAJYjDAAAYDnCAAAAliMMAABgOcIAAACWIwwAAGA5wgAAAJYjDAAAYDnCAAAAliMMAABgOcIAAACWIwwAAGA5wgAAAJYjDAAAYDnCAAAAliMMAABgOcIAAACWIwwAAGA5wgAAAJYjDAAAYDnCAAAAliMMAABgOcIAAACWIwwAAGC5LpGuAADEGq/Xq6qqKqWmpsrtdke6OkCLCAMAEEalpaUqLy/33c7JyZHH44lgjYCW0U0AAGHi9Xr9goAklZeXy+v1RqhGQGgIAwAQIq/Xq4qKiiY/3KuqqlpVDjgF3QQAEIJQmv9TU1ODPrapcsApaBkAgBaE2vzvdruVk5PjV5abm8sgQjgeLQMA0ILmmv8bf9B7PB5lZ2czmwBRhTAAAC1obfO/2+0mBCCq0E0AAC2g+R+xzmWMMZGuBABEAxYTQqwiDAAAYDm6CQAAsBwDCIEoQRM1gI5CGACiQOMFb4YOHar8/HxCQQcjgMEWjBkAHM7r9aq4uDjofWyC03HYcAg2YcwA4HDNrWvPJjgdgw2HYBvCAOBwLa1rzyY44ceGQ7ANYQBwuGAL3jTEJjjhx4ZDsA1hAIgCHo9HRUVFGjp0qF85q+B1DFYchG0YQAhEGUa4dx7ea9iCMAAAgOXoJgAAwHKEAQAALEcYAADAcoQBAAAsRxgAAMByhAEAACxHGAAAwHJsYQzEKBbMARAqwgAQg9h+F0Br0E0AxBi23wXQWoQBIMaw/S6A1iIMADGG7XcBtBZhAIgxbL8LoLXYtRCIUZGcTcBMBiC6EAYAhBUzGYDoQzcBgLAJ50wGr9eriooKZkEAnYB1BgCETXMzGVrTXUDrAtC5aBkAEDbhmMnAOglA5yMMAAibcMxkYJ0EoPPRTQAgrDwej7Kzs9s8m4B1EoDOx2wCAI7TeMxAbm6uCgoKIlgjILYRBgA4EmsVAJ2HMAAAgOUYQAgAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlCAMAAFiOMAAAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlCAMAAFiOMAAAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlCAMAAFiOMAAAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlCAMAAFiOMAAAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlCAMAAFiOMAAAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlCAMAAFiOMAAAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlCAMAAFiOMAAAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlCAMAAFiOMAAAgOUIAwAAWI4wAACA5QgDAABYjjAAAIDlukS6AkC08Hq9qqqqUmpqqtxud6SrAwBhQxgAQlBaWqry8nLf7ZycHHk8nrCcm5ABINIIA0ALvF6vXxCQpPLycmVnZ7f7w7sjQwYAhIoxA0ALqqqqgpa/9dZb8nq9bT5vUyGjPecEgLYgDAAtqKmpCVr+5ptvqri4WKWlpW06b1Mho6lyAOgohAGgGaWlpXr++eebPaat3+ZTU1NbVQ4AHYUwADQhWDN+U9rybd7tdisnJ8evLDc3l0GEADodAwiBJrTmA76t3+Y9Ho+ys7OZTQAgoggDQBOa+oA/66yztGXLFt/t9n6bd7vdhAAAEeUyxphIVwJwqsZT/3Jzc1VQUMDaAABiCmEAaAEf/ABiHWEAAADLMZsAAADLEQYAALAcYQAAAMsRBgAAsBzrDCDmMRsAAJpHGEBMY4tgAGgZ3QSIWWwRDAChIQwgZrFFMACEhjCAmPXhhx8GLWeLYADwRxhATPJ6vX6bCdU7++yzGUQIAI0QBhCTmuoKyMzM7OSaAIDzEQYQk5rqCqCLAAACEQYQk9xut3JycvzKcnNz6SLoQF6vVxUVFczWAKIQuxYiprHgUOdgPQcguhEGALSL1+tVcXFxQHlRUREBDIgSdBMAaBfWcwCiH2EAQLswWBOIfoQBRB0GqjkLgzWB6MeYAUQVBqo5F4M1gehFGEDUYKAaAHQMugkQNRioBgAdgzCAqMFANQDoGIQBRA0GqgFAx2DMAKIOA9UAILwIAwAAWI5uAgAALEcYAADAcl0iXQEgFIwTAICOQxiA47HqIAB0LLoJ4Gher9cvCEhSeXk5+xJECfaRAKIDLQNwtOZWHaS7wNlo0QGiBy0DcDRWHYxOtOgA0YUwAEdj1cHoxD4SQHShmwCO5/F4lJ2dzWyCKEKLDhBdWIEQjsM0wtjQeMxAbm6uCgoKIlgjAE0hDMBRGHQWWwh2QHQgDMAxvF6viouLA8qLior4IAGADsQAQjgGg85iC2sMANGDAYRwDAadxQ66e4DoQssAHINphLGBNQaA6EPLAByFaYTRj1UjgehDGIDjuN1uPjSiGN09QPShmwBAWNHdA0QfphYC6BCsMQBED8IAAACWo5sAAADLEQYAALAcYQAAAMsRBgAAsBxhAAAAy7HoEDodU84AwFkIA+hUbGADAM5DNwE6DRvYAIAzEQbQaZrbwAYAEDmEAXQaNrABAGciDKDTsIENADgTexOg0zGbAACchTAAAIDl6CYAAMByhAEAACxHGAAAwHKEAQAALEcYAADAcoQBAAAsRxgAAMByhAEAACxHGAAAwHKEAQAALEcYAADAcoQBAAAsRxgAAMByXSJdAQDOxXbTgB0IAwCCKi0tVXl5ue92Tk6OPB5PBGsEoKPQTQAggNfr9QsCklReXi6v1xuhGgHoSIQBAAGqqqpaVQ4guhEGAARITU1tVTmA6EYYABDA7XYrJyfHryw3N5dBhECMchljTKQrASCympo1wGwCwA6EAcByzBoAQDcBYDFmDQCQCAOA1Zg1AEAiDABWY9YAAIkwAFiNWQMAJAYQAhCzBgDbEQYAALAc3QQAAFiOMAAAgOUIAwAAWI4wALTWiy9KN98sVVcHv7+6uu7+F1/s3HoBQBsxgBBojfXrpUmTpBMnpAkTpJUrpaSkr+6vrpYuuURau1ZKSJBWr5bGjYtcfQEgBIQBIFT1QaCmRqqtleLi6j7o6wNBfRBYv/6r++PjwxYImP4HoKMQBmCdp7Y8pd9u/K12HNyhk7UnlRSfpDHuMfrjpD/q9NTTgz+ocRCoVx8Ili2TLrvsqyDQ8P4wBAI2EwLQkQgDsMZnRz/TyD+N1P8e/t8mjxmXOU5/v+LviotrMJzmxRfrugQaB4F6cXFSr17S4cNN3x8fX9d1cOGFIde3viWgpqZGzz//fMD9RUVFtBAACAsGEMIaZ//xbF8QGNhzoDZO2ygzx2jbrG0a1HuQJGn9jvUa/9fx/g9cs6ZujECwD3qprrypIFB//4kTdecJUWlpqYqLi7Vq1aqgQUBiMyEA4UMYgBVK3i7Rx0c+liSdknyKdt24S5ldMvXSSy/pv1v/q40/2Kj07ulyyaUNOzdowzsbVFFRUbeV791317UMxDXz69JUEJBk4uJ0OCdH3muvDamuwbYVDobNhACEC90EsMKwR4bpg6oPZGS07NJl6vtJ34AP3E8zPtXjux6XJGUpSz/RTyT9f/98Xp7/4MAQGZdLlZmZeuayy1STkBBSX39FRYVWrVrV7DG5ubkqKCgIuR4A0Jwuka4A0Bn2HNmjxC6JOl5zXLkpuSp+tjjgmLRdaZKkOMVpn/b5ysvLy5WdnS33ypWtCgQmLk6VQ4b4goDfuZrp62/qG/9FF12k+Ph4ZhMACDu6CWCF+Lh41ZpaGWO0b9++oMcc0zHf/7vk8ruvqqqqbvrgsmV1gwWb6zKQpLg41fTooWenTvUFAb9zNaOpbYXPOeccjRgxgiAAIOxoGYAVRvYfqZd3vyxJWl+1PugxG7RBklSrWg3QAL/7UlNT69YRuOyy5gcL1qutVfwXX+jS5cv9WgZ852qBx+NRdnY26woA6BS0DMAKC8cv9P3/79/4vUaPGe13/z7tU4WrQvGueEnSBE3w3Zebmyt3Wlqrxwy4ams1dMcO/XDZMsWfOPHVuUL8YHe73bQEAOgUDCCENQqXF2r5tuWSpB4JPXR3zt3KPJGp/9n5P1rz3zWqMTWSpB8O/6HuH3P/V9/K2xAEGjJxcToyZowOl5TIPXRoWF8TAIQDYQBWuXb1tfrTv/8ko+A/9leefaVKLinxL7z5ZumBB5o/cVxcy0HhppukhQubPwYAIoAwAOscP3lcvy37rZ559xnt/2K/Urul6sIhF+p33/md+nbvG/iACK1ACACdhTAAhCLCexMAQEdiACEQinHj6j7Q4+O/mlbYcNfC3r3r/jtunP/9BAEAUYAwAISqYSCov12/fbFU99/6QCARBABEDboJgNZ68cW6TYfuvvurINBQdbX0619LEycyRgBAVCAMAABgOboJAACwHGEAAADLEQYAALAcYQAAAMsRBgAAsBxhAAAAyxEGAACwHGEAAADLEQYAALAcYQAAAMsRBgAAsBxhAAAAyxEGAACwHGEAAADLEQYAALAcYQAAAMv9H5OkPiDSovryAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualize_query_with_base_embeddings(\"Tell me about Japan's Industrial Growth\", embeddings, retriever, umap_transformer, base_umap_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGbCAYAAABZBpPkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvG0lEQVR4nO3deXxTZaL/8W+6IzsFqjUsUhQqIsrmkAJFbFkExI1lUBYHGPdtFBSdK5tzGRkRBXUYsYpCcYHr4E9RpA4glSLjHbEqjAoIamAUKALK3vb5/dGb2DRJm0DbLOfzfr18Yc45OXlOluabZ7UZY4wAAIBlxYS6AAAAILQIAwAAWBxhAAAAiyMMAABgcYQBAAAsjjAAAIDFEQYAALA4wgAAABZHGAAAwOIsHQZsNpvuuOOOUBej2rVu3Vrjxo0LdTHcpk2bJpvNFupinJEff/xR119/vZKTk2Wz2fTkk09q3bp1stlsWrduXZX379Onj/r06VPj5awJ48aNU7169UJdjBpls9k0bdq0Gn2MRYsWyWazadeuXTX6OOFk165dstlsWrRoUUDHjxs3Tq1bt6728/oSzGcy0HJFsogMA6+//rpsNpv+/ve/e+3r1KmTbDab1q5d67WvZcuWcjgctVFEN9cXhs1m05IlS3wek5GRIZvNposuuijo8/fp0+e07ofg3HvvvXrvvfc0ZcoULV68WAMGDAh1kbwUFBRo2rRpOnjwYKiLUm2i8ZpCwRXIY2Ji9P3333vtP3z4sOrUqRN1P5C2bt2qadOmBRTA9uzZo2nTpunTTz+t8XKFo4gMAz179pQkffjhhx7bDx8+rC+++EJxcXHasGGDx77vv/9e33//vfu+tS0pKUlLly712r5r1y4VFBQoKSkpBKWqHX/84x917NixUBfjjKxZs0ZDhw7V/fffrxtvvFHt27dX7969dezYMfXu3TvUxZNU9sU5ffr0qPrirK1rOnbsmP74xz/W6GOMHj1ax44dU6tWrWr0cSqTmJioV155xWv7G2+8EYLSeFu4cKG++uqrajvf1q1bNX36dJ9hYPXq1Vq9erX79p49ezR9+nTCQCRJTU3Veeed5xUGNm7cKGOMhg0b5rXPdTtUYeDKK69UXl6e9u/f77F96dKlSklJUdeuXWu1PEeOHKm1x4qLi4v4sLN37141atTIY1tMTIySkpIUExORHyOUk5SUpLi4uBp9jNjYWCUlJYW0yezKK6/0GQaWLl2qQYMGhaBEZVx/j+Lj45WYmFgrj5mQkKCEhIRaeaxIELF/xXr27KnNmzd7/OLcsGGDOnTooIEDB+qjjz5SaWmpxz6bzaaMjAyvc61YsUIXXXSREhMT1aFDB61atcq9b+3atX6bJJYuXSqbzaaNGzdWWd6hQ4cqMTFRy5Yt8zrH8OHDFRsb6/N+S5YsUffu3XXWWWepcePG6t27t0eaDYSrrfKDDz7QbbfdpubNm8tut7v3P/vss+rQoYMSExOVmpqq22+/3eOX2NSpUxUfH699+/Z5nfv3v/+9GjVqpOPHj/t9fF99BvLy8tSzZ081atRI9erVU7t27fTQQw8FdD2BPCdVXZP0axPL1q1bdfnll+uss87Sueeeq9mzZ7uPcT13xhg988wz7iYfSX77DDz33HNKS0tTnTp11L17d+Xn5/u8jhMnTmjq1Klq27atEhMT1aJFC02ePFknTpzwOM5VdVvZ+3TatGmaNGmSJOm8885zl7Oy6tH8/HwNGzZMLVu2dD/+vffe67cW55tvvlH//v1Vt25dpaamasaMGaq46OmRI0d03333qUWLFkpMTFS7du30+OOPexxXWXtv+bb7YK9p3rx5io2N9Xid58yZI5vNpj/84Q/ubSUlJapfv74eeOABn4/remybzabt27dr3LhxatSokRo2bKibbrpJR48e9SpzVa+P5LvPQOvWrTV48GB9+OGH6t69u5KSktSmTRu9/PLLXtf32WefKTMzU3Xq1JHdbtejjz6qF198Mah+CKNGjdKnn36qL7/80r3thx9+0Jo1azRq1Civ40+ePKlHHnlEXbp0UcOGDVW3bl316tXLZzPswYMHNW7cODVs2FCNGjXS2LFjfdbouPqg7NixQ1deeaXq16+vG264wb2vYtt8oOetaNGiRRo2bJgk6fLLL3e/f1yf1/J9BtatW6du3bpJkm666Sb3sZX1SSgtLdWTTz6pDh06KCkpSSkpKbr55pv1008/VVm2cBTRYeDUqVPatGmTe9uGDRvkcDjkcDh06NAhffHFFx772rdvr+TkZI/zfPjhh7rttts0cuRIzZ49W8ePH9d1112noqIiSWVvmBYtWig3N9erDLm5uUpLS1OPHj2qLO9ZZ52loUOHeqTywsJCbdmyxeeHUJKmT5+u0aNHKz4+XjNmzND06dPVokULrVmzpsrH8+W2227T1q1b9cgjj+jBBx+UVPZH7/bbb1dqaqrmzJmj6667Tn/729/Ur18/nTp1SlJZ9WZxcbFee+01j/OdPHlSy5cv13XXXRfUL/8tW7Zo8ODBOnHihGbMmKE5c+boqquu8mra8SWQ5ySQa3L56aefNGDAAHXq1Elz5sxR+/bt9cADD+jdd9+VJPXu3VuLFy+WJGVnZ2vx4sXu277k5OTo5ptv1tlnn63Zs2crIyNDV111lVc7bWlpqa666io9/vjjGjJkiObPn6+rr75ac+fO1YgRI7zOW9X79Nprr9Vvf/tbSdLcuXPd5WzWrJnfsi5btkxHjx7Vrbfeqvnz56t///6aP3++xowZ43VsSUmJBgwYoJSUFM2ePVtdunTR1KlTNXXqVPcxxhhdddVVmjt3rgYMGKAnnnhC7dq106RJkzy+jAMV7DX16tVLpaWlHrWC+fn5iomJ8Qhkmzdv1i+//BJQ887w4cP1888/a9asWRo+fLgWLVqk6dOnex1X1etTme3bt+v6669Xdna25syZo8aNG2vcuHHasmWL+5jdu3fr8ssv15YtWzRlyhTde++9ys3N1VNPPVXl+cvr3bu37Ha7R5Pla6+9pnr16vmsGTh8+LCef/559enTR4899pimTZumffv2qX///h7V6cYYDR06VIsXL9aNN96oRx99VE6nU2PHjvVZjuLiYvXv31/NmzfX448/ruuuu87nccGet+K13nXXXZKkhx56yP3+SU9P9zo2PT1dM2bMkFT2A8d1bGXvkZtvvlmTJk1SRkaGnnrqKd10003Kzc1V//79vf7ORAQTobZs2WIkmZkzZxpjjDl16pSpW7eueemll4wxxqSkpJhnnnnGGGPM4cOHTWxsrJk4caLHOSSZhIQEs337dve2wsJCI8nMnz/fvW3KlCkmMTHRHDx40L1t7969Ji4uzkydOrXScq5du9ZIMsuWLTNvv/22sdls5rvvvjPGGDNp0iTTpk0bY4wxmZmZpkOHDu77bdu2zcTExJhrrrnGlJSUeJyztLTU/f8V72eMMa1atTJjx451337xxReNJNOzZ09TXFzscQ0JCQmmX79+Ho/x9NNPG0nmhRdecG/r0aOHueyyyzwe54033jCSzNq1ayt9DqZOnWrKv9Xmzp1rJJl9+/ZVer+KAnlOgrmmzMxMI8m8/PLL7m0nTpwwZ599trnuuus8zi/J3H777R7bXK+t6/pPnjxpmjdvbi655BJz4sQJ93HPPfeckWQyMzPd2xYvXmxiYmJMfn6+xzkXLFhgJJkNGzZ4PHYg79O//OUvRpLZuXOnz+evoqNHj3ptmzVrlrHZbObbb791bxs7dqyRZO688073ttLSUjNo0CCTkJDgfh1XrFhhJJlHH33U45zXX3+9sdls7vLv3LnTSDIvvvii1+NL8vhMBXNNJSUlpkGDBmby5MnuMiYnJ5thw4aZ2NhY8/PPPxtjjHniiSdMTEyM+emnn/w+rus9+7vf/c7jMa655hqTnJzsVeZAXh/X57D8tbRq1cpIMuvXr3dv27t3r0lMTDT33Xefe9udd95pbDab2bx5s3tbUVGRadKkSUDPj+t69u3bZ+6//37Ttm1b975u3bqZm266yX0t5d/nxcXFHu9lY4z56aefTEpKisdz43rtZ8+e7XHfXr16eb3WrvfTgw8+6FXOsWPHmlatWp3WeX1ZtmyZ379RmZmZHp/Jjz/+2O85K5YrPz/fSDK5ubkex61atcrn9kgQsTUD6enpSk5Odv8KKCws1JEjR9yjBRwOh/uX5saNG1VSUuKzv0BWVpbS0tLcty+++GI1aNBA33zzjXvbmDFjdOLECS1fvty97bXXXlNxcbFuvPHGgMvcr18/NWnSRK+++qqMMXr11Vfdv3wqWrFihUpLS/XII494tUmfbpvjxIkTPZoj3n//fZ08eVL33HOPx2NMnDhRDRo00MqVK93bxowZo02bNmnHjh3ubbm5uWrRooUyMzODKoer7f3NN9/0aMqpSiDPSTDXJEn16tXzeA0TEhLUvXt3j9c/UP/7v/+rvXv36pZbbvFoi3RVcZa3bNkypaenq3379tq/f7/7v759+0qSVzVsIO/TYNWpU8f9/0eOHNH+/fvlcDhkjNHmzZu9ji/fy9xVNX7y5Em9//77kqR33nlHsbGx7l9jLvfdd5+MMe7alpoSExMjh8Oh9evXS5L+/e9/q6ioSA8++KCMMe7mvPz8fF100UVefUB8ueWWWzxu9+rVS0VFRTp8+LDH9jN5fS688EL16tXLfbtZs2Zq166dx31XrVqlHj166JJLLnFva9Kkibt6PRijRo3S9u3b9fHHH7v/9Vc7GRsb634vl5aW6sCBAyouLlbXrl31ySefuI975513FBcXp1tvvdXjvnfeeaffcpQ/1p/TOW9tWLZsmRo2bKjs7GyPz2+XLl1Ur149n80o4S5iw4DNZpPD4XD3DdiwYYOaN2+utm3bSvIMA65/fYWBli1bem1r3LixR7tP+/bt1a1bN4+mgtzcXP3mN79xP14g4uPjNWzYMC1dulTr16/X999/7/dDuGPHDsXExOjCCy8M+PxVOe+88zxuf/vtt5Kkdu3aeWxPSEhQmzZt3PslacSIEUpMTHQ/B4cOHdLbb7+tG264IehwMmLECGVkZGjChAlKSUnRyJEj9frrr1cZDAJ5ToK5Jkmy2+1e5a/4+gfKde7zzz/fY3t8fLzatGnjsW3btm3asmWLmjVr5vHfBRdcIKmsw2J5gbxPg/Xdd99p3LhxatKkierVq6dmzZq5g92hQ4c8jo2JifG6BldZXe3V3377rVJTU1W/fn2P41zVshWf+5rQq1cv/etf/9KxY8eUn5+vc845R507d1anTp3cTQUffvihx5dvZSo+740bN5Ykr+f9TF6fQO777bff+vxbE8zfH5dLL71U7du319KlS5Wbm6uzzz7bHUJ9eemll3TxxRcrKSlJycnJatasmVauXOnxHvn22291zjnneM1HUfFz6BIXF+fRb8mfYM9bW7Zt26ZDhw6pefPmXp/hX375xevzGwlqtvtsDevZs6feeustff755+7+Ai4Oh0OTJk3S7t279eGHHyo1NdXrj5kkvx33TIWOUWPGjNHdd98tp9OpEydO6KOPPtLTTz8ddJlHjRqlBQsWaNq0aerUqVO1ftlXpfwvwWA1btxYgwcPVm5urh555BEtX75cJ06cCKpmpHw51q9fr7Vr12rlypVatWqVXnvtNfXt21erV6/2+5rUhEBf/+pWWlqqjh076oknnvC5v0WLFh63q7ucJSUlys7O1oEDB/TAAw+offv2qlu3rnbv3q1x48YFVWMTLH/hsaSk5IzP7epLtHHjRuXn57u/9Hv16qX8/Hx9+eWX2rdvX8BhINDn/Uxen1C8B0eNGqW//vWvql+/vkaMGOF3RMySJUs0btw4XX311Zo0aZKaN2+u2NhYzZo1y6OWMFiJiYkRPQqntLRUzZs399mXTFKlfXXCVeS+GvKcb2DDhg0eIwW6dOmixMRErVu3Tps2bfI5iiAYI0eOVGxsrF555RXl5uYqPj7eZ0evQMrcsmVLrVu3zm+tgCSlpaWptLRUW7duPZNiV8o13rniuN6TJ09q586dXuOhx4wZo6+//loff/yxcnNzdemll6pDhw6n9dgxMTG64oor9MQTT2jr1q3605/+pDVr1lRavRbIcxLsNVUn17m3bdvmsf3UqVPauXOnx7a0tDQdOHBAV1xxhbKysrz+O51fPsHU0Hz++ef6+uuvNWfOHD3wwAMaOnSosrKylJqa6vP40tJSryrvr7/+WpLcvb9btWqlPXv26Oeff/Y4ztVz3fX8uH5dV+wR7qvmINhap+7duyshIUH5+fkeYaB3797atGmT/vGPf7hvR5JWrVpp+/btXtt9bQvEqFGj9J///Edff/11pX+Hli9frjZt2uiNN97Q6NGj1b9/f2VlZXmNHmrVqpX+85//6JdffvHYfqZzBpzpeYN5/wRzbFpamoqKipSRkeHz89upU6eAzxUuIjoMdO3aVUlJScrNzdXu3bs9agYSExPVuXNnPfPMMzpy5MgZzy/QtGlTDRw4UEuWLFFubq4GDBigpk2bBn0em82mefPmaerUqRo9erTf466++mrFxMRoxowZXr/SqusXQ1ZWlhISEjRv3jyPc+bk5OjQoUNevYsHDhyopk2b6rHHHtMHH3xwWrUCknTgwAGvba620IrD6soL5DkJ9pqqU9euXdWsWTMtWLBAJ0+edG9ftGiR1xff8OHDtXv3bi1cuNDrPMeOHTuteSDq1q0ryftL1hfXr9Hyz5ExptLe6eVrwowxevrppxUfH68rrrhCUtkY9pKSEq8as7lz58pms2ngwIGSpAYNGqhp06butn2XZ5999oyuSSqbL6Bbt2565ZVX9N1333nUDBw7dkzz5s1TWlqazjnnnIDOFy769++vjRs3evTgP3DggN9fplVJS0vTk08+qVmzZql79+5+j/P1Ptm0aZPXcOorr7xSxcXF+utf/+reVlJSovnz559W+arrvMG8f4I5dvjw4SopKdHMmTO99hUXF0fkxF8R3UyQkJCgbt26KT8/X4mJierSpYvHfofDoTlz5kiqnsmGxowZo+uvv16SfL4JAjV06FANHTq00mPatm2rhx9+WDNnzlSvXr107bXXKjExUR9//LFSU1M1a9as0358l2bNmmnKlCmaPn26BgwYoKuuukpfffWVnn32WXXr1s3ryz4+Pl4jR47U008/rdjYWL+dH6syY8YMrV+/XoMGDVKrVq20d+9ePfvss7Lb7ZW+ToE8J8FeU3WKj4/Xo48+qptvvll9+/bViBEjtHPnTr344oteTVSjR4/W66+/rltuuUVr165VRkaGSkpK9OWXX+r111/Xe++9F/REVK73/8MPP6yRI0cqPj5eQ4YMcf+RK699+/ZKS0vT/fffr927d6tBgwb6n//5H79t3ElJSVq1apXGjh2ryy67TO+++65Wrlyphx56yF0lOmTIEF1++eV6+OGHtWvXLnXq1EmrV6/Wm2++qXvuucejg92ECRP05z//WRMmTFDXrl21fv16d03D6V6TS69evfTnP/9ZDRs2VMeOHSVJzZs3V7t27fTVV1+F1bodgZo8ebKWLFmi7Oxs3Xnnnapbt66ef/55tWzZUgcOHDitTsV33313lccMHjxYb7zxhq655hoNGjRIO3fu1IIFC3ThhRd6/FofMmSIMjIy9OCDD2rXrl268MIL9cYbb3j1PQnWmZ73kksuUWxsrB577DEdOnRIiYmJ6tu3r5o3b+51bFpamho1aqQFCxaofv36qlu3ri677DKvvlaSlJmZqZtvvlmzZs3Sp59+qn79+ik+Pl7btm3TsmXL9NRTT7m/KyJGbQ9fqG5TpkwxkozD4fDa5xr6Vr9+fY8hdS7yMVzMGO+heS4nTpwwjRs3Ng0bNjTHjh0LqHzlhxZWxtcQQWOMeeGFF8yll15qEhMTTePGjU1mZqbJy8ur9H7+hhZ+/PHHPh/76aefNu3btzfx8fEmJSXF3HrrrR7Drsr75z//aSSZfv36VXo95VUcWviPf/zDDB061KSmppqEhASTmppqfvvb35qvv/46oPNV9ZwEek3+nvOKw4iMCWxoocuzzz5rzjvvPJOYmGi6du1q1q9f7zWMyZiyoYiPPfaY6dChg/taunTpYqZPn24OHTpU6WMb4/t9OnPmTHPuueeamJiYKoecbd261WRlZZl69eqZpk2bmokTJ7qHxFUcCla3bl2zY8cO069fP3PWWWeZlJQUM3XqVK8hnj///LO59957TWpqqomPjzfnn3+++ctf/uIxHNaYsmGN48ePNw0bNjT169c3w4cPN3v37vUa4hfsNRljzMqVK40kM3DgQI/tEyZMMJJMTk6O130qPm75oXjl+RoeGOjr429o4aBBg7zu6+v9snnzZtOrVy+TmJho7Ha7mTVrlpk3b56RZH744Qcfz8Sv/F1PRRWvpbS01Pz3f/+3adWqlUlMTDSXXnqpefvtt31+RoqKiszo0aNNgwYNTMOGDc3o0aPN5s2b/b6ffDmT8/qzcOFC06ZNGxMbG+vxefX1HL/55pvmwgsvNHFxcR7n91UuY8qGDXfp0sXUqVPH1K9f33Ts2NFMnjzZ7Nmzp8pyhRubMTXcUyqKFBcXKzU1VUOGDFFOTk6oixMShYWFuuSSS/Tyyy9X2swBoObdc889+tvf/qZffvmlVjveIvpEdJ+B2rZixQrt27fP5wxtVrFw4ULVq1dP1157baiLAlhKxWmii4qKtHjxYvXs2ZMggDMW0X0GasumTZv02WefaebMmbr00kuDnmQnGrz11lvaunWrnnvuOd1xxx2VttkCqH49evRQnz59lJ6erh9//FE5OTk6fPiw/uu//ivURUMUoJkgAOPGjdOSJUt0ySWXaNGiRbroootCXaRa17p1a/3444/q37+/Fi9e7DWxDICa9dBDD2n58uVyOp2y2Wzq3Lmzpk6dqqysrFAXDVGAMAAAgMXRZwAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAat+aNdJ990nHj/vef/x42f41a2q3XIBF2YwxJtSFABDdnE6nioqKlJycLPvWrdLgwdKpU9KAAdLf/y4lJf168PHj0jXXSKtWSfHx0ttvS/36ha7wgAUQBgDUqLy8PBUUFEiS2mzfrhtefVUxpaVSaakUE1P2Re8KBK4gsHr1r/tjYwkEQA2jmQBAjXE6nR5BYNTSpVJxcdkXvVT27+rVZQHg0CHPIODaX1JSVpOwenWIrgKIfoQBADWmqKhIktT6m280aulS2UpLvf/ouAJB69aeQaD8flcgoA8BUCMIAwBqTHJysiTpgm3bFOsrCLiUlkqHD3sHgfL7T52SVq6skXICVhcX6gIAiHzlOwhK+rWzoN0uh8Ohf5w6paZ79yrtm28U46+bkr8gIP3at+BPf6qB0gOgAyGAM1K+g2BFDodD2dnZcjqdOrBnj86fPFl18vMr/+KvqGInQwDVjmYCAH6VlpbKedipg8cP+txfvoOgLwUFBXI6nbLb7bq4e3fVWbWq7Is9JsA/PQQBoFYQBgB4+eQ/n6jH8z0UNzNOLea2UOPHGqvhnxvq7nfvVnFpsfs4VwfByngck5Qkvfqq1KBB1YEgJqbsuFdfJQgANYwwAMDD8q3L1W1hN320+yNdnHKx7up+l0Z3HK0YW4zm/XOeLph/gU4Wn5T0awfByngcc/y4NHJk5Z0FXVydCkeO9D9TIYBqQZ8BAG77j+7X2Y+frYTYBP3r9/9SerN0j/13v3u35v1znrLOy1LemDxJlfcZyMjIUFZWVtmNihMKBcpPU8F3h77T1HVT9d3B79QwqaHuuuwu9WndJ6jrBVCGMADA7Xdv/k4vfvqi3h/9vq5oc4WkClMJ2+26+K8X64u9X+jgAwfVIKmB1zGS52gCSacfBFzKBYLjcVLmi5n6555/SpJsssmo7M9YSt0UvT/6fV2UctEZPhOAtRAGALglz/6/L/PJZe38FX/1OxwOHW1xVFe/drWmZU7T1D5TAzvxffdJTzxR+TExMVUGhdI/3KO0liu06+Au9WzRU08NfEqdz+ms/Uf3a3LeZL1U+JLibHHaevtWpTVJC6xsAOgzAOBXR08d1Tn1zpHke6RAQUGB2iW0K9t/2Bn4iQcNKlt0yF+nQVdnwcr2x8fr0Yt+0q6Du3RHtzuU/7t8dT6ns5xOp3Zv260ZXWZo9Y2rdar0lIYtGxZ42QAQBgD8KikuSXuP7JXkf6TA2u1rJUnN6zUP/MR9+5YtNhQb6/2F72oC2LXL97DDcosVPVP0rurE1dFTA56SVFZzkZOToxUrVignJ0elO0rVLbWbPv3hU+0/uj/w8gEWRxgA4Daw7UDtO7pPG7/f6HekwIJtC2STTff+5t7gTt6vn3cgKN85sGHDsn/LB4IKqxbuO7JPPVv2VExMjN+aixFpI2Rk9M62d4K9fMCyCAMA3B7Pflw22dR/SX+V1i+Vw+Hw2L+jxQ59tv8z9bD3UNOzmgb/AOUDget2+VECSUm/BgLJa/liI6OE2ARJ/msuSo+X9Ts4VXIq+PIBFsXaBIBFVRwlIEmpDVK1cMhCTXxros576jxltMjQb7r8RrsP7lbenjzt+36fzq53tvJG553+A/frJ61aVbbo0J/+5D2hkCsQPPxwWV+Dvn3du+ol1NNHzo8k+Z/j4L0f3pMk9W7V+/TLCFgMowkAC/I1SiA7O9t9e83ONbrr3bu0Zd8W97ak2CRdm36tcobmKCkuNDMCTvx/E/X85uf10tUvaUynMV7XcX6X8zV281jZG9i18+6dQZ/fV0ACrIAwAFiM0+lUTk6O1/bx48d7fQEePH5Q/973bzVIbKD0pumKCXRNgRpy+PhhpT6RqqOnjmpKryma3me6ftjzg4qKirTp8Cb9If8POnLqiN4Z9Y4Gnj8wqHNXFZCAaEYYACymsLBQK1as8Np+9dVXq1OnTrVfoCBtK9qm7s9318HjBxUXE6cmSU30y6lfdPTUUcXaYvXC0Bc0ptOYoM4ZTEACohEdCAGLcDqdKiwsVElJic/9gawzEA7OTz5fRZOKtHDIQl3Q5ALZbDY1O6uZ/tDjDzr84OGgg4DkvzNiIAsxAdGADoRAlHM6nfrggw+0fft297bU1FTt2bPHfTsjIyOifgHHxMRoQucJmtB5QrWcz18QipSABJwpmgmAKFbZIkJDhgxRbGwsneX+T8XnymORJSDKUTMARClfk/KUFxsbGxF9BGpLdna20tPTAxpNwKgDRBvCABClPvjgg0r3UwXuzW63V/nlzqgDRCM6EAJRyOl0evQRqCjS+giEC39TIDudQSzaBIQhagaAKOSvF3xKSooGDx5MEDhNlY064DlFJKNmAIhC/poACAJnhlEHiFaEASAK2e12r0WGaBo4c5U9r655HGgyQCRiaCEQxej1XjMqPq90KkSkIwwAwBlgKmNEA5oJAOAMMJUxogFhAADOAJ0KEQ0IAwBwBuisiWhAnwEgCtBRMPR4DRDJCANAhKMnO4AzRTMBEMGYHhdAdSAMABGMnuwAqgNhAIhg9GQHUB0IA0AEoyc7gOpAB0IgCtCTHcCZIAwAAGBxNBMAAGBxhAEAACyOMAAAgMURBgAAsDjCAAAAFhcX6gIACBxDCAHUBMIAECFYkAhATaGZAIgALEgU/ZxOpwoLC3lNERLUDAARoLIFiWguiHzU+iDUqBkAIgALEkUvan0QDggDQASw2+3q2LGjxzYWJIoOLEONcEAYACJAXl6ePv/8c/ftjh07KisrK4QlQnWh1gfhgDAAhDlf1ciff/451chRgmWoEQ7oQAiEOToPRr/s7Gylp6czhwRChjAAhDmqka3BbrcTAhAyNBMAYY5qZAA1zWaMMaEuBICqMRUxgJpCGAAAwOJoJgAAwOIIAwAAWBxhAAAAiyMMAABgcYQBAAAsjjAAAIDFMQMhEOaYXwBATSMMAGEsLy/PY5Eih8Oh7OzsEJYIQDSimQAIU75WKywoKGC1QgDVjjAAhKnKVisEgOpEGADCFKsVAqgthAEgTLFaIYDawkJFQJhjNAGAmkYYAADA4mgmAADA4ggDAABYHGEAAACLIwwAAGBxTEcMAGGE0SMIBcIAAIQJ1qJAqNBMAABhgLUoEEqEAQAIA6xFgVAiDABAGGAtCoQSYQAAwgBrUSCUmI4YAMIIowkQCoQBAAAsjqGFQAThV2N04fVEuCAMABGCMejRhdcT4YQOhEAEYAx6dOH1RLghDAARgDHo0YXXE+GGMABEAMagRxdeT4QbwgAQARiDHl14PRFuGFoIRBB6n0cXXk+EC8IAAAAWx9BCAKgl1AQgXBEGAKAWVJxXoG3btsrMzCQUICzQTAAANczpdConJ8fnPkIBwgE1A0CEoIo5clU2f8D27du1fft2ZiBESBEGgAjA1LWRLZD5AwoKCpSenk7QQ0gwzwAQ5pi6NvL5mlfAF2YgRKgQBoAwx9S10SE7O1vjx49X27Zt/R7DDIQIFZoJgDBXUlIS1HaEL7vdrhtuuEFOp1MffPCBtm/f7t7HDIQIJcIAEOZiY2OD2o7wVz4U0CkU4YAwAIQ5FrWJXna7nRCAsECfASDMsagNgJrGpENAhKBKGUBNIQwAAGBxNBMAAGBxhAEAACyOMAAAgMURBgAAsDjCAAAAFkcYAADA4ggDAABYHGEAAACLIwwAAGBxhAEAACyOMAAAgMURBgAAsDjCAAAAFkcYAADA4ggDAABYHGEAAACLIwwAAGBxhAEAACyOMAAAgMURBgAAsDjCAAAAFkcYAADA4ggDAABYHGEAAACLIwwAAGBxcaEuAABEG6fTqaKiIiUnJ8tut4e6OECVCAMAUI3y8vJUUFDgvu1wOJSdnR3CEgFVo5kAAKqJ0+n0CAKSVFBQIKfTGaISAYEhDABAgJxOpwoLC/1+uRcVFQW1HQgXNBMAQAACqf5PTk72eV9/24FwQc0AAFQh0Op/u90uh8PhsS0jI4NOhAh71AwAQBUqq/6v+EWfnZ2t9PR0RhMgohAGAKAKwVb/2+12QgAiCs0EAFAFqv8R7WzGGBPqQgBAJGAyIUQrwgAAABZHMwEAABZHB0IgQlBFDaCmEAaACFBxwpu2bdsqMzOTUFDDCGCwCvoMAGHO6XQqJyfH5z4Wwak5LDgEK6HPABDmKpvXnkVwagYLDsFqCANAmKtqXnsWwal+LDgEqyEMAGHO14Q35bEITvVjwSFYDWEAiADZ2dkaP3682rZt67GdWfBqBjMOwmroQAhEGHq41x6ea1gFYQAAAIujmQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiWMIYiFJMmAMgUIQBIAqx/C6AYNBMAEQZlt8FECzCABBlWH4XQLAIA0CUYfldAMEiDABRhuV3AQSLVQuBKBXK0QSMZAAiC2EAQLViJAMQeWgmAFBtqnMkg9PpVGFhIaMggFrAPAMAqk1lIxmCaS6gdgGoXdQMAKg21TGSgXkSgNpHGABQbapjJAPzJAC1j2YCANUqOztb6enppz2agHkSgNrHaAIAYadin4GMjAxlZWWFsERAdCMMAAhLzFUA1B7CAAAAFkcHQgAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDi4kJdACBSOJ1OFRUVKTk5WXa7PdTFAYBqQxgAApCXl6eCggL3bYfDoezs7Go5NyEDQKgRBoAqOJ1OjyAgSQUFBUpPTz/jL++aDBkAECj6DABVKCoq8rl98+bNcjqdp31efyHjTM4JAKeDMABUoaSkxOf2Tz75RDk5OcrLyzut8/oLGf62A0BNIQwAlcjLy9Nbb71V6TGn+2s+OTk5qO0AUFMIA4Afvqrx/TmdX/N2u10Oh8NjW0ZGBp0IAdQ6OhACfgTzBX+6v+azs7OVnp7OaAIAIUUYAPzw9wXfsWNHff755+7bZ/pr3m63EwIAhJTNGGNCXQggXFUc+peRkaGsrCzmBgAQVQgDQBX44gcQ7QgDAABYHKMJAACwOMIAAAAWRxgAAMDiCAMAAFgc8wwg6jEaAAAqRxhAVGOJYACoGs0EiFosEQwAgSEMIGqxRDAABIYwgKi1Y8cOn9tZIhgAPBEGEJWcTqfHYkIuF198MZ0IAaACwgCikr+mgDZt2tRySQAg/BEGEJX8NQXQRAAA3ggDiEp2u10Oh8NjW0ZGBk0ENcjpdKqwsJDRGkAEYtVCRDUmHKodzOcARDbCAIAz4nQ6lZOT47V9/PjxBDAgQtBMAOCMMJ8DEPkIAwDOCJ01gchHGEDEoaNaeKGzJhD56DOAiEJHtfBFZ00gchEGEDHoqAYANYNmAkQMOqoBQM0gDCBi0FENAGoGYQARg45qAFAz6DOAiENHNQCoXoQBAAAsjmYCAAAsjjAAAIDFxYW6AEAg6CcAADWHMICwx6yDAFCzaCZAWHM6nR5BQJIKCgpYlyBCsI4EEBmoGUBYq2zWQZoLwhs1OkDkoGYAYY1ZByMTNTpAZCEMIKwx62BkYh0JILLQTICwl52drfT0dEYTRBBqdIDIwgyECDsMI4wOFfsMZGRkKCsrK4QlAuAPYQBhhU5n0YVgB0QGwgDChtPpVE5Ojtf28ePH80UCADWIDoQIG3Q6iy7MMQBEDjoQImzQ6Sx60NwDRBZqBhA2GEYYHZhjAIg81AwgrDCMMPIxayQQeQgDCDt2u50vjQhGcw8QeWgmAFCtaO4BIg9DCwHUCOYYACIHYQAAAIujmQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOCYdQq1jyBkAhBfCAGoVC9gAQPihmQC1hgVsACA8EQZQaypbwAYAEDqEAdQaFrABgPBEGECtYQEbAAhPrE2AWsdoAgAIL4QBAAAsjmYCAAAsjjAAAIDFEQYAALA4wgAAABZHGAAAwOIIAwAAWBxhAAAAiyMMAABgcYQBAAAsjjAAAIDFEQYAALA4wgAAABZHGAAAwOLiQl0AAOGL5aYBayAMAPApLy9PBQUF7tsOh0PZ2dkhLBGAmkIzAQAvTqfTIwhIUkFBgZxOZ4hKBKAmEQYAeCkqKgpqO4DIRhgA4CU5OTmo7QAiG2EAgBe73S6Hw+GxLSMjg06EQJSyGWNMqAsBILT8jRpgNAFgDYQBwOIYNQCAZgLAwhg1AEAiDACWxqgBABJhALA0Rg0AkAgDgKUxagCARAdCAGLUAGB1hAEAACyOZgIAACyOMAAAgMURBgAAsDjCAAAAFkcYAADA4uJCXQAAgWH4H4CaQhgAIgCLCQGoSYQBIEy5agJKSkp8LiaUnp5ODQGAakEYAMJQxZoAX4qKiggDAKoFYQCW5XQ6tW3bNknS+eef7/HFWp3t88Gey9eywr6wmBCA6kIYgCVV/OW9fv16dzt8dbbPn865Alk+mMWEAFQnwgAsx98v74KCAiUnJ1db+7yvxwnkXP5+8Q8ZMkSxsbGMJgBQ7ZhnAJZT2S/v3bt3B32fYB+nqnP5W1a4c+fO6tSpE0EAQLWjZgCWU1lb+7nnnqtPPvkkqPsE+ziBnCs7O1vp6enMKwCgVlAzAMvx9ctb+vXXt69f5afzZezvF36g57Lb7dQEAKgVNmOMCXUhgFAI19EEAFDbCAMAAFgczQQAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwOMIAAAAWRxgAAMDiCAMAAFgcYQAAAIsjDAAAYHGEAQAALI4wAACAxREGAACwuP8P6M8K82WMVyMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualize_query_with_base_embeddings(\"Why McIlroy is confident about winning Madrid title\", embeddings, retriever, umap_transformer, base_umap_vectors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing Togther API Mistal Model using the utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.llm import LLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = LLM().get_llm_together()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pydantic Parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.output_parsers import PydanticOutputParser,ListOutputParser,StructuredOutputParser\n",
    "from langchain.schema.output_parser import StrOutputParser\n",
    "from langchain.prompts import PromptTemplate,ChatPromptTemplate\n",
    "from pydantic import BaseModel, Field\n",
    "from typing import List\n",
    "\n",
    "\n",
    "class LineList(BaseModel):\n",
    "    lines: list[str] = Field(description=\"Lines of text\")\n",
    "\n",
    "\n",
    "class LineListOutputParser(PydanticOutputParser):\n",
    "    def __init__(self) -> None:\n",
    "        super().__init__(pydantic_object=LineList)\n",
    "\n",
    "    def parse(self, text: str) -> list[str]:\n",
    "        lines = text.strip().split(\"\\n\")\n",
    "        return lines\n",
    "\n",
    "\n",
    "output_parser = LineListOutputParser()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Query Rewriter or Augmented Queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_rewrite(question:str):\n",
    "    \n",
    "    template=\"\"\"You are an AI language model assistant. Your task is to generate five\n",
    "    different versions of the given user question to retrieve relevant documents from a vector\n",
    "    database. By generating multiple perspectives on the user question, your goal is to help\n",
    "    the user overcome some of the limitations of the distance-based similarity search.\n",
    "    Provide these alternative questions separated by newlines. \n",
    "    Original question: {question}\n",
    "    Only provide the query, do not do numbering at the start of the questions.\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    prompt = ChatPromptTemplate.from_template(template)\n",
    "\n",
    "    #output_parser = ListOutputParser\n",
    "\n",
    "    chain = prompt | llm | output_parser\n",
    "\n",
    "    result = chain.invoke({\"question\" : question})\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"Tell me about Japan's Industrial Growth\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "augmented_queries = query_rewrite(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"1. What are the key factors that contributed to Japan's industrial development?\",\n",
       " \"    2. Describe the historical evolution of Japan's industrial sector.\",\n",
       " \"    3. Provide an overview of Japan's industrial achievements throughout history.\",\n",
       " \"    4. Discuss the major milestones in Japan's industrial progression.\",\n",
       " '    5. Explain the significant industrial transformations Japan has undergone.']"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "augmented_queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"What are the key factors that contributed to Japan's industrial development?\",\n",
       " \"Describe the historical evolution of Japan's industrial sector\",\n",
       " \"Provide an overview of Japan's industrial achievements throughout history\",\n",
       " \"Discuss the major milestones in Japan's industrial progression\",\n",
       " 'Explain the significant industrial transformations Japan has undergone']"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "pattern = r'[0-9]'\n",
    "\n",
    "aug_queries = []\n",
    "for line in augmented_queries:\n",
    "    aug_queries.append(re.sub(pattern, '', line).replace('.','').strip())\n",
    "\n",
    "aug_queries \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve the docouments for Augmented Queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = [retriever.get_relevant_documents(query) for query in aug_queries]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Removal of Duplicate Documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_contents = set()\n",
    "unique_docs = []\n",
    "for sublist in docs:\n",
    "    for doc in sublist:\n",
    "        if doc.page_content not in unique_contents:\n",
    "            unique_docs.append(doc)\n",
    "            unique_contents.add(doc.page_content)\n",
    "unique_contents = list(unique_contents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import CrossEncoder\n",
    "cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creation of Query(Original Query) and Doc Pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairs = []\n",
    "for doc in unique_contents:\n",
    "    pairs.append([query, doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting Scores on Pairs using Cross Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.09827161, -4.024235  , -2.9265568 ], dtype=float32)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = cross_encoder.predict(pairs)\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.09827161,\n",
       "  \"Japanese industry is growing faster than expected, boosting hopes that the country's retreat back\"),\n",
       " (-2.9265568, 'Industrial revival hope for Japan'),\n",
       " (-4.024235,\n",
       "  \"exports, normally the engine for Japan's economy in the face of weak domestic demand, had helped\")]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scored_docs = zip(scores, unique_contents)\n",
    "sorted_docs = sorted(scored_docs, reverse=True)\n",
    "sorted_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reranking the documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Japanese industry is growing faster than expected, boosting hopes that the country's retreat back\",\n",
       " 'Industrial revival hope for Japan',\n",
       " \"exports, normally the engine for Japan's economy in the face of weak domestic demand, had helped\"]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reranked_docs = [doc for _, doc in sorted_docs]\n",
    "reranked_docs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}