flant5_rag.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7c490dde-f73c-4acc-9571-5a7afbc9d841",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:29:39.783140Z",
     "iopub.status.busy": "2024-08-02T08:29:39.782716Z",
     "iopub.status.idle": "2024-08-02T08:29:43.114780Z",
     "shell.execute_reply": "2024-08-02T08:29:43.113954Z",
     "shell.execute_reply.started": "2024-08-02T08:29:39.783111Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting python-dotenv\n",
      "  Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n",
      "Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
      "Installing collected packages: python-dotenv\n",
      "Successfully installed python-dotenv-1.0.1\n"
     ]
    }
   ],
   "source": [
    "!pip install python-dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "721d0523-ebd6-479d-8c30-9742d5e48c9d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:29:44.963381Z",
     "iopub.status.busy": "2024-08-02T08:29:44.962947Z",
     "iopub.status.idle": "2024-08-02T08:29:48.457246Z",
     "shell.execute_reply": "2024-08-02T08:29:48.456606Z",
     "shell.execute_reply.started": "2024-08-02T08:29:44.963353Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "from minsearch import Index\n",
    "\n",
    "load_dotenv()\n",
    "os.environ[\"HF_HOME\"] = \"/run/cache/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "acee66c5-a2c8-419f-b88c-c5398908588d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T07:57:00.847961Z",
     "iopub.status.busy": "2024-08-02T07:57:00.847573Z",
     "iopub.status.idle": "2024-08-02T07:59:30.394171Z",
     "shell.execute_reply": "2024-08-02T07:59:30.393537Z",
     "shell.execute_reply.started": "2024-08-02T07:57:00.847936Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfd40153a98344e098629d0c0030754b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/2.54k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "768dd965d3b4469f97fcd662889f3839",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e9ae0e28c514aa5ad0c0365e6b8adc4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/2.20k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "894113d8ca374581a91a393994ce84ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/2.42M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1534910942e942babf563e7bd1cd8fb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/1.44k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0485b2b7ae041ee97f71d32bc9c8650",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors.index.json:   0%|          | 0.00/53.0k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c643e910b1c84dcdb842a87c4c211095",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7692eb7b02dc434fa9674e150fab55ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00001-of-00002.safetensors:   0%|          | 0.00/9.45G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d07c5ec9809e47039f84f9631ff323cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00002-of-00002.safetensors:   0%|          | 0.00/1.95G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8959fbe04164434c9cb7144b494269dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ded280d28f314801967bc11da3c2f855",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
    "\n",
    "tokenizer = T5Tokenizer.from_pretrained(\"google/flan-t5-xl\")\n",
    "model = T5ForConditionalGeneration.from_pretrained(\"google/flan-t5-xl\", device_map=\"auto\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9c6de6b0-c1cd-422f-a5c5-d8e3592219f2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:21:07.537847Z",
     "iopub.status.busy": "2024-08-02T08:21:07.537373Z",
     "iopub.status.idle": "2024-08-02T08:21:07.549551Z",
     "shell.execute_reply": "2024-08-02T08:21:07.548637Z",
     "shell.execute_reply.started": "2024-08-02T08:21:07.537821Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[13959,  1566,    12,  2968,    10,   571,   625,    33,    25,    58,\n",
      "             1]], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "input_text = \"translate English to German: How old are you?\"\n",
    "input_ids = tokenizer(input_text, return_tensors=\"pt\").input_ids.to(\"cuda\")\n",
    "\n",
    "print(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d006d7e8-3703-4d12-9e5e-46238175ac4e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:21:12.897858Z",
     "iopub.status.busy": "2024-08-02T08:21:12.897467Z",
     "iopub.status.idle": "2024-08-02T08:21:14.908093Z",
     "shell.execute_reply": "2024-08-02T08:21:14.907369Z",
     "shell.execute_reply.started": "2024-08-02T08:21:12.897833Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/saturncloud/envs/saturn/lib/python3.9/site-packages/transformers/generation/utils.py:1259: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Wie alt sind Sie?</s>\n"
     ]
    }
   ],
   "source": [
    "outputs = model.generate(input_ids)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "495f2b7f-a25f-4c3b-b5e3-2966e83ca36b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:21:38.747836Z",
     "iopub.status.busy": "2024-08-02T08:21:38.747443Z",
     "iopub.status.idle": "2024-08-02T08:21:38.752658Z",
     "shell.execute_reply": "2024-08-02T08:21:38.751751Z",
     "shell.execute_reply.started": "2024-08-02T08:21:38.747810Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[   0, 2739, 4445,  436,  292,   58,    1]], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "print(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6f272af5-7e3d-4877-8a84-6b1c06b26214",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:30:04.943081Z",
     "iopub.status.busy": "2024-08-02T08:30:04.942525Z",
     "iopub.status.idle": "2024-08-02T08:30:04.950846Z",
     "shell.execute_reply": "2024-08-02T08:30:04.950154Z",
     "shell.execute_reply.started": "2024-08-02T08:30:04.943054Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "documents = []\n",
    "\n",
    "with open('documents.json', 'r') as file:\n",
    "\n",
    "    docs = json.load(file)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b1a97a00-39c9-437e-9653-12a7f518c39d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:30:07.062874Z",
     "iopub.status.busy": "2024-08-02T08:30:07.062476Z",
     "iopub.status.idle": "2024-08-02T08:30:07.067256Z",
     "shell.execute_reply": "2024-08-02T08:30:07.066295Z",
     "shell.execute_reply.started": "2024-08-02T08:30:07.062849Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for doc in docs:\n",
    "\n",
    "    for document in doc['documents']:\n",
    "\n",
    "        document['course'] = doc['course'] \n",
    "        documents.append(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0ce75afb-a676-4d4b-8ff9-fcc3539926d2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:30:08.288595Z",
     "iopub.status.busy": "2024-08-02T08:30:08.288202Z",
     "iopub.status.idle": "2024-08-02T08:30:08.393384Z",
     "shell.execute_reply": "2024-08-02T08:30:08.392579Z",
     "shell.execute_reply.started": "2024-08-02T08:30:08.288571Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<minsearch.Index at 0x7f1b68b61550>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textfields = [\"text\", \"section\", \"question\"]\n",
    "\n",
    "\n",
    "indobject = Index(text_fields = textfields, keyword_fields = ['course'])\n",
    "indobject.fit(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4b201e90-5752-450b-a77d-0d70238e7977",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:35:18.192458Z",
     "iopub.status.busy": "2024-08-02T08:35:18.192063Z",
     "iopub.status.idle": "2024-08-02T08:35:18.196821Z",
     "shell.execute_reply": "2024-08-02T08:35:18.196100Z",
     "shell.execute_reply.started": "2024-08-02T08:35:18.192432Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def search(query, boost_dict = {\"question\": 3}, filter_dict =  {\"course\":\"mlops-zoomcamp\"}, num_res = 5 ):\n",
    "\n",
    "    \n",
    "    context = indobject.search(query = query, boost_dict = boost_dict, filter_dict = filter_dict,\n",
    "                               num_results = num_res\n",
    "                    )\n",
    "    \n",
    "    return context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c7c71ae1-4916-4290-8b54-ec48ed9860c2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:30:19.763055Z",
     "iopub.status.busy": "2024-08-02T08:30:19.762606Z",
     "iopub.status.idle": "2024-08-02T08:30:19.768259Z",
     "shell.execute_reply": "2024-08-02T08:30:19.767304Z",
     "shell.execute_reply.started": "2024-08-02T08:30:19.763023Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def build_prompt(query, related_docs):\n",
    "    \n",
    "    \n",
    "    \n",
    "    prompt = \"\"\"\n",
    "    You are an assistant for teaching online courses answer the question based on the context provided.\n",
    "    Only the information available in the provided context should be used strictly adhere to this\n",
    "    \n",
    "    question:{query}\n",
    "    \n",
    "    context:{context}\n",
    "    \n",
    "    \n",
    "    \"\"\".strip()\n",
    "    \n",
    "    \n",
    "    context = \"\"\n",
    "    for doc in related_docs:\n",
    "        \n",
    "        context += f\"section: {doc['section']}\\nquestion: {doc['question']}\\nanswer: {doc['text']} \\n\\n\"\n",
    "        \n",
    "    \n",
    "    prompt = prompt.format(query = query, context = context).strip()\n",
    "    \n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7a0a39d1-f8c5-4ab8-b55a-bcf750c9a3e8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:42:59.357750Z",
     "iopub.status.busy": "2024-08-02T08:42:59.357355Z",
     "iopub.status.idle": "2024-08-02T08:42:59.363460Z",
     "shell.execute_reply": "2024-08-02T08:42:59.362459Z",
     "shell.execute_reply.started": "2024-08-02T08:42:59.357725Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def chat(prompt,generate_params = {}):\n",
    "    \n",
    "    input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids.to(\"cuda\")\n",
    "    \n",
    "    outputs = model.generate(input_ids,\n",
    "                            max_length = generate_params.get(\"max_length\", 150),\n",
    "                             \n",
    "                            top_k = generate_params.get(\"top_k\", 50),\n",
    "                             \n",
    "                            top_p = generate_params.get(\"top_p\", 0.95),\n",
    "                             \n",
    "                            do_sample = generate_params.get(\"do_sample\",False),\n",
    "                             \n",
    "                            num_beams = generate_params.get(\"num_beams\", 5),\n",
    "                             \n",
    "                            temperature = generate_params.get(\"temperature\", 1.0))\n",
    "    \n",
    "    answer = tokenizer.decode(outputs[0])\n",
    "    return answer\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c1a1549d-69da-42a2-8511-2e9b746678ad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:40:40.133073Z",
     "iopub.status.busy": "2024-08-02T08:40:40.132648Z",
     "iopub.status.idle": "2024-08-02T08:40:40.137415Z",
     "shell.execute_reply": "2024-08-02T08:40:40.136702Z",
     "shell.execute_reply.started": "2024-08-02T08:40:40.133047Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def rag(query, boost_dict = {\"question\": 3}, course_filter =  {\"course\": \"data-engineering-zoomcamp\"}, generate_params ={}):\n",
    "    \n",
    "    context = search(query = query, boost_dict = boost_dict, filter_dict = course_filter)\n",
    "    \n",
    "    prompt = build_prompt(query, context)\n",
    "    \n",
    "    answer = chat(prompt, generate_params = generate_params)\n",
    "    \n",
    "    return answer\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5c1538e7-8381-444f-9c70-6f628c4507b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:43:04.717876Z",
     "iopub.status.busy": "2024-08-02T08:43:04.717501Z",
     "iopub.status.idle": "2024-08-02T08:43:14.477718Z",
     "shell.execute_reply": "2024-08-02T08:43:14.477083Z",
     "shell.execute_reply.started": "2024-08-02T08:43:04.717850Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "query = \"I just found that course has already started can i still enroll in it?\"\n",
    "answer = rag(query = query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "d75af29a-5ac4-465c-af38-65e469f522e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-02T08:43:16.047802Z",
     "iopub.status.busy": "2024-08-02T08:43:16.047405Z",
     "iopub.status.idle": "2024-08-02T08:43:16.053009Z",
     "shell.execute_reply": "2024-08-02T08:43:16.051993Z",
     "shell.execute_reply.started": "2024-08-02T08:43:16.047776Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"<pad> Yes, even if you don't register, you're still eligible to submit the homeworks. Be aware, however, that there will be deadlines for turning in the final projects. So don't leave everything for the last minute.</s>\""
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c711177a-736b-4d21-b6dd-ca9100c6ccb1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "saturn (Python 3)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}