0_RAG_v0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import json\n",
    "\n",
    "# import\n",
    "from langchain_community.document_loaders import DirectoryLoader\n",
    "from langchain_community.embeddings.sentence_transformer import (\n",
    "    SentenceTransformerEmbeddings\n",
    ")\n",
    "from langchain_community.embeddings import OpenAIEmbeddings\n",
    "from langchain_community.vectorstores import Chroma\n",
    "from langchain_text_splitters import CharacterTextSplitter\n",
    "from openai import OpenAI, AzureOpenAI\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = json.load(open('./config.json'))\n",
    "\n",
    "organization = \"\"\n",
    "\n",
    "api_source = 'OpenAI'\n",
    "\n",
    "api_key = config[api_source]['openai_api_key'] #constants.AZURE_OPENAI_KEY\n",
    "api_endpoint = config[api_source]['openai_api_endpoint']  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "Directory not found: './reference_labels'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[6], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m loader \u001b[38;5;241m=\u001b[39m DirectoryLoader(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m./reference_labels\u001b[39m\u001b[38;5;124m'\u001b[39m, glob \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m*.txt\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m documents  \u001b[38;5;241m=\u001b[39m loader\u001b[38;5;241m.\u001b[39mload()\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llm_cpus/lib/python3.11/site-packages/langchain_community/document_loaders/directory.py:146\u001b[0m, in \u001b[0;36mDirectoryLoader.load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    144\u001b[0m p \u001b[38;5;241m=\u001b[39m Path(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpath)\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m p\u001b[38;5;241m.\u001b[39mexists():\n\u001b[0;32m--> 146\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDirectory not found: \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m p\u001b[38;5;241m.\u001b[39mis_dir():\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpected directory, got file: \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: Directory not found: './reference_labels'"
     ]
    }
   ],
   "source": [
    "loader = DirectoryLoader('./reference_labels', glob = '*.txt')\n",
    "documents  = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/undinagisladottir/opt/anaconda3/envs/llm_cpus/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The class `langchain_community.embeddings.openai.OpenAIEmbeddings` was deprecated in langchain-community 0.0.9 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAIEmbeddings`.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "## saving embeddings\n",
    "# text_splitter = CharacterTextSplitter()\n",
    "# chunks = text_splitter.split_documents(documents)\n",
    "# db = Chroma.from_documents(chunks, OpenAIEmbeddings(openai_api_key=api_key), persist_directory=\"../Chroma\")\n",
    "# db.persist()\n",
    "\n",
    "## loading them\n",
    "db = Chroma(persist_directory=\"../Chroma\", embedding_function=OpenAIEmbeddings(openai_api_key=api_key))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = OpenAI(api_key=api_key)\n",
    "chat_completion = client.chat.completions.create(\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": \"you are a fact finder and you answer in yes, no or unsure\"},\n",
    "        {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": \"\"\"\n",
    "            Is there any evidence for the association between the use of Lisinopril and Acute Kidney Injury?\"\"\"\n",
    "        }\n",
    "    ],\n",
    "    model='gpt-4-1106-preview',\n",
    "    temperature=0,\n",
    ")\n",
    "term = chat_completion.choices[0].message.content\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Yes.'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "term"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm_cpus",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}