fundamentals/document_analysis/notebooks/09-unstructure-data-to-structured-data.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieve Information from Specific Data Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up Azure OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import openai\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "# Set up Azure OpenAI\n",
    "load_dotenv()\n",
    "openai.api_type = \"azure\"\n",
    "openai.api_base = \"\" # Api base is the 'Endpoint' which can be found in Azure Portal where Azure OpenAI is created. It looks like https://xxxxxx.openai.azure.com/\n",
    "openai.api_version = \"2022-12-01\"\n",
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deploy a Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text-davinci-003\n",
      "Found a succeeded deployment that supports embeddings with id: Text-davinci-003.\n"
     ]
    }
   ],
   "source": [
    "# list models deployed with embeddings capability\n",
    "deployment_id = None\n",
    "result = openai.Deployment.list()\n",
    "desired_model = 'text-davinci-003'\n",
    "\n",
    "# check if desired model is already deployed\n",
    "for deployment in result.data:\n",
    "    if deployment[\"status\"] != \"succeeded\":\n",
    "        continue\n",
    "    \n",
    "    if deployment['model'] != desired_model:\n",
    "        continue\n",
    "    \n",
    "    deployment_id = deployment[\"id\"]; print(deployment_id)\n",
    "    break\n",
    "\n",
    "# if not model deployed, deploy one\n",
    "if not deployment_id:\n",
    "    print('No deployment with status: succeeded found.')\n",
    "    model = desired_model\n",
    "\n",
    "    # Now let's create the deployment\n",
    "    print(f'Creating a new deployment with model: {model}')\n",
    "    result = openai.Deployment.create(model=model, scale_settings={\"scale_type\":\"standard\"})\n",
    "    deployment_id = result[\"id\"]\n",
    "    print(f'Successfully created {model} with deployment_id {deployment_id}')\n",
    "else:\n",
    "    print(f'Found a succeeded deployment that supports embeddings with id: {deployment_id}.')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "fname = '../data/bbc-news-data.csv'\n",
    "df_orig = pd.read_csv(fname, delimiter='\\t', index_col=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>filename</th>\n",
       "      <th>title</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>business</td>\n",
       "      <td>001.txt</td>\n",
       "      <td>Ad sales boost Time Warner profit</td>\n",
       "      <td>Quarterly profits at US media giant TimeWarne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>business</td>\n",
       "      <td>002.txt</td>\n",
       "      <td>Dollar gains on Greenspan speech</td>\n",
       "      <td>The dollar has hit its highest level against ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>business</td>\n",
       "      <td>003.txt</td>\n",
       "      <td>Yukos unit buyer faces loan claim</td>\n",
       "      <td>The owners of embattled Russian oil giant Yuk...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>business</td>\n",
       "      <td>004.txt</td>\n",
       "      <td>High fuel prices hit BA's profits</td>\n",
       "      <td>British Airways has blamed high fuel prices f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>business</td>\n",
       "      <td>005.txt</td>\n",
       "      <td>Pernod takeover talk lifts Domecq</td>\n",
       "      <td>Shares in UK drinks and food firm Allied Dome...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2220</th>\n",
       "      <td>tech</td>\n",
       "      <td>397.txt</td>\n",
       "      <td>BT program to beat dialler scams</td>\n",
       "      <td>BT is introducing two initiatives to help bea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2221</th>\n",
       "      <td>tech</td>\n",
       "      <td>398.txt</td>\n",
       "      <td>Spam e-mails tempt net shoppers</td>\n",
       "      <td>Computer users across the world continue to i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2222</th>\n",
       "      <td>tech</td>\n",
       "      <td>399.txt</td>\n",
       "      <td>Be careful how you code</td>\n",
       "      <td>A new European directive could put software w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2223</th>\n",
       "      <td>tech</td>\n",
       "      <td>400.txt</td>\n",
       "      <td>US cyber security chief resigns</td>\n",
       "      <td>The man making sure US computer networks are ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2224</th>\n",
       "      <td>tech</td>\n",
       "      <td>401.txt</td>\n",
       "      <td>Losing yourself in online gaming</td>\n",
       "      <td>Online role playing games are time-consuming,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2225 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      category filename                              title  \\\n",
       "0     business  001.txt  Ad sales boost Time Warner profit   \n",
       "1     business  002.txt   Dollar gains on Greenspan speech   \n",
       "2     business  003.txt  Yukos unit buyer faces loan claim   \n",
       "3     business  004.txt  High fuel prices hit BA's profits   \n",
       "4     business  005.txt  Pernod takeover talk lifts Domecq   \n",
       "...        ...      ...                                ...   \n",
       "2220      tech  397.txt   BT program to beat dialler scams   \n",
       "2221      tech  398.txt    Spam e-mails tempt net shoppers   \n",
       "2222      tech  399.txt            Be careful how you code   \n",
       "2223      tech  400.txt    US cyber security chief resigns   \n",
       "2224      tech  401.txt   Losing yourself in online gaming   \n",
       "\n",
       "                                                content  \n",
       "0      Quarterly profits at US media giant TimeWarne...  \n",
       "1      The dollar has hit its highest level against ...  \n",
       "2      The owners of embattled Russian oil giant Yuk...  \n",
       "3      British Airways has blamed high fuel prices f...  \n",
       "4      Shares in UK drinks and food firm Allied Dome...  \n",
       "...                                                 ...  \n",
       "2220   BT is introducing two initiatives to help bea...  \n",
       "2221   Computer users across the world continue to i...  \n",
       "2222   A new European directive could put software w...  \n",
       "2223   The man making sure US computer networks are ...  \n",
       "2224   Online role playing games are time-consuming,...  \n",
       "\n",
       "[2225 rows x 4 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "DEVELOPMENT = False  # Set this to True for development on small subset of data\n",
    "\n",
    "if DEVELOPMENT:\n",
    "    # Sub-sample for development\n",
    "    df = df_orig.sample(n=20, replace=False, random_state=9).copy() # Set sample size\n",
    "else:\n",
    "    df = df_orig.copy()\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>filename</th>\n",
       "      <th>title</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>510</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>001.txt</td>\n",
       "      <td>Gallery unveils interactive tree</td>\n",
       "      <td>A Christmas tree that can receive text messag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>002.txt</td>\n",
       "      <td>Jarre joins fairytale celebration</td>\n",
       "      <td>French musician Jean-Michel Jarre is to perfo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>003.txt</td>\n",
       "      <td>Musical treatment for Capra film</td>\n",
       "      <td>The classic film It's A Wonderful Life is to ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>004.txt</td>\n",
       "      <td>Richard and Judy choose top books</td>\n",
       "      <td>The 10 authors shortlisted for a Richard and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>005.txt</td>\n",
       "      <td>Poppins musical gets flying start</td>\n",
       "      <td>The stage adaptation of children's film Mary ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>891</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>382.txt</td>\n",
       "      <td>Last Star Wars 'not for children'</td>\n",
       "      <td>The sixth and final Star Wars movie may not b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>892</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>383.txt</td>\n",
       "      <td>French honour for director Parker</td>\n",
       "      <td>British film director Sir Alan Parker has bee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>893</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>384.txt</td>\n",
       "      <td>Robots march to US cinema summit</td>\n",
       "      <td>Animated movie Robots has opened at the top o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>894</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>385.txt</td>\n",
       "      <td>Hobbit picture 'four years away'</td>\n",
       "      <td>Lord of the Rings director Peter Jackson has ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>895</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>386.txt</td>\n",
       "      <td>Buffy creator joins Wonder Woman</td>\n",
       "      <td>The creator of Buffy the Vampire Slayer is to...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>386 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          category filename                              title  \\\n",
       "510  entertainment  001.txt   Gallery unveils interactive tree   \n",
       "511  entertainment  002.txt  Jarre joins fairytale celebration   \n",
       "512  entertainment  003.txt   Musical treatment for Capra film   \n",
       "513  entertainment  004.txt  Richard and Judy choose top books   \n",
       "514  entertainment  005.txt  Poppins musical gets flying start   \n",
       "..             ...      ...                                ...   \n",
       "891  entertainment  382.txt  Last Star Wars 'not for children'   \n",
       "892  entertainment  383.txt  French honour for director Parker   \n",
       "893  entertainment  384.txt   Robots march to US cinema summit   \n",
       "894  entertainment  385.txt   Hobbit picture 'four years away'   \n",
       "895  entertainment  386.txt   Buffy creator joins Wonder Woman   \n",
       "\n",
       "                                               content  \n",
       "510   A Christmas tree that can receive text messag...  \n",
       "511   French musician Jean-Michel Jarre is to perfo...  \n",
       "512   The classic film It's A Wonderful Life is to ...  \n",
       "513   The 10 authors shortlisted for a Richard and ...  \n",
       "514   The stage adaptation of children's film Mary ...  \n",
       "..                                                 ...  \n",
       "891   The sixth and final Star Wars movie may not b...  \n",
       "892   British film director Sir Alan Parker has bee...  \n",
       "893   Animated movie Robots has opened at the top o...  \n",
       "894   Lord of the Rings director Peter Jackson has ...  \n",
       "895   The creator of Buffy the Vampire Slayer is to...  \n",
       "\n",
       "[386 rows x 4 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['category']=='entertainment']"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unstrcutured data to structured data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def retrieve_structured_data(prompt):\n",
    "    try:\n",
    "        # Request API\n",
    "        response = openai.Completion.create(\n",
    "            deployment_id= deployment_id, \n",
    "            prompt=prompt,\n",
    "            temperature=1,\n",
    "            max_tokens=300,\n",
    "            top_p=1.0,\n",
    "            frequency_penalty=0.0,\n",
    "            presence_penalty=1\n",
    "        )\n",
    "\n",
    "        # response\n",
    "        result = response['choices'][0]['text']; print(result)\n",
    "    except Exception as err:\n",
    "        print(f\"Unexpected {err=}, {type(err)=}\")\n",
    "\n",
    "    return "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample Queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The 10 authors shortlisted for a Richard and Judy book award in 2005 are hoping for a boost in sales following the success of this year's winner.  The TV couple's interest in the book world coined the term \"the Richard & Judy effect\" and created the top two best-selling paperbacks of 2004 so far. The finalists for 2005 include Andrew Taylor's The American Boy and Robbie Williams' autobiography Feel. This year's winner, Alice Sebold's The Lovely Bones, sold over one million. Joseph O'Connor's Star of the Sea came second and saw sales increase by 350%. The best read award, on Richard Madeley and Judy Finnigan's Channel 4 show, is part of the British Book Awards. David Mitchell's Booker-shortlisted novel, Cloud Atlas, makes it into this year's top 10 along with several lesser known works.  \"There's no doubt that this year's selection of book club entries is the best yet. If anything, the choice is even wider than last time,\" said Madeley. \"It was very hard to follow last year's extremely successful list, but we think this year's books will do even better,\" said Richard and Judy executive producer Amanda Ross. \"We were spoiled for choice and it was tough getting down to only 10 from the 301 submitted.\"  \n",
      "  \n",
      "\n",
      " Extract author and books from the text above in a table. \n",
      "\n",
      " \n",
      " \n",
      "Author          | Book           \n",
      "----------------|-----------------\n",
      "Andrew Taylor   | The American Boy \n",
      "Robbie Williams | Feel \n",
      "Alice Sebold    | The Lovely Bones \n",
      "Joseph O'Connor | Star of the Sea\n",
      "David Mitchell  | Cloud Atlas\n"
     ]
    }
   ],
   "source": [
    "idx = 513 #index of the selected text\n",
    "\n",
    "# prompt postifx\n",
    "prompt_postfix = \"\"\" \n",
    "  \\n\\n Extract author and books from the text above in a table. \n",
    "\"\"\"\n",
    "# build prompt\n",
    "prompt = df['content'].loc[idx] + prompt_postfix; print(prompt)\n",
    "\n",
    "# query\n",
    "retrieve_structured_data(prompt=prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The sixth and final Star Wars movie may not be suitable for young children, film-maker George Lucas has said.  He told US TV show 60 Minutes that Revenge of the Sith would be the darkest and most violent of the series. \"I don't think I would take a five or six-year-old to this,\" he told the CBS programme, to be aired on Sunday. Lucas predicted the film would get a US rating advising parents some scenes may be unsuitable for under-13s. It opens in the UK and US on 19 May. He said he expected the film would be classified PG-13 - roughly equivalent to a British 12A rating.  The five previous Star Wars films have all carried less restrictive PG - parental guidance - ratings in the US. In the UK, they have all been passed U - suitable for all - with the exception of Attack of The Clones, which got a PG rating in 2002. Revenge of the Sith - the third prequel to the original 1977 Star Wars film - chronicles the transformation of the heroic Anakin Skywalker into the evil Darth Vader as he travels to a Hell-like planet composed of erupting volcanoes and molten lava. \"We're going to watch him make a pact with the devil,\" Lucas said. \"The film is much more dark, more emotional. It's much more of a tragedy.\"  \n",
      "  \n",
      "\n",
      " Extract Star Wars movie series and associated ratings from the text above into a table. \n",
      "\n",
      "\n",
      "Star Wars Movie  | US Rating | UK Rating \n",
      "-------------------------------------------\n",
      "Revenge of the Sith | PG-13    | 12A \n",
      "Attack of the Clones | PG       | PG \n",
      "Original 1977 Star Wars Film | PG      | U\n"
     ]
    }
   ],
   "source": [
    "idx = 891 #index of the selected text\n",
    "\n",
    "# prompt postifx\n",
    "prompt_postfix = \"\"\" \n",
    "  \\n\\n Extract Star Wars movie series and associated ratings from the text above into a table. \n",
    "\"\"\"\n",
    "# build prompt\n",
    "prompt = df['content'].loc[idx] + prompt_postfix; print(prompt)\n",
    "\n",
    "# query\n",
    "retrieve_structured_data(prompt=prompt)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "azureml_py38",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6d65a8c07f5b6469e0fc613f182488c0dccce05038bbda39e5ac9075c0454d11"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}