OpenAI/Movie-recommender/part1Embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "18da7b67",
   "metadata": {},
   "source": [
    "# Movies recommendation with Azure Open AI & Azure Cognitive Search\n",
    "## Part 1: Embeddings generation with Azure Open AI and Azure Cognitive Search ingestion"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f20a08b3",
   "metadata": {},
   "source": [
    "<img src=\"https://github.com/retkowsky/images/blob/master/movies_search.png?raw=true\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "649fb619",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%pip install azure-search-documents --pre --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6682834a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import math\n",
    "import openai\n",
    "import os\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import pytz\n",
    "import requests\n",
    "import sys\n",
    "import tiktoken\n",
    "import time\n",
    "\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.indexes import SearchIndexClient\n",
    "from azure.search.documents.indexes.models import (\n",
    "    VectorSearch,\n",
    "    SimpleField,\n",
    "    SemanticSettings,\n",
    "    SemanticField,\n",
    "    SemanticConfiguration,\n",
    "    SearchIndex,\n",
    "    SearchFieldDataType,\n",
    "    SearchField,\n",
    "    SearchableField,\n",
    "    PrioritizedFields,\n",
    "    HnswVectorSearchAlgorithmConfiguration,\n",
    ")\n",
    "\n",
    "from datetime import datetime\n",
    "#from dotenv import load_dotenv\n",
    "from openai.embeddings_utils import get_embedding, cosine_similarity\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "018fc6e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.11.5 (main, Sep 19 2023, 13:46:29) [GCC 10.2.1 20210110]'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9c70ed19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Local time: 19-Sep-2023 15:36:09\n"
     ]
    }
   ],
   "source": [
    "local_tz = pytz.timezone(requests.get(\"https://ipinfo.io\").json()[\"timezone\"])\n",
    "print(\"Local time:\", datetime.now(local_tz).strftime(\"%d-%b-%Y %H:%M:%S\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "28ca0f6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Open AI version: 0.28.0\n"
     ]
    }
   ],
   "source": [
    "print(\"Open AI version:\", openai.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a4752d9",
   "metadata": {},
   "source": [
    "Load configuration settings from GitHub Codespace secrets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "df6462e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load_dotenv(\"azure.env\")\n",
    "\n",
    "\n",
    "openai.api_type: str = \"azure\"\n",
    "# openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "openai.api_key = os.getenv(\"AZURE_OPENAI_API_KEY\")\n",
    "# openai.api_base = os.getenv(\"OPENAI_API_BASE\")\n",
    "openai.api_base = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "# openai.api_version = os.getenv(\"OPENAI_API_VERSION\")\n",
    "openai.api_version = os.getenv(\"AZURE_OPENAI_MODEL_CHAT_VERSION\")\n",
    "\n",
    "acs_endpoint = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "acs_key = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e68d463c",
   "metadata": {},
   "source": [
    "Make sure you include your embedding model deployment name (it may be different)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7b4281f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Azure Open AI embeddings model to use\n",
    "embeddings_engine = \"text-embedding-ada-002-Unai\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0463dd72",
   "metadata": {},
   "source": [
    "- Vector search is in public preview\n",
    "- Model name: text-embedding-ada-002\n",
    "- Model version: 2\n",
    "- API version: 2023-05-15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6f1ae436",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Azure Cognitive Search index name to create\n",
    "index_name = \"moviereview\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0bbefd2",
   "metadata": {},
   "source": [
    "## 0. Azure Cognitive Search vector store\n",
    "<img src=\"https://github.com/retkowsky/images/blob/master/vector_search_architecture.png?raw=true\">"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e725f2e0",
   "metadata": {},
   "source": [
    "## 1. Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "18707923",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-rw-rw- 1 vscode vscode 2.9M Sep 19 14:38 movies.xlsx\n"
     ]
    }
   ],
   "source": [
    "EXCEL_FILE = \"movies.xlsx\"\n",
    "\n",
    "!ls $EXCEL_FILE -lh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7d12599f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel(EXCEL_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5a164ddc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"title\"] = df[\"title\"].astype(str)\n",
    "df[\"year\"] = df[\"year\"].astype(str)\n",
    "\n",
    "columns_to_drop = [\"tagline\", \"website\"]\n",
    "df = df.drop(columns_to_drop, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "58e7a836",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>description</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tt0035423</td>\n",
       "      <td>Kate &amp; Leopold</td>\n",
       "      <td>Meg Ryan Hugh Jackman Liev Schreiber Breckin M...</td>\n",
       "      <td>James Mangold</td>\n",
       "      <td>When her scientist ex-boyfriend discovers a po...</td>\n",
       "      <td>Comedy Fantasy Romance Science Fiction</td>\n",
       "      <td>2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tt0052646</td>\n",
       "      <td>The Brain That Wouldn't Die</td>\n",
       "      <td>Jason Evers Virginia Leith Doris Brent Audrey ...</td>\n",
       "      <td>Joseph Green</td>\n",
       "      <td>Dr. Bill Cortner (Jason Evers) and his fiancÃ©...</td>\n",
       "      <td>Horror Science Fiction</td>\n",
       "      <td>1962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tt0053559</td>\n",
       "      <td>13 Ghosts</td>\n",
       "      <td>Charles Herbert Jo Morrow Martin Milner Rosema...</td>\n",
       "      <td>William Castle</td>\n",
       "      <td>Reclusive Dr. Zorba has died and left his mans...</td>\n",
       "      <td>Horror</td>\n",
       "      <td>1960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tt0053580</td>\n",
       "      <td>The Alamo</td>\n",
       "      <td>John Wayne Richard Widmark Laurence Harvey Fra...</td>\n",
       "      <td>John Wayne</td>\n",
       "      <td>The legendary true story of a small band of so...</td>\n",
       "      <td>Action Adventure Drama History Western</td>\n",
       "      <td>1960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tt0053604</td>\n",
       "      <td>The Apartment</td>\n",
       "      <td>Jack Lemmon Shirley MacLaine Fred MacMurray Ra...</td>\n",
       "      <td>Billy Wilder</td>\n",
       "      <td>Bud Baxter is a minor clerk in a huge New York...</td>\n",
       "      <td>Comedy Drama Romance</td>\n",
       "      <td>1960</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     imdb_id                        title  \\\n",
       "0  tt0035423               Kate & Leopold   \n",
       "1  tt0052646  The Brain That Wouldn't Die   \n",
       "2  tt0053559                    13 Ghosts   \n",
       "3  tt0053580                    The Alamo   \n",
       "4  tt0053604                The Apartment   \n",
       "\n",
       "                                                cast        director  \\\n",
       "0  Meg Ryan Hugh Jackman Liev Schreiber Breckin M...   James Mangold   \n",
       "1  Jason Evers Virginia Leith Doris Brent Audrey ...    Joseph Green   \n",
       "2  Charles Herbert Jo Morrow Martin Milner Rosema...  William Castle   \n",
       "3  John Wayne Richard Widmark Laurence Harvey Fra...      John Wayne   \n",
       "4  Jack Lemmon Shirley MacLaine Fred MacMurray Ra...    Billy Wilder   \n",
       "\n",
       "                                         description  \\\n",
       "0  When her scientist ex-boyfriend discovers a po...   \n",
       "1  Dr. Bill Cortner (Jason Evers) and his fiancÃ©...   \n",
       "2  Reclusive Dr. Zorba has died and left his mans...   \n",
       "3  The legendary true story of a small band of so...   \n",
       "4  Bud Baxter is a minor clerk in a huge New York...   \n",
       "\n",
       "                                   genres  year  \n",
       "0  Comedy Fantasy Romance Science Fiction  2001  \n",
       "1                  Horror Science Fiction  1962  \n",
       "2                                  Horror  1960  \n",
       "3  Action Adventure Drama History Western  1960  \n",
       "4                    Comedy Drama Romance  1960  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2c318304",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10785, 7)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "35da0218",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10784, 7)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop_duplicates()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b6a448e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>description</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tt0035423</td>\n",
       "      <td>Kate &amp; Leopold</td>\n",
       "      <td>Meg Ryan Hugh Jackman Liev Schreiber Breckin M...</td>\n",
       "      <td>James Mangold</td>\n",
       "      <td>When her scientist ex-boyfriend discovers a po...</td>\n",
       "      <td>Comedy Fantasy Romance Science Fiction</td>\n",
       "      <td>2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tt0052646</td>\n",
       "      <td>The Brain That Wouldn't Die</td>\n",
       "      <td>Jason Evers Virginia Leith Doris Brent Audrey ...</td>\n",
       "      <td>Joseph Green</td>\n",
       "      <td>Dr. Bill Cortner (Jason Evers) and his fiancÃ©...</td>\n",
       "      <td>Horror Science Fiction</td>\n",
       "      <td>1962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tt0053559</td>\n",
       "      <td>13 Ghosts</td>\n",
       "      <td>Charles Herbert Jo Morrow Martin Milner Rosema...</td>\n",
       "      <td>William Castle</td>\n",
       "      <td>Reclusive Dr. Zorba has died and left his mans...</td>\n",
       "      <td>Horror</td>\n",
       "      <td>1960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tt0053580</td>\n",
       "      <td>The Alamo</td>\n",
       "      <td>John Wayne Richard Widmark Laurence Harvey Fra...</td>\n",
       "      <td>John Wayne</td>\n",
       "      <td>The legendary true story of a small band of so...</td>\n",
       "      <td>Action Adventure Drama History Western</td>\n",
       "      <td>1960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tt0053604</td>\n",
       "      <td>The Apartment</td>\n",
       "      <td>Jack Lemmon Shirley MacLaine Fred MacMurray Ra...</td>\n",
       "      <td>Billy Wilder</td>\n",
       "      <td>Bud Baxter is a minor clerk in a huge New York...</td>\n",
       "      <td>Comedy Drama Romance</td>\n",
       "      <td>1960</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     imdb_id                        title  \\\n",
       "0  tt0035423               Kate & Leopold   \n",
       "1  tt0052646  The Brain That Wouldn't Die   \n",
       "2  tt0053559                    13 Ghosts   \n",
       "3  tt0053580                    The Alamo   \n",
       "4  tt0053604                The Apartment   \n",
       "\n",
       "                                                cast        director  \\\n",
       "0  Meg Ryan Hugh Jackman Liev Schreiber Breckin M...   James Mangold   \n",
       "1  Jason Evers Virginia Leith Doris Brent Audrey ...    Joseph Green   \n",
       "2  Charles Herbert Jo Morrow Martin Milner Rosema...  William Castle   \n",
       "3  John Wayne Richard Widmark Laurence Harvey Fra...      John Wayne   \n",
       "4  Jack Lemmon Shirley MacLaine Fred MacMurray Ra...    Billy Wilder   \n",
       "\n",
       "                                         description  \\\n",
       "0  When her scientist ex-boyfriend discovers a po...   \n",
       "1  Dr. Bill Cortner (Jason Evers) and his fiancÃ©...   \n",
       "2  Reclusive Dr. Zorba has died and left his mans...   \n",
       "3  The legendary true story of a small band of so...   \n",
       "4  Bud Baxter is a minor clerk in a huge New York...   \n",
       "\n",
       "                                   genres  year  \n",
       "0  Comedy Fantasy Romance Science Fiction  2001  \n",
       "1                  Horror Science Fiction  1962  \n",
       "2                                  Horror  1960  \n",
       "3  Action Adventure Drama History Western  1960  \n",
       "4                    Comedy Drama Romance  1960  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cb26c362",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removing some extra spaces\n",
    "df[\"description\"] = df[\"description\"].str.replace(\"  \", \" \")\n",
    "df[\"title\"] = df[\"title\"].str.replace(\"  \", \" \")\n",
    "df[\"cast\"] = df[\"cast\"].str.replace(\"  \", \" \")\n",
    "df[\"director\"] = df[\"director\"].str.replace(\"  \", \" \")\n",
    "df[\"genres\"] = df[\"genres\"].str.replace(\"  \", \" \")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c73d1649",
   "metadata": {},
   "source": [
    "Analyze number of tokens needed for description field embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "558a72c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10784"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n",
    "df[\"nb_tokens\"] = df[\"description\"].apply(lambda x: len(tokenizer.encode(x)))\n",
    "df = df[df.nb_tokens < 8192]\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "074d0e79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>description</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "      <th>nb_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tt0035423</td>\n",
       "      <td>Kate &amp; Leopold</td>\n",
       "      <td>Meg Ryan Hugh Jackman Liev Schreiber Breckin M...</td>\n",
       "      <td>James Mangold</td>\n",
       "      <td>When her scientist ex-boyfriend discovers a po...</td>\n",
       "      <td>Comedy Fantasy Romance Science Fiction</td>\n",
       "      <td>2001</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tt0052646</td>\n",
       "      <td>The Brain That Wouldn't Die</td>\n",
       "      <td>Jason Evers Virginia Leith Doris Brent Audrey ...</td>\n",
       "      <td>Joseph Green</td>\n",
       "      <td>Dr. Bill Cortner (Jason Evers) and his fiancÃ©...</td>\n",
       "      <td>Horror Science Fiction</td>\n",
       "      <td>1962</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tt0053559</td>\n",
       "      <td>13 Ghosts</td>\n",
       "      <td>Charles Herbert Jo Morrow Martin Milner Rosema...</td>\n",
       "      <td>William Castle</td>\n",
       "      <td>Reclusive Dr. Zorba has died and left his mans...</td>\n",
       "      <td>Horror</td>\n",
       "      <td>1960</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tt0053580</td>\n",
       "      <td>The Alamo</td>\n",
       "      <td>John Wayne Richard Widmark Laurence Harvey Fra...</td>\n",
       "      <td>John Wayne</td>\n",
       "      <td>The legendary true story of a small band of so...</td>\n",
       "      <td>Action Adventure Drama History Western</td>\n",
       "      <td>1960</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tt0053604</td>\n",
       "      <td>The Apartment</td>\n",
       "      <td>Jack Lemmon Shirley MacLaine Fred MacMurray Ra...</td>\n",
       "      <td>Billy Wilder</td>\n",
       "      <td>Bud Baxter is a minor clerk in a huge New York...</td>\n",
       "      <td>Comedy Drama Romance</td>\n",
       "      <td>1960</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     imdb_id                        title  \\\n",
       "0  tt0035423               Kate & Leopold   \n",
       "1  tt0052646  The Brain That Wouldn't Die   \n",
       "2  tt0053559                    13 Ghosts   \n",
       "3  tt0053580                    The Alamo   \n",
       "4  tt0053604                The Apartment   \n",
       "\n",
       "                                                cast        director  \\\n",
       "0  Meg Ryan Hugh Jackman Liev Schreiber Breckin M...   James Mangold   \n",
       "1  Jason Evers Virginia Leith Doris Brent Audrey ...    Joseph Green   \n",
       "2  Charles Herbert Jo Morrow Martin Milner Rosema...  William Castle   \n",
       "3  John Wayne Richard Widmark Laurence Harvey Fra...      John Wayne   \n",
       "4  Jack Lemmon Shirley MacLaine Fred MacMurray Ra...    Billy Wilder   \n",
       "\n",
       "                                         description  \\\n",
       "0  When her scientist ex-boyfriend discovers a po...   \n",
       "1  Dr. Bill Cortner (Jason Evers) and his fiancÃ©...   \n",
       "2  Reclusive Dr. Zorba has died and left his mans...   \n",
       "3  The legendary true story of a small band of so...   \n",
       "4  Bud Baxter is a minor clerk in a huge New York...   \n",
       "\n",
       "                                   genres  year  nb_tokens  \n",
       "0  Comedy Fantasy Romance Science Fiction  2001         82  \n",
       "1                  Horror Science Fiction  1962        106  \n",
       "2                                  Horror  1960         53  \n",
       "3  Action Adventure Drama History Western  1960         36  \n",
       "4                    Comedy Drama Romance  1960         67  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "58e58e19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    10784.000000\n",
       "mean        62.621384\n",
       "std         35.517581\n",
       "min          2.000000\n",
       "25%         35.000000\n",
       "50%         57.000000\n",
       "75%         81.000000\n",
       "max        232.000000\n",
       "Name: nb_tokens, dtype: float64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"nb_tokens\"].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1f3ddb18",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(\"nb_tokens\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "85db3ca2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10784, 7)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e22b7044",
   "metadata": {},
   "source": [
    "## 2. Generating text embeddings with Azure Open AI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdf7bd17",
   "metadata": {},
   "source": [
    "### Vectors embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "eafef145",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding engine: text-embedding-ada-002-Unai\n"
     ]
    }
   ],
   "source": [
    "print(\"Embedding engine:\", embeddings_engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3225abc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def openai_text_embeddings(text):\n",
    "    \"\"\"\n",
    "    Generating embeddings from text using Azure Open AI\n",
    "    Input: text\n",
    "    Output: text embeddings\n",
    "    \"\"\"\n",
    "    embeddings = openai.Embedding.create(\n",
    "        input=text,\n",
    "        deployment_id=embeddings_engine,\n",
    "    )\n",
    "    embeddings = embeddings[\"data\"][0][\"embedding\"]\n",
    "\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a42c3f8c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-0.03617486730217934,\n",
       " -0.005520837381482124,\n",
       " -0.007070655468851328,\n",
       " -0.030174769461154938,\n",
       " 0.0020399712957441807]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb = openai_text_embeddings(\"My name is James Bond\")\n",
    "emb[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "51db08ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of the embeddings = 1536\n"
     ]
    }
   ],
   "source": [
    "print(\"Size of the embeddings =\", len(emb))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b89ce44",
   "metadata": {},
   "source": [
    "### Running the embedding for the 'description/overview' column (for me it took 20 minutes , maybe faster on other machines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "39cce04f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running the embedding process...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10784/10784 [20:14<00:00,  8.88it/s] \n"
     ]
    }
   ],
   "source": [
    "print(\"Running the embedding process...\")\n",
    "df[\"embed_overview\"] = None\n",
    "\n",
    "with tqdm(total=len(df)) as pbar:\n",
    "    def apply_embedding(x):\n",
    "        \"\"\"\n",
    "        Azure Open AI text embedding\n",
    "        \"\"\"\n",
    "        global pbar\n",
    "        embedding = get_embedding(x[\"description\"], engine=embeddings_engine)\n",
    "        pbar.update(1)  # Update the progress bar\n",
    "        return embedding\n",
    "    df[\"embed_overview\"] = df.apply(apply_embedding, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c69f5a16",
   "metadata": {},
   "source": [
    "### Running the embedding for the 'title' column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "31564a51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running the embedding process...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 1438/10784 [02:37<05:40, 27.41it/s]  "
     ]
    }
   ],
   "source": [
    "print(\"Running the embedding process...\")\n",
    "df[\"embed_title\"] = None\n",
    "\n",
    "with tqdm(total=len(df)) as pbar:\n",
    "    def apply_embedding(x):\n",
    "        \"\"\"\n",
    "        Azure Open AI text embedding\n",
    "        \"\"\"\n",
    "        global pbar\n",
    "        embedding = get_embedding(x[\"title\"], engine=embeddings_engine)\n",
    "        pbar.update(1)  # Update the progress bar\n",
    "        return embedding\n",
    "    df[\"embed_title\"] = df.apply(apply_embedding, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5e7e197",
   "metadata": {},
   "source": [
    "### Saving the documents (initial data + embeddings) into a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "bb151f8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>description</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "      <th>embed_overview</th>\n",
       "      <th>embed_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tt0035423</td>\n",
       "      <td>Kate &amp; Leopold</td>\n",
       "      <td>Meg Ryan Hugh Jackman Liev Schreiber Breckin M...</td>\n",
       "      <td>James Mangold</td>\n",
       "      <td>When her scientist ex-boyfriend discovers a po...</td>\n",
       "      <td>Comedy Fantasy Romance Science Fiction</td>\n",
       "      <td>2001</td>\n",
       "      <td>[0.021089695394039154, -0.010973065160214901, ...</td>\n",
       "      <td>[0.012436236254870892, 0.017795050516724586, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tt0052646</td>\n",
       "      <td>The Brain That Wouldn't Die</td>\n",
       "      <td>Jason Evers Virginia Leith Doris Brent Audrey ...</td>\n",
       "      <td>Joseph Green</td>\n",
       "      <td>Dr. Bill Cortner (Jason Evers) and his fiancÃ©...</td>\n",
       "      <td>Horror Science Fiction</td>\n",
       "      <td>1962</td>\n",
       "      <td>[-0.012167753651738167, -0.011197278276085854,...</td>\n",
       "      <td>[-0.030167536810040474, 0.009392774663865566, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tt0053559</td>\n",
       "      <td>13 Ghosts</td>\n",
       "      <td>Charles Herbert Jo Morrow Martin Milner Rosema...</td>\n",
       "      <td>William Castle</td>\n",
       "      <td>Reclusive Dr. Zorba has died and left his mans...</td>\n",
       "      <td>Horror</td>\n",
       "      <td>1960</td>\n",
       "      <td>[0.010904512368142605, -0.0042160083539783955,...</td>\n",
       "      <td>[-0.00030440607224591076, -0.01510559953749179...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tt0053580</td>\n",
       "      <td>The Alamo</td>\n",
       "      <td>John Wayne Richard Widmark Laurence Harvey Fra...</td>\n",
       "      <td>John Wayne</td>\n",
       "      <td>The legendary true story of a small band of so...</td>\n",
       "      <td>Action Adventure Drama History Western</td>\n",
       "      <td>1960</td>\n",
       "      <td>[-0.013121142983436584, -0.021301323547959328,...</td>\n",
       "      <td>[-0.016000520437955856, -0.018589602783322334,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tt0053604</td>\n",
       "      <td>The Apartment</td>\n",
       "      <td>Jack Lemmon Shirley MacLaine Fred MacMurray Ra...</td>\n",
       "      <td>Billy Wilder</td>\n",
       "      <td>Bud Baxter is a minor clerk in a huge New York...</td>\n",
       "      <td>Comedy Drama Romance</td>\n",
       "      <td>1960</td>\n",
       "      <td>[-0.00418621813878417, -0.010639390908181667, ...</td>\n",
       "      <td>[-0.0017813920276239514, -0.002438942203298211...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     imdb_id                        title   \n",
       "0  tt0035423               Kate & Leopold  \\\n",
       "1  tt0052646  The Brain That Wouldn't Die   \n",
       "2  tt0053559                    13 Ghosts   \n",
       "3  tt0053580                    The Alamo   \n",
       "4  tt0053604                The Apartment   \n",
       "\n",
       "                                                cast        director   \n",
       "0  Meg Ryan Hugh Jackman Liev Schreiber Breckin M...   James Mangold  \\\n",
       "1  Jason Evers Virginia Leith Doris Brent Audrey ...    Joseph Green   \n",
       "2  Charles Herbert Jo Morrow Martin Milner Rosema...  William Castle   \n",
       "3  John Wayne Richard Widmark Laurence Harvey Fra...      John Wayne   \n",
       "4  Jack Lemmon Shirley MacLaine Fred MacMurray Ra...    Billy Wilder   \n",
       "\n",
       "                                         description   \n",
       "0  When her scientist ex-boyfriend discovers a po...  \\\n",
       "1  Dr. Bill Cortner (Jason Evers) and his fiancÃ©...   \n",
       "2  Reclusive Dr. Zorba has died and left his mans...   \n",
       "3  The legendary true story of a small band of so...   \n",
       "4  Bud Baxter is a minor clerk in a huge New York...   \n",
       "\n",
       "                                   genres  year   \n",
       "0  Comedy Fantasy Romance Science Fiction  2001  \\\n",
       "1                  Horror Science Fiction  1962   \n",
       "2                                  Horror  1960   \n",
       "3  Action Adventure Drama History Western  1960   \n",
       "4                    Comedy Drama Romance  1960   \n",
       "\n",
       "                                      embed_overview   \n",
       "0  [0.021089695394039154, -0.010973065160214901, ...  \\\n",
       "1  [-0.012167753651738167, -0.011197278276085854,...   \n",
       "2  [0.010904512368142605, -0.0042160083539783955,...   \n",
       "3  [-0.013121142983436584, -0.021301323547959328,...   \n",
       "4  [-0.00418621813878417, -0.010639390908181667, ...   \n",
       "\n",
       "                                         embed_title  \n",
       "0  [0.012436236254870892, 0.017795050516724586, 0...  \n",
       "1  [-0.030167536810040474, 0.009392774663865566, ...  \n",
       "2  [-0.00030440607224591076, -0.01510559953749179...  \n",
       "3  [-0.016000520437955856, -0.018589602783322334,...  \n",
       "4  [-0.0017813920276239514, -0.002438942203298211...  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d47eca6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10784, 9)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e555513a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents = 10784\n"
     ]
    }
   ],
   "source": [
    "documents = df.to_dict(orient=\"records\")\n",
    "print(\"Number of documents =\", len(documents))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f49910d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving documents...\n",
      "Done\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['movies.pkl']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Saving the documents into a pkl file\n",
    "PKL_DIR = \"embeddings\"\n",
    "PKL_FILE = \"movies.pkl\"\n",
    "\n",
    "os.makedirs(PKL_DIR, exist_ok=True)\n",
    "\n",
    "print(\"Saving documents...\")\n",
    "with open(os.path.join(PKL_DIR, PKL_FILE), 'wb') as f:\n",
    "    pickle.dump(documents, f)\n",
    "print(\"Done\")\n",
    "\n",
    "os.listdir(PKL_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2f6662f",
   "metadata": {},
   "source": [
    "## 3. Cosine similarity principles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1b679336",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cosine_similarity(vector1, vector2):\n",
    "    \"\"\"\n",
    "    Get cosine similarity value between two embedded vectors\n",
    "    Inputs: 2 embedded vectors\n",
    "    Output: cosine similarity value\n",
    "    \"\"\"\n",
    "    if len(vector1) != len(vector2):\n",
    "        return None\n",
    "    \n",
    "    dot_product = sum(x * y for x, y in zip(vector1, vector2))\n",
    "    magnitude1 = math.sqrt(sum(x * x for x in vector1))\n",
    "    magnitude2 = math.sqrt(sum(x * x for x in vector2))\n",
    "    cosine_similarity = round(dot_product / (magnitude1 * magnitude2), 15)\n",
    "\n",
    "    if cosine_similarity == 1:\n",
    "        print(\n",
    "            \"\\033[1;31;34mStrictly identical text: Cosine similarity =\",\n",
    "            cosine_similarity,\n",
    "        )\n",
    "    \n",
    "    elif cosine_similarity >= 0.8:\n",
    "        print(\"\\033[1;31;32mSame semantic text: Cosine similarity =\", cosine_similarity)\n",
    "    \n",
    "    else:\n",
    "        print(\n",
    "            \"\\033[1;31;91mDifferent semantic text: Cosine similarity =\",\n",
    "            cosine_similarity,\n",
    "        )\n",
    "    \n",
    "    return cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8384f299",
   "metadata": {},
   "outputs": [],
   "source": [
    "emb1 = openai_text_embeddings(\"My name is James Bond\")\n",
    "emb2 = openai_text_embeddings(\"Sean Connery.\")\n",
    "emb3 = openai_text_embeddings(\"Azure Open AI is great!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c8cc80e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;31;34mStrictly identical text: Cosine similarity = 1.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_cosine_similarity(emb1, emb1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4dec7e96",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;31;32mSame semantic text: Cosine similarity = 0.83354490270083\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.83354490270083"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_cosine_similarity(emb1, emb2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "86c84ffc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;31;91mDifferent semantic text: Cosine similarity = 0.723586104586534\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.723586104586534"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_cosine_similarity(emb1, emb3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "74cb8fff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;31;32mSame semantic text: Cosine similarity = 0.83354490270083\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.83354490270083"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_cosine_similarity(emb2, emb1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2ffca17b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;31;91mDifferent semantic text: Cosine similarity = 0.718070263122291\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.718070263122291"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_cosine_similarity(emb2, emb3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "658b7886",
   "metadata": {},
   "source": [
    "## 4. Quick local tests (without Azure Cognitive Search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c88f027b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def quick_search(df, user_query, top_n=3):\n",
    "    \"\"\"\n",
    "    Searching documents\n",
    "    Inputs: dataframe, query and topn\n",
    "    Output: results\n",
    "    \"\"\"\n",
    "    embedding = get_embedding(\n",
    "        user_query,\n",
    "        engine=embeddings_engine,\n",
    "    )\n",
    "    df[\"cosine_similarity\"] = df.embed_overview.apply(\n",
    "        lambda x: cosine_similarity(x, embedding)\n",
    "    )\n",
    "    results = df.sort_values(\"cosine_similarity\", ascending=False).head(top_n)\n",
    "    display(results)\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "0e5cdb3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>description</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "      <th>embed_overview</th>\n",
       "      <th>embed_title</th>\n",
       "      <th>cosine_similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1276</th>\n",
       "      <td>tt0088247</td>\n",
       "      <td>The Terminator</td>\n",
       "      <td>Arnold Schwarzenegger Michael Biehn Linda Hami...</td>\n",
       "      <td>James Cameron</td>\n",
       "      <td>In the post-apocalyptic future reigning tyrann...</td>\n",
       "      <td>Action Thriller Science Fiction</td>\n",
       "      <td>1984</td>\n",
       "      <td>[-0.02188941463828087, -0.044750526547431946, ...</td>\n",
       "      <td>[-0.025060655549168587, -0.0402563251554966, -...</td>\n",
       "      <td>0.850306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2159</th>\n",
       "      <td>tt0103064</td>\n",
       "      <td>Terminator 2 Judgment Day</td>\n",
       "      <td>Arnold Schwarzenegger Linda Hamilton Edward Fu...</td>\n",
       "      <td>James Cameron</td>\n",
       "      <td>Nearly 10 years have passed since Sarah Connor...</td>\n",
       "      <td>Action Thriller Science Fiction</td>\n",
       "      <td>1991</td>\n",
       "      <td>[-0.013668149709701538, -0.0405806303024292, -...</td>\n",
       "      <td>[-0.02478639781475067, -0.0283088106662035, -0...</td>\n",
       "      <td>0.829719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3755</th>\n",
       "      <td>tt0181852</td>\n",
       "      <td>Terminator 3 Rise of the Machines</td>\n",
       "      <td>Arnold Schwarzenegger Nick Stahl Claire Danes ...</td>\n",
       "      <td>Jonathan Mostow</td>\n",
       "      <td>It's been 10 years since John Connor saved Ear...</td>\n",
       "      <td>Action Thriller Science Fiction</td>\n",
       "      <td>2003</td>\n",
       "      <td>[0.007508266717195511, -0.04598238691687584, -...</td>\n",
       "      <td>[-0.0275761429220438, -0.058681610971689224, -...</td>\n",
       "      <td>0.826460</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        imdb_id                              title   \n",
       "1276  tt0088247                     The Terminator  \\\n",
       "2159  tt0103064          Terminator 2 Judgment Day   \n",
       "3755  tt0181852  Terminator 3 Rise of the Machines   \n",
       "\n",
       "                                                   cast         director   \n",
       "1276  Arnold Schwarzenegger Michael Biehn Linda Hami...    James Cameron  \\\n",
       "2159  Arnold Schwarzenegger Linda Hamilton Edward Fu...    James Cameron   \n",
       "3755  Arnold Schwarzenegger Nick Stahl Claire Danes ...  Jonathan Mostow   \n",
       "\n",
       "                                            description   \n",
       "1276  In the post-apocalyptic future reigning tyrann...  \\\n",
       "2159  Nearly 10 years have passed since Sarah Connor...   \n",
       "3755  It's been 10 years since John Connor saved Ear...   \n",
       "\n",
       "                               genres  year   \n",
       "1276  Action Thriller Science Fiction  1984  \\\n",
       "2159  Action Thriller Science Fiction  1991   \n",
       "3755  Action Thriller Science Fiction  2003   \n",
       "\n",
       "                                         embed_overview   \n",
       "1276  [-0.02188941463828087, -0.044750526547431946, ...  \\\n",
       "2159  [-0.013668149709701538, -0.0405806303024292, -...   \n",
       "3755  [0.007508266717195511, -0.04598238691687584, -...   \n",
       "\n",
       "                                            embed_title  cosine_similarity  \n",
       "1276  [-0.025060655549168587, -0.0402563251554966, -...           0.850306  \n",
       "2159  [-0.02478639781475067, -0.0283088106662035, -0...           0.829719  \n",
       "3755  [-0.0275761429220438, -0.058681610971689224, -...           0.826460  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results = quick_search(df, \"I want to see some Terminator movies\", top_n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "c57d0172",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>description</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "      <th>embed_overview</th>\n",
       "      <th>embed_title</th>\n",
       "      <th>cosine_similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>tt0062512</td>\n",
       "      <td>You Only Live Twice</td>\n",
       "      <td>Sean Connery Akiko Wakabayashi Karin Dor Mie H...</td>\n",
       "      <td>Lewis Gilbert</td>\n",
       "      <td>A mysterious space craft kidnaps a Russian and...</td>\n",
       "      <td>Action Thriller Adventure</td>\n",
       "      <td>1967</td>\n",
       "      <td>[-0.008935574442148209, -0.03285738453269005, ...</td>\n",
       "      <td>[-0.02086525410413742, -0.007323090452700853, ...</td>\n",
       "      <td>0.818913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>tt0070328</td>\n",
       "      <td>Live and Let Die</td>\n",
       "      <td>Roger Moore Yaphet Kotto Jane Seymour Clifton ...</td>\n",
       "      <td>Guy Hamilton</td>\n",
       "      <td>James Bond must investigate a mysterious murde...</td>\n",
       "      <td>Adventure Action Thriller</td>\n",
       "      <td>1973</td>\n",
       "      <td>[-0.01073690690100193, -0.02086213417351246, -...</td>\n",
       "      <td>[-0.032646261155605316, -0.016837764531373978,...</td>\n",
       "      <td>0.817240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1845</th>\n",
       "      <td>tt0097742</td>\n",
       "      <td>Licence to Kill</td>\n",
       "      <td>Timothy Dalton Carey Lowell Robert Davi Talisa...</td>\n",
       "      <td>John Glen</td>\n",
       "      <td>James Bond and his American colleague Felix Le...</td>\n",
       "      <td>Adventure Action Thriller</td>\n",
       "      <td>1989</td>\n",
       "      <td>[0.002153094159439206, 0.004642711021006107, -...</td>\n",
       "      <td>[-0.0041930824518203735, -0.005286786705255508...</td>\n",
       "      <td>0.811900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        imdb_id                title   \n",
       "290   tt0062512  You Only Live Twice  \\\n",
       "522   tt0070328     Live and Let Die   \n",
       "1845  tt0097742      Licence to Kill   \n",
       "\n",
       "                                                   cast       director   \n",
       "290   Sean Connery Akiko Wakabayashi Karin Dor Mie H...  Lewis Gilbert  \\\n",
       "522   Roger Moore Yaphet Kotto Jane Seymour Clifton ...   Guy Hamilton   \n",
       "1845  Timothy Dalton Carey Lowell Robert Davi Talisa...      John Glen   \n",
       "\n",
       "                                            description   \n",
       "290   A mysterious space craft kidnaps a Russian and...  \\\n",
       "522   James Bond must investigate a mysterious murde...   \n",
       "1845  James Bond and his American colleague Felix Le...   \n",
       "\n",
       "                         genres  year   \n",
       "290   Action Thriller Adventure  1967  \\\n",
       "522   Adventure Action Thriller  1973   \n",
       "1845  Adventure Action Thriller  1989   \n",
       "\n",
       "                                         embed_overview   \n",
       "290   [-0.008935574442148209, -0.03285738453269005, ...  \\\n",
       "522   [-0.01073690690100193, -0.02086213417351246, -...   \n",
       "1845  [0.002153094159439206, 0.004642711021006107, -...   \n",
       "\n",
       "                                            embed_title  cosine_similarity  \n",
       "290   [-0.02086525410413742, -0.007323090452700853, ...           0.818913  \n",
       "522   [-0.032646261155605316, -0.016837764531373978,...           0.817240  \n",
       "1845  [-0.0041930824518203735, -0.005286786705255508...           0.811900  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results = quick_search(df, \"Je veux voir un film de James Bond\", top_n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9c11c1c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>description</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "      <th>embed_overview</th>\n",
       "      <th>embed_title</th>\n",
       "      <th>cosine_similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4128</th>\n",
       "      <td>tt0254199</td>\n",
       "      <td>CQ</td>\n",
       "      <td>Jeremy Davies Angela Lindvall Ã‰lodie Bouchez ...</td>\n",
       "      <td>Roman Coppola</td>\n",
       "      <td>A young filmmaker in 1960s Paris juggles direc...</td>\n",
       "      <td>Comedy Drama Science Fiction</td>\n",
       "      <td>2001</td>\n",
       "      <td>[-0.0043552410788834095, -0.016719359904527664...</td>\n",
       "      <td>[0.0024958874564617872, 0.005884502083063126, ...</td>\n",
       "      <td>0.814828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9116</th>\n",
       "      <td>tt2049543</td>\n",
       "      <td>Synchronicity</td>\n",
       "      <td>Chad McKnight Brianne Davis AJ Bowen Scott Poy...</td>\n",
       "      <td>Jacob Gentry</td>\n",
       "      <td>In this mind-bending 'Sci-Fi Noir' a daring ph...</td>\n",
       "      <td>Thriller Mystery Science Fiction</td>\n",
       "      <td>2015</td>\n",
       "      <td>[-0.0016540847718715668, -0.018224244937300682...</td>\n",
       "      <td>[-0.01355673186480999, -0.02561151422560215, 0...</td>\n",
       "      <td>0.799714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3936</th>\n",
       "      <td>tt0216216</td>\n",
       "      <td>The 6th Day</td>\n",
       "      <td>Arnold Schwarzenegger Michael Rapaport Tony Go...</td>\n",
       "      <td>Roger Spottiswoode</td>\n",
       "      <td>Futuristic action about a man who meets a clon...</td>\n",
       "      <td>Action Mystery Science Fiction Thriller</td>\n",
       "      <td>2000</td>\n",
       "      <td>[-0.015566646121442318, -0.03223968297243118, ...</td>\n",
       "      <td>[0.0033594027627259493, 0.0012303648982197046,...</td>\n",
       "      <td>0.797103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1792</th>\n",
       "      <td>tt0097100</td>\n",
       "      <td>Communion</td>\n",
       "      <td>Christopher Walken Lindsay Crouse Frances Ster...</td>\n",
       "      <td>Philippe Mora</td>\n",
       "      <td>A novelist's wife and son see him changed by a...</td>\n",
       "      <td>Drama Horror Science Fiction Thriller</td>\n",
       "      <td>1989</td>\n",
       "      <td>[0.021728629246354103, -0.012683791108429432, ...</td>\n",
       "      <td>[0.01540224440395832, -0.007430095691233873, 0...</td>\n",
       "      <td>0.795096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4932</th>\n",
       "      <td>tt0366627</td>\n",
       "      <td>The Jacket</td>\n",
       "      <td>Adrien Brody Keira Knightley Kris Kristofferso...</td>\n",
       "      <td>John Maybury</td>\n",
       "      <td>A military veteran goes on a journey into the ...</td>\n",
       "      <td>Drama Mystery Thriller Fantasy</td>\n",
       "      <td>2005</td>\n",
       "      <td>[0.0058812107890844345, -0.02817738987505436, ...</td>\n",
       "      <td>[-0.016100479289889336, -0.0138565544039011, -...</td>\n",
       "      <td>0.793296</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        imdb_id          title   \n",
       "4128  tt0254199             CQ  \\\n",
       "9116  tt2049543  Synchronicity   \n",
       "3936  tt0216216    The 6th Day   \n",
       "1792  tt0097100      Communion   \n",
       "4932  tt0366627     The Jacket   \n",
       "\n",
       "                                                   cast            director   \n",
       "4128  Jeremy Davies Angela Lindvall Ã‰lodie Bouchez ...       Roman Coppola  \\\n",
       "9116  Chad McKnight Brianne Davis AJ Bowen Scott Poy...        Jacob Gentry   \n",
       "3936  Arnold Schwarzenegger Michael Rapaport Tony Go...  Roger Spottiswoode   \n",
       "1792  Christopher Walken Lindsay Crouse Frances Ster...       Philippe Mora   \n",
       "4932  Adrien Brody Keira Knightley Kris Kristofferso...        John Maybury   \n",
       "\n",
       "                                            description   \n",
       "4128  A young filmmaker in 1960s Paris juggles direc...  \\\n",
       "9116  In this mind-bending 'Sci-Fi Noir' a daring ph...   \n",
       "3936  Futuristic action about a man who meets a clon...   \n",
       "1792  A novelist's wife and son see him changed by a...   \n",
       "4932  A military veteran goes on a journey into the ...   \n",
       "\n",
       "                                       genres  year   \n",
       "4128             Comedy Drama Science Fiction  2001  \\\n",
       "9116         Thriller Mystery Science Fiction  2015   \n",
       "3936  Action Mystery Science Fiction Thriller  2000   \n",
       "1792    Drama Horror Science Fiction Thriller  1989   \n",
       "4932           Drama Mystery Thriller Fantasy  2005   \n",
       "\n",
       "                                         embed_overview   \n",
       "4128  [-0.0043552410788834095, -0.016719359904527664...  \\\n",
       "9116  [-0.0016540847718715668, -0.018224244937300682...   \n",
       "3936  [-0.015566646121442318, -0.03223968297243118, ...   \n",
       "1792  [0.021728629246354103, -0.012683791108429432, ...   \n",
       "4932  [0.0058812107890844345, -0.02817738987505436, ...   \n",
       "\n",
       "                                            embed_title  cosine_similarity  \n",
       "4128  [0.0024958874564617872, 0.005884502083063126, ...           0.814828  \n",
       "9116  [-0.01355673186480999, -0.02561151422560215, 0...           0.799714  \n",
       "3936  [0.0033594027627259493, 0.0012303648982197046,...           0.797103  \n",
       "1792  [0.01540224440395832, -0.007430095691233873, 0...           0.795096  \n",
       "4932  [-0.016100479289889336, -0.0138565544039011, -...           0.793296  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results = quick_search(df, \"Quiero ver películas de ciencia ficción\", top_n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8b42cb2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>description</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "      <th>embed_overview</th>\n",
       "      <th>embed_title</th>\n",
       "      <th>cosine_similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2883</th>\n",
       "      <td>tt0115899</td>\n",
       "      <td>Il ciclone</td>\n",
       "      <td>Leonardo Pieraccioni Lorena Forteza Barbara En...</td>\n",
       "      <td>Leonardo Pieraccioni</td>\n",
       "      <td>La vita di una tranquilla famiglia toscana di ...</td>\n",
       "      <td>Drama Comedy Romance Foreign</td>\n",
       "      <td>1996</td>\n",
       "      <td>[0.0005137093830853701, -0.008648240938782692,...</td>\n",
       "      <td>[-0.012413086369633675, -0.006416610442101955,...</td>\n",
       "      <td>0.825148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3389</th>\n",
       "      <td>tt0120910</td>\n",
       "      <td>Fantasia 2000</td>\n",
       "      <td>Steve Martin Quincy Jones Bette Midler James E...</td>\n",
       "      <td>Paul Brizzi Hendel Butoy Francis Glebas Eric G...</td>\n",
       "      <td>An update of the original film with new interp...</td>\n",
       "      <td>Music Animation Family Fantasy</td>\n",
       "      <td>1999</td>\n",
       "      <td>[-0.02040269412100315, -0.01043155137449503, -...</td>\n",
       "      <td>[0.017556900158524513, -0.0321248397231102, -0...</td>\n",
       "      <td>0.803908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1183</th>\n",
       "      <td>tt0086850</td>\n",
       "      <td>Acqua e sapone</td>\n",
       "      <td>Carlo Verdone Natasha Hovey Glenn Saxson Elena...</td>\n",
       "      <td>Carlo Verdone</td>\n",
       "      <td>La madre-manager di una giovanissima modella a...</td>\n",
       "      <td>Drama Family</td>\n",
       "      <td>1983</td>\n",
       "      <td>[-0.011183799244463444, -0.019681930541992188,...</td>\n",
       "      <td>[0.024926507845520973, 0.004666802939027548, 0...</td>\n",
       "      <td>0.803762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2833</th>\n",
       "      <td>tt0114844</td>\n",
       "      <td>Viaggi di nozze</td>\n",
       "      <td>Carlo Verdone Claudia Gerini Veronica Pivetti ...</td>\n",
       "      <td>Carlo Verdone</td>\n",
       "      <td>Le vicessitudini di tre coppie di novelli spos...</td>\n",
       "      <td>Non available</td>\n",
       "      <td>1995</td>\n",
       "      <td>[-0.008751184679567814, -0.01806781068444252, ...</td>\n",
       "      <td>[-0.009643185883760452, -0.012746378779411316,...</td>\n",
       "      <td>0.797610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1515</th>\n",
       "      <td>tt0092276</td>\n",
       "      <td>Yuppies 2</td>\n",
       "      <td>Massimo Boldi Jerry CalÃ  Christian De Sica Ez...</td>\n",
       "      <td>Enrico Oldoini</td>\n",
       "      <td>Continuano le avventure degli yuppies milanesi...</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>1986</td>\n",
       "      <td>[-0.008874254301190376, -0.03015108034014702, ...</td>\n",
       "      <td>[-0.0203807782381773, -0.029809894040226936, -...</td>\n",
       "      <td>0.797188</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        imdb_id            title   \n",
       "2883  tt0115899       Il ciclone  \\\n",
       "3389  tt0120910    Fantasia 2000   \n",
       "1183  tt0086850   Acqua e sapone   \n",
       "2833  tt0114844  Viaggi di nozze   \n",
       "1515  tt0092276        Yuppies 2   \n",
       "\n",
       "                                                   cast   \n",
       "2883  Leonardo Pieraccioni Lorena Forteza Barbara En...  \\\n",
       "3389  Steve Martin Quincy Jones Bette Midler James E...   \n",
       "1183  Carlo Verdone Natasha Hovey Glenn Saxson Elena...   \n",
       "2833  Carlo Verdone Claudia Gerini Veronica Pivetti ...   \n",
       "1515  Massimo Boldi Jerry CalÃ  Christian De Sica Ez...   \n",
       "\n",
       "                                               director   \n",
       "2883                               Leonardo Pieraccioni  \\\n",
       "3389  Paul Brizzi Hendel Butoy Francis Glebas Eric G...   \n",
       "1183                                      Carlo Verdone   \n",
       "2833                                      Carlo Verdone   \n",
       "1515                                     Enrico Oldoini   \n",
       "\n",
       "                                            description   \n",
       "2883  La vita di una tranquilla famiglia toscana di ...  \\\n",
       "3389  An update of the original film with new interp...   \n",
       "1183  La madre-manager di una giovanissima modella a...   \n",
       "2833  Le vicessitudini di tre coppie di novelli spos...   \n",
       "1515  Continuano le avventure degli yuppies milanesi...   \n",
       "\n",
       "                              genres  year   \n",
       "2883    Drama Comedy Romance Foreign  1996  \\\n",
       "3389  Music Animation Family Fantasy  1999   \n",
       "1183                    Drama Family  1983   \n",
       "2833                   Non available  1995   \n",
       "1515                          Comedy  1986   \n",
       "\n",
       "                                         embed_overview   \n",
       "2883  [0.0005137093830853701, -0.008648240938782692,...  \\\n",
       "3389  [-0.02040269412100315, -0.01043155137449503, -...   \n",
       "1183  [-0.011183799244463444, -0.019681930541992188,...   \n",
       "2833  [-0.008751184679567814, -0.01806781068444252, ...   \n",
       "1515  [-0.008874254301190376, -0.03015108034014702, ...   \n",
       "\n",
       "                                            embed_title  cosine_similarity  \n",
       "2883  [-0.012413086369633675, -0.006416610442101955,...           0.825148  \n",
       "3389  [0.017556900158524513, -0.0321248397231102, -0...           0.803908  \n",
       "1183  [0.024926507845520973, 0.004666802939027548, 0...           0.803762  \n",
       "2833  [-0.009643185883760452, -0.012746378779411316,...           0.797610  \n",
       "1515  [-0.0203807782381773, -0.029809894040226936, -...           0.797188  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results = quick_search(df, \"Voglio vedere dei film musicali\", top_n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "0467c207",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>description</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "      <th>embed_overview</th>\n",
       "      <th>embed_title</th>\n",
       "      <th>cosine_similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3389</th>\n",
       "      <td>tt0120910</td>\n",
       "      <td>Fantasia 2000</td>\n",
       "      <td>Steve Martin Quincy Jones Bette Midler James E...</td>\n",
       "      <td>Paul Brizzi Hendel Butoy Francis Glebas Eric G...</td>\n",
       "      <td>An update of the original film with new interp...</td>\n",
       "      <td>Music Animation Family Fantasy</td>\n",
       "      <td>1999</td>\n",
       "      <td>[-0.02040269412100315, -0.01043155137449503, -...</td>\n",
       "      <td>[0.017556900158524513, -0.0321248397231102, -0...</td>\n",
       "      <td>0.803740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4961</th>\n",
       "      <td>tt0368667</td>\n",
       "      <td>Interstella 5555 The 5tory of the 5ecret 5tar ...</td>\n",
       "      <td>Romanthony</td>\n",
       "      <td>Kazuhisa TakenÃ´chi</td>\n",
       "      <td>A sci-fi anime House-musical movie collaborati...</td>\n",
       "      <td>Animation Science Fiction Music</td>\n",
       "      <td>2003</td>\n",
       "      <td>[0.017697621136903763, -0.011214802972972393, ...</td>\n",
       "      <td>[0.004670271649956703, -0.014423966407775879, ...</td>\n",
       "      <td>0.787416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4704</th>\n",
       "      <td>tt0335266</td>\n",
       "      <td>Lost in Translation</td>\n",
       "      <td>Bill Murray Scarlett Johansson Anna Faris Giov...</td>\n",
       "      <td>Sofia Coppola</td>\n",
       "      <td>Two lost souls visiting Tokyo -- the young neg...</td>\n",
       "      <td>Drama</td>\n",
       "      <td>2003</td>\n",
       "      <td>[0.005371313542127609, -0.017871160060167313, ...</td>\n",
       "      <td>[-0.013685127720236778, -0.029664143919944763,...</td>\n",
       "      <td>0.784041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2971</th>\n",
       "      <td>tt0116922</td>\n",
       "      <td>Lost Highway</td>\n",
       "      <td>Bill Pullman Patricia Arquette John Roselius L...</td>\n",
       "      <td>David Lynch</td>\n",
       "      <td>A tormented jazz musician finds himself lost i...</td>\n",
       "      <td>Drama Thriller Mystery</td>\n",
       "      <td>1997</td>\n",
       "      <td>[0.0029709336813539267, -0.032269012182950974,...</td>\n",
       "      <td>[-0.008940287865698338, -0.01946103200316429, ...</td>\n",
       "      <td>0.782554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5209</th>\n",
       "      <td>tt0397535</td>\n",
       "      <td>Memoirs of a Geisha</td>\n",
       "      <td>Zhang Ziyi Gong Li Youki Kudoh Tsai Chin Suzuk...</td>\n",
       "      <td>Rob Marshall</td>\n",
       "      <td>A sweeping romantic epic set in Japan in the y...</td>\n",
       "      <td>Drama History Romance</td>\n",
       "      <td>2005</td>\n",
       "      <td>[-0.02323296293616295, -0.024124089628458023, ...</td>\n",
       "      <td>[-0.022762073203921318, -0.011697894893586636,...</td>\n",
       "      <td>0.778716</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        imdb_id                                              title   \n",
       "3389  tt0120910                                      Fantasia 2000  \\\n",
       "4961  tt0368667  Interstella 5555 The 5tory of the 5ecret 5tar ...   \n",
       "4704  tt0335266                                Lost in Translation   \n",
       "2971  tt0116922                                       Lost Highway   \n",
       "5209  tt0397535                                Memoirs of a Geisha   \n",
       "\n",
       "                                                   cast   \n",
       "3389  Steve Martin Quincy Jones Bette Midler James E...  \\\n",
       "4961                                         Romanthony   \n",
       "4704  Bill Murray Scarlett Johansson Anna Faris Giov...   \n",
       "2971  Bill Pullman Patricia Arquette John Roselius L...   \n",
       "5209  Zhang Ziyi Gong Li Youki Kudoh Tsai Chin Suzuk...   \n",
       "\n",
       "                                               director   \n",
       "3389  Paul Brizzi Hendel Butoy Francis Glebas Eric G...  \\\n",
       "4961                                Kazuhisa TakenÃ´chi   \n",
       "4704                                      Sofia Coppola   \n",
       "2971                                        David Lynch   \n",
       "5209                                       Rob Marshall   \n",
       "\n",
       "                                            description   \n",
       "3389  An update of the original film with new interp...  \\\n",
       "4961  A sci-fi anime House-musical movie collaborati...   \n",
       "4704  Two lost souls visiting Tokyo -- the young neg...   \n",
       "2971  A tormented jazz musician finds himself lost i...   \n",
       "5209  A sweeping romantic epic set in Japan in the y...   \n",
       "\n",
       "                               genres  year   \n",
       "3389   Music Animation Family Fantasy  1999  \\\n",
       "4961  Animation Science Fiction Music  2003   \n",
       "4704                            Drama  2003   \n",
       "2971           Drama Thriller Mystery  1997   \n",
       "5209            Drama History Romance  2005   \n",
       "\n",
       "                                         embed_overview   \n",
       "3389  [-0.02040269412100315, -0.01043155137449503, -...  \\\n",
       "4961  [0.017697621136903763, -0.011214802972972393, ...   \n",
       "4704  [0.005371313542127609, -0.017871160060167313, ...   \n",
       "2971  [0.0029709336813539267, -0.032269012182950974,...   \n",
       "5209  [-0.02323296293616295, -0.024124089628458023, ...   \n",
       "\n",
       "                                            embed_title  cosine_similarity  \n",
       "3389  [0.017556900158524513, -0.0321248397231102, -0...           0.803740  \n",
       "4961  [0.004670271649956703, -0.014423966407775879, ...           0.787416  \n",
       "4704  [-0.013685127720236778, -0.029664143919944763,...           0.784041  \n",
       "2971  [-0.008940287865698338, -0.01946103200316429, ...           0.782554  \n",
       "5209  [-0.022762073203921318, -0.011697894893586636,...           0.778716  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results = quick_search(df, \"音楽映画が観たい\", top_n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "436ab9f7",
   "metadata": {},
   "source": [
    "## 5. Azure Cognitive Search functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "8e549a54",
   "metadata": {},
   "outputs": [],
   "source": [
    "def delete_index(index_name):\n",
    "    \"\"\"\n",
    "    Deleting an Azure Cognitive Search index\n",
    "    Input: Azure Cognitive Search index\n",
    "    Output: None\n",
    "    \"\"\"\n",
    "    start = time.time()\n",
    "    search_client = SearchIndexClient(\n",
    "        endpoint=acs_endpoint, credential=AzureKeyCredential(acs_key)\n",
    "    )\n",
    "    \n",
    "    try:\n",
    "        print(\"Deleting the Azure Cognitive Search index:\", index_name)\n",
    "        search_client.delete_index(index_name)\n",
    "        print(\"Done. Elapsed time:\", round(time.time() - start, 2), \"secs\")\n",
    "    except:\n",
    "        print(\"Cannot delete index. Check the index name.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "4e61eead",
   "metadata": {},
   "outputs": [],
   "source": [
    "def index_stats(index_name):\n",
    "    \"\"\"\n",
    "    Get statistics about Azure Cognitive Search index\n",
    "    Input: Azure Cognitive Search index\n",
    "    Output: Get Azure Cognitive Search index stats\n",
    "    \"\"\"\n",
    "    url = (\n",
    "        acs_endpoint\n",
    "        + \"/indexes/\"\n",
    "        + index_name\n",
    "        + \"/stats?api-version=2021-04-30-Preview\"\n",
    "    )\n",
    "    headers = {\n",
    "        \"Content-Type\": \"application/json\",\n",
    "        \"api-key\": acs_key,\n",
    "    }\n",
    "    response = requests.get(url, headers=headers)\n",
    "    print(\"Azure Cognitive Search index status for:\", index_name, \"\\n\")\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        res = response.json()\n",
    "        print(json.dumps(res, indent=2))\n",
    "        document_count = res[\"documentCount\"]\n",
    "        storage_size = res[\"storageSize\"]\n",
    "\n",
    "    else:\n",
    "        print(\"Request failed with status code:\", response.status_code)\n",
    "\n",
    "    return document_count, storage_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "29eaac49",
   "metadata": {},
   "outputs": [],
   "source": [
    "def index_status(index_name):\n",
    "    \"\"\"\n",
    "    Azure Cognitive Search index status\n",
    "    Input: Azure Cognitive Search index\n",
    "    Output: Get Azure Cognitive Search index status\n",
    "    \"\"\"\n",
    "    print(\"Azure Cognitive Search Index:\", index_name, \"\\n\")\n",
    "\n",
    "    headers = {\"Content-Type\": \"application/json\", \"api-key\": acs_key}\n",
    "    params = {\"api-version\": \"2021-04-30-Preview\"}\n",
    "    index_status = requests.get(\n",
    "        acs_endpoint + \"/indexes/\" + index_name, headers=headers, params=params\n",
    "    )\n",
    "\n",
    "    try:\n",
    "        print(json.dumps((index_status.json()), indent=5))\n",
    "    except:\n",
    "        print(\"Request failed with status code:\", response.status_code)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0da537a",
   "metadata": {},
   "source": [
    "## 6. Creating an Azure Cognitive Search index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "85f1e803",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting the Azure Cognitive Search client\n",
      "Done. Azure Cognitive Search client defined.\n",
      "<azure.search.documents.indexes._search_index_client.SearchIndexClient object at 0x7fbee4cb70a0>\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    # Setting the Azure Cognitive Search client\n",
    "    print(\"Setting the Azure Cognitive Search client\")\n",
    "    search_client = SearchIndexClient(\n",
    "        endpoint=acs_endpoint,\n",
    "        credential=AzureKeyCredential(acs_key)\n",
    "    )\n",
    "    print(\"Done. Azure Cognitive Search client defined.\")\n",
    "    print(search_client)\n",
    "\n",
    "except:\n",
    "    print(\"Request failed. Cannot create Azure Cognitive Search client:\", acs_endpoint)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ff3bd13",
   "metadata": {},
   "source": [
    "### Removing any existing index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "70a493bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting the Azure Cognitive Search index: moviereview\n",
      "Done. Elapsed time: 0.86 secs\n"
     ]
    }
   ],
   "source": [
    "delete_index(index_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e1e0e6a",
   "metadata": {},
   "source": [
    "### Creating search index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "0c551775",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vector embeddings size = 1536\n"
     ]
    }
   ],
   "source": [
    "vector_search_dim = len(openai_text_embeddings(\"Hello\"))\n",
    "print(\"Vector embeddings size =\", vector_search_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "19fc42da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done. The moviereview Azure Cognitive Search index has been created!\n"
     ]
    }
   ],
   "source": [
    "# Create a search index\n",
    "index_client = SearchIndexClient(\n",
    "    endpoint=acs_endpoint, credential=AzureKeyCredential(acs_key)\n",
    ")\n",
    "fields = [\n",
    "    # Index\n",
    "    SimpleField(\n",
    "        name=\"imdb_id\",\n",
    "        type=SearchFieldDataType.String,\n",
    "        key=True,\n",
    "        sortable=True,\n",
    "        filterable=True,\n",
    "        facetable=True,\n",
    "    ),\n",
    "    # Searchable fields\n",
    "    SearchableField(name=\"title\", type=SearchFieldDataType.String, filterable=True),\n",
    "    SearchableField(name=\"cast\", type=SearchFieldDataType.String, filterable=True),\n",
    "    SearchableField(name=\"director\", type=SearchFieldDataType.Single, filterable=True),\n",
    "    SearchableField(name=\"description\", type=SearchFieldDataType.String),\n",
    "    SearchableField(name=\"genres\", type=SearchFieldDataType.String, filterable=True),\n",
    "    SearchableField(name=\"year\", type=SearchFieldDataType.String, filterable=True),\n",
    "    # Vectors embeddings\n",
    "    SearchField(\n",
    "        name=\"embed_overview\",\n",
    "        type=SearchFieldDataType.Collection(SearchFieldDataType.Single),\n",
    "        searchable=True,\n",
    "        vector_search_dimensions=vector_search_dim,\n",
    "        vector_search_configuration=\"my-vector-config\",\n",
    "    ),\n",
    "    SearchField(\n",
    "        name=\"embed_title\",\n",
    "        type=SearchFieldDataType.Collection(SearchFieldDataType.Single),\n",
    "        searchable=True,\n",
    "        vector_search_dimensions=vector_search_dim,\n",
    "        vector_search_configuration=\"my-vector-config\",\n",
    "    ),\n",
    "]\n",
    "\n",
    "\n",
    "# Configuration\n",
    "vector_search = VectorSearch(\n",
    "    algorithm_configurations=[\n",
    "        HnswVectorSearchAlgorithmConfiguration(\n",
    "            name=\"my-vector-config\",\n",
    "            # HNSW is a graph-based Approximate Nearest Neighbors (ANN)\n",
    "            # algorithm optimized for high-recall, low-latency applications\n",
    "            kind=\"hnsw\",\n",
    "            parameters={\n",
    "                \"m\": 4,\n",
    "                \"efConstruction\": 400,\n",
    "                \"efSearch\": 500,\n",
    "                \"metric\": \"cosine\",  # Cosine similarity metric\n",
    "            },\n",
    "        )\n",
    "    ]\n",
    ")\n",
    "\n",
    "semantic_config = SemanticConfiguration(\n",
    "    name=\"my-semantic-config\",\n",
    "    prioritized_fields=PrioritizedFields(\n",
    "        title_field=SemanticField(field_name=\"title\"),\n",
    "        prioritized_keywords_fields=[SemanticField(field_name=\"genres\")],\n",
    "        prioritized_content_fields=[SemanticField(field_name=\"description\")],\n",
    "    ),\n",
    ")\n",
    "\n",
    "# Create the semantic settings with the configuration\n",
    "semantic_settings = SemanticSettings(configurations=[semantic_config])\n",
    "\n",
    "# Create the search index with the semantic settings\n",
    "index = SearchIndex(\n",
    "    name=index_name,\n",
    "    fields=fields,\n",
    "    vector_search=vector_search,\n",
    "    semantic_settings=semantic_settings,\n",
    ")\n",
    "\n",
    "try:\n",
    "    result = index_client.create_or_update_index(index)\n",
    "    print(f\"Done. The {result.name} Azure Cognitive Search index has been created!\")\n",
    "\n",
    "except:\n",
    "    print(f\"Error. The {result.name} Azure Cognitive Search index cannot be created.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f975c827",
   "metadata": {},
   "source": [
    "## 7. Uploading the documents into the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "a6491d06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents to load = 10784\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of documents to load =\", len(documents))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "c5b89b0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def upload_documents(docs):\n",
    "    \"\"\"\n",
    "    Uploading documents into the Azure Cognitive Search index\n",
    "    Inputs: documents\n",
    "    Outputs: loading documents to Azure Cognitive Search index\n",
    "    \"\"\"\n",
    "    search_client = SearchClient(\n",
    "        endpoint=acs_endpoint,\n",
    "        index_name=index_name,\n",
    "        credential=AzureKeyCredential(acs_key),\n",
    "    )\n",
    "    result = search_client.upload_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "d421458b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def chunk_list(input_list, chunk_size):\n",
    "    \"\"\"\n",
    "    Chunk a list according to the chunk_size value\n",
    "    Inputs: documents (list), chunk size list\n",
    "    Outputs: chunk list of documents\n",
    "    \"\"\"\n",
    "    return [\n",
    "        input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size)\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "d49c63fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading the documents into the Azure Cognitive Search index...\n",
      "Total number of documents to load = 10784\n",
      "\n",
      "Processing chunk 001 | Number of loaded documents = 000500 of 10784 | Done: 5 %\n",
      "Processing chunk 002 | Number of loaded documents = 001000 of 10784 | Done: 9 %\n",
      "Processing chunk 003 | Number of loaded documents = 001500 of 10784 | Done: 14 %\n",
      "Processing chunk 004 | Number of loaded documents = 002000 of 10784 | Done: 19 %\n",
      "Processing chunk 005 | Number of loaded documents = 002500 of 10784 | Done: 23 %\n",
      "Processing chunk 006 | Number of loaded documents = 003000 of 10784 | Done: 28 %\n",
      "Processing chunk 007 | Number of loaded documents = 003500 of 10784 | Done: 32 %\n",
      "Processing chunk 008 | Number of loaded documents = 004000 of 10784 | Done: 37 %\n",
      "Processing chunk 009 | Number of loaded documents = 004500 of 10784 | Done: 42 %\n",
      "Processing chunk 010 | Number of loaded documents = 005000 of 10784 | Done: 46 %\n",
      "Processing chunk 011 | Number of loaded documents = 005500 of 10784 | Done: 51 %\n",
      "Processing chunk 012 | Number of loaded documents = 006000 of 10784 | Done: 56 %\n",
      "Processing chunk 013 | Number of loaded documents = 006500 of 10784 | Done: 60 %\n",
      "Processing chunk 014 | Number of loaded documents = 007000 of 10784 | Done: 65 %\n",
      "Processing chunk 015 | Number of loaded documents = 007500 of 10784 | Done: 70 %\n",
      "Processing chunk 016 | Number of loaded documents = 008000 of 10784 | Done: 74 %\n",
      "Processing chunk 017 | Number of loaded documents = 008500 of 10784 | Done: 79 %\n",
      "Processing chunk 018 | Number of loaded documents = 009000 of 10784 | Done: 83 %\n",
      "Processing chunk 019 | Number of loaded documents = 009500 of 10784 | Done: 88 %\n",
      "Processing chunk 020 | Number of loaded documents = 010000 of 10784 | Done: 93 %\n",
      "Processing chunk 021 | Number of loaded documents = 010500 of 10784 | Done: 97 %\n",
      "Processing chunk 022 | Number of loaded documents = 011000 of 10784 | Done: 100 %\n",
      "\n",
      "Done\n",
      "Elapsed time: 00:03:29.065358\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "chunk_size = 500  # We will load documents chunk by chunk\n",
    "chunks = chunk_list(documents, chunk_size)\n",
    "idx = 1\n",
    "\n",
    "print(\"Loading the documents into the Azure Cognitive Search index...\")\n",
    "print(\"Total number of documents to load =\", len(documents))\n",
    "print()\n",
    "\n",
    "loaded_docs = chunk_size\n",
    "\n",
    "for chunk in chunks:\n",
    "    pct_done = round(loaded_docs / len(documents) * 100)\n",
    "    if pct_done >= 100:\n",
    "        pct_done = 100\n",
    "\n",
    "    print(\n",
    "        f\"Processing chunk {idx:03}\",\n",
    "        f\"| Number of loaded documents = {loaded_docs:06}\",\n",
    "        \"of\",\n",
    "        len(documents),\n",
    "        \"| Done:\",\n",
    "        pct_done,\n",
    "        \"%\",\n",
    "    )\n",
    "    upload_documents(chunk)\n",
    "    loaded_docs += chunk_size\n",
    "    idx += 1\n",
    "\n",
    "elapsed = time.time() - start\n",
    "print(\"\\nDone\")\n",
    "print(\n",
    "    \"Elapsed time: \"\n",
    "    + time.strftime(\n",
    "        \"%H:%M:%S.{}\".format(str(elapsed % 1)[2:])[:15], time.gmtime(elapsed)\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "f256de96",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed time to process 10784 documents = 209 seconds\n",
      "Time per processed document in second = 0.01939\n",
      "Number of processed documents per second = 51\n"
     ]
    }
   ],
   "source": [
    "print(f\"Elapsed time to process {len(documents)} documents = {round(elapsed)} seconds\")\n",
    "print(f\"Time per processed document in second = {round(elapsed / len(documents), 5)}\")\n",
    "print(f\"Number of processed documents per second = {int(len(documents) / elapsed)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e29fd702",
   "metadata": {},
   "source": [
    "## 8. Azure Cognitive Search Index informations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "9abec83e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'moviereview'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "70612697",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Azure Cognitive Search Index: moviereview \n",
      "\n",
      "{\n",
      "     \"@odata.context\": \"https://azurecogsearcheastussr.search.windows.net/$metadata#indexes/$entity\",\n",
      "     \"@odata.etag\": \"\\\"0x8DBAE03DD8B3409\\\"\",\n",
      "     \"name\": \"moviereview\",\n",
      "     \"defaultScoringProfile\": null,\n",
      "     \"fields\": [\n",
      "          {\n",
      "               \"name\": \"imdb_id\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": false,\n",
      "               \"filterable\": true,\n",
      "               \"retrievable\": true,\n",
      "               \"sortable\": true,\n",
      "               \"facetable\": true,\n",
      "               \"key\": true,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"normalizer\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"title\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": true,\n",
      "               \"retrievable\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"normalizer\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"cast\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": true,\n",
      "               \"retrievable\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"normalizer\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"director\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": true,\n",
      "               \"retrievable\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"normalizer\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"description\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": false,\n",
      "               \"retrievable\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"normalizer\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"genres\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": true,\n",
      "               \"retrievable\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"normalizer\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"year\",\n",
      "               \"type\": \"Edm.String\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": true,\n",
      "               \"retrievable\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"normalizer\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"embed_overview\",\n",
      "               \"type\": \"Collection(Edm.Single)\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": false,\n",
      "               \"retrievable\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"normalizer\": null,\n",
      "               \"synonymMaps\": []\n",
      "          },\n",
      "          {\n",
      "               \"name\": \"embed_title\",\n",
      "               \"type\": \"Collection(Edm.Single)\",\n",
      "               \"searchable\": true,\n",
      "               \"filterable\": false,\n",
      "               \"retrievable\": true,\n",
      "               \"sortable\": false,\n",
      "               \"facetable\": false,\n",
      "               \"key\": false,\n",
      "               \"indexAnalyzer\": null,\n",
      "               \"searchAnalyzer\": null,\n",
      "               \"analyzer\": null,\n",
      "               \"normalizer\": null,\n",
      "               \"synonymMaps\": []\n",
      "          }\n",
      "     ],\n",
      "     \"scoringProfiles\": [],\n",
      "     \"corsOptions\": null,\n",
      "     \"suggesters\": [],\n",
      "     \"analyzers\": [],\n",
      "     \"normalizers\": [],\n",
      "     \"tokenizers\": [],\n",
      "     \"tokenFilters\": [],\n",
      "     \"charFilters\": [],\n",
      "     \"encryptionKey\": null,\n",
      "     \"similarity\": {\n",
      "          \"@odata.type\": \"#Microsoft.Azure.Search.BM25Similarity\",\n",
      "          \"k1\": null,\n",
      "          \"b\": null\n",
      "     },\n",
      "     \"semantic\": {\n",
      "          \"defaultConfiguration\": null,\n",
      "          \"configurations\": [\n",
      "               {\n",
      "                    \"name\": \"my-semantic-config\",\n",
      "                    \"prioritizedFields\": {\n",
      "                         \"titleField\": {\n",
      "                              \"fieldName\": \"title\"\n",
      "                         },\n",
      "                         \"prioritizedContentFields\": [\n",
      "                              {\n",
      "                                   \"fieldName\": \"description\"\n",
      "                              }\n",
      "                         ],\n",
      "                         \"prioritizedKeywordsFields\": [\n",
      "                              {\n",
      "                                   \"fieldName\": \"genres\"\n",
      "                              }\n",
      "                         ]\n",
      "                    }\n",
      "               }\n",
      "          ]\n",
      "     }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "index_status(index_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "291d88e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Azure Cognitive Search index status for: moviereview \n",
      "\n",
      "{\n",
      "  \"@odata.context\": \"https://azurecogsearcheastussr.search.windows.net/$metadata#Microsoft.Azure.Search.V2021_04_30_Preview.IndexStatistics\",\n",
      "  \"documentCount\": 9000,\n",
      "  \"storageSize\": 472209194\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "document_count, storage_size = index_stats(index_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "3ba394a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents in the index = 9,000\n",
      "Size of the index = 450.33 MB\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of documents in the index =\", f\"{document_count:,}\")\n",
    "print(\"Size of the index =\", round(storage_size / (1024 * 1024), 2), \"MB\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59c3db49",
   "metadata": {},
   "source": [
    "Note: Please wait some time in order to have the updated results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "306abe6b",
   "metadata": {},
   "source": [
    "> Go to the next notebook"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}