Information_Retreival_System.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import string\n",
    "import numpy as np\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File 1 created\n",
      "File 2 created\n",
      "File 3 created\n"
     ]
    }
   ],
   "source": [
    "url = \"https://newyork.craigslist.org/search/sss?\"\n",
    "count = 0\n",
    "no_of_ads_to_be_fetched = 3\n",
    "flag = True\n",
    "while True:\n",
    "    response = requests.get(url)  # we got to the website\n",
    "    data = response.text  # we fetched the HTML code / source code of the website\n",
    "    # cleaned up the code (Parsing HTML of the website, aka making soup)\n",
    "    soup = BeautifulSoup(data, \"html.parser\")\n",
    "    ads = soup.find_all(\"div\", {\"class\": \"result-info\"})\n",
    "    for ad in ads:\n",
    "        if count >= no_of_ads_to_be_fetched:\n",
    "            flag = False\n",
    "            break\n",
    "        title = ad.find(\"a\", {\"class\": \"result-title\"}).text\n",
    "\n",
    "        price_tag = ad.find(\"span\", {\"class\": \"result-price\"})\n",
    "        if (price_tag):\n",
    "            price = price_tag.text\n",
    "        else:\n",
    "            price = \"Not Listed Publicly\"\n",
    "\n",
    "        link = ad.find(\"a\", {\"class\": \"result-title\"}).get('href')\n",
    "        # working on each individual link found and printing the contents of its web page\n",
    "        # repeating the process that we did for the homepage:\n",
    "        ad_response = requests.get(link)\n",
    "        ad_data = ad_response.text\n",
    "        ad_soup = BeautifulSoup(ad_data, \"html.parser\")\n",
    "        if(ad_soup.find(\"section\", {\"id\": \"postingbody\"})):\n",
    "            ad_description = ad_soup.find(\n",
    "                \"section\", {\"id\": \"postingbody\"}).text\n",
    "        else:\n",
    "            ad_description = \"Descripton not available.\"\n",
    "        # ad_description = os.linesep.join([s for s in ad_description.splitlines() if s]) #just to remove empty lines\n",
    "        # import os to use the upper line\n",
    "\n",
    "        # all_info\n",
    "        all_info = title + \"\\n\" + price + \"\\n\" + ad_description\n",
    "\n",
    "        # saving the ads in files\n",
    "        filename = str(count + 1) + \".txt\"\n",
    "        fout = open(\"ads\\\\\" + filename, \"w\", encoding='utf-8')\n",
    "        fout.write(all_info)\n",
    "        fout.close()\n",
    "        print(\"File\", count + 1, \"created\")\n",
    "        count += 1\n",
    "\n",
    "    if flag:  # if 10 ads have NOT been fetched\n",
    "        url_tag = soup.find(\"a\", {\"title\": \"next page\"})\n",
    "        # if all the \"next pages\" of the site end before we reach target number of ads, then this will handle it.\n",
    "        if url_tag.get('href'):\n",
    "            url = \"https://newyork.craigslist.org\" + url_tag.get('href')\n",
    "        else:\n",
    "            break\n",
    "    else:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Frequency dictionary\n",
    "frequencies = dict()\n",
    "for i in range(0, no_of_ads_to_be_fetched):\n",
    "    filename = str(i + 1) + \".txt\"\n",
    "    try: \n",
    "        fin = open(\"ads\\\\\" + filename, encoding = 'utf-8')\n",
    "    except:\n",
    "        print(\"Failed to open\", filename)\n",
    "        \n",
    "    #read, case-normalize and tokenize:\n",
    "    words = fin.read().lower().split() #all words from file in lower case but CONTAMINATED w/punctuations\n",
    "    words = str(words).translate(str.maketrans(string.punctuation, \" \" * len(string.punctuation))) #de-contaminated STRING\n",
    "    words = words.split() #de-contaminated LIST\n",
    "    \n",
    "    for word in words: #store words in dictionary\n",
    "        if word not in frequencies:\n",
    "            frequencies[word] = 1\n",
    "        else:\n",
    "            frequencies[word] += 1\n",
    "    fin.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#assigning unique id to each word\n",
    "w2n = dict()\n",
    "n2w = dict()\n",
    "i = 0\n",
    "for k,v in frequencies.items():\n",
    "    w2n[k] = i\n",
    "    n2w[i] = k\n",
    "\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating document's matrix\n",
    "cols = len(frequencies)\n",
    "doc_matrix = np.zeros((no_of_ads_to_be_fetched, cols))\n",
    "\n",
    "#assigning 1 in doc matrix in the cell whose string is present in that particular file\n",
    "for i in range(0, no_of_ads_to_be_fetched):\n",
    "    for n in n2w:\n",
    "        filename = str(i + 1) + \".txt\"\n",
    "        try: \n",
    "            fin = open(\"ads\\\\\" + filename, encoding = 'utf-8')\n",
    "        except:\n",
    "            print(\"Failed to open\", filename)\n",
    "\n",
    "        reader = fin.read().lower()\n",
    "        if n2w[n] in reader:\n",
    "            doc_matrix[i, n] = 1   \n",
    "    fin.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Scraper Ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "What do you want to buy? refrigerated box\n",
      "Your results were matched in following files:\n",
      "3.txt\n",
      "\n",
      "**Search again? [y / any key]: y\n",
      "\n",
      "What do you want to buy? _+)*(*&^^%%^#@[]refrigerated+_))*()*(^box../';'\n",
      "Your results were matched in following files:\n",
      "3.txt\n",
      "\n",
      "**Search again? [y / any key]: y\n",
      "\n",
      "What do you want to buy? chAiRs\n",
      "Your results were matched in following files:\n",
      "2.txt\n",
      "\n",
      "**Search again? [y / any key]: y\n",
      "\n",
      "What do you want to buy? jumperoo\n",
      "Your results were matched in following files:\n",
      "1.txt\n",
      "\n",
      "**Search again? [y / any key]: n\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "0",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[1;31mSystemExit\u001b[0m\u001b[1;31m:\u001b[0m 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Aamna\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3334: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "# Query handling\n",
    "while True:\n",
    "    query = input(\"\\nWhat do you want to buy? \")\n",
    "    query = query.lower().split()\n",
    "    query = str(query).translate(str.maketrans(string.punctuation,\n",
    "                                            \" \" * len(string.punctuation)))  # de-contaminated STRING\n",
    "    query = query.split()  # de-contaminated LIST\n",
    "\n",
    "    # Creating query matrix\n",
    "    query_matrix = np.zeros((cols))\n",
    "    # Obtaining id of the queried word from w2n dictionary\n",
    "    count = 0\n",
    "    for token in query:\n",
    "        if token in w2n:\n",
    "            uid = w2n[token]\n",
    "            query_matrix[uid] = 1\n",
    "            count += 1\n",
    "    if count == 0:\n",
    "        print(\"Your search \", query, \"did not match any documents.\")\n",
    "    else:\n",
    "        # Dot Product\n",
    "        transpose = doc_matrix.T\n",
    "        dot_prod = query_matrix.dot(transpose)\n",
    "\n",
    "        # Used in elimination\n",
    "        descending_scores = np.sort(dot_prod)[::-1]\n",
    "\n",
    "        # Ranking the pages\n",
    "        descending_filenos = np.argsort(dot_prod)[::-1][:no_of_ads_to_be_fetched]\n",
    "\n",
    "        # Eliminating files with 0 matches\n",
    "        count = 0\n",
    "        for score in descending_scores:\n",
    "            if score < 1:\n",
    "                break\n",
    "            else:\n",
    "                count += 1\n",
    "\n",
    "        # Printing the matched results\n",
    "        print(\"Your results were matched in following files:\")\n",
    "        for i in range(0, count):\n",
    "            filename = str(descending_filenos[i] + 1) + \".txt\"\n",
    "            print(filename)\n",
    "    again = \"\"\n",
    "    again = input(\"\\n**Search again? [y / any key]: \")\n",
    "    if again.lower() == 'y':\n",
    "        continue\n",
    "        \n",
    "    else:\n",
    "        sys.exit(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regarding the SystemExit Error\n",
    "`sys.exit` just raises SystemExit to terminate the interperter.\n",
    "\n",
    "ipython catches `sys.exit` when it executes a script in interactive mode, so **this is NOT acutally an error but a feature of ipython to avoid the interactive interpreter from being shutdown when a script is executed**, as that's not what you usually want in an interactive session.\n",
    "\n",
    "https://stackoverflow.com/a/10888822/11917891"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}