Text_Web_Crawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c935828e",
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################\n",
    "#Get page and import libraries\n",
    "##############################\n",
    "#import re\n",
    "#import string\n",
    "from collections import Counter\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "MainUrl = \"https://community.dur.ac.uk/hubert.shum/comp42315/publicationfull_year_characteranimation.htm\"\n",
    "page = requests.get(MainUrl)\n",
    "soup = BeautifulSoup(page.content, \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a93003f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting the links through the divs holding images\n",
    "body = soup.find(\"body\", class_=\"PageDefault\")\n",
    "#div = body.find(\"div\", class_ =\"w3-container w3-cell w3-mobile w3-cell-top\")\n",
    "imageDiv = body.find(\"div\", class_ =\"ImgIconPublicationDiv\")\n",
    "url = imageDiv.find(\"a\")\n",
    "print(url)\n",
    "\n",
    "if (url != None):\n",
    "    urlPublication = url[\"href\"]\n",
    "    print(urlPublication)\n",
    "else:\n",
    "    print(\"Cannot find the link in the publication page!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36d6c40e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#define variable which finds the body and the class we are interested in\n",
    "body = soup.find(\"body\", class_=\"PageDefault\")\n",
    "#Create empty List\n",
    "urlPublications = []\n",
    "#Define variable that finds the title of the movie(the section we are interested in) through tag and class\n",
    "imageDivs = body.find_all(\"div\", class_ =\"ImgIconPublicationDiv\")\n",
    "\n",
    "for imageDiv in imageDivs:\n",
    "    url = imageDiv.find(\"a\")\n",
    "    if (url != None):\n",
    "        urlPublications.append(url[\"href\"])\n",
    "        print(urlPublications[len(urlPublications)-1])\n",
    "    else:\n",
    "        print(\"Cannot find the link in the publication page!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d79b1e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting the host name\n",
    "hostUrl = \"https://community.dur.ac.uk/hubert.shum/comp42315/publicationfull_year_characteranimation.htm\"\n",
    "urlWithoutHttps = hostUrl[8:] # \"www.imdb.com/chart/moviemeter/?sort=i\n",
    "iFirstSlash = urlWithoutHttps.find(\"/\")\n",
    "hostname = \"/hubert.shum/comp42315/\"\n",
    "urlFolder = hostUrl[:8] + urlWithoutHttps[:iFirstSlash]\n",
    "\n",
    "urlGlobal = []\n",
    "for urlPublication in urlPublications:\n",
    "    urlGlobal.append(urlFolder + hostname + urlPublication)\n",
    "    print(urlGlobal[len(urlGlobal)-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2c974de",
   "metadata": {},
   "outputs": [],
   "source": [
    "for link in urlGlobal:\n",
    "    print(link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cedde628",
   "metadata": {},
   "outputs": [],
   "source": [
    "for link in urlGlobal:\n",
    "    # Getting the url upon each iteration\n",
    "    page = requests.get(link)\n",
    "    soup = BeautifulSoup(page.content, \"html.parser\")\n",
    "    \n",
    "    h1title = soup.find(\"h1\")\n",
    "    titleWithCapitals = h1title.text\n",
    "    title = titleWithCapitals.lower()\n",
    "    \n",
    "    #find the abstract \n",
    "    abstractDiv = soup.find(\"div\", attrs = {\"style\": \"margin-left: var(--size-marginleft)\"})\n",
    "    #print(abstractDiv) # check this works\n",
    "    abstractText = abstractDiv.p.text\n",
    "    abstract = abstractText.lower()\n",
    "    \n",
    "    \n",
    "    print(title)\n",
    "    print(abstract)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6390bbc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "TitlesAbstract = []\n",
    "for link in urlGlobal:\n",
    "    # Getting the url upon each iteration\n",
    "    page = requests.get(link)\n",
    "    soup = BeautifulSoup(page.content, \"html.parser\")\n",
    "    \n",
    "    h1title = soup.find(\"h1\")\n",
    "    titleWithCapitals = h1title.text\n",
    "    titlelower = titleWithCapitals.lower()\n",
    "    titleWithoutPunctutation = titlelower.replace(\"-\", \" \")\n",
    "    title = titleWithoutPunctutation.replace(\":\", \" \")\n",
    "    TitlesAbstract.append(title)\n",
    "    \n",
    "    #find the abstract \n",
    "    abstractDiv = soup.find(\"div\", attrs = {\"style\": \"margin-left: var(--size-marginleft)\"})\n",
    "    #print(abstractDiv) # check this works\n",
    "    abstractText = abstractDiv.p.text\n",
    "    abstract = abstractText.lower()\n",
    "    abstract = abstract.replace(\"-\", \"\")\n",
    "    abstract = abstract.replace(\":\", \"\")\n",
    "    abstract = abstract.replace(\")\", \"\")\n",
    "    abstract = abstract.replace(\"(\", \"\")\n",
    "    abstract = abstract.replace(\".\", \"\")\n",
    "    abstract = abstract.replace(\"[\", \"\")\n",
    "    abstract = abstract.replace(\"]\", \"\")\n",
    "    TitlesAbstract.append(abstract)\n",
    "    \n",
    "\n",
    "print(TitlesAbstract)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caffdcbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting the list of titles into one string\n",
    "s = TitlesAbstract\n",
    "AsString = ' '.join([str(elem) for elem in s])\n",
    "print(AsString)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66860606",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = AsString\n",
    "# Articles, sum exist, Pronouns excpet self unsure as this may be conceptually important in autonomy , Coordinating conjunctions, Correlative conjunctions, Conjunctions of time, Modal Nouns, Modal Adjectives, Verbs of perception, subjunctives\n",
    "\n",
    "\n",
    "stopwords = ['the', 'a','an','it',\n",
    "             'be','is','will','am','are','been','have','were','has','was',\n",
    "             'mine', 'he', 'his', 'her', 'hers', 'you', 'your', 'they', 'theyre', 'their', 'theirs', 'we', 'our','that','this','these',\n",
    "             'for', 'and', 'nor', 'but', 'or', 'yet', 'so', 'by', 'in', 'with', 'of', 'via', 'to','into','due',\n",
    "             'either', 'or', 'not', 'only', 'both', 'whether', 'just', 'the', 'as', 'much', 'sooner', 'rather', 'than', 'which','such','other',\n",
    "             'after', 'soon', 'long', 'now', 'once', 'since', 'till', 'untill', 'when', 'whenever', 'while', 'from','while','during'\n",
    "             'necessity', 'requirement', 'obligation', 'possibility',\n",
    "             'perhaps', 'possibly', 'certainly', 'definitely', 'given',\n",
    "             'suggest', 'seem', 'appear', 'suppose', 'need', 'indicate', 'imply', 'can', 'called','show','reveal','however','demonstrate','propose','proposed',\n",
    "             'also','aditionallly', 'moreover','furthermore','more','however',\n",
    "             'there', 'here','over','near','at','on','by','where','between',\n",
    "             'used','use','using','well','useful','able',\n",
    "             'could','should','would','might', 'may', 'shall',\n",
    "             'many', 'one', 'two', 'three', 'four',\n",
    "             'new',\n",
    "            ]\n",
    "\n",
    "querywords = query.split()\n",
    "\n",
    "resultwords  = [word for word in querywords if word not in stopwords]\n",
    "result = ' '.join(resultwords)\n",
    "\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f90607f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = result.split()\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd65b7a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "Counter = Counter(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77cbfa0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "most_occur = Counter.most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2758fd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(most_occur)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}