Chapter04_FirstWebScraper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Writing Your First Web Scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'<html>\\n<head>\\n<title>A Useful Page</title>\\n</head>\\n<body>\\n<h1>An Interesting Title</h1>\\n<div>\\nLorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\\n</div>\\n</body>\\n</html>\\n'\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "\n",
    "html = urlopen('http://pythonscraping.com/pages/page1.html')\n",
    "print(html.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h1>An Interesting Title</h1>\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "html = urlopen('http://www.pythonscraping.com/pages/page1.html')\n",
    "bs = BeautifulSoup(html.read(), 'html.parser')\n",
    "print(bs.h1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "html = urlopen('https://en.wikipedia.org/wiki/Iron_Gwazi')\n",
    "bs = BeautifulSoup(html.read(), 'html.parser')\n",
    "# 'class':['mw-file-description']\n",
    "#bs.find_all(attrs={'class': ['mw-ui-icon-wikimedia-listBullet', 'vector-icon']})\n",
    "\n",
    "bs.find_all(_class='mw-ui-icon-wikimedia-listBullet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The server could not be found!\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from urllib.error import HTTPError\n",
    "from urllib.error import URLError\n",
    "\n",
    "try:\n",
    "    html = urlopen(\"https://pythonscrapingthisurldoesnotexist.com\")\n",
    "except HTTPError as e:\n",
    "    print(\"The server returned an HTTP error\")\n",
    "except URLError as e:\n",
    "    print(\"The server could not be found!\")\n",
    "else:\n",
    "    print(html.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h1>An Interesting Title</h1>\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from urllib.error import HTTPError\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\n",
    "def getTitle(url):\n",
    "    try:\n",
    "        html = urlopen(url)\n",
    "    except HTTPError as e:\n",
    "        return None\n",
    "    try:\n",
    "        bsObj = BeautifulSoup(html.read(), \"lxml\")\n",
    "        title = bsObj.body.h1\n",
    "    except AttributeError as e:\n",
    "        return None\n",
    "    return title\n",
    "\n",
    "\n",
    "title = getTitle(\"http://www.pythonscraping.com/pages/page1.html\")\n",
    "if title == None:\n",
    "    print(\"Title could not be found\")\n",
    "else:\n",
    "    print(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}