Chapter19_WebScrapingInParallel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "68d570a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 - Counter\n",
      "2 - Counter\n",
      "3 - Fizz\n",
      "3 - Counter\n",
      "4 - Counter\n",
      "5 - Buzz\n",
      "5 - Counter\n",
      "6 - Fizz\n",
      "6 - Counter\n",
      "7 - Counter\n",
      "8 - Counter\n",
      "9 - Fizz\n",
      "9 - Counter\n",
      "10 - Buzz\n",
      "10 - Counter\n",
      "11 - Counter\n",
      "12 - Fizz\n",
      "12 - Counter\n",
      "13 - Counter\n",
      "14 - Counter\n",
      "15 - Buzz\n",
      "15 - Fizz\n",
      "15 - Counter\n",
      "16 - Counter\n",
      "17 - Counter\n",
      "18 - Fizz\n",
      "18 - Counter\n",
      "19 - Counter\n",
      "20 - Buzz\n",
      "20 - Counter\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None, None]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import threading\n",
    "import time\n",
    "\n",
    "\n",
    "def print_time(threadName, delay, iterations):\n",
    "    start = int(time.time())\n",
    "    for i in range(0,iterations):\n",
    "        time.sleep(delay)\n",
    "        print(f'{int(time.time() - start)} - {threadName}')\n",
    "\n",
    "threads = [\n",
    "    threading.Thread(target=print_time, args=('Fizz', 3, 6)),\n",
    "    threading.Thread(target=print_time, args=('Buzz', 5, 4)),\n",
    "    threading.Thread(target=print_time, args=('Counter', 1, 20))\n",
    "]\n",
    "\n",
    "[t.start() for t in threads]\n",
    "[t.join() for t in threads]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6f87fa23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thread 2: Scraping /wiki/Lifeboat_sketch\n",
      "Thread 1: Scraping /wiki/Knowledge_Discovery_Metamodel\n",
      "Thread 1: Scraping /wiki/Pinellas_County,_Florida\n",
      "Thread 2: Scraping /wiki/Fane_Aircraft_Company\n",
      "Thread 1: Scraping /wiki/University_Press_of_America\n",
      "Thread 1: Scraping /wiki/Kevin_Bacon\n",
      "Thread 2: Scraping /wiki/Monty_Python\n",
      "Thread 2: Scraping /wiki/List_of_IEC_standards\n",
      "Thread 2: Scraping /wiki/North_Yorkshire_Moors_Railway\n",
      "Thread 1: Scraping /wiki/Pacific_Tigers_men%27s_basketball_statistical_leaders\n",
      "Thread 2: Scraping /wiki/Monty_Python_v._American_Broadcasting_Companies,_Inc.\n",
      "Thread 1: Scraping /wiki/ISO/IEC_38500\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [5]\u001b[0m, in \u001b[0;36m<cell line: 28>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m threads \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m     24\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 1\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Kevin_Bacon\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     25\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 2\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Monty_Python\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     26\u001b[0m ]\n\u001b[1;32m     27\u001b[0m [t\u001b[38;5;241m.\u001b[39mstart() \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n\u001b[0;32m---> 28\u001b[0m [t\u001b[38;5;241m.\u001b[39mjoin() \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n",
      "Input \u001b[0;32mIn [5]\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     23\u001b[0m threads \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m     24\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 1\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Kevin_Bacon\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     25\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 2\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Monty_Python\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     26\u001b[0m ]\n\u001b[1;32m     27\u001b[0m [t\u001b[38;5;241m.\u001b[39mstart() \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n\u001b[0;32m---> 28\u001b[0m [\u001b[43mt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/threading.py:1053\u001b[0m, in \u001b[0;36mThread.join\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1050\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcannot join current thread\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m   1052\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1053\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_wait_for_tstate_lock\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1054\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1055\u001b[0m     \u001b[38;5;66;03m# the behavior of a negative timeout isn't documented, but\u001b[39;00m\n\u001b[1;32m   1056\u001b[0m     \u001b[38;5;66;03m# historically .join(timeout=x) for x<0 has acted as if timeout=0\u001b[39;00m\n\u001b[1;32m   1057\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_wait_for_tstate_lock(timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mmax\u001b[39m(timeout, \u001b[38;5;241m0\u001b[39m))\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/threading.py:1073\u001b[0m, in \u001b[0;36mThread._wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1070\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m   1072\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1073\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[43mlock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43mblock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m   1074\u001b[0m         lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[1;32m   1075\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stop()\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import random\n",
    "import threading\n",
    "import time\n",
    "\n",
    "# Recursively find links on a Wikipedia page, \n",
    "# then follow a random link, with artificial 5 sec delay\n",
    "def scrape_article(thread_name, path):\n",
    "    time.sleep(5)\n",
    "    print(f'{thread_name}: Scraping {path}')\n",
    "    html = urlopen('http://en.wikipedia.org{}'.format(path))\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    title = bs.find('h1').get_text()\n",
    "    links = bs.find('div', {'id':'bodyContent'}).find_all('a',\n",
    "        href=re.compile('^(/wiki/)((?!:).)*$'))\n",
    "    if len(links) > 0:\n",
    "        newArticle = links[random.randint(0, len(links)-1)].attrs['href']\n",
    "        scrape_article(thread_name, newArticle)\n",
    "\n",
    "           \n",
    "threads = [\n",
    "    threading.Thread(target=scrape_article, args=('Thread 1', '/wiki/Kevin_Bacon',)),\n",
    "    threading.Thread(target=scrape_article, args=('Thread 2', '/wiki/Monty_Python',)),\n",
    "]\n",
    "[t.start() for t in threads]\n",
    "[t.join() for t in threads]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6b6ae604",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thread 2: Scraping /wiki/List_of_office_suites\n",
      "Thread 1: Scraping /wiki/Meta-Object_Facility\n",
      "Thread 1: Scraping /wiki/Cedric_Gibbons\n",
      "Thread 1: Scraping /wiki/Fujifilm_FinePix_Real_3D\n",
      "Thread 2: Scraping /wiki/Draft_horse\n",
      "Thread 2: Scraping /wiki/PDP-9\n",
      "Thread 2: Scraping /wiki/Sidley_United_F.C.\n",
      "Thread 1: Scraping /wiki/The_Bulletin_(Bend)\n",
      "Thread 2: Scraping /wiki/Utagawa_Kunisada_II\n",
      "Thread 2: Scraping /wiki/Monty_PythonThread 1: Scraping /wiki/Kevin_Bacon\n",
      "\n",
      "Thread 1: Scraping /wiki/Geographic_coordinate_system\n",
      "Thread 2: Scraping /wiki/ConceptDraw_Office\n",
      "Getting links in Thread 1\n",
      "Getting links in Thread 2\n",
      "Getting links in Thread 1\n",
      "Getting links in Thread 2\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [6]\u001b[0m, in \u001b[0;36m<cell line: 37>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     32\u001b[0m threads \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m     33\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 1\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Kevin_Bacon\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     34\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 2\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Monty_Python\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     35\u001b[0m ]\n\u001b[1;32m     36\u001b[0m [t\u001b[38;5;241m.\u001b[39mstart() \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n\u001b[0;32m---> 37\u001b[0m [t\u001b[38;5;241m.\u001b[39mjoin() \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n",
      "Input \u001b[0;32mIn [6]\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     32\u001b[0m threads \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m     33\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 1\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Kevin_Bacon\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     34\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 2\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Monty_Python\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     35\u001b[0m ]\n\u001b[1;32m     36\u001b[0m [t\u001b[38;5;241m.\u001b[39mstart() \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n\u001b[0;32m---> 37\u001b[0m [\u001b[43mt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/threading.py:1053\u001b[0m, in \u001b[0;36mThread.join\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1050\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcannot join current thread\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m   1052\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1053\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_wait_for_tstate_lock\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1054\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1055\u001b[0m     \u001b[38;5;66;03m# the behavior of a negative timeout isn't documented, but\u001b[39;00m\n\u001b[1;32m   1056\u001b[0m     \u001b[38;5;66;03m# historically .join(timeout=x) for x<0 has acted as if timeout=0\u001b[39;00m\n\u001b[1;32m   1057\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_wait_for_tstate_lock(timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mmax\u001b[39m(timeout, \u001b[38;5;241m0\u001b[39m))\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/threading.py:1073\u001b[0m, in \u001b[0;36mThread._wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1070\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m   1072\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1073\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[43mlock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43mblock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m   1074\u001b[0m         lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[1;32m   1075\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stop()\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import random\n",
    "import threading\n",
    "import time\n",
    "\n",
    "\n",
    "visited = []\n",
    "\n",
    "def get_links(thread_name, bs):\n",
    "    print('Getting links in {}'.format(thread_name))\n",
    "    links = bs.find('div', {'id':'bodyContent'}).find_all('a',\n",
    "        href=re.compile('^(/wiki/)((?!:).)*$'))\n",
    "    return [link for link in links if link not in visited]\n",
    "\n",
    "# Recursively find links on a Wikipedia page, \n",
    "# then follow a random link, with artificial 5 sec delay\n",
    "def scrape_article(thread_name, path):\n",
    "    time.sleep(5)\n",
    "    visited.append(path)\n",
    "    print(f'{thread_name}: Scraping {path}')\n",
    "    html = urlopen('http://en.wikipedia.org{}'.format(path))\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    title = bs.find('h1').get_text()\n",
    "    links = get_links(thread_name, bs)\n",
    "    if len(links) > 0:\n",
    "        newArticle = links[random.randint(0, len(links)-1)].attrs['href']\n",
    "        scrape_article(thread_name, newArticle)\n",
    "\n",
    "           \n",
    "threads = [\n",
    "    threading.Thread(target=scrape_article, args=('Thread 1', '/wiki/Kevin_Bacon',)),\n",
    "    threading.Thread(target=scrape_article, args=('Thread 2', '/wiki/Monty_Python',)),\n",
    "]\n",
    "[t.start() for t in threads]\n",
    "[t.join() for t in threads]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f666895f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Article already exists: /wiki/United_States\n",
      "Article already exists: /wiki/United_States\n",
      "Article already exists: /wiki/United_States\n",
      "Article already exists: /wiki/United_States\n",
      "Article already exists: /wiki/United_States\n",
      "Article already exists: /wiki/United_States\n",
      "Article already exists: /wiki/United_States\n",
      "Storing article /wiki/1959_in_music\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Article already exists: /wiki/Billboard_Hot_100\n",
      "Storing article /wiki/Fred_Bronson\n",
      "Article already exists: /wiki/ISBN_(identifier)\n",
      "Storing article /wiki/Joel_Whitburn\n",
      "Article already exists: /wiki/ISBN_(identifier)Thread 2: Scraping /wiki/Monty_PythonThread 1: Scraping /wiki/Kevin_Bacon\n",
      "\n",
      "\n",
      "Article already exists: /wiki/ISBN_(identifier)\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1940\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1941\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1942\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1943\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1944\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1945\n",
      "Getting links in Thread 1\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1946\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1947\n",
      "Article already exists: /wiki/Kevin_Bacon_(disambiguation)\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1948\n",
      "Article already exists: /wiki/Philadelphia\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1949\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-23:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/threading.py\", line 973, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/threading.py\", line 910, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_77749/3538858134.py\", line 42, in scrape_article\n",
      "AttributeError: 'str' object has no attribute 'attrs'\n",
      "Exception in thread Thread-24:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/threading.py\", line 973, in _bootstrap_inner\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Article already exists: /wiki/Kevin_Bacon_filmography\n",
      "Article already exists: /wiki/Kyra_SedgwickStoring article /wiki/List_of_Billboard_number-one_singles_of_1950\n",
      "\n",
      "Storing article /wiki/Sosie_Bacon\n",
      "Storing article /wiki/Edmund_Bacon_(architect)Storing article /wiki/List_of_Billboard_number-one_singles_of_1951\n",
      "\n",
      "Getting links in Thread 2\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1952\n",
      "Storing article /wiki/Michael_Bacon_(musician)Storing article /wiki/List_of_Billboard_number-one_singles_of_1953\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1954\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/threading.py\", line 910, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_77749/3538858134.py\", line 42, in scrape_article\n",
      "AttributeError: "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Storing article /wiki/Holly_NearStoring article /wiki/List_of_Billboard_number-one_singles_of_1955\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1956\n",
      "Storing article /wiki/Leading_man\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1957\n",
      "Storing article /wiki/Character_actor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'str' object has no attribute 'attrs'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_1958\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1960\n",
      "Storing article /wiki/Golden_Globe_Award\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1961Storing article /wiki/Screen_Actors_Guild_Award\n",
      "\n",
      "Storing article /wiki/Primetime_Emmy_Award\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1962\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1963Article already exists: /wiki/The_Guardian\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1964Storing article /wiki/Academy_Award\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1965\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1966\n",
      "Storing article /wiki/Hollywood_Walk_of_FameStoring article /wiki/List_of_Billboard_Hot_100_number_ones_of_1967\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1968\n",
      "Storing article /wiki/National_Lampoon%27s_Animal_HouseStoring article /wiki/List_of_Billboard_Hot_100_number_ones_of_1969\n",
      "\n",
      "Storing article /wiki/Footloose_(1984_film)\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1970\n",
      "Storing article /wiki/Diner_(1982_film)\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1971\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1972Storing article /wiki/JFK_(film)\n",
      "\n",
      "Storing article /wiki/A_Few_Good_Men\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1973\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1974Storing article /wiki/Apollo_13_(film)\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1975Storing article /wiki/Mystic_River_(film)\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1976Storing article /wiki/Frost/Nixon_(film)\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1977\n",
      "Storing article /wiki/Friday_the_13th_(1980_film)\n",
      "Storing article /wiki/Tremors_(1990_film)Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1978\n",
      "\n",
      "Storing article /wiki/The_River_Wild\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1979\n",
      "Storing article /wiki/Sleepers_(film)\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1980\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1981Storing article /wiki/Wild_Things_(film)\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1982Storing article /wiki/The_Woodsman_(2004_film)\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1983\n",
      "Storing article /wiki/Flatliners\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1984Storing article /wiki/Crazy,_Stupid,_Love\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1985\n",
      "Storing article /wiki/Black_Mass_(film)Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1986\n",
      "\n",
      "Storing article /wiki/Patriots_Day_(film)Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1987\n",
      "\n",
      "Storing article /wiki/Losing_Chase\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1988\n",
      "Storing article /wiki/Loverboy_(2005_film)\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1989Article already exists: /wiki/Fox_Broadcasting_Company\n",
      "\n",
      "Storing article /wiki/The_Following\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1990\n",
      "Storing article /wiki/Michael_Strobl\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1991\n",
      "Storing article /wiki/HBO\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1992\n",
      "Storing article /wiki/Taking_Chance\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1993\n",
      "Storing article /wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1994Storing article /wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1995\n",
      "Storing article /wiki/Amazon_Prime_VideoStoring article /wiki/List_of_Billboard_Hot_100_number_ones_of_1996\n",
      "\n",
      "Storing article /wiki/I_Love_Dick_(TV_series)Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1997\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1998\n",
      "Storing article /wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_1999\n",
      "Storing article /wiki/Showtime_(TV_network)\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2000\n",
      "Storing article /wiki/City_on_a_Hill_(TV_series)\n",
      "Storing article /wiki/Six_Degrees_of_Kevin_BaconStoring article /wiki/List_of_Billboard_Hot_100_number_ones_of_2001\n",
      "\n",
      "Storing article /wiki/SixDegrees.org\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2002\n",
      "Storing article /wiki/EE_Limited\n",
      "Article already exists: /wiki/Philadelphia\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2003\n",
      "Article already exists: /wiki/Edmund_Bacon_(architect)\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2004\n",
      "Storing article /wiki/Urban_planning\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2005\n",
      "Storing article /wiki/Design_of_CitiesStoring article /wiki/List_of_Billboard_Hot_100_number_ones_of_2006\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2007\n",
      "Storing article /wiki/Julia_R._Masterman_School\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2008\n",
      "Storing article /wiki/Spring_Garden,_Philadelphia\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2009\n",
      "Storing article /wiki/Pennsylvania_Governor%27s_School_for_the_Arts\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2010\n",
      "Storing article /wiki/Bucknell_University\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2011Storing article /wiki/Lewisburg,_Pennsylvania\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2012Storing article /wiki/Glory_Van_Scott\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2013\n",
      "Article already exists: /wiki/Kevin_Bacon_filmography\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2014Storing article /wiki/Circle_in_the_Square\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2015\n",
      "Storing article /wiki/Nancy_Mills\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2016\n",
      "Storing article /wiki/Cosmopolitan_(magazine)\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2017\n",
      "Storing article /wiki/Fraternities_and_sororities\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2018\n",
      "Storing article /wiki/Animal_House\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2019\n",
      "Storing article /wiki/Search_for_Tomorrow\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2020\n",
      "Storing article /wiki/Guiding_Light\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2021\n",
      "Article already exists: /wiki/Friday_the_13th_(1980_film)\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2022\n",
      "Storing article /wiki/Getting_Out\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number_ones_of_2023\n",
      "Storing article /wiki/Phoenix_Theater\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_of_the_1940s\n",
      "Storing article /wiki/Flux\n",
      "Storing article /wiki/Second_Stage_Theatre\n",
      "Storing article /wiki/List_of_Billboard_number-one_singles_from_1950_to_1958\n",
      "Storing article /wiki/Obie_AwardStoring article /wiki/List_of_Billboard_Hot_100_number-one_singles_from_1958_to_1969\n",
      "\n",
      "Storing article /wiki/Forty_Deuce\n",
      "Storing article /wiki/Slab_BoysStoring article /wiki/List_of_Billboard_Hot_100_number-one_singles_of_the_1970s\n",
      "\n",
      "Storing article /wiki/Sean_Penn\n",
      "Storing article /wiki/Val_KilmerStoring article /wiki/List_of_Billboard_Hot_100_number-one_singles_of_the_1980s\n",
      "\n",
      "Storing article /wiki/Barry_Levinson\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number-one_singles_of_the_1990s\n",
      "Article already exists: /wiki/Diner_(1982_film)\n",
      "Storing article /wiki/Steve_Guttenberg\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number-one_singles_of_the_2000s\n",
      "Storing article /wiki/Daniel_Stern_(actor)Storing article /wiki/List_of_Billboard_Hot_100_number-one_singles_of_the_2010s\n",
      "\n",
      "Storing article /wiki/List_of_Billboard_Hot_100_number-one_singles_of_the_2020sStoring article /wiki/Mickey_Rourke\n",
      "\n",
      "Storing article /wiki/Tim_Daly\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Storing article /wiki/List_of_Your_Hit_Parade_number-one_songs\n",
      "Storing article /wiki/List_of_Billboard_top-ten_singlesStoring article /wiki/Ellen_Barkin\n",
      "\n",
      "Storing article /wiki/Billboard_Year-EndArticle already exists: /wiki/Footloose_(1984_film)\n",
      "\n",
      "Storing article /wiki/James_DeanStoring article /wiki/List_of_Billboard_Hot_100_chart_achievements_and_milestones\n",
      "\n",
      "Storing article /wiki/Rebel_Without_a_Cause\n",
      "Article already exists: /wiki/Kevin_Bacon_(disambiguation)\n",
      "Storing article /wiki/Mickey_RooneyArticle already exists: /wiki/Philadelphia\n",
      "\n",
      "Article already exists: /wiki/Kevin_Bacon_filmography\n",
      "Storing article /wiki/Judy_Garland\n",
      "Article already exists: /wiki/Kyra_Sedgwick\n",
      "Article already exists: /wiki/Sosie_Bacon\n",
      "Article already exists: /wiki/Edmund_Bacon_(architect)\n",
      "Article already exists: /wiki/Michael_Bacon_(musician)Storing article /wiki/Typecasting_(acting)\n",
      "\n",
      "Article already exists: /wiki/Holly_Near\n",
      "Storing article /wiki/John_Hughes_(filmmaker)\n",
      "Article already exists: /wiki/Leading_man\n",
      "Article already exists: /wiki/Character_actor\n",
      "Storing article /wiki/Planes,_Trains_and_Automobiles\n",
      "Storing article /wiki/She%27s_Having_a_BabyArticle already exists: /wiki/Golden_Globe_Award\n",
      "\n",
      "Article already exists: /wiki/Screen_Actors_Guild_AwardStoring article /wiki/The_Big_Picture_(1989_film)\n",
      "\n",
      "Article already exists: /wiki/Primetime_Emmy_AwardArticle already exists: /wiki/Tremors_(1990_film)\n",
      "\n",
      "Article already exists: /wiki/The_Guardian\n",
      "Article already exists: /wiki/Academy_Award\n",
      "Storing article /wiki/Joel_Schumacher\n",
      "Article already exists: /wiki/Hollywood_Walk_of_Fame\n",
      "Article already exists: /wiki/National_Lampoon%27s_Animal_House\n",
      "Article already exists: /wiki/FlatlinersArticle already exists: /wiki/Footloose_(1984_film)\n",
      "\n",
      "Article already exists: /wiki/Diner_(1982_film)Storing article /wiki/Elizabeth_Perkins\n",
      "\n",
      "Article already exists: /wiki/JFK_(film)\n",
      "Storing article /wiki/He_Said,_She_Said_(film)\n",
      "Article already exists: /wiki/A_Few_Good_Men\n",
      "Article already exists: /wiki/The_New_York_Times\n",
      "Article already exists: /wiki/Apollo_13_(film)\n",
      "Article already exists: /wiki/Mystic_River_(film)Storing article /wiki/Oliver_Stone\n",
      "\n",
      "Article already exists: /wiki/Frost/Nixon_(film)\n",
      "Article already exists: /wiki/Friday_the_13th_(1980_film)\n",
      "Article already exists: /wiki/Tremors_(1990_film)Article already exists: /wiki/JFK_(film)\n",
      "\n",
      "Storing article /wiki/A_Few_Good_Men_(film)Article already exists: /wiki/The_River_Wild\n",
      "\n",
      "Article already exists: /wiki/Sleepers_(film)\n",
      "Storing article /wiki/Michael_Greif\n",
      "Article already exists: /wiki/Wild_Things_(film)\n",
      "Article already exists: /wiki/Golden_Globe_AwardArticle already exists: /wiki/The_Woodsman_(2004_film)\n",
      "\n",
      "Article already exists: /wiki/The_River_WildArticle already exists: /wiki/Flatliners\n",
      "\n",
      "Article already exists: /wiki/Crazy,_Stupid,_Love\n",
      "Storing article /wiki/Meryl_Streep\n",
      "Article already exists: /wiki/Black_Mass_(film)\n",
      "Article already exists: /wiki/Patriots_Day_(film)Storing article /wiki/Murder_in_the_First_(film)\n",
      "\n",
      "Article already exists: /wiki/Losing_Chase\n",
      "Article already exists: /wiki/Loverboy_(2005_film)\n",
      "Storing article /wiki/Blockbuster_(entertainment)\n",
      "Article already exists: /wiki/Fox_Broadcasting_Company\n",
      "Article already exists: /wiki/Apollo_13_(film)Article already exists: /wiki/The_Following\n",
      "\n",
      "Article already exists: /wiki/Michael_Strobl\n",
      "Article already exists: /wiki/Sleepers_(film)Article already exists: /wiki/HBO\n",
      "\n",
      "Article already exists: /wiki/Taking_ChanceStoring article /wiki/Picture_Perfect_(1997_film)\n",
      "\n",
      "Article already exists: /wiki/Losing_ChaseArticle already exists: /wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "\n",
      "Storing article /wiki/Digging_to_ChinaArticle already exists: /wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "\n",
      "Article already exists: /wiki/Amazon_Prime_Video\n",
      "Storing article /wiki/PayolaArticle already exists: /wiki/I_Love_Dick_(TV_series)\n",
      "\n",
      "Article already exists: /wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\n",
      "Storing article /wiki/Telling_Lies_in_America_(film)\n",
      "Article already exists: /wiki/Showtime_(TV_network)\n",
      "Article already exists: /wiki/Wild_Things_(film)\n",
      "Article already exists: /wiki/City_on_a_Hill_(TV_series)\n",
      "Storing article /wiki/Stir_of_Echoes\n",
      "Article already exists: /wiki/Six_Degrees_of_Kevin_Bacon\n",
      "Article already exists: /wiki/SixDegrees.orgStoring article /wiki/David_Koepp\n",
      "\n",
      "Article already exists: /wiki/EE_Limited\n",
      "Storing article /wiki/Cannes_Film_FestivalArticle already exists: /wiki/Philadelphia\n",
      "\n",
      "Article already exists: /wiki/Edmund_Bacon_(architect)\n",
      "Storing article /wiki/Paul_Verhoeven\n",
      "Article already exists: /wiki/Urban_planning\n",
      "Storing article /wiki/Hollow_Man\n",
      "Article already exists: /wiki/Design_of_Cities\n",
      "Article already exists: /wiki/Julia_R._Masterman_SchoolStoring article /wiki/Colin_Firth\n",
      "\n",
      "Article already exists: /wiki/Spring_Garden,_Philadelphia\n",
      "Storing article /wiki/Rachel_Blanchard\n",
      "Article already exists: /wiki/Pennsylvania_Governor%27s_School_for_the_Arts\n",
      "Article already exists: /wiki/Bucknell_University\n",
      "Article already exists: /wiki/Lewisburg,_Pennsylvania\n",
      "Storing article /wiki/M%C3%A9nage_%C3%A0_trois\n",
      "Article already exists: /wiki/Glory_Van_Scott\n",
      "Storing article /wiki/Where_the_Truth_Lies\n",
      "Article already exists: /wiki/Kevin_Bacon_filmography\n",
      "Article already exists: /wiki/Circle_in_the_SquareStoring article /wiki/Atom_Egoyan\n",
      "\n",
      "Article already exists: /wiki/Nancy_Mills\n",
      "Storing article /wiki/MPAA\n",
      "Article already exists: /wiki/Cosmopolitan_(magazine)\n",
      "Article already exists: /wiki/Fraternities_and_sororitiesStoring article /wiki/MPAA_film_rating_system\n",
      "\n",
      "Article already exists: /wiki/Animal_House\n",
      "Article already exists: /wiki/Sean_Penn\n",
      "Article already exists: /wiki/Search_for_Tomorrow\n",
      "Storing article /wiki/Tim_RobbinsArticle already exists: /wiki/Guiding_Light\n",
      "\n",
      "Article already exists: /wiki/Friday_the_13th_(1980_film)\n",
      "Storing article /wiki/Clint_EastwoodArticle already exists: /wiki/Getting_Out\n",
      "\n",
      "Article already exists: /wiki/Phoenix_Theater\n",
      "Article already exists: /wiki/Mystic_River_(film)\n",
      "Article already exists: /wiki/Flux\n",
      "Article already exists: /wiki/The_Woodsman_(2004_film)\n",
      "Article already exists: /wiki/Second_Stage_Theatre\n",
      "Storing article /wiki/HBO_Films\n",
      "Article already exists: /wiki/Obie_Award\n",
      "Article already exists: /wiki/Forty_Deuce\n",
      "Article already exists: /wiki/Taking_ChanceArticle already exists: /wiki/Slab_Boys\n",
      "\n",
      "Article already exists: /wiki/Michael_Strobl\n",
      "Storing article /wiki/Desert_Storm\n",
      "Article already exists: /wiki/Sean_Penn\n",
      "Article already exists: /wiki/Val_Kilmer\n",
      "Article already exists: /wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "Storing article /wiki/Toronto_Film_FestivalArticle already exists: /wiki/Barry_Levinson\n",
      "\n",
      "Storing article /wiki/Matthew_VaughnArticle already exists: /wiki/Diner_(1982_film)\n",
      "\n",
      "Article already exists: /wiki/Steve_Guttenberg\n",
      "Storing article /wiki/Sebastian_Shaw_(comics)\n",
      "Article already exists: /wiki/Daniel_Stern_(actor)\n",
      "Storing article /wiki/Dustin_Lance_Black\n",
      "Article already exists: /wiki/Mickey_Rourke\n",
      "Article already exists: /wiki/Tim_Daly\n",
      "Article already exists: /wiki/Ellen_Barkin\n",
      "Storing article /wiki/8_(play)\n",
      "Article already exists: /wiki/Footloose_(1984_film)\n",
      "Storing article /wiki/Perry_v._Brown\n",
      "Article already exists: /wiki/James_Dean\n",
      "Storing article /wiki/Proposition_8\n",
      "Storing article /wiki/Rebel_Without_a_Cause\n",
      "Storing article /wiki/Charles_J._Cooper\n",
      "Storing article /wiki/Wilshire_Ebell_Theatre\n",
      "Article already exists: /wiki/Mickey_Rooney\n",
      "Storing article /wiki/American_Foundation_for_Equal_Rights\n",
      "Article already exists: /wiki/Judy_Garland\n",
      "Article already exists: /wiki/The_Following\n",
      "Article already exists: /wiki/Typecasting_(acting)\n",
      "Storing article /wiki/Saturn_Award_for_Best_Actor_on_TelevisionArticle already exists: /wiki/John_Hughes_(filmmaker)\n",
      "\n",
      "Article already exists: /wiki/Planes,_Trains_and_Automobiles\n",
      "Storing article /wiki/Huffington_PostArticle already exists: /wiki/She%27s_Having_a_Baby\n",
      "\n",
      "Article already exists: /wiki/The_Big_Picture_(1989_film)\n",
      "Article already exists: /wiki/Tremors_(1990_film)\n",
      "Storing article /wiki/The_Bacon_Brothers\n",
      "Article already exists: /wiki/Tremors_(1990_film)\n",
      "Article already exists: /wiki/Joel_Schumacher\n",
      "Article already exists: /wiki/Michael_Bacon_(musician)Article already exists: /wiki/Flatliners\n",
      "\n",
      "Storing article /wiki/InstagramArticle already exists: /wiki/Elizabeth_Perkins\n",
      "\n",
      "Storing article /wiki/Old_97%27s\n",
      "Article already exists: /wiki/He_Said,_She_Said_(film)\n",
      "Storing article /wiki/The_Guardians_of_the_Galaxy_Holiday_SpecialArticle already exists: /wiki/The_New_York_Times\n",
      "\n",
      "Article already exists: /wiki/Oliver_Stone\n",
      "Article already exists: /wiki/JFK_(film)Storing article /wiki/EE_(telecommunications_company)\n",
      "\n",
      "Article already exists: /wiki/A_Few_Good_Men_(film)\n",
      "Article already exists: /wiki/Six_Degrees_of_Kevin_BaconArticle already exists: /wiki/Michael_Greif\n",
      "\n",
      "Article already exists: /wiki/Golden_Globe_AwardStoring article /wiki/Trivia\n",
      "\n",
      "Article already exists: /wiki/The_River_Wild\n",
      "Article already exists: /wiki/Meryl_StreepStoring article /wiki/Big_screen\n",
      "\n",
      "Storing article /wiki/Six_degrees_of_separationArticle already exists: /wiki/Murder_in_the_First_(film)\n",
      "\n",
      "Article already exists: /wiki/Blockbuster_(entertainment)\n",
      "Storing article /wiki/Internet_meme\n",
      "Article already exists: /wiki/Apollo_13_(film)Article already exists: /wiki/SixDegrees.org\n",
      "\n",
      "Storing article /wiki/Social_networking_service\n",
      "Article already exists: /wiki/Sleepers_(film)\n",
      "Article already exists: /wiki/Six_Degrees_of_Kevin_BaconArticle already exists: /wiki/Picture_Perfect_(1997_film)\n",
      "\n",
      "Article already exists: /wiki/IMDbArticle already exists: /wiki/Losing_Chase\n",
      "\n",
      "Storing article /wiki/Paul_Erd%C5%91sArticle already exists: /wiki/Digging_to_China\n",
      "\n",
      "Article already exists: /wiki/Payola\n",
      "Article already exists: /wiki/Telling_Lies_in_America_(film)\n",
      "Storing article /wiki/Erd%C5%91s_number\n",
      "Article already exists: /wiki/Wild_Things_(film)\n",
      "Article already exists: /wiki/Paul_Erd%C5%91sArticle already exists: /wiki/Stir_of_Echoes\n",
      "\n",
      "Article already exists: /wiki/David_KoeppArticle already exists: /wiki/Erd%C5%91s_number\n",
      "\n",
      "Article already exists: /wiki/Cannes_Film_Festival\n",
      "Storing article /wiki/Erd%C5%91s%E2%80%93Bacon_number\n",
      "Article already exists: /wiki/Paul_Verhoeven\n",
      "Article already exists: /wiki/Hollow_ManArticle already exists: /wiki/Kyra_Sedgwick\n",
      "\n",
      "Article already exists: /wiki/PBS\n",
      "Article already exists: /wiki/Colin_Firth\n",
      "Storing article /wiki/Lanford_Wilson\n",
      "Storing article /wiki/Lemon_Sky\n",
      "Article already exists: /wiki/Rachel_Blanchard\n",
      "Storing article /wiki/PyratesArticle already exists: /wiki/M%C3%A9nage_%C3%A0_trois\n",
      "\n",
      "Article already exists: /wiki/Where_the_Truth_Lies\n",
      "Article already exists: /wiki/Atom_Egoyan\n",
      "Article already exists: /wiki/Murder_in_the_First_(film)Article already exists: /wiki/MPAA\n",
      "\n",
      "Article already exists: /wiki/The_Woodsman_(2004_film)\n",
      "Article already exists: /wiki/MPAA_film_rating_system\n",
      "Article already exists: /wiki/Loverboy_(2005_film)Article already exists: /wiki/Sean_Penn\n",
      "\n",
      "Article already exists: /wiki/Tim_Robbins\n",
      "Article already exists: /wiki/Sosie_Bacon\n",
      "Article already exists: /wiki/Clint_Eastwood\n",
      "Storing article /wiki/Upper_West_Side\n",
      "Article already exists: /wiki/Mystic_River_(film)\n",
      "Article already exists: /wiki/The_Woodsman_(2004_film)\n",
      "Article already exists: /wiki/Manhattan\n",
      "Article already exists: /wiki/HBO_Films\n",
      "Storing article /wiki/Tracy_PollanArticle already exists: /wiki/Taking_Chance\n",
      "\n",
      "Article already exists: /wiki/Separation_of_church_and_state_in_the_United_States\n",
      "Article already exists: /wiki/Michael_Strobl\n",
      "Storing article /wiki/The_Times\n",
      "Article already exists: /wiki/Desert_Storm\n",
      "Storing article /wiki/Atheism\n",
      "Article already exists: /wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "Article already exists: /wiki/Toronto_Film_Festival\n",
      "Storing article /wiki/AntireligionArticle already exists: /wiki/Matthew_Vaughn\n",
      "\n",
      "Article already exists: /wiki/Sebastian_Shaw_(comics)\n",
      "Storing article /wiki/Will.i.am\n",
      "Article already exists: /wiki/Dustin_Lance_Black\n",
      "Article already exists: /wiki/8_(play)\n",
      "Storing article /wiki/It%27s_a_New_Day_(Will.i.am_song)\n",
      "Article already exists: /wiki/Perry_v._Brown\n",
      "Article already exists: /wiki/Proposition_8Storing article /wiki/Barack_Obama\n",
      "\n",
      "Article already exists: /wiki/Charles_J._Cooper\n",
      "Storing article /wiki/Ponzi_scheme\n",
      "Storing article /wiki/Wilshire_Ebell_Theatre\n",
      "Article already exists: /wiki/American_Foundation_for_Equal_RightsStoring article /wiki/Bernie_Madoff\n",
      "\n",
      "Article already exists: /wiki/The_Following\n",
      "Article already exists: /wiki/Saturn_Award_for_Best_Actor_on_TelevisionStoring article /wiki/Finding_Your_Roots\n",
      "\n",
      "Article already exists: /wiki/Huffington_Post\n",
      "Storing article /wiki/Henry_Louis_Gates\n",
      "Article already exists: /wiki/Tremors_(1990_film)\n",
      "Article already exists: /wiki/Apollo_13_(film)\n",
      "Article already exists: /wiki/The_Bacon_BrothersArticle already exists: /wiki/Mystic_River_(film)\n",
      "\n",
      "Storing article /wiki/Blockbuster_Entertainment_Awards\n",
      "Article already exists: /wiki/Michael_Bacon_(musician)\n",
      "Article already exists: /wiki/Instagram\n",
      "Article already exists: /wiki/Blockbuster_Entertainment_Awards\n",
      "Article already exists: /wiki/Old_97%27s\n",
      "Article already exists: /wiki/Hollow_Man\n",
      "Article already exists: /wiki/The_Guardians_of_the_Galaxy_Holiday_Special\n",
      "Storing article /wiki/Boston_Society_of_Film_Critics\n",
      "Article already exists: /wiki/EE_(telecommunications_company)\n",
      "Article already exists: /wiki/Six_Degrees_of_Kevin_Bacon\n",
      "Storing article /wiki/Boston_Society_of_Film_Critics_Award_for_Best_Cast\n",
      "Article already exists: /wiki/Trivia\n",
      "Article already exists: /wiki/Mystic_River_(film)\n",
      "Article already exists: /wiki/Big_screen\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [11]\u001b[0m, in \u001b[0;36m<cell line: 55>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     49\u001b[0m threads \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m     50\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 1\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Kevin_Bacon\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     51\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 2\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Monty_Python\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     52\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mstorage, args\u001b[38;5;241m=\u001b[39m(queue,))\n\u001b[1;32m     53\u001b[0m ]\n\u001b[1;32m     54\u001b[0m [t\u001b[38;5;241m.\u001b[39mstart() \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n\u001b[0;32m---> 55\u001b[0m [t\u001b[38;5;241m.\u001b[39mjoin() \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n",
      "Input \u001b[0;32mIn [11]\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     49\u001b[0m threads \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m     50\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 1\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Kevin_Bacon\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     51\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mscrape_article, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThread 2\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/wiki/Monty_Python\u001b[39m\u001b[38;5;124m'\u001b[39m,)),\n\u001b[1;32m     52\u001b[0m     threading\u001b[38;5;241m.\u001b[39mThread(target\u001b[38;5;241m=\u001b[39mstorage, args\u001b[38;5;241m=\u001b[39m(queue,))\n\u001b[1;32m     53\u001b[0m ]\n\u001b[1;32m     54\u001b[0m [t\u001b[38;5;241m.\u001b[39mstart() \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n\u001b[0;32m---> 55\u001b[0m [\u001b[43mt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m threads]\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/threading.py:1053\u001b[0m, in \u001b[0;36mThread.join\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1050\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcannot join current thread\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m   1052\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1053\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_wait_for_tstate_lock\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1054\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1055\u001b[0m     \u001b[38;5;66;03m# the behavior of a negative timeout isn't documented, but\u001b[39;00m\n\u001b[1;32m   1056\u001b[0m     \u001b[38;5;66;03m# historically .join(timeout=x) for x<0 has acted as if timeout=0\u001b[39;00m\n\u001b[1;32m   1057\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_wait_for_tstate_lock(timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mmax\u001b[39m(timeout, \u001b[38;5;241m0\u001b[39m))\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/threading.py:1073\u001b[0m, in \u001b[0;36mThread._wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1070\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m   1072\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1073\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[43mlock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43mblock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m   1074\u001b[0m         lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[1;32m   1075\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stop()\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Article already exists: /wiki/Six_degrees_of_separationStoring article /wiki/Bravo_Otto\n",
      "\n",
      "Article already exists: /wiki/Internet_meme\n",
      "Article already exists: /wiki/SixDegrees.org\n",
      "Article already exists: /wiki/Bravo_Otto\n",
      "Article already exists: /wiki/Social_networking_service\n",
      "Article already exists: /wiki/Footloose_(1984_film)\n",
      "Article already exists: /wiki/Six_Degrees_of_Kevin_Bacon\n",
      "Article already exists: /wiki/IMDbStoring article /wiki/CableACE_Award\n",
      "\n",
      "Article already exists: /wiki/Paul_Erd%C5%91sArticle already exists: /wiki/CableACE_Award\n",
      "\n",
      "Article already exists: /wiki/Losing_ChaseArticle already exists: /wiki/Erd%C5%91s_number\n",
      "\n",
      "Storing article /wiki/Chlotrudis_Awards\n",
      "Article already exists: /wiki/Paul_Erd%C5%91s\n",
      "Article already exists: /wiki/Erd%C5%91s_number\n",
      "Article already exists: /wiki/The_Woodsman_(2004_film)\n",
      "Article already exists: /wiki/Erd%C5%91s%E2%80%93Bacon_numberStoring article /wiki/Critics%27_Choice_Movie_Awards\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import random\n",
    "import threading\n",
    "from queue import Queue\n",
    "import time\n",
    "import pymysql\n",
    "\n",
    "\n",
    "def storage(queue):\n",
    "    conn = pymysql.connect(host='127.0.0.1', unix_socket='/tmp/mysql.sock',\n",
    "    user='root', passwd='password', db='mysql', charset='utf8')\n",
    "    cur = conn.cursor()\n",
    "    cur.execute('USE wikipedia')\n",
    "    while 1:\n",
    "        if not queue.empty():\n",
    "            path = queue.get()\n",
    "            cur.execute('SELECT * FROM pages WHERE url = %s', (path))\n",
    "            if cur.rowcount == 0:\n",
    "                print(f'Storing article {path}')\n",
    "                cur.execute('INSERT INTO pages (url) VALUES (%s)', (path))\n",
    "                conn.commit()\n",
    "            else:\n",
    "                print(\"Article already exists: {}\".format(path))\n",
    "\n",
    "visited = set()\n",
    "def get_links(thread_name, bs):\n",
    "    print('Getting links in {}'.format(thread_name))\n",
    "    links = bs.find('div', {'id':'bodyContent'}).find_all('a', href=re.compile('^(/wiki/)((?!:).)*$'))\n",
    "    links = [link.get('href') for link in links]\n",
    "    return [link for link in links if link and link not in visited]\n",
    "\n",
    "def scrape_article(thread_name, path):\n",
    "    time.sleep(5)\n",
    "    visited.add(path)\n",
    "    print(f'{thread_name}: Scraping {path}')\n",
    "    bs = BeautifulSoup(urlopen('http://en.wikipedia.org{}'.format(path)), 'html.parser')\n",
    "    links = get_links(thread_name, bs)\n",
    "    if len(links) > 0:\n",
    "        [queue.put(link) for link in links]\n",
    "        newArticle = links[random.randint(0, len(links)-1)].attrs['href']\n",
    "        scrape_article(thread_name, newArticle)\n",
    "\n",
    "\n",
    "queue = Queue()\n",
    "\n",
    "\n",
    "threads = [\n",
    "    threading.Thread(target=scrape_article, args=('Thread 1', '/wiki/Kevin_Bacon',)),\n",
    "    threading.Thread(target=scrape_article, args=('Thread 2', '/wiki/Monty_Python',)),\n",
    "    threading.Thread(target=storage, args=(queue,))\n",
    "]\n",
    "[t.start() for t in threads]\n",
    "[t.join() for t in threads]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e14a0392",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Counter\n",
      "2 Counter\n",
      "3 Fizz\n",
      "3 Counter\n",
      "4 Counter\n",
      "5 Buzz\n",
      "6 Counter\n",
      "6 Fizz\n",
      "7 Counter\n"
     ]
    }
   ],
   "source": [
    "import threading\n",
    "import time\n",
    "\n",
    "def print_time(threadName, delay, iterations):\n",
    "    start = int(time.time())\n",
    "    for i in range(0,iterations):\n",
    "        time.sleep(delay)\n",
    "        seconds_elapsed = str(int(time.time()) - start)\n",
    "        print ('{} {}'.format(seconds_elapsed, threadName))\n",
    "\n",
    "threading.Thread(target=print_time, args=('Fizz', 3, 33)).start()\n",
    "threading.Thread(target=print_time, args=('Buzz', 5, 20)).start()\n",
    "threading.Thread(target=print_time, args=('Counter', 1, 100)).start()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8d9bab6",
   "metadata": {},
   "source": [
    "## Processes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1cc7a04",
   "metadata": {},
   "source": [
    "## IMPORTANT NOTE FOR JUPYTER NOTEBOOK\n",
    "\n",
    "The multiprocessing package does not play nicely with Jupyter Notebooks. Fortunately, there exists the multiprocess package that does work well with it -- and they have the exact same interface. (note: \"multiprocessing\" vs. \"multiprocess\" -- two different packages, keep them straight!)\n",
    "You can install multiprocess, which works with Jupyter notebooks, using:<br>\n",
    "<code>$ pip install multiprocess</code>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f676616b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter\n",
      "Counter\n",
      "Fizz\n",
      "Counter\n",
      "Counter\n",
      "Buzz\n",
      "Counter\n",
      "Fizz\n",
      "Counter\n",
      "Counter\n",
      "Counter\n",
      "Fizz\n",
      "Counter\n",
      "Buzz\n",
      "Counter\n",
      "Counter\n",
      "Fizz\n",
      "Counter\n",
      "Counter\n",
      "Counter\n",
      "Buzz\n",
      "Fizz\n",
      "Counter\n",
      "Counter\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-8:\n",
      "Process Process-6:\n",
      "Process Process-7:\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m<cell line: 23>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     20\u001b[0m processes\u001b[38;5;241m.\u001b[39mappend(Process(target\u001b[38;5;241m=\u001b[39mprint_time, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mBuzz\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;241m5\u001b[39m, \u001b[38;5;241m20\u001b[39m)))\n\u001b[1;32m     22\u001b[0m [p\u001b[38;5;241m.\u001b[39mstart() \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m processes]\n\u001b[0;32m---> 23\u001b[0m [p\u001b[38;5;241m.\u001b[39mjoin() \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m processes]\n",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     20\u001b[0m processes\u001b[38;5;241m.\u001b[39mappend(Process(target\u001b[38;5;241m=\u001b[39mprint_time, args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mBuzz\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;241m5\u001b[39m, \u001b[38;5;241m20\u001b[39m)))\n\u001b[1;32m     22\u001b[0m [p\u001b[38;5;241m.\u001b[39mstart() \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m processes]\n\u001b[0;32m---> 23\u001b[0m [\u001b[43mp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m processes]\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py:149\u001b[0m, in \u001b[0;36mBaseProcess.join\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_parent_pid \u001b[38;5;241m==\u001b[39m os\u001b[38;5;241m.\u001b[39mgetpid(), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcan only join a child process\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    148\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_popen \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcan only join a started process\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m--> 149\u001b[0m res \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_popen\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    150\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m res \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    151\u001b[0m     _children\u001b[38;5;241m.\u001b[39mdiscard(\u001b[38;5;28mself\u001b[39m)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/multiprocess/popen_fork.py:43\u001b[0m, in \u001b[0;36mPopen.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m     41\u001b[0m             \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     42\u001b[0m     \u001b[38;5;66;03m# This shouldn't block if wait() returned successfully.\u001b[39;00m\n\u001b[0;32m---> 43\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpoll\u001b[49m\u001b[43m(\u001b[49m\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mWNOHANG\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0.0\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     44\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturncode\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/multiprocess/popen_fork.py:27\u001b[0m, in \u001b[0;36mPopen.poll\u001b[0;34m(self, flag)\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturncode \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m     26\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 27\u001b[0m         pid, sts \u001b[38;5;241m=\u001b[39m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwaitpid\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpid\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mflag\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     28\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m:\n\u001b[1;32m     29\u001b[0m         \u001b[38;5;66;03m# Child process not yet created. See #1731717\u001b[39;00m\n\u001b[1;32m     30\u001b[0m         \u001b[38;5;66;03m# e.errno == errno.ECHILD == 10\u001b[39;00m\n\u001b[1;32m     31\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 315, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 315, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 315, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 108, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 108, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 108, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1252522224.py\", line 12, in print_time\n",
      "    time.sleep(delay)\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1252522224.py\", line 12, in print_time\n",
      "    time.sleep(delay)\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1252522224.py\", line 12, in print_time\n",
      "    time.sleep(delay)\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "# Uncomment this for non-Notebooks\n",
    "#from multiprocessing import Process \n",
    "\n",
    "# Comment this for non-Notebooks\n",
    "from multiprocess import Process\n",
    "\n",
    "import time\n",
    "\n",
    "def print_time(threadName, delay, iterations):\n",
    "    start = int(time.time())\n",
    "    for i in range(0,iterations):\n",
    "        time.sleep(delay)\n",
    "        seconds_elapsed = str(int(time.time()) - start)\n",
    "        print (threadName if threadName else seconds_elapsed)\n",
    "\n",
    "processes = [\n",
    "    Process(target=print_time, args=('Counter', 1, 100)),\n",
    "    Process(target=print_time, args=('Fizz', 3, 33)),\n",
    "    Process(target=print_time, args=('Buzz', 5, 20)) \n",
    "]\n",
    "\n",
    "[p.start() for p in processes]\n",
    "[p.join() for p in processes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bbdd4962",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping Kevin Bacon in process 4067\n",
      "Scraping Monty Python in process 4068\n",
      "Scraping Ewan McGregor in process 4067\n",
      "Scraping Charisma Records in process 4068\n",
      "Scraping Renée Zellweger in process 4067\n",
      "Scraping Genesis (band) in process 4068\n",
      "Scraping Alana Haim in process 4067\n",
      "Scraping Maroon 5 in process 4068\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-19:\n",
      "Process Process-18:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 315, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 315, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 108, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 108, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1953871450.py\", line 28, in scrape_article\n",
      "    scrape_article(links[random.randint(0, len(links)-1)].attrs['href'])\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1953871450.py\", line 28, in scrape_article\n",
      "    scrape_article(links[random.randint(0, len(links)-1)].attrs['href'])\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1953871450.py\", line 28, in scrape_article\n",
      "    scrape_article(links[random.randint(0, len(links)-1)].attrs['href'])\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1953871450.py\", line 28, in scrape_article\n",
      "    scrape_article(links[random.randint(0, len(links)-1)].attrs['href'])\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1953871450.py\", line 28, in scrape_article\n",
      "    scrape_article(links[random.randint(0, len(links)-1)].attrs['href'])\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1953871450.py\", line 28, in scrape_article\n",
      "    scrape_article(links[random.randint(0, len(links)-1)].attrs['href'])\n",
      "  [Previous line repeated 1 more time]\n",
      "  [Previous line repeated 1 more time]\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1953871450.py\", line 23, in scrape_article\n",
      "    time.sleep(5)\n",
      "KeyboardInterrupt\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/1953871450.py\", line 23, in scrape_article\n",
      "    time.sleep(5)\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import random\n",
    "\n",
    "# Uncomment this for non-Notebooks\n",
    "#from multiprocessing import Process \n",
    "\n",
    "# Comment this for non-Notebooks\n",
    "from multiprocess import Process\n",
    "\n",
    "import os\n",
    "import time\n",
    "\n",
    "visited = []\n",
    "def get_links(bs):\n",
    "    links = bs.find('div', {'id':'bodyContent'}).find_all('a', href=re.compile('^(/wiki/)((?!:).)*$'))\n",
    "    return [link for link in links if link not in visited]\n",
    "\n",
    "def scrape_article(path):\n",
    "    visited.append(path)\n",
    "    html = urlopen('http://en.wikipedia.org{}'.format(path))\n",
    "    time.sleep(5)\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    print(f'Scraping {bs.find(\"h1\").get_text()} in process {os.getpid()}')\n",
    "    links = get_links(bs)\n",
    "    if len(links) > 0:\n",
    "        scrape_article(links[random.randint(0, len(links)-1)].attrs['href'])\n",
    "\n",
    "\n",
    "processes = [\n",
    "    Process(target=scrape_article, args=('/wiki/Kevin_Bacon',)),\n",
    "    Process(target=scrape_article, args=('/wiki/Monty_Python',)) \n",
    "]\n",
    "\n",
    "[p.start() for p in processes]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0f0c5b00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None, None]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping Kevin Bacon in process 97023\n",
      "Scraping Monty Python in process 97024\n",
      "Scraping Kevin Bacon (disambiguation) in process 97023\n",
      "Scraping Philadelphia in process 97024\n",
      "Scraping Kevin Bacon filmography in process 97023\n",
      "Scraping Kyra Sedgwick in process 97024\n",
      "Scraping Sosie Bacon in process 97023\n",
      "Scraping Edmund Bacon (architect) in process 97024\n",
      "Scraping Michael Bacon (musician) in process 97023\n",
      "Scraping Holly Near in process 97024\n",
      "Scraping Leading actor in process 97023\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-17:\n",
      "Process Process-16:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 315, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 315, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 108, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 108, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/3812966439.py\", line 43, in scrape_article\n",
      "    time.sleep(5)\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/3812966439.py\", line 43, in scrape_article\n",
      "    time.sleep(5)\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "Process Process-15:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 315, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 108, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/var/folders/y6/jnf4yrtx1pg3y9tqb8fmhnrr0000gp/T/ipykernel_92257/3812966439.py\", line 24, in task_delegator\n",
      "    if not urlsQueue.empty():\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/queues.py\", line 132, in empty\n",
      "    return not self._poll()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/connection.py\", line 260, in poll\n",
      "    return self._poll(timeout)\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/connection.py\", line 427, in _poll\n",
      "    r = wait([self], timeout)\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/connection.py\", line 934, in wait\n",
      "    ready = selector.select(timeout)\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/selectors.py\", line 416, in select\n",
      "    fd_event_list = self._selector.poll(timeout)\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/process.py\", line 318, in _bootstrap\n",
      "    util._exit_function()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/util.py\", line 360, in _exit_function\n",
      "    _run_finalizers()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/util.py\", line 300, in _run_finalizers\n",
      "    finalizer()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/util.py\", line 224, in __call__\n",
      "    res = self._callback(*self._args, **self._kwargs)\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/site-packages/multiprocess/queues.py\", line 202, in _finalize_join\n",
      "    thread.join()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/threading.py\", line 1053, in join\n",
      "    self._wait_for_tstate_lock()\n",
      "  File \"/Users/RSpecht/opt/anaconda3/lib/python3.9/threading.py\", line 1073, in _wait_for_tstate_lock\n",
      "    if lock.acquire(block, timeout):\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import random\n",
    "\n",
    "# Uncomment this for non-Notebooks\n",
    "#from multiprocessing import Process, Queue\n",
    "\n",
    "# Comment this for non-Notebooks\n",
    "from multiprocess import Process, Queue\n",
    "import os\n",
    "import time\n",
    "\n",
    "\n",
    "def task_delegator(taskQueue, urlsQueue):\n",
    "    #Initialize with a task for each process\n",
    "    visited = ['/wiki/Kevin_Bacon', '/wiki/Monty_Python']\n",
    "    taskQueue.put('/wiki/Kevin_Bacon')\n",
    "    taskQueue.put('/wiki/Monty_Python')\n",
    "\n",
    "    while 1:\n",
    "        # Check to see if there are new links in the urlsQueue\n",
    "        # for processing\n",
    "        if not urlsQueue.empty():\n",
    "            links = [link for link in urlsQueue.get() if link not in visited]\n",
    "            for link in links:\n",
    "                #Add new link to the taskQueue\n",
    "                taskQueue.put(link)\n",
    "\n",
    "def get_links(bs):\n",
    "    links = bs.find('div', {'id':'bodyContent'}).find_all('a',\n",
    "        href=re.compile('^(/wiki/)((?!:).)*$'))\n",
    "    return [link.attrs['href'] for link in links]\n",
    "\n",
    "def scrape_article(taskQueue, urlsQueue):\n",
    "    while 1:\n",
    "        while taskQueue.empty():\n",
    "            #Sleep 100 ms while waiting for the task queue\n",
    "            #This should be rare\n",
    "            time.sleep(.1)\n",
    "        path = taskQueue.get()\n",
    "        html = urlopen('http://en.wikipedia.org{}'.format(path))\n",
    "        time.sleep(5)\n",
    "        bs = BeautifulSoup(html, 'html.parser')\n",
    "        print(f'Scraping {bs.find('h1').get_text()} in process {os.getpid()}')\n",
    "        links = get_links(bs)\n",
    "        #Send these to the delegator for processing\n",
    "        urlsQueue.put(links)\n",
    "\n",
    "taskQueue = Queue()\n",
    "urlsQueue = Queue()\n",
    "\n",
    "processes = [\n",
    "    Process(target=task_delegator, args=(taskQueue, urlsQueue,)),\n",
    "    Process(target=scrape_article, args=(taskQueue, urlsQueue,)),\n",
    "    Process(target=scrape_article, args=(taskQueue, urlsQueue,))\n",
    "]\n",
    "\n",
    "[p.start() for p in processes]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cacd33c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}