lecture6.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.robotparser\n",
    "url_prefix = \"localhost:8888\" # 根据当前地址栏的端口设置端口，jupyter默认为8888\n",
    "rp = urllib.robotparser.RobotFileParser()\n",
    "robots_url = \"http://%s/view/robots.txt\" % url_prefix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'http://localhost:8888/view/robots.txt'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "robots_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rp.set_url(robots_url)\n",
    "rp.read()\n",
    "rp.can_fetch(\"Baiduspider\", \"http://%s/\" % url_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rp.can_fetch(\"Baiduspider\", \"http://%s/article/\" % url_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE HTML>\n",
      "<html>\n",
      "\n",
      "<head>\n",
      "    <meta charset=\"utf-8\">\n",
      "\n",
      "    <title>Jupyter Notebook</title>\n",
      "    <link id=\"favicon\" rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/static/base/images/favicon.ico?v=97c6417ed01bdc0ae3ef32ae4894fd03\">\n",
      "    <meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\" />\n",
      "    <link rel=\"stylesheet\" href=\"/static/components/jquery-ui/themes/smoothness/jquery-ui.min.css?v=3c2a865c832a1322285c55c6ed99abb2\" type=\"text/css\" />\n",
      "    <link rel=\"stylesheet\" href=\"/static/components/jquery-typeahead/dist/jquery.typeahead.min.css?v=7afb461de36accb1aa133a1710f5bc56\" type=\"text/css\" />\n",
      "    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n",
      "    \n",
      "    \n",
      "\n",
      "    <link rel=\"stylesheet\" href=\"/static/style/style.min.css?v=e91a43337d7c294cc9fab2938fa723b3\" type=\"text/css\"/>\n",
      "    \n",
      "<link rel=\"stylesheet\" href=\"/static/auth/css/override.css?v=ae668279c8a80c1d2e81fc28345169ee\" type=\"text/css\" />\n",
      "\n",
      "    <link rel=\"stylesheet\" href=\"/custom/custom.css\" type=\"text/css\" />\n",
      "    <script src=\"/static/components/es6-promise/promise.min.js?v=f004a16cb856e0ff11781d01ec5ca8fe\" type=\"text/javascript\" charset=\"utf-8\"></script>\n",
      "    <script src=\"/static/components/react/react.production.min.js?v=34f96ffc962a7deecc83037ccb582b58\" type=\"text/javascript\"></script>\n",
      "    <script src=\"/static/components/react/react-dom.production.min.js?v=b14d91fb641317cda38dbc9dbf985ab4\" type=\"text/javascript\"></script>\n",
      "    <script src=\"/static/components/create-react-class/index.js?v=94feb9971ce6d26211729abc43f96cd2\" type=\"text/javascript\"></script>\n",
      "    <script src=\"/static/components/requirejs/require.js?v=951f856e81496aaeec2e71a1c2c0d51f\" type=\"text/javascript\" charset=\"utf-8\"></script>\n",
      "    <script>\n",
      "      require.config({\n",
      "          \n",
      "          urlArgs: \"v=20191228195158\",\n",
      "          \n",
      "          baseUrl: '/static/',\n",
      "          paths: {\n",
      "            'auth/js/main': 'auth/js/main.min',\n",
      "            custom : '/custom',\n",
      "            nbextensions : '/nbextensions',\n",
      "            kernelspecs : '/kernelspecs',\n",
      "            underscore : 'components/underscore/underscore-min',\n",
      "            backbone : 'components/backbone/backbone-min',\n",
      "            jed: 'components/jed/jed',\n",
      "            jquery: 'components/jquery/jquery.min',\n",
      "            json: 'components/requirejs-plugins/src/json',\n",
      "            text: 'components/requirejs-text/text',\n",
      "            bootstrap: 'components/bootstrap/dist/js/bootstrap.min',\n",
      "            bootstraptour: 'components/bootstrap-tour/build/js/bootstrap-tour.min',\n",
      "            'jquery-ui': 'components/jquery-ui/jquery-ui.min',\n",
      "            moment: 'components/moment/min/moment-with-locales',\n",
      "            codemirror: 'components/codemirror',\n",
      "            termjs: 'components/xterm.js/xterm',\n",
      "            typeahead: 'components/jquery-typeahead/dist/jquery.typeahead.min',\n",
      "          },\n",
      "          map: { // for backward compatibility\n",
      "              \"*\": {\n",
      "                  \"jqueryui\": \"jquery-ui\",\n",
      "              }\n",
      "          },\n",
      "          shim: {\n",
      "            typeahead: {\n",
      "              deps: [\"jquery\"],\n",
      "              exports: \"typeahead\"\n",
      "            },\n",
      "            underscore: {\n",
      "              exports: '_'\n",
      "            },\n",
      "            backbone: {\n",
      "              deps: [\"underscore\", \"jquery\"],\n",
      "              exports: \"Backbone\"\n",
      "            },\n",
      "            bootstrap: {\n",
      "              deps: [\"jquery\"],\n",
      "              exports: \"bootstrap\"\n",
      "            },\n",
      "            bootstraptour: {\n",
      "              deps: [\"bootstrap\"],\n",
      "              exports: \"Tour\"\n",
      "            },\n",
      "            \"jquery-ui\": {\n",
      "              deps: [\"jquery\"],\n",
      "              exports: \"$\"\n",
      "            }\n",
      "          },\n",
      "          waitSeconds: 30,\n",
      "      });\n",
      "\n",
      "      require.config({\n",
      "          map: {\n",
      "              '*':{\n",
      "                'contents': 'services/contents',\n",
      "              }\n",
      "          }\n",
      "      });\n",
      "\n",
      "      // error-catching custom.js shim.\n",
      "      define(\"custom\", function (require, exports, module) {\n",
      "          try {\n",
      "              var custom = require('custom/custom');\n",
      "              console.debug('loaded custom.js');\n",
      "              return custom;\n",
      "          } catch (e) {\n",
      "              console.error(\"error loading custom.js\", e);\n",
      "              return {};\n",
      "          }\n",
      "      })\n",
      "\n",
      "    document.nbjs_translations = {\"domain\": \"nbjs\", \"locale_data\": {\"nbjs\": {\"\": {\"domain\": \"nbjs\"}}}};\n",
      "    document.documentElement.lang = navigator.language.toLowerCase();\n",
      "    </script>\n",
      "\n",
      "    \n",
      "    \n",
      "\n",
      "</head>\n",
      "\n",
      "<body class=\"\"\n",
      " \n",
      "  \n",
      " \n",
      "dir=\"ltr\">\n",
      "\n",
      "<noscript>\n",
      "    <div id='noscript'>\n",
      "      Jupyter Notebook需要的JavaScript.<br>\n",
      "      请允许它继续.\n",
      "  </div>\n",
      "</noscript>\n",
      "\n",
      "<div id=\"header\" role=\"navigation\" aria-label=\"Top Menu\">\n",
      "  <div id=\"header-container\" class=\"container\">\n",
      "  <div id=\"ipython_notebook\" class=\"nav navbar-brand\"><a href=\"/tree\" title='指示板'>\n",
      "      <img src='/static/base/images/logo.png?v=641991992878ee24c6f3826e81054a0f' alt='Jupyter Notebook'/>\n",
      "  </a></div>\n",
      "\n",
      "  \n",
      "  \n",
      "  \n",
      "  \n",
      "  \n",
      "  \n",
      "\n",
      "\n",
      "  \n",
      "  \n",
      "  </div>\n",
      "  <div class=\"header-bar\"></div>\n",
      "\n",
      "  \n",
      "  \n",
      "</div>\n",
      "\n",
      "<div id=\"site\">\n",
      "\n",
      "\n",
      "<div id=\"ipython-main-app\" class=\"container\">\n",
      "\n",
      "    \n",
      "    \n",
      "    <div class=\"row\">\n",
      "    <div class=\"navbar col-sm-8\">\n",
      "      <div class=\"navbar-inner\">\n",
      "        <div class=\"container\">\n",
      "          <div class=\"center-nav\">\n",
      "            <form action=\"/login?next=%2Fview%2Fcrawlme.html\" method=\"post\" class=\"navbar-form pull-left\">\n",
      "              <input type=\"hidden\" name=\"_xsrf\" value=\"2|ddfea313|117b8c4dda745abc7f40ce62064e53c7|1577542029\"/>\n",
      "              \n",
      "                <label for=\"password_input\"><strong>密码或者token:</strong></label>\n",
      "              \n",
      "              <input type=\"password\" name=\"password\" id=\"password_input\" class=\"form-control\">\n",
      "              <button type=\"submit\" class=\"btn btn-default\" id=\"login_submit\">登录</button>\n",
      "            </form>\n",
      "          </div>\n",
      "        </div>\n",
      "      </div>\n",
      "    </div>\n",
      "    </div>\n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    <div class=\"col-sm-6 col-sm-offset-3 text-left rendered_html\">\n",
      "      <h3>\n",
      "        Token authentication is enabled\n",
      "      </h3>\n",
      "      <p>\n",
      "        If no password has been configured, you need to open the notebook\n",
      "        server with its login token in the URL, or paste it above.\n",
      "        This requirement will be lifted if you\n",
      "        <b><a href='https://jupyter-notebook.readthedocs.io/en/stable/public_server.html'>\n",
      "            enable a password</a></b>.\n",
      "      </p>\n",
      "      <p>\n",
      "        The command:\n",
      "        <pre>jupyter notebook list</pre>\n",
      "        will show you the URLs of running servers with their tokens,\n",
      "        which you can copy and paste into your browser. For example:\n",
      "      </p>\n",
      "      <pre>Currently running servers:\n",
      "http://localhost:8888/?token=c8de56fa... :: /Users/you/notebooks\n",
      "</pre>\n",
      "      <p>\n",
      "        or you can paste just the token value into the password field on this\n",
      "        page.\n",
      "      </p>\n",
      "      <p>\n",
      "        See\n",
      "        <b><a\n",
      "         href='https://jupyter-notebook.readthedocs.io/en/stable/public_server.html'>\n",
      "                the documentation on how to enable a password</a>\n",
      "        </b>\n",
      "        in place of token authentication,\n",
      "        if you would like to avoid dealing with random tokens.\n",
      "      </p>\n",
      "      <p>\n",
      "        Cookies are required for authenticated access to notebooks.\n",
      "      </p>\n",
      "      \n",
      "        <h3>Setup a Password</h3>\n",
      "        <p> You can also setup a password by entering your token and a new password\n",
      "        on the fields below:</p>\n",
      "        <form action=\"/login?next=%2Fview%2Fcrawlme.html\" method=\"post\" class=\"\">\n",
      "                <input type=\"hidden\" name=\"_xsrf\" value=\"2|ddfea313|117b8c4dda745abc7f40ce62064e53c7|1577542029\"/>\n",
      "          <div class=\"form-group\">\n",
      "            <label for=\"token_input\"><h4>Token</h4></label>\n",
      "            <input type=\"password\" name=\"password\" id=\"token_input\" class=\"form-control\">\n",
      "          </div>\n",
      "          <div class=\"form-group\">\n",
      "            <label for=\"new_password_input\"><h4>New Password</h4></label>\n",
      "            <input type=\"password\" name=\"new_password\" id=\"new_password_input\" class=\"form-control\" required>\n",
      "          </div>\n",
      "          <div class=\"form-group\">\n",
      "            <button type=\"submit\" class=\"btn btn-default\" id=\"login_new_pass_submit\">Log in and set new password</button>\n",
      "          </div>\n",
      "        </form>\n",
      "      \n",
      "\n",
      "    </div>\n",
      "    \n",
      "    \n",
      "</div>\n",
      "\n",
      "\n",
      "</div>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "<script type=\"text/javascript\">\n",
      "  require([\"auth/js/main\"], function (auth) {\n",
      "    auth.login_main();\n",
      "  });\n",
      "</script>\n",
      "\n",
      "\n",
      "\n",
      "<script type='text/javascript'>\n",
      "  function _remove_token_from_url() {\n",
      "    if (window.location.search.length <= 1) {\n",
      "      return;\n",
      "    }\n",
      "    var search_parameters = window.location.search.slice(1).split('&');\n",
      "    for (var i = 0; i < search_parameters.length; i++) {\n",
      "      if (search_parameters[i].split('=')[0] === 'token') {\n",
      "        // remote token from search parameters\n",
      "        search_parameters.splice(i, 1);\n",
      "        var new_search = '';\n",
      "        if (search_parameters.length) {\n",
      "          new_search = '?' + search_parameters.join('&');\n",
      "        }\n",
      "        var new_url = window.location.origin + \n",
      "                      window.location.pathname + \n",
      "                      new_search + \n",
      "                      window.location.hash;\n",
      "        window.history.replaceState({}, \"\", new_url);\n",
      "        return;\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "  _remove_token_from_url();\n",
      "</script>\n",
      "</body>\n",
      "\n",
      "</html>\n",
      "UTF-8\n",
      "http://localhost:8888/login?next=%2Fview%2Fcrawlme.html\n",
      "200\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "r = requests.get(\"http://%s/view/crawlme.html\" % url_prefix)\n",
    "print(r.text)\n",
    "print(r.encoding)\n",
    "print(r.url)\n",
    "print(r.status_code)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "正则表达式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foot', 'fell', 'fastest']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "re.findall(r'\\bf[a-z]*', 'which foot or hand fell fastest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cat in the hat'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub(r'(\\b[a-z]+) \\1', r'\\1', 'cat in the the hat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = '''\\\n",
    "<html>\n",
    "    <head>\n",
    "    </head>\n",
    "    <body>\n",
    "        <h1>Title</h1>\n",
    "        <p> This is a paragraph</p>\n",
    "        <table>\n",
    "            <tr>\n",
    "                <td> <a href='a.html'>link a</a> </td>\n",
    "            </tr>\n",
    "        </table>\n",
    "        <a href='b.html'>link b</a>\n",
    "    </body>\n",
    "</html>\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a.html', 'b.html']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(\"href='(.*?)'\", html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'abbbb'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.match(r'ab*', 'abbbb').group()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.match(r'ab*?', 'abbbb').group()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a1234'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.match(r'a\\d{3,4}', 'a1234567').group()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'abc'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#使用|表示或，\\|表示字面值\n",
    "re.match(r'(abc)|(def)', 'abc').group()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['12', '11', '10']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = re.compile('\\d+')\n",
    "txt = '12 drummers drumming, 11 pipers piping, 10 lords a-leaping'\n",
    "p.findall(txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'abcd'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = re.compile('(a(b)c)d')\n",
    "m = p.match('abcd')\n",
    "m.group(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'abc'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.group(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'b'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.group(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['明', '亮', '蛋']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'(.)\\1', '明明亮亮蛋蛋')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}