averaging_rouge.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the implementation of Algorithm 1 found in project paper that calculates the average of Rouge scores of each models using the first 100 BBC business news articles.  In order to perform an experimental comparison with four extractive summarization techniques, we extract six sentences from each article in order to compose the summary. \n",
    "1. the value of the evaluation measure F1, Recall, Precision were calculated for each article\n",
    "2. take the average of those scores to arrive at a consolidated F1, Recall and Precision scores for each Rouge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:05:01.240275Z",
     "start_time": "2020-08-24T13:04:05.238941Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Gj-j8iTVmXPF",
    "outputId": "5828b319-fa61-49f8-f8c1-1dab18fedb64"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: bert-extractive-summarizer in c:\\users\\marti\\anaconda3\\lib\\site-packages (0.5.0)\n",
      "Requirement already satisfied: transformers in c:\\users\\marti\\anaconda3\\lib\\site-packages (from bert-extractive-summarizer) (2.11.0)\n",
      "Requirement already satisfied: spacy in c:\\users\\marti\\anaconda3\\lib\\site-packages (from bert-extractive-summarizer) (2.2.2)\n",
      "Requirement already satisfied: scikit-learn in c:\\users\\marti\\anaconda3\\lib\\site-packages (from bert-extractive-summarizer) (0.22.1)\n",
      "Requirement already satisfied: packaging in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (20.1)\n",
      "Requirement already satisfied: filelock in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (3.0.12)\n",
      "Requirement already satisfied: tqdm>=4.27 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (4.42.1)\n",
      "Requirement already satisfied: numpy in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (1.18.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (2020.7.14)\n",
      "Requirement already satisfied: sentencepiece in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (0.1.91)\n",
      "Requirement already satisfied: sacremoses in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (0.0.43)\n",
      "Requirement already satisfied: requests in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (2.22.0)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (0.7)\n",
      "Requirement already satisfied: tokenizers==0.7.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers->bert-extractive-summarizer) (0.7.0)\n",
      "Requirement already satisfied: plac<1.2.0,>=0.9.6 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (0.9.6)\n",
      "Requirement already satisfied: setuptools in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (45.2.0.post20200210)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (1.0.2)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (2.0.3)\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (1.5.0)\n",
      "Requirement already satisfied: srsly<1.1.0,>=0.1.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (1.0.2)\n",
      "Requirement already satisfied: blis<0.5.0,>=0.4.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (0.4.1)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (3.0.2)\n",
      "Requirement already satisfied: thinc<7.4.0,>=7.3.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (7.3.1)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.3.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy->bert-extractive-summarizer) (0.6.0)\n",
      "Requirement already satisfied: scipy>=0.17.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from scikit-learn->bert-extractive-summarizer) (1.4.1)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from scikit-learn->bert-extractive-summarizer) (0.14.1)\n",
      "Requirement already satisfied: six in c:\\users\\marti\\anaconda3\\lib\\site-packages (from packaging->transformers->bert-extractive-summarizer) (1.14.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from packaging->transformers->bert-extractive-summarizer) (2.4.6)\n",
      "Requirement already satisfied: click in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sacremoses->transformers->bert-extractive-summarizer) (7.0)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers->bert-extractive-summarizer) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers->bert-extractive-summarizer) (2020.4.5.1)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers->bert-extractive-summarizer) (1.25.8)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers->bert-extractive-summarizer) (2.8)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->spacy->bert-extractive-summarizer) (2.2.0)\n",
      "Collecting spacy==2.1.3\n",
      "  Using cached spacy-2.1.3-cp36-cp36m-win_amd64.whl (26.9 MB)\n",
      "Requirement already satisfied: plac<1.0.0,>=0.9.6 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.1.3) (0.9.6)\n",
      "Collecting preshed<2.1.0,>=2.0.1\n",
      "  Using cached preshed-2.0.1-cp36-cp36m-win_amd64.whl (73 kB)\n",
      "Requirement already satisfied: srsly<1.1.0,>=0.0.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.1.3) (1.0.2)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.1.3) (2.0.3)\n",
      "Requirement already satisfied: jsonschema<3.0.0,>=2.6.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.1.3) (2.6.0)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.1.3) (1.0.2)\n",
      "Collecting blis<0.3.0,>=0.2.2\n",
      "  Using cached blis-0.2.4-cp36-cp36m-win_amd64.whl (3.1 MB)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.2.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.1.3) (0.6.0)\n",
      "Requirement already satisfied: numpy>=1.15.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.1.3) (1.18.1)\n",
      "Collecting thinc<7.1.0,>=7.0.2\n",
      "  Using cached thinc-7.0.8-cp36-cp36m-win_amd64.whl (1.9 MB)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.1.3) (2.22.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from thinc<7.1.0,>=7.0.2->spacy==2.1.3) (4.42.1)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy==2.1.3) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy==2.1.3) (2020.4.5.1)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy==2.1.3) (2.8)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy==2.1.3) (1.25.8)\n",
      "Installing collected packages: preshed, blis, thinc, spacy\n",
      "  Attempting uninstall: preshed\n",
      "    Found existing installation: preshed 3.0.2\n",
      "    Uninstalling preshed-3.0.2:\n",
      "      Successfully uninstalled preshed-3.0.2\n",
      "  Attempting uninstall: blis\n",
      "    Found existing installation: blis 0.4.1\n",
      "    Uninstalling blis-0.4.1:\n",
      "      Successfully uninstalled blis-0.4.1\n",
      "  Attempting uninstall: thinc\n",
      "    Found existing installation: thinc 7.3.1\n",
      "    Uninstalling thinc-7.3.1:\n",
      "      Successfully uninstalled thinc-7.3.1\n",
      "  Attempting uninstall: spacy\n",
      "    Found existing installation: spacy 2.2.2\n",
      "    Uninstalling spacy-2.2.2:\n",
      "      Successfully uninstalled spacy-2.2.2\n",
      "Successfully installed blis-0.2.4 preshed-2.0.1 spacy-2.1.3 thinc-7.0.8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: en-core-web-sm 2.2.5 has requirement spacy>=2.2.2, but you'll have spacy 2.1.3 which is incompatible.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers==2.2.2\n",
      "  Using cached transformers-2.2.2-py3-none-any.whl (387 kB)\n",
      "Requirement already satisfied: requests in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.2.2) (2.22.0)\n",
      "Requirement already satisfied: boto3 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.2.2) (1.14.47)\n",
      "Requirement already satisfied: tqdm in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.2.2) (4.42.1)\n",
      "Requirement already satisfied: sacremoses in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.2.2) (0.0.43)\n",
      "Requirement already satisfied: regex in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.2.2) (2020.7.14)\n",
      "Requirement already satisfied: numpy in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.2.2) (1.18.1)\n",
      "Requirement already satisfied: sentencepiece in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.2.2) (0.1.91)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers==2.2.2) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers==2.2.2) (2020.4.5.1)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers==2.2.2) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers==2.2.2) (1.25.8)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from boto3->transformers==2.2.2) (0.10.0)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from boto3->transformers==2.2.2) (0.3.3)\n",
      "Requirement already satisfied: botocore<1.18.0,>=1.17.47 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from boto3->transformers==2.2.2) (1.17.47)\n",
      "Requirement already satisfied: six in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sacremoses->transformers==2.2.2) (1.14.0)\n",
      "Requirement already satisfied: click in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sacremoses->transformers==2.2.2) (7.0)\n",
      "Requirement already satisfied: joblib in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sacremoses->transformers==2.2.2) (0.14.1)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from botocore<1.18.0,>=1.17.47->boto3->transformers==2.2.2) (0.15.2)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from botocore<1.18.0,>=1.17.47->boto3->transformers==2.2.2) (2.8.1)\n",
      "Installing collected packages: transformers\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 2.11.0\n",
      "    Uninstalling transformers-2.11.0:\n",
      "      Successfully uninstalled transformers-2.11.0\n",
      "Successfully installed transformers-2.2.2\n",
      "Requirement already satisfied: neuralcoref in c:\\users\\marti\\anaconda3\\lib\\site-packages (4.0)\n",
      "Requirement already satisfied: spacy>=2.1.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from neuralcoref) (2.1.3)\n",
      "Requirement already satisfied: numpy>=1.15.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from neuralcoref) (1.18.1)\n",
      "Requirement already satisfied: boto3 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from neuralcoref) (1.14.47)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from neuralcoref) (2.22.0)\n",
      "Requirement already satisfied: srsly<1.1.0,>=0.0.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy>=2.1.0->neuralcoref) (1.0.2)\n",
      "Requirement already satisfied: jsonschema<3.0.0,>=2.6.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy>=2.1.0->neuralcoref) (2.6.0)\n",
      "Requirement already satisfied: blis<0.3.0,>=0.2.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy>=2.1.0->neuralcoref) (0.2.4)\n",
      "Requirement already satisfied: thinc<7.1.0,>=7.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy>=2.1.0->neuralcoref) (7.0.8)\n",
      "Requirement already satisfied: preshed<2.1.0,>=2.0.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy>=2.1.0->neuralcoref) (2.0.1)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy>=2.1.0->neuralcoref) (1.0.2)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.2.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy>=2.1.0->neuralcoref) (0.6.0)\n",
      "Requirement already satisfied: plac<1.0.0,>=0.9.6 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy>=2.1.0->neuralcoref) (0.9.6)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy>=2.1.0->neuralcoref) (2.0.3)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from boto3->neuralcoref) (0.10.0)\n",
      "Requirement already satisfied: botocore<1.18.0,>=1.17.47 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from boto3->neuralcoref) (1.17.47)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from boto3->neuralcoref) (0.3.3)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (2.8)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (1.25.8)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (2020.4.5.1)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from thinc<7.1.0,>=7.0.2->spacy>=2.1.0->neuralcoref) (4.42.1)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from botocore<1.18.0,>=1.17.47->boto3->neuralcoref) (0.15.2)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from botocore<1.18.0,>=1.17.47->boto3->neuralcoref) (2.8.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.18.0,>=1.17.47->boto3->neuralcoref) (1.14.0)\n",
      "Requirement already satisfied: rouge in c:\\users\\marti\\anaconda3\\lib\\site-packages (1.0.0)\n",
      "Requirement already satisfied: six in c:\\users\\marti\\anaconda3\\lib\\site-packages (from rouge) (1.14.0)\n",
      "Requirement already satisfied: sumy in c:\\users\\marti\\anaconda3\\lib\\site-packages (0.8.1)\n",
      "Requirement already satisfied: nltk>=3.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sumy) (3.4.5)\n",
      "Requirement already satisfied: breadability>=0.1.20 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sumy) (0.1.20)\n",
      "Requirement already satisfied: docopt<0.7,>=0.6.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sumy) (0.6.2)\n",
      "Requirement already satisfied: requests>=2.7.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sumy) (2.22.0)\n",
      "Requirement already satisfied: pycountry>=18.2.23 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sumy) (20.7.3)\n",
      "Requirement already satisfied: six in c:\\users\\marti\\anaconda3\\lib\\site-packages (from nltk>=3.0.2->sumy) (1.14.0)\n",
      "Requirement already satisfied: lxml>=2.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from breadability>=0.1.20->sumy) (4.5.0)\n",
      "Requirement already satisfied: chardet in c:\\users\\marti\\anaconda3\\lib\\site-packages (from breadability>=0.1.20->sumy) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests>=2.7.0->sumy) (2.8)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests>=2.7.0->sumy) (1.25.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests>=2.7.0->sumy) (2020.4.5.1)\n",
      "Requirement already satisfied: newspaper3k in c:\\users\\marti\\anaconda3\\lib\\site-packages (0.2.8)\n",
      "Requirement already satisfied: beautifulsoup4>=4.4.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (4.8.2)\n",
      "Requirement already satisfied: tinysegmenter==0.3 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (0.3)\n",
      "Requirement already satisfied: jieba3k>=0.35.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (0.35.1)\n",
      "Requirement already satisfied: feedparser>=5.2.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (5.2.1)\n",
      "Requirement already satisfied: requests>=2.10.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (2.22.0)\n",
      "Requirement already satisfied: Pillow>=3.3.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (7.0.0)\n",
      "Requirement already satisfied: lxml>=3.6.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (4.5.0)\n",
      "Requirement already satisfied: nltk>=3.2.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (3.4.5)\n",
      "Requirement already satisfied: tldextract>=2.0.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (2.2.3)\n",
      "Requirement already satisfied: PyYAML>=3.11 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (5.3)\n",
      "Requirement already satisfied: python-dateutil>=2.5.3 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (2.8.1)\n",
      "Requirement already satisfied: cssselect>=0.9.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (1.1.0)\n",
      "Requirement already satisfied: feedfinder2>=0.0.4 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from newspaper3k) (0.0.4)\n",
      "Requirement already satisfied: soupsieve>=1.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from beautifulsoup4>=4.4.1->newspaper3k) (1.9.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests>=2.10.0->newspaper3k) (2020.4.5.1)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests>=2.10.0->newspaper3k) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests>=2.10.0->newspaper3k) (1.25.8)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests>=2.10.0->newspaper3k) (2.8)\n",
      "Requirement already satisfied: six in c:\\users\\marti\\anaconda3\\lib\\site-packages (from nltk>=3.2.1->newspaper3k) (1.14.0)\n",
      "Requirement already satisfied: requests-file>=1.4 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from tldextract>=2.0.1->newspaper3k) (1.5.1)\n"
     ]
    }
   ],
   "source": [
    "! pip install bert-extractive-summarizer\n",
    "! pip install spacy==2.1.3\n",
    "! pip install transformers==2.2.2\n",
    "! pip install neuralcoref\n",
    "! pip install rouge\n",
    "! pip install sumy\n",
    "! pip install newspaper3k"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Averaging algorithm on first 100 BBC business news articles dataset using different extractive summarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:05:02.320354Z",
     "start_time": "2020-08-24T13:05:01.240275Z"
    }
   },
   "outputs": [],
   "source": [
    "import os,glob\n",
    "from statistics import mean \n",
    "#from summarizer import Summarizer\n",
    "from rouge import Rouge \n",
    "from sumy.parsers.plaintext import PlaintextParser\n",
    "from sumy.nlp.tokenizers import Tokenizer\n",
    "from sumy.summarizers.lsa import LsaSummarizer\n",
    "from sumy.summarizers.luhn import LuhnSummarizer\n",
    "from sumy.summarizers.lex_rank import LexRankSummarizer \n",
    "from sumy.nlp.stemmers import Stemmer\n",
    "from sumy.utils import get_stop_words\n",
    "\n",
    "\n",
    "directory = 'BBC News Summary/News Articles/business'\n",
    "directory_gs = 'BBC News Summary/Summaries/business'\n",
    "\n",
    "rouge = Rouge()\n",
    "language = \"english\"\n",
    "news_list = []\n",
    "gs_list = []\n",
    "\n",
    "for filename in glob.glob(os.path.join(directory, '*.txt')):\n",
    "    with open(filename, 'r') as f:\n",
    "        text = f.read()\n",
    "        news_list.append(text)\n",
    "        \n",
    "for filename in glob.glob(os.path.join(directory_gs, '*.txt')):\n",
    "    with open(filename, 'r') as f:\n",
    "        text = f.read()\n",
    "        gs_list.append(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:05:02.358084Z",
     "start_time": "2020-08-24T13:05:02.320354Z"
    }
   },
   "outputs": [],
   "source": [
    "def avg_rouge(model, sentence_cnt):\n",
    "    r1_f = [] \n",
    "    r1_p = []\n",
    "    r1_r = []\n",
    "    r2_f = []\n",
    "    r2_p = []\n",
    "    r2_r = []\n",
    "    rl_f = []\n",
    "    rl_p = []\n",
    "    rl_r = []\n",
    "    summary = \"\"\n",
    "    stemmer = Stemmer(language) \n",
    "    if model == 'lsa':\n",
    "        sumy_model = LsaSummarizer()\n",
    "        #sumy_model = LsaSummarizer(stemmer)\n",
    "    elif model == 'luhn':\n",
    "        sumy_model = LuhnSummarizer()\n",
    "        #sumy_model = LuhnSummarizer(stemmer)\n",
    "    elif model == 'lex':\n",
    "        sumy_model = LexRankSummarizer()\n",
    "        #sumy_model = LexRankSummarizer(stemmer)\n",
    "    elif model == 'bert':\n",
    "        model_bert = Summarizer()\n",
    "        \n",
    "    for news, gs in zip(news_list, gs_list):\n",
    "        if model == 'lsa' or 'luhn' or 'lex':\n",
    "            parser = PlaintextParser(news, Tokenizer(language))\n",
    "                   \n",
    "            #sumy_model.stop_words = get_stop_words(language)\n",
    "            result = sumy_model(parser.document, sentence_cnt)\n",
    "            for i in result:\n",
    "                summary = summary + str(i) + \" \" \n",
    "        elif model == 'bert':\n",
    "            result = model_bert(news, num_sentences = sentence_cnt)\n",
    "            summary = \"\".join(result)   \n",
    "\n",
    "        scores = rouge.get_scores(summary, gs)\n",
    "        r1_f.append(scores[0]['rouge-1']['f'])\n",
    "        r1_p.append(scores[0]['rouge-1']['p'])\n",
    "        r1_r.append(scores[0]['rouge-1']['r'])\n",
    "        r2_f.append(scores[0]['rouge-2']['f'])\n",
    "        r2_p.append(scores[0]['rouge-2']['p'])\n",
    "        r2_r.append(scores[0]['rouge-2']['r'])\n",
    "        rl_f.append(scores[0]['rouge-l']['f'])\n",
    "        rl_p.append(scores[0]['rouge-l']['p'])\n",
    "        rl_r.append(scores[0]['rouge-l']['r'])\n",
    "\n",
    "    print(model + \": \" + \"Average of Rouge-1 F1 =\", round(mean(r1_f), 3))  \n",
    "    print(model + \": \" + \"Average of Rouge-1 p =\", round(mean(r1_p), 3))  \n",
    "    print(model + \": \" + \"Average of Rouge-1 r =\", round(mean(r1_r), 3))  \n",
    "    print(model + \": \" + \"Average of Rouge-2 F1 =\", round(mean(r2_f), 3))  \n",
    "    print(model + \": \" + \"Average of Rouge-2 p =\", round(mean(r2_p), 3))  \n",
    "    print(model + \": \" + \"Average of Rouge-2 r =\", round(mean(r2_r), 3))  \n",
    "    print(model + \": \" + \"Average of Rouge-l F1 =\", round(mean(rl_f), 3))  \n",
    "    print(model + \": \" + \"Average of Rouge-l p =\", round(mean(rl_p), 3))  \n",
    "    print(model + \": \" + \"Average of Rouge-l r =\", round(mean(rl_r), 3))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:06:11.442815Z",
     "start_time": "2020-08-24T13:05:02.360710Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lsa: Average of Rouge-1 F1 = 0.064\n",
      "lsa: Average of Rouge-1 p = 0.04\n",
      "lsa: Average of Rouge-1 r = 0.834\n",
      "lsa: Average of Rouge-2 F1 = 0.04\n",
      "lsa: Average of Rouge-2 p = 0.025\n",
      "lsa: Average of Rouge-2 r = 0.54\n",
      "lsa: Average of Rouge-l F1 = 0.085\n",
      "lsa: Average of Rouge-l p = 0.051\n",
      "lsa: Average of Rouge-l r = 0.799\n"
     ]
    }
   ],
   "source": [
    "avg_rouge('lsa', 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:07:23.645993Z",
     "start_time": "2020-08-24T13:06:11.442815Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "luhn: Average of Rouge-1 F1 = 0.061\n",
      "luhn: Average of Rouge-1 p = 0.036\n",
      "luhn: Average of Rouge-1 r = 0.906\n",
      "luhn: Average of Rouge-2 F1 = 0.048\n",
      "luhn: Average of Rouge-2 p = 0.027\n",
      "luhn: Average of Rouge-2 r = 0.739\n",
      "luhn: Average of Rouge-l F1 = 0.093\n",
      "luhn: Average of Rouge-l p = 0.054\n",
      "luhn: Average of Rouge-l r = 0.884\n"
     ]
    }
   ],
   "source": [
    "avg_rouge('luhn', 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:35.033685Z",
     "start_time": "2020-08-24T13:07:23.648150Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lex: Average of Rouge-1 F1 = 0.067\n",
      "lex: Average of Rouge-1 p = 0.041\n",
      "lex: Average of Rouge-1 r = 0.865\n",
      "lex: Average of Rouge-2 F1 = 0.048\n",
      "lex: Average of Rouge-2 p = 0.029\n",
      "lex: Average of Rouge-2 r = 0.623\n",
      "lex: Average of Rouge-l F1 = 0.094\n",
      "lex: Average of Rouge-l p = 0.056\n",
      "lex: Average of Rouge-l r = 0.834\n"
     ]
    }
   ],
   "source": [
    "avg_rouge('lex', 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.378080Z",
     "start_time": "2020-08-24T13:08:35.035746Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting spacy==2.2.2\n",
      "  Using cached spacy-2.2.2-cp36-cp36m-win_amd64.whl (9.4 MB)\n",
      "Requirement already satisfied: setuptools in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.2.2) (45.2.0.post20200210)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.2.2) (2.0.3)\n",
      "Requirement already satisfied: numpy>=1.15.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.2.2) (1.18.1)\n",
      "Requirement already satisfied: plac<1.2.0,>=0.9.6 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.2.2) (0.9.6)\n",
      "Requirement already satisfied: srsly<1.1.0,>=0.1.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.2.2) (1.0.2)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.2.2) (1.0.2)\n",
      "Collecting thinc<7.4.0,>=7.3.0\n",
      "  Using cached thinc-7.3.1-cp36-cp36m-win_amd64.whl (2.0 MB)\n",
      "Collecting blis<0.5.0,>=0.4.0\n",
      "  Using cached blis-0.4.1-cp36-cp36m-win_amd64.whl (5.0 MB)\n",
      "Collecting preshed<3.1.0,>=3.0.2\n",
      "  Using cached preshed-3.0.2-cp36-cp36m-win_amd64.whl (105 kB)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.2.2) (2.22.0)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.3.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.2.2) (0.6.0)\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy==2.2.2) (1.5.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from thinc<7.4.0,>=7.3.0->spacy==2.2.2) (4.42.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy==2.2.2) (2020.4.5.1)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy==2.2.2) (2.8)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy==2.2.2) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy==2.2.2) (1.25.8)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->spacy==2.2.2) (2.2.0)\n",
      "Installing collected packages: blis, preshed, thinc, spacy\n",
      "  Attempting uninstall: blis\n",
      "    Found existing installation: blis 0.2.4\n",
      "    Uninstalling blis-0.2.4:\n",
      "      Successfully uninstalled blis-0.2.4\n",
      "  Attempting uninstall: preshed\n",
      "    Found existing installation: preshed 2.0.1\n",
      "    Uninstalling preshed-2.0.1:\n",
      "      Successfully uninstalled preshed-2.0.1\n",
      "  Attempting uninstall: thinc\n",
      "    Found existing installation: thinc 7.0.8\n",
      "    Uninstalling thinc-7.0.8:\n",
      "      Successfully uninstalled thinc-7.0.8\n",
      "  Attempting uninstall: spacy\n",
      "    Found existing installation: spacy 2.1.3\n",
      "    Uninstalling spacy-2.1.3:\n",
      "      Successfully uninstalled spacy-2.1.3\n",
      "Successfully installed blis-0.4.1 preshed-3.0.2 spacy-2.2.2 thinc-7.3.1\n",
      "Collecting transformers==2.11.0\n",
      "  Using cached transformers-2.11.0-py3-none-any.whl (674 kB)\n",
      "Requirement already satisfied: tokenizers==0.7.0 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (0.7.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (2020.7.14)\n",
      "Requirement already satisfied: filelock in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (3.0.12)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (0.7)\n",
      "Requirement already satisfied: numpy in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (1.18.1)\n",
      "Requirement already satisfied: packaging in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (20.1)\n",
      "Requirement already satisfied: sentencepiece in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (0.1.91)\n",
      "Requirement already satisfied: sacremoses in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (0.0.43)\n",
      "Requirement already satisfied: tqdm>=4.27 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (4.42.1)\n",
      "Requirement already satisfied: requests in c:\\users\\marti\\anaconda3\\lib\\site-packages (from transformers==2.11.0) (2.22.0)\n",
      "Requirement already satisfied: six in c:\\users\\marti\\anaconda3\\lib\\site-packages (from packaging->transformers==2.11.0) (1.14.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from packaging->transformers==2.11.0) (2.4.6)\n",
      "Requirement already satisfied: click in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sacremoses->transformers==2.11.0) (7.0)\n",
      "Requirement already satisfied: joblib in c:\\users\\marti\\anaconda3\\lib\\site-packages (from sacremoses->transformers==2.11.0) (0.14.1)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers==2.11.0) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers==2.11.0) (2020.4.5.1)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers==2.11.0) (1.25.8)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from requests->transformers==2.11.0) (2.8)\n",
      "Installing collected packages: transformers\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 2.2.2\n",
      "    Uninstalling transformers-2.2.2:\n",
      "      Successfully uninstalled transformers-2.2.2\n",
      "Successfully installed transformers-2.11.0\n",
      "Collecting spacy-transformers[cuda100]==0.5.1\n",
      "  Using cached spacy-transformers-0.5.1.tar.gz (59 kB)\n",
      "Requirement already satisfied: spacy<2.3.0,>=2.2.1 in c:\\users\\marti\\anaconda3\\lib\\site-packages (from spacy-transformers[cuda100]==0.5.1) (2.2.2)\n",
      "Collecting transformers<2.1.0,>=2.0.0\n",
      "  Using cached transformers-2.0.0-py3-none-any.whl (290 kB)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: Could not find a version that satisfies the requirement torch>=1.0.0 (from spacy-transformers[cuda100]==0.5.1) (from versions: 0.1.2, 0.1.2.post1, 0.1.2.post2)\n",
      "ERROR: No matching distribution found for torch>=1.0.0 (from spacy-transformers[cuda100]==0.5.1)\n"
     ]
    }
   ],
   "source": [
    "! pip install spacy==2.2.2\n",
    "! pip install transformers==2.11.0\n",
    "! pip install spacy-transformers[cuda100]==0.5.1\n",
    "#! pip install spacy==2.1.3\n",
    "#! pip install transformers==2.2.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.393738Z",
     "start_time": "2020-08-24T13:08:57.378080Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 168,
     "referenced_widgets": [
      "1c99a6eebb1a49a78251b34ba765a007",
      "ab7f66be5c194594a16e736691fb5216",
      "d1398b430783406e950869fc15204a7d",
      "a3f6630af33b4198ae5d34e59b677804",
      "6e6b3a6c1aa24e468a0518567520fb9a",
      "71cf774e675c417fba016f04f2e81b27",
      "52664d40719741e087651d0b1b1b9a75",
      "0fc1e186a2a742d8aa05f3fd8b415d62",
      "aac7ecbcb75e45f9af511f3e0c3b6346",
      "0afedae2d9554d6daa8cb0aea50fee88",
      "282e84fc347e4ff3834486d9cc33cddb",
      "3e55da1538f84739a11ea512544dbc25",
      "ae4c4dc1728e44379fc9a6bac25cb6c1",
      "f8a711e32763495fa5da4803153f9612",
      "6a7ecd4666654c689bd1639c74094f91",
      "971466fa948c4930bad03c39c1b12f4e",
      "b5d81e669cf64c13b9a731a99a70fac5",
      "ca9f7b0dc4934365a42b8d6ab34d5c04",
      "17d4d52df9864ce48937aa6273c085a6",
      "62fc91b285cc4f34b27b5c3785918f2d",
      "893a49dfa86941119d6775f0ebb60e3d",
      "f91f1dfb0c5949fe9a9121c76c98492f",
      "952c355bec9544a7b9cb5cadedbab7d2",
      "9f0aebd6d3a942dd9a1e5bdbc6f5faa5"
     ]
    },
    "colab_type": "code",
    "id": "udU0BJkImbZG",
    "outputId": "788dd146-16b9-4c7d-f96c-2cdbd8884a08"
   },
   "outputs": [],
   "source": [
    "# from summarizer import Summarizer\n",
    "\n",
    "# model_bert = Summarizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.409327Z",
     "start_time": "2020-08-24T13:08:57.393738Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 271
    },
    "colab_type": "code",
    "id": "DcomKkhiUggj",
    "outputId": "da660d80-cb27-45b5-acf3-4d6908232f6d"
   },
   "outputs": [],
   "source": [
    "# sentence_cnt = 6\n",
    "# result = model_bert(text, num_sentences = sentence_cnt - 1)\n",
    "# summary_bert = \"\".join(result)\n",
    "# pprint(summary_bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.594252Z",
     "start_time": "2020-08-24T13:08:57.409327Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "tVZPaCxvgJlw"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from newspaper import fulltext\n",
    "from sumy.parsers.html import HtmlParser\n",
    "from sumy.nlp.stemmers import Stemmer\n",
    "from sumy.utils import get_stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.609874Z",
     "start_time": "2020-08-24T13:08:57.594252Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "DoNW4Hi2HjgR"
   },
   "outputs": [],
   "source": [
    "# def bert_summarize(url, num_sentences_out):\n",
    "\n",
    "#     text = fulltext(requests.get(url).text)\n",
    "#     model = Summarizer()\n",
    "#     result = model(text, num_sentences = num_sentences_out)\n",
    "#     summary = \"\".join(result)\n",
    "#     return summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.625461Z",
     "start_time": "2020-08-24T13:08:57.609874Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "MnIOS4xp19td"
   },
   "outputs": [],
   "source": [
    "def lsa_summarize(url, num_sentences_out):\n",
    "    # for plain text files\n",
    "    # text = fulltext(requests.get(url).text)\n",
    "    summary_lsa = \"\"\n",
    "    language = \"english\"\n",
    "    parser = HtmlParser.from_url(url, Tokenizer(language))\n",
    "    # for plain text files\n",
    "    # parser = PlaintextParser(text, Tokenizer(\"english\"))\n",
    "    stemmer = Stemmer(language)\n",
    "\n",
    "    model_lsa = LsaSummarizer(stemmer)\n",
    "    #model_lsa.stop_words = get_stop_words(language)\n",
    "    result_lsa = model_lsa(parser.document, num_sentences_out)\n",
    "    \n",
    "    for i in result_lsa:\n",
    "        summary_lsa = summary_lsa + str(i) + \" \" \n",
    "    return summary_lsa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.641082Z",
     "start_time": "2020-08-24T13:08:57.625461Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "zNme9WPx1_B2"
   },
   "outputs": [],
   "source": [
    "def luhn_summarize(url, num_sentences_out):\n",
    "    # for plain text files\n",
    "    #text = fulltext(requests.get(url).text)\n",
    "    summary_luhn = \"\"\n",
    "    language = \"english\"\n",
    "    parser = HtmlParser.from_url(url, Tokenizer(language))\n",
    "    # for plain text files\n",
    "    #parser = PlaintextParser(text, Tokenizer(\"english\"))\n",
    "    stemmer = Stemmer(language)\n",
    "\n",
    "    model_luhn = LuhnSummarizer(stemmer)\n",
    "    #model_luhn.stop_words = get_stop_words(language)\n",
    "    result_luhn = model_luhn(parser.document, num_sentences_out)\n",
    "    \n",
    "    for i in result_luhn:\n",
    "        summary_luhn = summary_luhn + str(i) + \" \"\n",
    "    return summary_luhn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.656735Z",
     "start_time": "2020-08-24T13:08:57.641082Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "kXQMmBVm2ABT"
   },
   "outputs": [],
   "source": [
    "def lex_summarize(url, num_sentences_out):\n",
    "\n",
    "    summary_lex = \"\"\n",
    "    language = \"english\"\n",
    "    parser = HtmlParser.from_url(url, Tokenizer(language))\n",
    "    # for plain text files\n",
    "    #parser = PlaintextParser(text, Tokenizer(\"english\"))\n",
    "    stemmer = Stemmer(language)\n",
    "    \n",
    "    model_lex = LexRankSummarizer(stemmer)\n",
    "    #model_lex.stop_words = get_stop_words(language)\n",
    "    result_lex = model_lex(parser.document, num_sentences_out)\n",
    "    \n",
    "    for i in result_lex:\n",
    "        summary_lex = summary_lex + str(i) + \" \"\n",
    "    return summary_lex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.672360Z",
     "start_time": "2020-08-24T13:08:57.656735Z"
    }
   },
   "outputs": [],
   "source": [
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:08:57.694524Z",
     "start_time": "2020-08-24T13:08:57.672360Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 399
    },
    "colab_type": "code",
    "id": "aww8kTJLI8_x",
    "outputId": "75cc64ba-3e5a-44fc-90dc-3376c0896d65"
   },
   "outputs": [],
   "source": [
    "# bert_summary = bert_summarize('https://edition.cnn.com/2020/08/16/asia/new-zealand-ardern-election-delay-coronavirus/index.html', 10)\n",
    "# pprint(bert_summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:09:03.839002Z",
     "start_time": "2020-08-24T13:08:57.694524Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 562
    },
    "colab_type": "code",
    "id": "Y5pxosO63HxV",
    "outputId": "e6c04152-73fc-48fc-993f-7d696c6182ef"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Ardern said while the decision to change the election date rested solely '\n",
      " 'with her as Prime Minister, she consulted with other party leaders as '\n",
      " '\"moving an election date especially this late in an electoral cycle is a '\n",
      " 'significant decision.\" Ardern said New Zealand\\'s Electoral Commission had '\n",
      " 'been preparing for a range of circumstances, such as holding an election in '\n",
      " 'level two or three lockdown, and that she did not intend to change the '\n",
      " \"election date again. New Zealand's Parliament will now reconvene Tuesday and \"\n",
      " 'be dissolved on September 6 ahead of the October poll. The commission said '\n",
      " 'that it had always prepared for the election to be run as if under Alert '\n",
      " 'Level 2 lockdown restrictions, with planned measures including contact '\n",
      " 'tracing, provision of hand sanitizer and physical distancing. In a statement '\n",
      " 'after Ardern\\'s announcement Tuesday she said: \"It was always National\\'s '\n",
      " 'view that to have a fair, democratic election, we needed to deal with this '\n",
      " 'second wave of Covid-19 so politicians from all parties had a reasonable '\n",
      " 'chance to present their policies, and the public felt comfortable engaging '\n",
      " 'with the campaign without putting their health at risk.\" New Zealand First '\n",
      " \"Leader Winston Peters -- who is Deputy Prime Minister in Ardern's coalition \"\n",
      " 'government -- had also called publicly for the election to be delayed and '\n",
      " 'issued a statement welcoming the move. \"Holding an election during a Covid '\n",
      " 'outbreak has the risk of serious interference in our democracy. Voters would '\n",
      " 'be expected to exercise their electoral rights with a dearth of information '\n",
      " 'and that is unacceptable,\" Peters said. It returned to level one on June 9 , '\n",
      " 'with border controls remaining in place but most citizens living life as '\n",
      " 'normal -- until last week. The rest of the country was put into level two '\n",
      " 'lockdown, with both lockdown periods extended until at least August 26 as '\n",
      " 'further cases of coronavirus were confirmed. ')\n"
     ]
    }
   ],
   "source": [
    "lsa_summary = lsa_summarize('https://edition.cnn.com/2020/08/16/asia/new-zealand-ardern-election-delay-coronavirus/index.html', 10)\n",
    "pprint(lsa_summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:09:06.678445Z",
     "start_time": "2020-08-24T13:09:03.840845Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 598
    },
    "colab_type": "code",
    "id": "eshrTAu53H8p",
    "outputId": "3b226e4f-8b2f-435c-9220-94fbaea2a212"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('\"Ultimately I want to ensure we have a well-run election that gives all '\n",
      " 'voters the best chance to receive all the information about parties and '\n",
      " 'candidates and delivers certainty for the future,\" she said. Ardern said '\n",
      " 'while the decision to change the election date rested solely with her as '\n",
      " 'Prime Minister, she consulted with other party leaders as \"moving an '\n",
      " 'election date especially this late in an electoral cycle is a significant '\n",
      " 'decision.\" Ardern said New Zealand\\'s Electoral Commission had been '\n",
      " 'preparing for a range of circumstances, such as holding an election in level '\n",
      " 'two or three lockdown, and that she did not intend to change the election '\n",
      " \"date again. In a statement after Ardern's announcement Tuesday she said: \"\n",
      " '\"It was always National\\'s view that to have a fair, democratic election, we '\n",
      " 'needed to deal with this second wave of Covid-19 so politicians from all '\n",
      " 'parties had a reasonable chance to present their policies, and the public '\n",
      " 'felt comfortable engaging with the campaign without putting their health at '\n",
      " 'risk.\" New Zealand First Leader Winston Peters -- who is Deputy Prime '\n",
      " \"Minister in Ardern's coalition government -- had also called publicly for \"\n",
      " 'the election to be delayed and issued a statement welcoming the move. As of '\n",
      " 'Monday afternoon local time, New Zealand had 58 confirmed cases believed to '\n",
      " 'be related to the most recent community cluster and an additional 20 active '\n",
      " 'cases linked to overseas travel, with returning residents still facing a '\n",
      " \"mandatory 14-day quarantine on arrival in the country. Announcing Monday's \"\n",
      " 'numbers in a live-streamed update, Director-General of Health Dr. Ashley '\n",
      " 'Bloomfield said that record levels of testing were taking place, with nearly '\n",
      " \"100,000 of New Zealand's 597,956 Covid-19 tests completed in the last week \"\n",
      " 'or so. \"The sense we have is that we are dealing at the moment with a single '\n",
      " 'cluster -- a single outbreak -- and thanks to the efforts of the community '\n",
      " 'in coming forward for that testing we are getting ... a full idea of the '\n",
      " 'contours of the outbreak.\" But Bloomfield said the origin of the outbreak '\n",
      " 'remained a mystery: \"Somehow the virus has come in across the border -- I '\n",
      " \"think we're confident that it wasn't brewing away or a sort of slow-burning \"\n",
      " \"ember in the community -- the question of where it's come across the border \"\n",
      " 'and how is what we\\'re trying to answer.\" The announcement on Tuesday that '\n",
      " 'locally acquired cases of coronavirus had been confirmed in the New '\n",
      " \"Zealand's biggest city, Auckland, prompted the government to introduce \"\n",
      " 'strict level three lockdown measures on August 12. ')\n"
     ]
    }
   ],
   "source": [
    "luhn_summary = luhn_summarize('https://edition.cnn.com/2020/08/16/asia/new-zealand-ardern-election-delay-coronavirus/index.html', 10)\n",
    "pprint(luhn_summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T13:09:13.602958Z",
     "start_time": "2020-08-24T13:09:06.678445Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 326
    },
    "colab_type": "code",
    "id": "jJWgxkp03IVl",
    "outputId": "c50a522c-fb88-4f72-9a1b-89e431417386"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(\"Ardern said that New Zealand's Electoral Commission had assured her that a \"\n",
      " 'safe and accessible election would be possible on the new date. \"Ultimately '\n",
      " 'I want to ensure we have a well-run election that gives all voters the best '\n",
      " 'chance to receive all the information about parties and candidates and '\n",
      " 'delivers certainty for the future,\" she said. Ardern said while the decision '\n",
      " 'to change the election date rested solely with her as Prime Minister, she '\n",
      " 'consulted with other party leaders as \"moving an election date especially '\n",
      " 'this late in an electoral cycle is a significant decision.\" Here in New '\n",
      " 'Zealand, we are all working as hard as we can to make sure that our new '\n",
      " 'normal disrupts our lives as little as possible.\" In a statement after '\n",
      " 'Ardern\\'s announcement Tuesday she said: \"It was always National\\'s view '\n",
      " 'that to have a fair, democratic election, we needed to deal with this second '\n",
      " 'wave of Covid-19 so politicians from all parties had a reasonable chance to '\n",
      " 'present their policies, and the public felt comfortable engaging with the '\n",
      " 'campaign without putting their health at risk.\" It returned to level one on '\n",
      " 'June 9 , with border controls remaining in place but most citizens living '\n",
      " 'life as normal -- until last week. \"The sense we have is that we are dealing '\n",
      " 'at the moment with a single cluster -- a single outbreak -- and thanks to '\n",
      " 'the efforts of the community in coming forward for that testing we are '\n",
      " 'getting ... a full idea of the contours of the outbreak.\" But Bloomfield '\n",
      " 'said the origin of the outbreak remained a mystery: \"Somehow the virus has '\n",
      " \"come in across the border -- I think we're confident that it wasn't brewing \"\n",
      " 'away or a sort of slow-burning ember in the community -- the question of '\n",
      " 'where it\\'s come across the border and how is what we\\'re trying to answer.\" '\n",
      " 'Since March, New Zealand has reported a total of 1,280 confirmed and 351 '\n",
      " 'probable coronavirus cases and 22 deaths. The announcement on Tuesday that '\n",
      " 'locally acquired cases of coronavirus had been confirmed in the New '\n",
      " \"Zealand's biggest city, Auckland, prompted the government to introduce \"\n",
      " 'strict level three lockdown measures on August 12. ')\n"
     ]
    }
   ],
   "source": [
    "lex_summary = lex_summarize('https://edition.cnn.com/2020/08/16/asia/new-zealand-ardern-election-delay-coronavirus/index.html', 10)\n",
    "pprint(lex_summary)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "BertSum.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0afedae2d9554d6daa8cb0aea50fee88": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0fc1e186a2a742d8aa05f3fd8b415d62": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "17d4d52df9864ce48937aa6273c085a6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f91f1dfb0c5949fe9a9121c76c98492f",
      "max": 231508,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_893a49dfa86941119d6775f0ebb60e3d",
      "value": 231508
     }
    },
    "1c99a6eebb1a49a78251b34ba765a007": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d1398b430783406e950869fc15204a7d",
       "IPY_MODEL_a3f6630af33b4198ae5d34e59b677804"
      ],
      "layout": "IPY_MODEL_ab7f66be5c194594a16e736691fb5216"
     }
    },
    "282e84fc347e4ff3834486d9cc33cddb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f8a711e32763495fa5da4803153f9612",
      "max": 1344997306,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ae4c4dc1728e44379fc9a6bac25cb6c1",
      "value": 1344997306
     }
    },
    "3e55da1538f84739a11ea512544dbc25": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_971466fa948c4930bad03c39c1b12f4e",
      "placeholder": "​",
      "style": "IPY_MODEL_6a7ecd4666654c689bd1639c74094f91",
      "value": " 1.34G/1.34G [01:04&lt;00:00, 20.8MB/s]"
     }
    },
    "52664d40719741e087651d0b1b1b9a75": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "62fc91b285cc4f34b27b5c3785918f2d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9f0aebd6d3a942dd9a1e5bdbc6f5faa5",
      "placeholder": "​",
      "style": "IPY_MODEL_952c355bec9544a7b9cb5cadedbab7d2",
      "value": " 232k/232k [00:00&lt;00:00, 720kB/s]"
     }
    },
    "6a7ecd4666654c689bd1639c74094f91": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6e6b3a6c1aa24e468a0518567520fb9a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "71cf774e675c417fba016f04f2e81b27": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "893a49dfa86941119d6775f0ebb60e3d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "952c355bec9544a7b9cb5cadedbab7d2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "971466fa948c4930bad03c39c1b12f4e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9f0aebd6d3a942dd9a1e5bdbc6f5faa5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a3f6630af33b4198ae5d34e59b677804": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0fc1e186a2a742d8aa05f3fd8b415d62",
      "placeholder": "​",
      "style": "IPY_MODEL_52664d40719741e087651d0b1b1b9a75",
      "value": " 434/434 [00:00&lt;00:00, 1.33kB/s]"
     }
    },
    "aac7ecbcb75e45f9af511f3e0c3b6346": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_282e84fc347e4ff3834486d9cc33cddb",
       "IPY_MODEL_3e55da1538f84739a11ea512544dbc25"
      ],
      "layout": "IPY_MODEL_0afedae2d9554d6daa8cb0aea50fee88"
     }
    },
    "ab7f66be5c194594a16e736691fb5216": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ae4c4dc1728e44379fc9a6bac25cb6c1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "b5d81e669cf64c13b9a731a99a70fac5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_17d4d52df9864ce48937aa6273c085a6",
       "IPY_MODEL_62fc91b285cc4f34b27b5c3785918f2d"
      ],
      "layout": "IPY_MODEL_ca9f7b0dc4934365a42b8d6ab34d5c04"
     }
    },
    "ca9f7b0dc4934365a42b8d6ab34d5c04": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d1398b430783406e950869fc15204a7d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_71cf774e675c417fba016f04f2e81b27",
      "max": 434,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_6e6b3a6c1aa24e468a0518567520fb9a",
      "value": 434
     }
    },
    "f8a711e32763495fa5da4803153f9612": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f91f1dfb0c5949fe9a9121c76c98492f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}