Benchmark_Rouge.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Benchmark-Rouge.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1zI0cMICgLQTIyZe77P1fsP1OTm4m16CQ",
      "authorship_tag": "ABX9TyOql9qPbIxjQiVKrtuMO1yB",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4d64fe9895674c278b6593dfc6409005": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_40eae545e8694b97bcd1b88bf9e4947e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1312f7550466404891ac0b7a8fd14fa4",
              "IPY_MODEL_82245ed06ff64bc49044a1b2f890939e",
              "IPY_MODEL_26afd0d4af8c4001aee57e9ce5974c03"
            ]
          }
        },
        "40eae545e8694b97bcd1b88bf9e4947e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1312f7550466404891ac0b7a8fd14fa4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0169dbbe4e654b17944a76d029243cb9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c291dde730744e5780a7084704ebfa53"
          }
        },
        "82245ed06ff64bc49044a1b2f890939e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ef254728c3c947599c86b36ddd013ba0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5622,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5622,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d47905ac216c4cba9d07c82443222791"
          }
        },
        "26afd0d4af8c4001aee57e9ce5974c03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d8d567d4c7274e9e808c0718af1ed34a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5622/5622 [03:33&lt;00:00, 37.56it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1e0e7b06710e4baa8b273581d47910b8"
          }
        },
        "0169dbbe4e654b17944a76d029243cb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c291dde730744e5780a7084704ebfa53": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef254728c3c947599c86b36ddd013ba0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d47905ac216c4cba9d07c82443222791": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d8d567d4c7274e9e808c0718af1ed34a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1e0e7b06710e4baa8b273581d47910b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/loudly-soft/ethnicolr/blob/master/Benchmark_Rouge.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tLaRRWIeKhh2"
      },
      "source": [
        "# Benchmark Text Summarisation with Rouge\n",
        "\n",
        "My self-learning notes on how to use Rouge to benchmark text summarisation and where to get ground truth data to test your model.\n",
        "\n",
        "**Synopsis**\n",
        "\n",
        "1. Download news summary dataset\n",
        "2. Implement pivoted QR algorithm [1] to summarize news from multiple sources\n",
        "3. Calculate precision, recall and F1-score with Rouge\n",
        "\n",
        "**Catalog of NLP Datasets**\n",
        "* https://metatext.io/datasets-list/summarization-task\n",
        "\n",
        "**Rouge Tutorials**\n",
        "* https://towardsdatascience.com/the-ultimate-performance-metric-in-nlp-111df6c64460\n",
        "* https://kavita-ganesan.com/what-is-rouge-and-how-it-works-for-evaluation-of-summaries/#.YVcnVZ1KiUk\n",
        "\n",
        "**References**\n",
        "\n",
        "[1] Conroy J, O’Leary D., Text Summarization via Hidden Markov Models and Pivoted QR Matrix Decomposition.  Technical Report, University of Maryland, College Park, Maryland, March, 2001."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ofzSgWyJ4dGA"
      },
      "source": [
        "## Download ground truth data for text summarisation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BJRjzyyD-gru"
      },
      "source": [
        "I will be using the multi-document summarisation dataset from:\n",
        "\n",
        "* https://github.com/Alex-Fabbri/Multi-News\n",
        "\n",
        "Each news summary is generated from multiple source articles.  There is a preprocessed and an unprocessed version of the data.  Both versions are divided into train, test and validation dataset with summaries and source articles.\n",
        "\n",
        "For this demo, I'll work with the pre-processed version of validation data:\n",
        "* https://drive.google.com/drive/folders/1uDarzpu2HFc-vjXNJCRv2NIHzakpSGOw\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "id": "syef00OL7-B0",
        "outputId": "79001db0-c591-4b08-8c06-f555def74526"
      },
      "source": [
        "!gdown --id 1fpLqEb4lQ2F0ooBzyBoVc-d2S1qh-euS\n",
        "!gdown --id 1RzmVVqVMNWhjNTUWKeiBS-HW1UIqnXeS"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1fpLqEb4lQ2F0ooBzyBoVc-d2S1qh-euS\n",
            "To: /content/val.txt.tgt\n",
            "7.55MB [00:00, 117MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1RzmVVqVMNWhjNTUWKeiBS-HW1UIqnXeS\n",
            "To: /content/val.txt.src\n",
            "62.8MB [00:00, 151MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bytTtFMj_qFH"
      },
      "source": [
        "The files are:\n",
        "* `val.txt.tgt` - summaries\n",
        "* `val.txt.src` - source articles\n",
        "\n",
        "Count number of lines in files for sanity check"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "-1Z4LGjw_qTR",
        "outputId": "840fdf44-adf5-49ed-bd25-ed6e822f9cff"
      },
      "source": [
        "!wc -l val.txt.tgt\n",
        "!wc -l val.txt.src"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5622 val.txt.tgt\n",
            "5622 val.txt.src\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vzoxPTvCBWwX"
      },
      "source": [
        "#### Disable horizontal scroll bar so long output in Colab is wrapped\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Of7gL78BBVc4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "72459a50-d8c1-43bc-ace9-44890619b2bb"
      },
      "source": [
        "from IPython.display import HTML, display\n",
        "\n",
        "def set_css():\n",
        "  display(HTML('''<style>pre { white-space: pre-wrap; }</style>'''))\n",
        "get_ipython().events.register('pre_run_cell', set_css)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bodVuASPAEc6"
      },
      "source": [
        "#### Extract source articles and summaries"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zh3qGtGuDvcd"
      },
      "source": [
        "The source file `val.txt.src` has multiple stories per line delimited by *'story_separator_special_tag'*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "5QeWT6V9Dulu",
        "outputId": "44a976d9-5938-4531-f234-d10484938cce"
      },
      "source": [
        "multi_sources = []\n",
        "\n",
        "with open('val.txt.src', 'r', encoding='UTF-8') as file:\n",
        "  for line in file:\n",
        "    articles = line.strip().split('story_separator_special_tag')\n",
        "    multi_sources.append(sorted([text.strip() for text in articles if text], key=len))\n",
        "\n",
        "# show the first source stories\n",
        "for i, article in enumerate(multi_sources[0]):\n",
        "  print('Source %d of %d' % (i+1, len(multi_sources[0])))\n",
        "  print('--------------')\n",
        "  print(article.replace('\\n', ' '))\n",
        "  print()\n",
        "  "
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Source 1 of 4\n",
            "--------------\n",
            "a charity shop is urging people to stop donating the da vinci code after becoming overwhelmed with copies .     the oxfam shop in swansea has been receiving an average of one copy of the dan brown novel a week for months , leaving them with little room for any other books .     staff who are struggling to sell copies of the book have put a note up in the store saying they would rather donors hand in their vinyl instead .\n",
            "\n",
            "Source 2 of 4\n",
            "--------------\n",
            "whether a sign of a good read ; or a comment on the ' pulp ' nature of some genres of fiction , the oxfam second-hand book charts have remained in the da vinci code author ' s favour for the past four years .     dan brown has topped oxfam ' s ' most donated ' list again , his fourth consecutive year . having sold more than 80 million copies of the da vinci code and had all four of his novels on the new york times bestseller list in the same week , it ' s hardly surprising that brown ' s hefty tomes are being donated to charity by readers keen to make some room on their shelves .     another cult crime writer responsible to heavy-weight hardbacks , stieg larsson , is oxfam ' s ' most sold ' author for the second time in a row . both the ' most donated ' and ' most sold ' lists are dominated by crime fiction , trilogies and fantasy , with jk rowling the only female author listed in either of the top fives .     click here or on \" view gallery \" to see both charts in pictures\n",
            "\n",
            "Source 3 of 4\n",
            "--------------\n",
            "a woman reads a copy of the newly released book ' ' the lost symbol ' ' by dan brown , at a speed reading book launch event in sydney , september 15 , 2009. reuters/tim wimborne     san francisco the latest novel from \" da vinci code \" author dan brown , \" the lost symbol , \" broke one-day sales records , its publisher and booksellers said .     readers snapped up over one million hardcover copies across the united states , canada and the united kingdom after it was released on tuesday , said publisher knopf doubleday , a division of random house inc.     \" we are seeing historic , record-breaking sales across all types of our accounts in north america for ' the lost symbol , \" said sonny mehta , editor in chief of knopf doubleday publishing group . knopf doubleday is a division of random house inc.     amazon.com inc , the world ' s largest online retailer , called the book its bestselling first-day adult fiction title ever , including pre-orders .     barnes &amp; noble inc said \" the lost symbol \" broke its previous one-day sales record for adult fiction .     the success of the dan brown ' s latest is a boost to publisher knopf doubleday and booksellers , which have endured sliding sales in the midst of the recession .     booksellers have anxiously awaited a popular title that will resonate with readers and fuel the same sort of frenzy seen earlier this decade with the \" harry potter \" series , from author j.k. rowling .     but the $ 25 billion domestic book market has wallowed in a slump in recent years as more readers make their purchases online , or forego books altogether for online games and other forms of media and entertainment .     now , digital books and electronic reading devices such as amazon ' s kindle or sony corp ' s reader are seen as both avenues of growth and sources of competition for traditional media , and publishers and booksellers are scrambling to respond .     the highly anticipated book from brown comes six years after the release of the american novelist ' s last book \" the da vinci code \" . it follows the adventures of harvard professor robert langdon and is set in the secret world of freemasons in washington d.c.     \" the da vinci code \" sold 80 million copies worldwide . it was made into a film starring tom hanks that grossed more than $ 758 million , according to tracking firm box office mojo .     shares of amazon closed up $ 7.15 to $ 90.70 on the nasdaq . amazon was upgraded to a \" buy \" rating by bank of america merrill lynch , which cited a resurgence in e-commerce that would benefit the seattle-based company .     barnes &amp; noble closed up 14 cents to $ 22.04 and book retailer borders group inc closed up 2 cents to $ 3.29 , on the new york stock exchange .     ( reporting by alexandria sage ; editing by carol bishopric )\n",
            "\n",
            "Source 4 of 4\n",
            "--------------\n",
            "bestselling author is also the most frequently given away to charity shops     dan brown might be one of the world ' s bestselling authors but it turns out that readers aren ' t too keen on keeping his special blend of religious conspiracy and scholarly derring-do on their shelves once they ' ve bought it .     brown , who has sold more than 81m copies of the da vinci code worldwide , has been revealed as the most donated author to oxfam ' s 700 high street shops . with just four books to his name – although his long-awaited fifth the lost symbol is published next month – brown did well to see off competition from john grisham , author of more than 20 and the second-most likely writer to be ditched in a charity shop by readers .     but as secondhand bookshop shelves flood with battered editions of angels and demons and digital fortress , brown can comfort himself with the fact that he ' s also oxfam ' s second most bought author : there are , apparently , still readers out there who have yet to follow the adventures of the dapper symbologist robert langdon . there ' s no such consolation for grisham , whose legal thrillers fail to make oxfam ' s bestseller charts at all .     \" there ' s no question that when you go into the back room of oxfam shops there are many dan brown books , \" said oxfam ' s director of trading david mccullough . \" but he ' s also very high on the bestseller list so there is a useful recycling exercise going on – it ' s not just people saying ' i ' ve read the da vinci code and now i must get rid of it ' . \"     ian rankin , whose dour , boozy detective john rebus is no robert langdon , tops oxfam ' s bestseller list , which the charity says is the first ever high-street secondhand bestseller chart . \" it ' s always good for an author to know that their books are popular , \" said the scottish author , who will unveil a new policeman hero , the teetotal malcolm fox , next month . \" with oxfam , it ' s also heartening to realise that each book donated and bought is helping such a worthwhile organisation . \"     stephenie meyer , author of the twilight series and instigator of myriad teenage crushes courtesy of her sparkly vampire hero edward cullen , is also sitting high in oxfam ' s charts , nestling between bernard cornwell and terry pratchett .     margaret atwood , meanwhile , winner of the booker prize and author of a host of critically acclaimed works of fiction , scrapes into the list in eighth place , keeping unlikely company with thriller powerhouse james patterson – currently producing at least eight books a year thanks to a horde of co-writers – and jodi picoult , never afraid to jerk a tear or pile on the plot twists .     \" we just need to dispel the idea that we are sitting there in oxfam with only first editions of literary gems – actually we ' ve got shelves of really good fiction , \" said mccullough . \" waterstone ' s might be more upset than secondhand booksellers , \" he added , referring to the recent slew of complaints from secondhand booksellers that the charity is stealing their business .     oxfam , europe ' s biggest high-street retailer of secondhand books and the third-biggest bookseller in the uk , launched a drive for book donations in may ahead of its first national book festival , bookfest , in july . authors including joanna trollope , philip pullman and jonathan coe all lent a hand in shops across the country as part of the festival , and the drive saw book donations rise 40 % , with sales up by more than 10 % .     rare books and first editions have also been pouring into shops since may . ten of the most sought-after editions have raised more than £ 4,500 for the charity between them . a first edition of lord of the rings sold for £ 800 , a first edition of watership down brought in £ 500 , sylvia plath ' s ariel sold for £ 350 , ian fleming ' s from russia with love for £ 300 and a second printing of martin chuzzlewit for £ 200 .     oxfam , which has more than 130 specialist bookshops and stocks books in almost all of its 700 stores , sells £ 1.6m-worth of books a month – equivalent , it says , to 50,000 emergency shelters , 64,000 goats or safe water for 2.1 million people .     the most donated authors to oxfam shops so far this year :     1. dan brown     2. john grisham     3. ian rankin     4. danielle steel     5. helen fielding     6. stephen king     7. jk rowling     8. catherine cookson     9. patricia cornwell     10. mills &amp; boon     the oxfam shop bestseller list :     1. ian rankin     2. dan brown     3. bernard cornwell     4. stephanie meyer     5. terry pratchett     6. khaled hosseini     7. helen fielding     8. margaret atwood     9. james patterson     10. jodi picoult     the top 10 most valuable donated books since may :     1. jrr tolkien , lord of the rings – first edition , sold for £ 800     2. don giovanni sheet music – first edition , sold for £ 750     3. sowerby ' s catalogue of shells – sold for £ 600     4. richard adams , watership down – first edition , sold for £ 500     5. handbook of indian dances - first edition with hand-blocked prints , sold for £ 500     6. richmal crompton , just william - first edition , sold for £ 440     7. sylvia plath , ariel – first edition , sold for £ 350     8. ian fleming , from russia with love – first edition , sold for £ 300     9. charles dickens , martin chuzzlewit – second print , sold for £ 200     10. we johns , biggles in australia – first edition , sold for £ 150\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "63ER3hUQDmTV"
      },
      "source": [
        "The summary file `val.txt.tgt` has 1 summary per line"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "id": "OWbgabzLAEmy",
        "outputId": "ac74eee1-aff4-43c6-bbd5-c0b441ace14c"
      },
      "source": [
        "summaries = []\n",
        "with open('val.txt.tgt', 'r', encoding='UTF-8') as file:\n",
        "  for line in file:\n",
        "    summaries.append(line.rstrip()[2:])\n",
        "\n",
        "# show the first summary\n",
        "print(summaries[0])\n"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "the da vinci code has sold so many copies — that would be at least 80 million — that it ' s bound to turn up in book donation piles . but at one charity shop in the uk , it ' s been donated so heavily that the shop has posted a sign propped up on a tower of da vinci code copies that reads : \" you could give us another da vinci code ... but we would rather have your vinyl ! \" the manager of the oxfam shop in swansea tells the telegraph that people are laughing and taking pictures of the sizable display : \" i would say that we get one copy of the book every day . \" he says people buy them \" occasionally , \" but with vinyl sales up 25 % in the past year , they ' d rather take records . dan brown ' s book isn ' t the only one that shops like oxfam struggle to re-sell . last year , oxfam was hit with a large and steady supply of fifty shades of grey , and it similarly begged donors : \" please — no more . \" but brown has a particular kind of staying power . the da vinci code was published in 2003 , and within six years brown had booted john grisham from the no . 1 slot on the list of writers whose books were most often donated to oxfam ' s 700 shops , reported the guardian at the time . the independent in 2012 reported brown ' s best-seller was the most-donated book for the fourth year running . ( see why dan brown took heat from the philippines. )\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "75Bz3tltaLLQ"
      },
      "source": [
        "## Generate extractive summaries\n",
        "\n",
        "The algorithm is QR decomposition with column pivoting from numerical linear algebra [1].  The algorithm selects key sentences and ranks them in order of importance.  The idea is to find the most important sentence (biggest L2 norm) and extract it for summary.  From the remaining sentences, extract the next sentence that is important and different (orthogonal) to the extracted ones.  Repeat until enough key sentences are extracted.\n",
        "\n",
        "I'm curious how this approach will fare on multi-document summary.  Would it be able to ignore conceptually similar sentences from different sources when the sources are combined into a single text for summarisation?\n",
        "\n",
        "**References**\n",
        "\n",
        "[1] Conroy J, O’Leary D., Text Summarization via Hidden Markov Models and Pivoted QR Matrix Decomposition.  Technical Report, University of Maryland, College Park, Maryland, March, 2001.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-I4Qfx0pglAH"
      },
      "source": [
        "#### Implement pivoted QR decomposition"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "wqrbS81dlHnF",
        "outputId": "3cf65fbe-842f-4f9e-d1c9-20103f8b30fb"
      },
      "source": [
        "from numpy import linalg as LA\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "def givens_vector(x, start_pos, eps=0.0000001):\n",
        "  \"\"\"Apply givens rotation to zero out elements of vector x from starting position\"\"\"\n",
        "\n",
        "  size = len(x)\n",
        "  G_acc = np.eye(size)\n",
        "  for i in range(size-1, start_pos-1, -1):\n",
        "    if x[i-1] < eps and x[i] < eps:\n",
        "      continue\n",
        "    # init givens matrix\n",
        "    c = x[i-1] / pow(pow(x[i-1], 2) + pow(x[i], 2), 0.5)\n",
        "    s = x[i] / pow(pow(x[i-1], 2) + pow(x[i], 2), 0.5)\n",
        "    G = np.eye(size)\n",
        "    G[i-1, i-1] = c\n",
        "    G[i-1, i] = s \n",
        "    G[i, i-1] = -s\n",
        "    G[i, i] = c\n",
        "    x = G.dot(x)\n",
        "    # stack the givens matrices\n",
        "    G_acc = np.matmul(G, G_acc)\n",
        "  return G_acc, x\n",
        "\n",
        "\n",
        "def extract_columns(W, max_cols, seed_col_index=None, eps=0.0000001):\n",
        "  \"\"\"Extract orthogonal columns with biggest L2 norm\"\"\"\n",
        "\n",
        "  selected_col_indices = []\n",
        "  col_indices = list(range(W.shape[1]))\n",
        "  while len(selected_col_indices) < max_cols and len(col_indices) > 0:\n",
        "    l2_norms = LA.norm(W, axis=0)\n",
        "    if seed_col_index is None:\n",
        "      # pick column of W with highest L2 norm\n",
        "      j = np.argmax(l2_norms)\n",
        "    else:\n",
        "      j = seed_col_index\n",
        "      seed_col_index = None\n",
        "    if l2_norms[j] < eps:\n",
        "      break\n",
        "    selected_col_indices.append(col_indices[j])\n",
        "    del col_indices[j]\n",
        "    # apply givens rotation to selected column\n",
        "    G, _ = givens_vector(W[:,j], len(selected_col_indices))\n",
        "    # apply givens rotation to remaining columns\n",
        "    W = np.matmul(G, np.delete(W, j, 1))\n",
        "  return selected_col_indices\n"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FmbJxHPtZKmL"
      },
      "source": [
        "A simple test to show that the heaviest (biggest L2 norm) column is chosen first, then the next heaviest column orthogonal to the chosen column is selected, then the next column orthogonal to all the chosen columns."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "Vgn1o0SlZLAr",
        "outputId": "9b0135fb-2651-461b-dd8e-c423b22262bb"
      },
      "source": [
        "X = np.array([\n",
        " [0, 0, 1],\n",
        " [0, 0, 1],\n",
        " [0, 0, 1],\n",
        " [0, 1, 0],\n",
        " [0, 1, 0],\n",
        " [1, 0, 0]])\n",
        "\n",
        "extract_columns(X, max_cols=3)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 1, 0]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s0W9yCRjSZyh"
      },
      "source": [
        "#### Functions to vectorize text to term-sentence matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "cmv2Ca3bSZ8h",
        "outputId": "2708b416-6438-4e33-b6f0-82977be3c589"
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "\n",
        "\n",
        "def vectorize(sentences, truncateSVD=True, threshold=0.8):\n",
        "  \"\"\"Convert sentences to term-sentence matrix and reduce noise with SVD\"\"\"\n",
        "\n",
        "  vectorizer = CountVectorizer(stop_words=\"english\", strip_accents='ascii')\n",
        "  W = vectorizer.fit_transform(sentences).toarray().transpose().astype(float)\n",
        "  \n",
        "  # truncate SVD to cover X% of singular values\n",
        "  if truncateSVD:\n",
        "    U, S, V = np.linalg.svd(W, full_matrices=True)\n",
        "    total_sigmas = sum(S)\n",
        "    subtotal = 0\n",
        "    for k, s in enumerate(S):\n",
        "      subtotal += s\n",
        "      if subtotal / total_sigmas > threshold:\n",
        "        break\n",
        "    W = V[0:k+1,:]\n",
        "    #print('%d out of %d sigmas selected' % (k+1, len(S)))\n",
        "  return W\n",
        "\n",
        "\n",
        "def parse_sentences(text):\n",
        "  \"\"\"Tokenize text into sentences\"\"\"\n",
        "\n",
        "  return [sentence.replace('\\n', ' ') for sentence in nltk.tokenize.sent_tokenize(text)]\n",
        "\n",
        "\n",
        "def summarise(W, sentences, max_sentences, seed_col_index=None):\n",
        "  \"\"\"Apply pivoted QR on term-sentence matrix\"\"\"\n",
        "\n",
        "  # sort extracted sentences by their original position in text\n",
        "  return ' '.join([sentences[i] for i in sorted(extract_columns(W, max_sentences, seed_col_index))])\n"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LZ3ZSqcZghxU"
      },
      "source": [
        "#### Apply algorithm to summarise news"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "4d64fe9895674c278b6593dfc6409005",
            "40eae545e8694b97bcd1b88bf9e4947e",
            "1312f7550466404891ac0b7a8fd14fa4",
            "82245ed06ff64bc49044a1b2f890939e",
            "26afd0d4af8c4001aee57e9ce5974c03",
            "0169dbbe4e654b17944a76d029243cb9",
            "c291dde730744e5780a7084704ebfa53",
            "ef254728c3c947599c86b36ddd013ba0",
            "d47905ac216c4cba9d07c82443222791",
            "d8d567d4c7274e9e808c0718af1ed34a",
            "1e0e7b06710e4baa8b273581d47910b8"
          ]
        },
        "id": "jS0BGIT_I9Qx",
        "outputId": "138e554f-6bfa-4e1b-9d07-ac9d173b3b2d"
      },
      "source": [
        "from tqdm.notebook import tqdm\n",
        "\n",
        "\n",
        "summaries_test = []\n",
        "summaries_pred = []\n",
        "\n",
        "for i in tqdm(range(len(summaries))):\n",
        "#for i in tqdm(range(100)):\n",
        "\n",
        "  # tokenize sentences from source articles and combine them\n",
        "  sentences = sum([parse_sentences(text) for text in multi_sources[i]], [])\n",
        "\n",
        "  # pivoted QR algo is slow so skip long text\n",
        "  if len(sentences) <= 200:\n",
        "\n",
        "    # choose only the longest source article to summarise and tokenize sentences\n",
        "    #sentences = parse_sentences(multi_sources[i][-1])\n",
        "\n",
        "    # vectorize sentences to term-sentence matrix\n",
        "    W = vectorize(sentences, truncateSVD=True, threshold=0.2)\n",
        "\n",
        "    # set max number of summary sentences to % of the longest article\n",
        "    #max_summary_sentences = max(1, int(len(parse_sentences(multi_sources[i][-1])) * 0.20))\n",
        "\n",
        "    # set max number of summary sentences\n",
        "    max_summary_sentences = 6\n",
        "\n",
        "    # generate summary\n",
        "    summary = summarise(W, sentences, max_summary_sentences)\n",
        "    summaries_pred.append(summary)\n",
        "\n",
        "    # keep ground truth summary\n",
        "    summaries_test.append(summaries[i])\n",
        "\n",
        "print(f'{len(summaries_pred)} out of {len(summaries)} summaries generated')\n"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4d64fe9895674c278b6593dfc6409005",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/5622 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5307 out of 5622 summaries generated\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WpqC7ILhbK7o"
      },
      "source": [
        "Show the first summary generated by model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 364
        },
        "id": "CrzXIzv7bIKf",
        "outputId": "2468bf25-9109-42fb-b9ee-35e43a6fbf1a"
      },
      "source": [
        "print(summaries_pred[0])"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "a woman reads a copy of the newly released book ' ' the lost symbol ' ' by dan brown , at a speed reading book launch event in sydney , september 15 , 2009. reuters/tim wimborne     san francisco the latest novel from \" da vinci code \" author dan brown , \" the lost symbol , \" broke one-day sales records , its publisher and booksellers said . readers snapped up over one million hardcover copies across the united states , canada and the united kingdom after it was released on tuesday , said publisher knopf doubleday , a division of random house inc.     \" we are seeing historic , record-breaking sales across all types of our accounts in north america for ' the lost symbol , \" said sonny mehta , editor in chief of knopf doubleday publishing group . but as secondhand bookshop shelves flood with battered editions of angels and demons and digital fortress , brown can comfort himself with the fact that he ' s also oxfam ' s second most bought author : there are , apparently , still readers out there who have yet to follow the adventures of the dapper symbologist robert langdon . margaret atwood , meanwhile , winner of the booker prize and author of a host of critically acclaimed works of fiction , scrapes into the list in eighth place , keeping unlikely company with thriller powerhouse james patterson – currently producing at least eight books a year thanks to a horde of co-writers – and jodi picoult , never afraid to jerk a tear or pile on the plot twists .     \" oxfam , which has more than 130 specialist bookshops and stocks books in almost all of its 700 stores , sells £ 1.6m-worth of books a month – equivalent , it says , to 50,000 emergency shelters , 64,000 goats or safe water for 2.1 million people . the most donated authors to oxfam shops so far this year :     1. dan brown     2. john grisham     3. ian rankin     4. danielle steel     5. helen fielding     6. stephen king     7. jk rowling     8. catherine cookson     9. patricia cornwell     10. mills &amp; boon     the oxfam shop bestseller list :     1. ian rankin     2. dan brown     3. bernard cornwell     4. stephanie meyer     5. terry pratchett     6. khaled hosseini     7. helen fielding     8. margaret atwood     9. james patterson     10. jodi picoult     the top 10 most valuable donated books since may :     1. jrr tolkien , lord of the rings – first edition , sold for £ 800     2. don giovanni sheet music – first edition , sold for £ 750     3. sowerby ' s catalogue of shells – sold for £ 600     4. richard adams , watership down – first edition , sold for £ 500     5. handbook of indian dances - first edition with hand-blocked prints , sold for £ 500     6. richmal crompton , just william - first edition , sold for £ 440     7. sylvia plath , ariel – first edition , sold for £ 350     8. ian fleming , from russia with love – first edition , sold for £ 300     9. charles dickens , martin chuzzlewit – second print , sold for £ 200     10. we johns , biggles in australia – first edition , sold for £ 150\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y0aCnnRJIT5x"
      },
      "source": [
        "## Benchmark with Rouge\n",
        "Install `rouge` library from https://github.com/pltrdy/rouge"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "VINIitOnIUEo",
        "outputId": "3244ae88-6177-48ba-b012-ff40f9f10a78"
      },
      "source": [
        "!pip install rouge"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: rouge in /usr/local/lib/python3.7/dist-packages (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from rouge) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r8GGmzIIhSRT"
      },
      "source": [
        "#### Calculate precision, recall and F1-score"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225
        },
        "id": "ZZnbN1aqduHC",
        "outputId": "4628e961-6419-4244-f47a-2420aa01eeb3"
      },
      "source": [
        "from rouge import Rouge\n",
        "\n",
        "# metric 'route-l' (with an L at the end) is slow, so ignore\n",
        "rouge = Rouge(metrics=['rouge-1', 'rouge-2'])\n",
        "scores = rouge.get_scores(summaries_pred, summaries_test, avg=True)\n",
        "\n",
        "# print rouge metrics\n",
        "metric_descs = {'f':'f1-score', 'p':'precision', 'r':'recall'}\n",
        "for score_name, metrics in scores.items():\n",
        "  print(score_name.upper())\n",
        "  print('-' * 2 * len(score_name))\n",
        "  for metric_name, val in metrics.items():\n",
        "    print(f'{metric_descs[metric_name]} = {val}')\n",
        "  print()\n"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>pre { white-space: pre-wrap; }</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ROUGE-1\n",
            "--------------\n",
            "recall = 0.3839146727188993\n",
            "precision = 0.3379472602044078\n",
            "f1-score = 0.350632333558966\n",
            "\n",
            "ROUGE-2\n",
            "--------------\n",
            "recall = 0.13253740035283365\n",
            "precision = 0.11745861062942464\n",
            "f1-score = 0.12041542842694532\n",
            "\n"
          ]
        }
      ]
    }
  ]
}