code/title_generation/prepare_title_gen_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "997eb788",
   "metadata": {},
   "source": [
    "#### Generate Train and Test data for the Title Generation Task. Segregate Clickbait and no-Clickbait paragraphs to evaluate on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e9d0b0f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5c9e0e58",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"./TitleGenData/\"\n",
    "os.makedirs(path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5516e010",
   "metadata": {},
   "outputs": [],
   "source": [
    "readp = './Webis17HeadlineArticleClsData/'\n",
    "\n",
    "df_train = pd.read_csv(readp+'train.csv')\n",
    "df_validation = pd.read_csv(readp+'validation.csv')\n",
    "df_test = pd.read_csv(readp+'test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d3eda97f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>postText</th>\n",
       "      <th>targetParagraphs</th>\n",
       "      <th>truthClass</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UK’s response to modern slavery leaving victim...</td>\n",
       "      <td>Thousands of modern slavery victims have not c...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>this is good</td>\n",
       "      <td>President Donald Trump has appointed the pro-l...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The \"forgotten\" Trump roast: Relive his brutal...</td>\n",
       "      <td>When the White House correspondents’ dinner is...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Meet the happiest #dog in the world!</td>\n",
       "      <td>Adorable is probably an understatement. This a...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tokyo's subway is shut down amid fears over an...</td>\n",
       "      <td>One of Tokyo's major subways systems says it s...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            postText  \\\n",
       "0  UK’s response to modern slavery leaving victim...   \n",
       "1                                       this is good   \n",
       "2  The \"forgotten\" Trump roast: Relive his brutal...   \n",
       "3               Meet the happiest #dog in the world!   \n",
       "4  Tokyo's subway is shut down amid fears over an...   \n",
       "\n",
       "                                    targetParagraphs    truthClass  \n",
       "0  Thousands of modern slavery victims have not c...  no-clickbait  \n",
       "1  President Donald Trump has appointed the pro-l...     clickbait  \n",
       "2  When the White House correspondents’ dinner is...  no-clickbait  \n",
       "3  Adorable is probably an understatement. This a...     clickbait  \n",
       "4  One of Tokyo's major subways systems says it s...  no-clickbait  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9186bc8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19538, 2459, 18979)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_train), len(df_validation), len(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b0d54c1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge train and val set to use of-the-shelf headline generation models \n",
    "train_val = df_train.append(df_validation, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d7b86f3c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>postText</th>\n",
       "      <th>targetParagraphs</th>\n",
       "      <th>truthClass</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UK’s response to modern slavery leaving victim...</td>\n",
       "      <td>Thousands of modern slavery victims have not c...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>this is good</td>\n",
       "      <td>President Donald Trump has appointed the pro-l...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The \"forgotten\" Trump roast: Relive his brutal...</td>\n",
       "      <td>When the White House correspondents’ dinner is...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Meet the happiest #dog in the world!</td>\n",
       "      <td>Adorable is probably an understatement. This a...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tokyo's subway is shut down amid fears over an...</td>\n",
       "      <td>One of Tokyo's major subways systems says it s...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            postText  \\\n",
       "0  UK’s response to modern slavery leaving victim...   \n",
       "1                                       this is good   \n",
       "2  The \"forgotten\" Trump roast: Relive his brutal...   \n",
       "3               Meet the happiest #dog in the world!   \n",
       "4  Tokyo's subway is shut down amid fears over an...   \n",
       "\n",
       "                                    targetParagraphs    truthClass  \n",
       "0  Thousands of modern slavery victims have not c...  no-clickbait  \n",
       "1  President Donald Trump has appointed the pro-l...     clickbait  \n",
       "2  When the White House correspondents’ dinner is...  no-clickbait  \n",
       "3  Adorable is probably an understatement. This a...     clickbait  \n",
       "4  One of Tokyo's major subways systems says it s...  no-clickbait  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_val.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "012ef43c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21997"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "64cef20b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>postText</th>\n",
       "      <th>targetParagraphs</th>\n",
       "      <th>truthClass</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>this is good</td>\n",
       "      <td>President Donald Trump has appointed the pro-l...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Meet the happiest #dog in the world!</td>\n",
       "      <td>Adorable is probably an understatement. This a...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>The top 10 best selling comic books of all time</td>\n",
       "      <td>More By Brian Prowse-Gany In celebration of Na...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Players to know, QBs to watch and more 👀nnGuid...</td>\n",
       "      <td>The first round of the 2017 NFL draft was nota...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>White House reporter @alivitali on her first 1...</td>\n",
       "      <td>The most consistent thing about my life for th...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21984</th>\n",
       "      <td>RT @capitalweather: The Derecho, from D to O: ...</td>\n",
       "      <td>This post was originally published on June 11,...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21987</th>\n",
       "      <td>Restaurant won't apologize for tasteless Caitl...</td>\n",
       "      <td>Whoever comes up with the menus for Nacho Mama...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21988</th>\n",
       "      <td>As a BASE jumper leaped, his girlfriend snappe...</td>\n",
       "      <td>Dean Potter jumped. Graham Hunt followed. Pott...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21991</th>\n",
       "      <td>Women scientists are tweeting \"sexy” photos of...</td>\n",
       "      <td>Marine biologists, archeologists, and many oth...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21992</th>\n",
       "      <td>Man who received world's first penis transplan...</td>\n",
       "      <td>The man who underwent the world's first succes...</td>\n",
       "      <td>clickbait</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5523 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                postText  \\\n",
       "1                                           this is good   \n",
       "3                   Meet the happiest #dog in the world!   \n",
       "10       The top 10 best selling comic books of all time   \n",
       "11     Players to know, QBs to watch and more 👀nnGuid...   \n",
       "18     White House reporter @alivitali on her first 1...   \n",
       "...                                                  ...   \n",
       "21984  RT @capitalweather: The Derecho, from D to O: ...   \n",
       "21987  Restaurant won't apologize for tasteless Caitl...   \n",
       "21988  As a BASE jumper leaped, his girlfriend snappe...   \n",
       "21991  Women scientists are tweeting \"sexy” photos of...   \n",
       "21992  Man who received world's first penis transplan...   \n",
       "\n",
       "                                        targetParagraphs truthClass  \n",
       "1      President Donald Trump has appointed the pro-l...  clickbait  \n",
       "3      Adorable is probably an understatement. This a...  clickbait  \n",
       "10     More By Brian Prowse-Gany In celebration of Na...  clickbait  \n",
       "11     The first round of the 2017 NFL draft was nota...  clickbait  \n",
       "18     The most consistent thing about my life for th...  clickbait  \n",
       "...                                                  ...        ...  \n",
       "21984  This post was originally published on June 11,...  clickbait  \n",
       "21987  Whoever comes up with the menus for Nacho Mama...  clickbait  \n",
       "21988  Dean Potter jumped. Graham Hunt followed. Pott...  clickbait  \n",
       "21991  Marine biologists, archeologists, and many oth...  clickbait  \n",
       "21992  The man who underwent the world's first succes...  clickbait  \n",
       "\n",
       "[5523 rows x 3 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_CB = train_val[train_val['truthClass'] == 'clickbait']\n",
    "df_CB #5523"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "37c5c675",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>postText</th>\n",
       "      <th>targetParagraphs</th>\n",
       "      <th>truthClass</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UK’s response to modern slavery leaving victim...</td>\n",
       "      <td>Thousands of modern slavery victims have not c...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The \"forgotten\" Trump roast: Relive his brutal...</td>\n",
       "      <td>When the White House correspondents’ dinner is...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tokyo's subway is shut down amid fears over an...</td>\n",
       "      <td>One of Tokyo's major subways systems says it s...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Ban lifted on Madrid doping laboratory</td>\n",
       "      <td>Share this with Madrid's Anti-Doping Laborator...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Despite the ‘Yuck Factor,’ Leeches Are Big in ...</td>\n",
       "      <td>MOSCOW — They are small as physician assistant...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21990</th>\n",
       "      <td>RT @WNTonight: Grand jury indicts South Caroli...</td>\n",
       "      <td>The police officer who fatally shot Walter Sco...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21993</th>\n",
       "      <td>RT @NYTSports: Abby didn't start, team couldn'...</td>\n",
       "      <td>WINNIPEG, Manitoba The moment Abby Wambach ste...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21994</th>\n",
       "      <td>Obama defends Affordable Care Act ahead of Sup...</td>\n",
       "      <td>With the Supreme Court set to issue a ruling w...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21995</th>\n",
       "      <td>New study of the Deflategate report concludes ...</td>\n",
       "      <td>BEFORE “Deflategate,” the National Football Le...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21996</th>\n",
       "      <td>Netflix secures rights to Brad Pitt Afghanista...</td>\n",
       "      <td>EXCLUSIVE: has acquired distribution rights to...</td>\n",
       "      <td>no-clickbait</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16474 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                postText  \\\n",
       "0      UK’s response to modern slavery leaving victim...   \n",
       "2      The \"forgotten\" Trump roast: Relive his brutal...   \n",
       "4      Tokyo's subway is shut down amid fears over an...   \n",
       "5                 Ban lifted on Madrid doping laboratory   \n",
       "6      Despite the ‘Yuck Factor,’ Leeches Are Big in ...   \n",
       "...                                                  ...   \n",
       "21990  RT @WNTonight: Grand jury indicts South Caroli...   \n",
       "21993  RT @NYTSports: Abby didn't start, team couldn'...   \n",
       "21994  Obama defends Affordable Care Act ahead of Sup...   \n",
       "21995  New study of the Deflategate report concludes ...   \n",
       "21996  Netflix secures rights to Brad Pitt Afghanista...   \n",
       "\n",
       "                                        targetParagraphs    truthClass  \n",
       "0      Thousands of modern slavery victims have not c...  no-clickbait  \n",
       "2      When the White House correspondents’ dinner is...  no-clickbait  \n",
       "4      One of Tokyo's major subways systems says it s...  no-clickbait  \n",
       "5      Share this with Madrid's Anti-Doping Laborator...  no-clickbait  \n",
       "6      MOSCOW — They are small as physician assistant...  no-clickbait  \n",
       "...                                                  ...           ...  \n",
       "21990  The police officer who fatally shot Walter Sco...  no-clickbait  \n",
       "21993  WINNIPEG, Manitoba The moment Abby Wambach ste...  no-clickbait  \n",
       "21994  With the Supreme Court set to issue a ruling w...  no-clickbait  \n",
       "21995  BEFORE “Deflategate,” the National Football Le...  no-clickbait  \n",
       "21996  EXCLUSIVE: has acquired distribution rights to...  no-clickbait  \n",
       "\n",
       "[16474 rows x 3 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_NCB = train_val[train_val['truthClass'] == 'no-clickbait']\n",
    "df_NCB #16473"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "85f9edeb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16390 5437\n"
     ]
    }
   ],
   "source": [
    "df_NCB.dropna(inplace=True)\n",
    "df_CB.dropna(inplace=True)\n",
    "print(len(df_NCB), len(df_CB))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "54ec2585",
   "metadata": {},
   "outputs": [],
   "source": [
    "savep = path + 'no_clickbait_train_data.csv'\n",
    "df_NCB.to_csv(savep, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "01d829ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "savep = path + 'clickbait_train_data.csv'\n",
    "df_CB.to_csv(savep, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f1e69271",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14455 4458\n"
     ]
    }
   ],
   "source": [
    "# test set\n",
    "\n",
    "df_CB = df_test[df_test['truthClass'] == 'clickbait']\n",
    "df_CB #4515\n",
    "\n",
    "df_NCB = df_test[df_test['truthClass'] == 'no-clickbait']\n",
    "df_NCB #14464\n",
    "\n",
    "df_NCB.dropna(inplace=True)\n",
    "df_CB.dropna(inplace=True)\n",
    "print(len(df_NCB), len(df_CB))\n",
    "\n",
    "savep = path + 'no_clickbait_test_data.csv'\n",
    "df_NCB.to_csv(savep, index=False)\n",
    "\n",
    "savep = path + 'clickbait_test_data.csv'\n",
    "df_CB.to_csv(savep, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39ad42a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad0185a0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}