parse_cnn_labeled_data_summary2csv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"C:/Users/koren/Downloads/cnn/training/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dirs, subdr, files in os.walk(path):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parsetext(path,filename):\n",
    "    with open(\"%s/%s\"%(path, filename),'r') as readin: \n",
    "        text=readin.read()\n",
    "    return text.lower()\n",
    "  \n",
    "def cleantext(text):\n",
    "    text=re.sub(r\"what's\",\"what is \",text)\n",
    "    text=re.sub(r\"it's\",\"it is \",text)\n",
    "    text=re.sub(r\"\\'ve\",\" have \",text)\n",
    "    text=re.sub(r\"i'm\",\"i am \",text)\n",
    "    text=re.sub(r\"\\'re\",\" are \",text)\n",
    "    text=re.sub(r\"n't\",\" not \",text)\n",
    "    text=re.sub(r\"\\'d\",\" would \",text)\n",
    "    text=re.sub(r\"\\'s\",\"s\",text)\n",
    "    text=re.sub(r\"\\'ll\",\" will \",text)\n",
    "    text=re.sub(r\"can't\",\" cannot \",text)\n",
    "    text=re.sub(r\" e g \",\" eg \",text)\n",
    "    text=re.sub(r\"e-mail\",\"email\",text)\n",
    "    text=re.sub(r\"9\\\\/11\",\" 911 \",text)\n",
    "    text=re.sub(r\" u.s\",\" american \",text)\n",
    "    text=re.sub(r\" u.n\",\" united nations \",text)\n",
    "    text=re.sub(r\"\\n\",\" \",text)\n",
    "    text=re.sub(r\":\",\" \",text)\n",
    "    text=re.sub(r\"-\",\" \",text)\n",
    "    text=re.sub(r\"\\_\",\" \",text)\n",
    "    text=re.sub(r\"\\d+\",\" \",text)\n",
    "    text=re.sub(r\"[$#@%&*!~?%{}()]\",\" \",text)\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data={\"articles\":[],\"summaries\":[]}\n",
    "for i, file_name in enumerate(files): \n",
    "    \n",
    "    if file_name.endswith(\"sent\"):\n",
    "        try:\n",
    "            article = cleantext(parsetext(path ,\"%s\"%file_name))\n",
    "            summary = cleantext(parsetext(path,\"%s\"%file_name.replace('sent' , 'summ')))\n",
    "\n",
    "            data[\"articles\"].append(article)\n",
    "            data[\"summaries\"].append(summary)\n",
    "\n",
    "        except Exception as e:\n",
    "            print('was error')\n",
    "            print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data_df = pd.DataFrame(zip(data['articles'], data['summaries']), columns = ['articles', 'summaries'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "90258"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data['articles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df.to_csv(\"non_relevant_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vocab, word_tensors, max_doc_length, label_tensors = \\\n",
    "        load_data(FLAGS.data_dir, FLAGS.max_doc_length, FLAGS.max_sen_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# reading labeled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"C:/Users/koren/Downloads/neuralsum/validation/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dirs, subdr, files in os.walk(path):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = pd.DataFrame(columns = ['sentence', 'labels', 'article'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, filename in enumerate(files):\n",
    "    with open(\"%s/%s\"%(path, filename),'r') as readin: \n",
    "            text=readin.read()\n",
    "            parsed_data = map(lambda x: x.split(\"\\t\\t\\t\"), text.split(\"\\n\\n\")[1].split(\"\\n\"))\n",
    "            df = pd.DataFrame(parsed_data, columns = ['sentence', 'labels'])\n",
    "            df['article'] = i\n",
    "            all_df = all_df.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df.to_csv(\"validation_labeled_cnn_data_full.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}