reuters-preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "from operator import itemgetter\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import reuters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package reuters to /home/ubuntu/nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('reuters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7769"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([_ for _ in reuters.fileids() if 'training/' in _])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['test/14843',\n",
       " 'test/14852',\n",
       " 'test/14865',\n",
       " 'test/14888',\n",
       " 'test/14900',\n",
       " 'test/14907',\n",
       " 'test/14909',\n",
       " 'test/14921',\n",
       " 'test/14932',\n",
       " 'test/14941',\n",
       " 'test/14943',\n",
       " 'test/14949',\n",
       " 'test/14978',\n",
       " 'test/14982',\n",
       " 'test/15001',\n",
       " 'test/15004',\n",
       " 'test/15024',\n",
       " 'test/15031',\n",
       " 'test/15037',\n",
       " 'test/15052',\n",
       " 'test/15060',\n",
       " 'test/15061',\n",
       " 'test/15063',\n",
       " 'test/15069',\n",
       " 'test/15091',\n",
       " 'test/15111',\n",
       " 'test/15119',\n",
       " 'test/15122',\n",
       " 'test/15124',\n",
       " 'test/15128',\n",
       " 'test/15138',\n",
       " 'test/15146',\n",
       " 'test/15149',\n",
       " 'test/15156',\n",
       " 'test/15161',\n",
       " 'test/15204',\n",
       " 'test/15210',\n",
       " 'test/15217',\n",
       " 'test/15239',\n",
       " 'test/15255',\n",
       " 'test/15276',\n",
       " 'test/15281',\n",
       " 'test/15294',\n",
       " 'test/15302',\n",
       " 'test/15306',\n",
       " 'test/15308',\n",
       " 'test/15311',\n",
       " 'test/15335',\n",
       " 'test/15337',\n",
       " 'test/15349',\n",
       " 'test/15363',\n",
       " 'test/15366',\n",
       " 'test/15372',\n",
       " 'test/15404',\n",
       " 'test/15421',\n",
       " 'test/15425',\n",
       " 'test/15464',\n",
       " 'test/15467',\n",
       " 'test/15468',\n",
       " 'test/15479',\n",
       " 'test/15487',\n",
       " 'test/15501',\n",
       " 'test/15542',\n",
       " 'test/15544',\n",
       " 'test/15545',\n",
       " 'test/15561',\n",
       " 'test/15565',\n",
       " 'test/15571',\n",
       " 'test/15575',\n",
       " 'test/15591',\n",
       " 'test/15595',\n",
       " 'test/15596',\n",
       " 'test/15600',\n",
       " 'test/15605',\n",
       " 'test/15616',\n",
       " 'test/15629',\n",
       " 'test/15664',\n",
       " 'test/15672',\n",
       " 'test/15682',\n",
       " 'test/15698',\n",
       " 'test/15713',\n",
       " 'test/15717',\n",
       " 'test/15739',\n",
       " 'test/15778',\n",
       " 'test/15785',\n",
       " 'test/15810',\n",
       " 'test/15861',\n",
       " 'test/15879',\n",
       " 'test/15885',\n",
       " 'test/15892',\n",
       " 'test/15894',\n",
       " 'test/15898',\n",
       " 'test/15900',\n",
       " 'test/15912',\n",
       " 'test/15918',\n",
       " 'test/15920',\n",
       " 'test/15956',\n",
       " 'test/15961',\n",
       " 'test/15963',\n",
       " 'test/15968',\n",
       " 'test/15970',\n",
       " 'test/15993',\n",
       " 'test/15995',\n",
       " 'test/15997',\n",
       " 'test/16002',\n",
       " 'test/16007',\n",
       " 'test/16021',\n",
       " 'test/16026',\n",
       " 'test/16029',\n",
       " 'test/16033',\n",
       " 'test/16037',\n",
       " 'test/16041',\n",
       " 'test/16091',\n",
       " 'test/16117',\n",
       " 'test/16130',\n",
       " 'test/16136',\n",
       " 'test/16145',\n",
       " 'test/16152',\n",
       " 'test/16155',\n",
       " 'test/16159',\n",
       " 'test/16163',\n",
       " 'test/16173',\n",
       " 'test/16176',\n",
       " 'test/16203',\n",
       " 'test/16210',\n",
       " 'test/16211',\n",
       " 'test/16215',\n",
       " 'test/16216',\n",
       " 'test/16223',\n",
       " 'test/16232',\n",
       " 'test/16233',\n",
       " 'test/16243',\n",
       " 'test/16247',\n",
       " 'test/16258',\n",
       " 'test/16277',\n",
       " 'test/16287',\n",
       " 'test/16302',\n",
       " 'test/16307',\n",
       " 'test/16320',\n",
       " 'test/16332',\n",
       " 'test/16370',\n",
       " 'test/16379',\n",
       " 'test/16395',\n",
       " 'test/16398',\n",
       " 'test/16402',\n",
       " 'test/16403',\n",
       " 'test/16404',\n",
       " 'test/16406',\n",
       " 'test/16409',\n",
       " 'test/16419',\n",
       " 'test/16420',\n",
       " 'test/16427',\n",
       " 'test/16443',\n",
       " 'test/16444',\n",
       " 'test/16449',\n",
       " 'test/16450',\n",
       " 'test/16457',\n",
       " 'test/16459',\n",
       " 'test/16460',\n",
       " 'test/16463',\n",
       " 'test/16465',\n",
       " 'test/16475',\n",
       " 'test/16479',\n",
       " 'test/16483',\n",
       " 'test/16493',\n",
       " 'test/16510',\n",
       " 'test/16512',\n",
       " 'test/16513',\n",
       " 'test/16519',\n",
       " 'test/16521',\n",
       " 'test/16523',\n",
       " 'test/16531',\n",
       " 'test/16555',\n",
       " 'test/16563',\n",
       " 'test/16570',\n",
       " 'test/16577',\n",
       " 'test/16583',\n",
       " 'test/16591',\n",
       " 'test/16597',\n",
       " 'test/16621',\n",
       " 'test/16622',\n",
       " 'test/16646',\n",
       " 'test/16648',\n",
       " 'test/16652',\n",
       " 'test/16656',\n",
       " 'test/16663',\n",
       " 'test/16667',\n",
       " 'test/16675',\n",
       " 'test/16676',\n",
       " 'test/16677',\n",
       " 'test/16680',\n",
       " 'test/16684',\n",
       " 'test/16685',\n",
       " 'test/16688',\n",
       " 'test/16695',\n",
       " 'test/16700',\n",
       " 'test/16707',\n",
       " 'test/16714',\n",
       " 'test/16718',\n",
       " 'test/16727',\n",
       " 'test/16734',\n",
       " 'test/16737',\n",
       " 'test/16747',\n",
       " 'test/16761',\n",
       " 'test/16796',\n",
       " 'test/16811',\n",
       " 'test/16833',\n",
       " 'test/16851',\n",
       " 'test/16872',\n",
       " 'test/16873',\n",
       " 'test/16874',\n",
       " 'test/16875',\n",
       " 'test/16917',\n",
       " 'test/16922',\n",
       " 'test/16923',\n",
       " 'test/16925',\n",
       " 'test/17039',\n",
       " 'test/17046',\n",
       " 'test/17048',\n",
       " 'test/17053',\n",
       " 'test/17479',\n",
       " 'test/17481',\n",
       " 'test/17484',\n",
       " 'test/17487',\n",
       " 'test/17488',\n",
       " 'test/17491',\n",
       " 'test/17493',\n",
       " 'test/17501',\n",
       " 'test/17502',\n",
       " 'test/17506',\n",
       " 'test/17507',\n",
       " 'test/17513',\n",
       " 'test/17518',\n",
       " 'test/17529',\n",
       " 'test/17534',\n",
       " 'test/17538',\n",
       " 'test/17540',\n",
       " 'test/17551',\n",
       " 'test/17559',\n",
       " 'test/17562',\n",
       " 'test/17567',\n",
       " 'test/17569',\n",
       " 'test/17573',\n",
       " 'test/17574',\n",
       " 'test/17576',\n",
       " 'test/17579',\n",
       " 'test/17580',\n",
       " 'test/17585',\n",
       " 'test/17589',\n",
       " 'test/17595',\n",
       " 'test/17596',\n",
       " 'test/17597',\n",
       " 'test/17601',\n",
       " 'test/17612',\n",
       " 'test/17625',\n",
       " 'test/17640',\n",
       " 'test/17643',\n",
       " 'test/17644',\n",
       " 'test/17651',\n",
       " 'test/17652',\n",
       " 'test/17659',\n",
       " 'test/17666',\n",
       " 'test/17671',\n",
       " 'test/17673',\n",
       " 'test/17677',\n",
       " 'test/17682',\n",
       " 'test/17683',\n",
       " 'test/17687',\n",
       " 'test/17688',\n",
       " 'test/17694',\n",
       " 'test/17700',\n",
       " 'test/17701',\n",
       " 'test/17706',\n",
       " 'test/17710',\n",
       " 'test/17716',\n",
       " 'test/17719',\n",
       " 'test/17723',\n",
       " 'test/17727',\n",
       " 'test/17728',\n",
       " 'test/17738',\n",
       " 'test/17743',\n",
       " 'test/17746',\n",
       " 'test/17749',\n",
       " 'test/17752',\n",
       " 'test/17760',\n",
       " 'test/17762',\n",
       " 'test/17763',\n",
       " 'test/17766',\n",
       " 'test/17770',\n",
       " 'test/17775',\n",
       " 'test/17776',\n",
       " 'test/17777',\n",
       " 'test/17782',\n",
       " 'test/17784',\n",
       " 'test/17785',\n",
       " 'test/17787',\n",
       " 'test/17792',\n",
       " 'test/17798',\n",
       " 'test/17809',\n",
       " 'test/17810',\n",
       " 'test/17812',\n",
       " 'test/17822',\n",
       " 'test/17825',\n",
       " 'test/17827',\n",
       " 'test/17835',\n",
       " 'test/17837',\n",
       " 'test/17839',\n",
       " 'test/17847',\n",
       " 'test/17849',\n",
       " 'test/17853',\n",
       " 'test/17857',\n",
       " 'test/17860',\n",
       " 'test/17863',\n",
       " 'test/17864',\n",
       " 'test/17866',\n",
       " 'test/17867',\n",
       " 'test/17872',\n",
       " 'test/17873',\n",
       " 'test/17876',\n",
       " 'test/17886',\n",
       " 'test/17907',\n",
       " 'test/17911',\n",
       " 'test/17913',\n",
       " 'test/17950',\n",
       " 'test/17956',\n",
       " 'test/17957',\n",
       " 'test/17958',\n",
       " 'test/17960',\n",
       " 'test/17968',\n",
       " 'test/17973',\n",
       " 'test/17978',\n",
       " 'test/17981',\n",
       " 'test/17984',\n",
       " 'test/17986',\n",
       " 'test/17993',\n",
       " 'test/18017',\n",
       " 'test/18029',\n",
       " 'test/18036',\n",
       " 'test/18037',\n",
       " 'test/18038',\n",
       " 'test/18040',\n",
       " 'test/18071',\n",
       " 'test/18075',\n",
       " 'test/18079',\n",
       " 'test/18080',\n",
       " 'test/18101',\n",
       " 'test/18111',\n",
       " 'test/18135',\n",
       " 'test/18138',\n",
       " 'test/18145',\n",
       " 'test/18149',\n",
       " 'test/18157',\n",
       " 'test/18182',\n",
       " 'test/18187',\n",
       " 'test/18188',\n",
       " 'test/18205',\n",
       " 'test/18210',\n",
       " 'test/18223',\n",
       " 'test/18229',\n",
       " 'test/18232',\n",
       " 'test/18236',\n",
       " 'test/18238',\n",
       " 'test/18239',\n",
       " 'test/18252',\n",
       " 'test/18300',\n",
       " 'test/18313',\n",
       " 'test/18317',\n",
       " 'test/18325',\n",
       " 'test/18334',\n",
       " 'test/18336',\n",
       " 'test/18342',\n",
       " 'test/18351',\n",
       " 'test/18467',\n",
       " 'test/18476',\n",
       " 'test/18479',\n",
       " 'test/18504',\n",
       " 'test/18530',\n",
       " 'test/18566',\n",
       " 'test/18567',\n",
       " 'test/18569',\n",
       " 'test/18570',\n",
       " 'test/18575',\n",
       " 'test/18576',\n",
       " 'test/18577',\n",
       " 'test/18585',\n",
       " 'test/18590',\n",
       " 'test/18592',\n",
       " 'test/18600',\n",
       " 'test/18605',\n",
       " 'test/18638',\n",
       " 'test/18643',\n",
       " 'test/18644',\n",
       " 'test/18654',\n",
       " 'test/18658',\n",
       " 'test/18660',\n",
       " 'test/18663',\n",
       " 'test/18667',\n",
       " 'test/18682',\n",
       " 'test/18686',\n",
       " 'test/18688',\n",
       " 'test/18730',\n",
       " 'test/18751',\n",
       " 'test/18778',\n",
       " 'test/18783',\n",
       " 'test/18811',\n",
       " 'test/18842',\n",
       " 'test/18856',\n",
       " 'test/18857',\n",
       " 'test/18859',\n",
       " 'test/18861',\n",
       " 'test/18863',\n",
       " 'test/18882',\n",
       " 'test/18895',\n",
       " 'test/18898',\n",
       " 'test/18906',\n",
       " 'test/18910',\n",
       " 'test/18921',\n",
       " 'test/18922',\n",
       " 'test/18938',\n",
       " 'test/18959',\n",
       " 'test/18962',\n",
       " 'test/18963',\n",
       " 'test/18975',\n",
       " 'test/18986',\n",
       " 'test/18988',\n",
       " 'test/19006',\n",
       " 'test/19042',\n",
       " 'test/19048',\n",
       " 'test/19050',\n",
       " 'test/19087',\n",
       " 'test/19113',\n",
       " 'test/19118',\n",
       " 'test/19127',\n",
       " 'test/19136',\n",
       " 'test/19137',\n",
       " 'test/19152',\n",
       " 'test/19155',\n",
       " 'test/19159',\n",
       " 'test/19180',\n",
       " 'test/19181',\n",
       " 'test/19202',\n",
       " 'test/19208',\n",
       " 'test/19240',\n",
       " 'test/19241',\n",
       " 'test/19255',\n",
       " 'test/19256',\n",
       " 'test/19258',\n",
       " 'test/19263',\n",
       " 'test/19270',\n",
       " 'test/19271',\n",
       " 'test/19272',\n",
       " 'test/19280',\n",
       " 'test/19281',\n",
       " 'test/19283',\n",
       " 'test/19286',\n",
       " 'test/19290',\n",
       " 'test/19297',\n",
       " 'test/19307',\n",
       " 'test/19312',\n",
       " 'test/19325',\n",
       " 'test/19327',\n",
       " 'test/19331',\n",
       " 'test/19334',\n",
       " 'test/19341',\n",
       " 'test/19343',\n",
       " 'test/19344',\n",
       " 'test/19351',\n",
       " 'test/19352',\n",
       " 'test/19356',\n",
       " 'test/19359',\n",
       " 'test/19360',\n",
       " 'test/19364',\n",
       " 'test/19368',\n",
       " 'test/19369',\n",
       " 'test/19378',\n",
       " 'test/19379',\n",
       " 'test/19380',\n",
       " 'test/19381',\n",
       " 'test/19383',\n",
       " 'test/19386',\n",
       " 'test/19406',\n",
       " 'test/19409',\n",
       " 'test/19411',\n",
       " 'test/19412',\n",
       " 'test/19419',\n",
       " 'test/19420',\n",
       " 'test/19426',\n",
       " 'test/19428',\n",
       " 'test/19432',\n",
       " 'test/19435',\n",
       " 'test/19441',\n",
       " 'test/19447',\n",
       " 'test/19448',\n",
       " 'test/19451',\n",
       " 'test/19453',\n",
       " 'test/19461',\n",
       " 'test/19462',\n",
       " 'test/19464',\n",
       " 'test/19470',\n",
       " 'test/19473',\n",
       " 'test/19482',\n",
       " 'test/19501',\n",
       " 'test/19507',\n",
       " 'test/19510',\n",
       " 'test/19572',\n",
       " 'test/19630',\n",
       " 'test/19631',\n",
       " 'test/19635',\n",
       " 'test/19645',\n",
       " 'test/19649',\n",
       " 'test/19654',\n",
       " 'test/19663',\n",
       " 'test/19666',\n",
       " 'test/19675',\n",
       " 'test/19679',\n",
       " 'test/19681',\n",
       " 'test/19683',\n",
       " 'test/19690',\n",
       " 'test/19697',\n",
       " 'test/19699',\n",
       " 'test/19702',\n",
       " 'test/19703',\n",
       " 'test/19706',\n",
       " 'test/19711',\n",
       " 'test/19713',\n",
       " 'test/19719',\n",
       " 'test/19726',\n",
       " 'test/19727',\n",
       " 'test/19731',\n",
       " 'test/19732',\n",
       " 'test/19734',\n",
       " 'test/19737',\n",
       " 'test/19744',\n",
       " 'test/19748',\n",
       " 'test/19751',\n",
       " 'test/19758',\n",
       " 'test/19760',\n",
       " 'test/19762',\n",
       " 'test/19763',\n",
       " 'test/19766',\n",
       " 'test/19767',\n",
       " 'test/19769',\n",
       " 'test/19773',\n",
       " 'test/19780',\n",
       " 'test/19796',\n",
       " 'test/19799',\n",
       " 'test/19810',\n",
       " 'test/19823',\n",
       " 'test/19826',\n",
       " 'test/19828',\n",
       " 'test/19830',\n",
       " 'test/19831',\n",
       " 'test/19833',\n",
       " 'test/19837',\n",
       " 'test/19839',\n",
       " 'test/19856',\n",
       " 'test/19860',\n",
       " 'test/19864',\n",
       " 'test/19866',\n",
       " 'test/19870',\n",
       " 'test/19872',\n",
       " 'test/19874',\n",
       " 'test/19880',\n",
       " 'test/19890',\n",
       " 'test/19892',\n",
       " 'test/19895',\n",
       " 'test/19900',\n",
       " 'test/19902',\n",
       " 'test/19906',\n",
       " 'test/19912',\n",
       " 'test/19913',\n",
       " 'test/19921',\n",
       " 'test/19930',\n",
       " 'test/19935',\n",
       " 'test/19939',\n",
       " 'test/19943',\n",
       " 'test/19946',\n",
       " 'test/19950',\n",
       " 'test/19955',\n",
       " 'test/19959',\n",
       " 'test/19962',\n",
       " 'test/19963',\n",
       " 'test/19965',\n",
       " 'test/19968',\n",
       " 'test/19975',\n",
       " 'test/19981',\n",
       " 'test/19983',\n",
       " 'test/19984',\n",
       " 'test/20035',\n",
       " 'test/20037',\n",
       " 'test/20039',\n",
       " 'test/20114',\n",
       " 'test/20137',\n",
       " 'test/20142',\n",
       " 'test/20193',\n",
       " 'test/20210',\n",
       " 'test/20216',\n",
       " 'test/20297',\n",
       " 'test/20298',\n",
       " 'test/20306',\n",
       " 'test/20311',\n",
       " 'test/20321',\n",
       " 'test/20328',\n",
       " 'test/20352',\n",
       " 'test/20379',\n",
       " 'test/20396',\n",
       " 'test/20398',\n",
       " 'test/20439',\n",
       " 'test/20457',\n",
       " 'test/20467',\n",
       " 'test/20485',\n",
       " 'test/20489',\n",
       " 'test/20525',\n",
       " 'test/20552',\n",
       " 'test/20584',\n",
       " 'test/20618',\n",
       " 'test/20641',\n",
       " 'test/20667',\n",
       " 'test/20679',\n",
       " 'test/20693',\n",
       " 'test/20698',\n",
       " 'test/20725',\n",
       " 'test/20734',\n",
       " 'test/20744',\n",
       " 'test/20754',\n",
       " 'test/20755',\n",
       " 'test/20766',\n",
       " 'test/20775',\n",
       " 'test/20792',\n",
       " 'test/20797',\n",
       " 'test/20811',\n",
       " 'test/20824',\n",
       " 'test/20834',\n",
       " 'test/20835',\n",
       " 'test/20840',\n",
       " 'test/20887',\n",
       " 'test/20898',\n",
       " 'test/20901',\n",
       " 'test/20905',\n",
       " 'test/20917',\n",
       " 'test/20921',\n",
       " 'test/20945',\n",
       " 'test/20946',\n",
       " 'test/20982',\n",
       " 'test/20995',\n",
       " 'test/21000',\n",
       " 'test/21003',\n",
       " 'test/21007',\n",
       " 'test/21019',\n",
       " 'test/21030',\n",
       " 'test/21039',\n",
       " 'test/21040',\n",
       " 'test/21041',\n",
       " 'test/21046',\n",
       " 'test/21050',\n",
       " 'test/21051',\n",
       " 'test/21059',\n",
       " 'test/21060',\n",
       " 'test/21061',\n",
       " 'test/21063',\n",
       " 'test/21065',\n",
       " 'test/21071',\n",
       " 'test/21073',\n",
       " 'test/21078',\n",
       " 'test/21082',\n",
       " 'test/21093',\n",
       " 'test/21097',\n",
       " 'test/21129',\n",
       " 'test/21133',\n",
       " 'test/21134',\n",
       " 'test/21140',\n",
       " 'test/21143',\n",
       " 'test/21201',\n",
       " 'test/21210',\n",
       " 'test/21228',\n",
       " 'test/21230',\n",
       " 'test/21251',\n",
       " 'test/21253',\n",
       " 'test/21261',\n",
       " 'test/21278',\n",
       " 'test/21282',\n",
       " 'test/21287',\n",
       " 'test/21291',\n",
       " 'test/21292',\n",
       " 'test/21297',\n",
       " 'test/21301',\n",
       " 'test/21307',\n",
       " 'test/21318',\n",
       " 'test/21319',\n",
       " 'test/21333',\n",
       " 'test/21357',\n",
       " 'test/21358',\n",
       " 'test/21375',\n",
       " 'test/21376',\n",
       " 'test/21382',\n",
       " 'test/21387',\n",
       " 'test/21393',\n",
       " 'test/21394',\n",
       " 'test/21401',\n",
       " 'test/21404',\n",
       " 'test/21408',\n",
       " 'test/21410',\n",
       " 'test/21411',\n",
       " 'test/21417',\n",
       " 'test/21419',\n",
       " 'test/21421',\n",
       " 'test/21427',\n",
       " 'test/21429',\n",
       " 'test/21440',\n",
       " 'test/21454',\n",
       " 'test/21462',\n",
       " 'test/21466',\n",
       " 'test/21469',\n",
       " 'test/21478',\n",
       " 'test/21503',\n",
       " 'test/21538',\n",
       " 'test/21550',\n",
       " 'test/21555',\n",
       " 'test/21571',\n",
       " 'training/10',\n",
       " 'training/1000',\n",
       " 'training/10005',\n",
       " 'training/10018',\n",
       " 'training/10025',\n",
       " 'training/10042',\n",
       " 'training/10049',\n",
       " 'training/10052',\n",
       " 'training/10058',\n",
       " 'training/10065',\n",
       " 'training/10066',\n",
       " 'training/10071',\n",
       " 'training/10073',\n",
       " 'training/10078',\n",
       " 'training/10086',\n",
       " 'training/10088',\n",
       " 'training/10098',\n",
       " 'training/10102',\n",
       " 'training/10106',\n",
       " 'training/10128',\n",
       " 'training/10129',\n",
       " 'training/10139',\n",
       " 'training/10146',\n",
       " 'training/10158',\n",
       " 'training/10168',\n",
       " 'training/10171',\n",
       " 'training/10190',\n",
       " 'training/1020',\n",
       " 'training/10213',\n",
       " 'training/10218',\n",
       " 'training/10219',\n",
       " 'training/10224',\n",
       " 'training/10238',\n",
       " 'training/1029',\n",
       " 'training/10310',\n",
       " 'training/10311',\n",
       " 'training/10313',\n",
       " 'training/1032',\n",
       " 'training/10335',\n",
       " 'training/10348',\n",
       " 'training/10361',\n",
       " 'training/10371',\n",
       " 'training/1039',\n",
       " 'training/10399',\n",
       " 'training/10404',\n",
       " 'training/10414',\n",
       " 'training/10419',\n",
       " 'training/10423',\n",
       " 'training/10429',\n",
       " 'training/10434',\n",
       " 'training/10435',\n",
       " 'training/10442',\n",
       " 'training/10446',\n",
       " 'training/10453',\n",
       " 'training/10454',\n",
       " 'training/10458',\n",
       " 'training/10462',\n",
       " 'training/10467',\n",
       " 'training/10473',\n",
       " 'training/10508',\n",
       " 'training/10518',\n",
       " 'training/10527',\n",
       " 'training/10541',\n",
       " 'training/10551',\n",
       " 'training/10555',\n",
       " 'training/10563',\n",
       " 'training/10576',\n",
       " 'training/10582',\n",
       " 'training/10585',\n",
       " 'training/10587',\n",
       " 'training/10596',\n",
       " 'training/10674',\n",
       " 'training/10676',\n",
       " 'training/10692',\n",
       " 'training/10694',\n",
       " 'training/10714',\n",
       " 'training/10721',\n",
       " 'training/1074',\n",
       " 'training/10744',\n",
       " 'training/1076',\n",
       " 'training/10768',\n",
       " 'training/10784',\n",
       " 'training/10786',\n",
       " 'training/10787',\n",
       " 'training/10788',\n",
       " 'training/10794',\n",
       " 'training/10815',\n",
       " 'training/10816',\n",
       " 'training/10831',\n",
       " 'training/10834',\n",
       " 'training/10843',\n",
       " 'training/10844',\n",
       " 'training/10848',\n",
       " 'training/10849',\n",
       " 'training/10867',\n",
       " 'training/10871',\n",
       " 'training/10872',\n",
       " 'training/10874',\n",
       " 'training/10883',\n",
       " 'training/10887',\n",
       " 'training/10892',\n",
       " 'training/10895',\n",
       " 'training/10899',\n",
       " 'training/10901',\n",
       " 'training/10908',\n",
       " 'training/10909',\n",
       " 'training/10918',\n",
       " 'training/10928',\n",
       " 'training/10941',\n",
       " 'training/10950',\n",
       " 'training/10960',\n",
       " 'training/10961',\n",
       " 'training/10962',\n",
       " 'training/10969',\n",
       " 'training/10971',\n",
       " 'training/10978',\n",
       " 'training/10985',\n",
       " 'training/10988',\n",
       " 'training/10989',\n",
       " 'training/1099',\n",
       " 'training/10992',\n",
       " 'training/10994',\n",
       " 'training/10998',\n",
       " 'training/110',\n",
       " 'training/11006',\n",
       " 'training/11029',\n",
       " 'training/1103',\n",
       " 'training/1104',\n",
       " 'training/11041',\n",
       " 'training/11048',\n",
       " 'training/11051',\n",
       " 'training/11056',\n",
       " 'training/11063',\n",
       " 'training/11066',\n",
       " 'training/11074',\n",
       " 'training/11078',\n",
       " 'training/11086',\n",
       " 'training/11088',\n",
       " 'training/11095',\n",
       " 'training/11101',\n",
       " 'training/11102',\n",
       " 'training/11113',\n",
       " 'training/11126',\n",
       " 'training/11133',\n",
       " 'training/11134',\n",
       " 'training/11136',\n",
       " 'training/11139',\n",
       " 'training/11143',\n",
       " 'training/11144',\n",
       " 'training/11161',\n",
       " 'training/11177',\n",
       " 'training/1118',\n",
       " 'training/11192',\n",
       " 'training/11201',\n",
       " 'training/11202',\n",
       " 'training/11210',\n",
       " 'training/11213',\n",
       " 'training/1123',\n",
       " 'training/11236',\n",
       " 'training/11237',\n",
       " 'training/11239',\n",
       " 'training/11242',\n",
       " 'training/11252',\n",
       " 'training/11253',\n",
       " 'training/11255',\n",
       " 'training/11257',\n",
       " 'training/11268',\n",
       " 'training/11274',\n",
       " 'training/11279',\n",
       " 'training/11285',\n",
       " 'training/11289',\n",
       " 'training/11292',\n",
       " 'training/11295',\n",
       " 'training/11299',\n",
       " 'training/11324',\n",
       " 'training/11344',\n",
       " 'training/11348',\n",
       " 'training/11349',\n",
       " 'training/11362',\n",
       " 'training/11369',\n",
       " 'training/11373',\n",
       " 'training/11374',\n",
       " 'training/11378',\n",
       " 'training/11386',\n",
       " 'training/11393',\n",
       " 'training/11396',\n",
       " 'training/11402',\n",
       " 'training/11405',\n",
       " 'training/11431',\n",
       " 'training/11441',\n",
       " 'training/11449',\n",
       " 'training/11464',\n",
       " 'training/11465',\n",
       " 'training/11468',\n",
       " 'training/11469',\n",
       " 'training/11477',\n",
       " 'training/11484',\n",
       " 'training/11488',\n",
       " 'training/11492',\n",
       " 'training/11504',\n",
       " 'training/11505',\n",
       " 'training/11508',\n",
       " 'training/11514',\n",
       " 'training/11515',\n",
       " 'training/11522',\n",
       " 'training/11523',\n",
       " 'training/11526',\n",
       " 'training/11527',\n",
       " 'training/1153',\n",
       " 'training/11539',\n",
       " 'training/11550',\n",
       " 'training/11553',\n",
       " 'training/11563',\n",
       " 'training/11579',\n",
       " 'training/11584',\n",
       " 'training/1159',\n",
       " 'training/11597',\n",
       " 'training/11598',\n",
       " 'training/11599',\n",
       " 'training/11602',\n",
       " 'training/11619',\n",
       " 'training/11623',\n",
       " 'training/1164',\n",
       " 'training/11657',\n",
       " 'training/11661',\n",
       " 'training/11679',\n",
       " 'training/11707',\n",
       " 'training/11722',\n",
       " 'training/11738',\n",
       " 'training/11743',\n",
       " 'training/11744',\n",
       " 'training/11748',\n",
       " 'training/11751',\n",
       " 'training/11755',\n",
       " 'training/11762',\n",
       " 'training/11763',\n",
       " 'training/11790',\n",
       " 'training/11791',\n",
       " 'training/11793',\n",
       " 'training/11808',\n",
       " 'training/11812',\n",
       " 'training/11816',\n",
       " 'training/11821',\n",
       " 'training/11822',\n",
       " 'training/11825',\n",
       " 'training/11829',\n",
       " 'training/11833',\n",
       " 'training/11837',\n",
       " 'training/11841',\n",
       " 'training/11847',\n",
       " 'training/11854',\n",
       " 'training/11860',\n",
       " 'training/11863',\n",
       " 'training/11867',\n",
       " 'training/11871',\n",
       " 'training/11872',\n",
       " 'training/11873',\n",
       " 'training/11876',\n",
       " 'training/11888',\n",
       " 'training/11891',\n",
       " 'training/11892',\n",
       " 'training/11895',\n",
       " 'training/11905',\n",
       " 'training/11909',\n",
       " 'training/11910',\n",
       " 'training/11915',\n",
       " 'training/11916',\n",
       " 'training/11920',\n",
       " 'training/11923',\n",
       " 'training/11954',\n",
       " 'training/11962',\n",
       " 'training/11968',\n",
       " 'training/11981',\n",
       " 'training/11982',\n",
       " 'training/11996',\n",
       " 'training/11997',\n",
       " 'training/11999',\n",
       " 'training/12',\n",
       " 'training/12007',\n",
       " 'training/12018',\n",
       " 'training/12028',\n",
       " ...]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reuters.fileids('acq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['earn',\n",
       " 'acq',\n",
       " 'money-fx',\n",
       " 'grain',\n",
       " 'crude',\n",
       " 'trade',\n",
       " 'interest',\n",
       " 'ship',\n",
       " 'wheat',\n",
       " 'corn']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts = [(_, len(reuters.fileids(_))) for _ in reuters.categories()]\n",
    "top_10_categories = sorted(counts, key=itemgetter(1), reverse=True)[:10]\n",
    "top_10_categories = [_[0] for _ in top_10_categories]\n",
    "top_10_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "earn 2877\n",
      "acq 1650\n",
      "money-fx 538\n",
      "grain 433\n",
      "crude 389\n",
      "trade 368\n",
      "interest 347\n",
      "ship 197\n",
      "wheat 212\n",
      "corn 181\n"
     ]
    }
   ],
   "source": [
    "with open('Reuters-R10-training.csv', 'wt') as stream:\n",
    "    writer = csv.writer(stream)\n",
    "    for category in top_10_categories:\n",
    "        files = [_ for _ in reuters.fileids(category) if 'training/' in _]\n",
    "        print(category, len(files))\n",
    "        for _file in files:\n",
    "            writer.writerow([category, ' '.join(reuters.words(_file))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "earn 1087\n",
      "acq 719\n",
      "money-fx 179\n",
      "grain 149\n",
      "crude 189\n",
      "trade 117\n",
      "interest 131\n",
      "ship 89\n",
      "wheat 71\n",
      "corn 56\n"
     ]
    }
   ],
   "source": [
    "with open('Reuters-R10-test.csv', 'wt') as stream:\n",
    "    writer = csv.writer(stream)\n",
    "    for category in top_10_categories:\n",
    "        files = [_ for _ in reuters.fileids(category) if 'test/' in _]\n",
    "        print(category, len(files))\n",
    "        for _file in files:\n",
    "            writer.writerow([category, ' '.join(reuters.words(_file))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Reuters-R10-training.csv') as stream:\n",
    "    reader = csv.reader(stream)\n",
    "    for row in reader:\n",
    "        assert(len(row) == 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Reuters-R10-test.csv') as stream:\n",
    "    reader = csv.reader(stream)\n",
    "    for row in reader:\n",
    "        assert(len(row) == 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}