citation_window.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "import csv\n",
    "import os, re, sys\n",
    "from collections import Counter\n",
    "from IPython.core.debugger import Tracer; debug_here = Tracer()\n",
    "from nltk import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5537\n"
     ]
    }
   ],
   "source": [
    "# Talk page usernames\n",
    "csvpath = '/home/michael/school/research/wp/wikipedia/data/talk/ipc_utf8_talkpages.csv'\n",
    "talk_counter = Counter()\n",
    "\n",
    "with open(csvpath, 'r') as csvfile:\n",
    "    reader = csv.reader(csvfile, delimiter=',')\n",
    "    for row in reader:\n",
    "        talk_counter[row[3]] += 1\n",
    "\n",
    "talk_eds = set(talk_counter.keys())\n",
    "print(len(talk_eds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(\"<ref name='\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess('<ref name=\"something\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'all these thing '"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess('all these thing <ref name=\"something\">')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' or nothing'"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess('<ref name=\"something\"> or nothing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess('{{cite web|url=|title=Peace talks set to begin after Israel agrees to free 104 Palestinian prisoners}}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Process text\n",
    "\n",
    "#urlpat = re.compile(r'https?:\\/\\/(?:www\\.)?[^ ]*')\n",
    "#urlhtml = re.compile(r'[^ ].*?\\.html?')\n",
    "#temppat = re.compile(r'\\{\\{.*?(?:\\}\\}|$)')\n",
    "#tempout = re.compile(r'^.*?\\}\\}')\n",
    "#linkpat = re.compile(r'\\[\\[.*?\\|')\n",
    "\n",
    "pats = []\n",
    "# urls\n",
    "pats.append(r'https?:\\/\\/(?:www\\.)?[^ ]*')\n",
    "pats.append(r'[^ ].*?\\.html?')\n",
    "\n",
    "# templates\n",
    "pats.append(r'\\{\\{.*?(?:\\}\\}|$)')\n",
    "pats.append(r'^.*?\\}\\}')\n",
    "\n",
    "# links\n",
    "pats.append(r'\\[\\[.*?\\|')\n",
    "\n",
    "# refs\n",
    "# pats.append(r'<ref name=.*?[>$]')\n",
    "pats.append(r'<ref name=.*?(?:>|$)')\n",
    "pat = re.compile('|'.join(pats))\n",
    "\n",
    "def preprocess(intext):\n",
    "    outtext = pat.sub('', intext)\n",
    "    # urls\n",
    "#    outtext = re.sub(r'https?:\\/\\/(?:www\\.)?[^ ]*', '', intext)\n",
    "##     outtext = re.sub(r'https?:\\/\\/(?:www\\.)?[^ ]*', '', intext, flags=re.MULTILINE)\n",
    "##     outtext = urlpat.sub('', intext)\n",
    "##     print('1 sub')\n",
    "##     sys.stdout.flush()\n",
    "#    outtext = re.sub(r'[^ ].*?\\.html?', '', outtext)\n",
    "##     outtext = re.sub(r'[^ ].*?\\.html?', '', outtext, flags=re.MULTILINE)\n",
    "##     outtext = urlhtml.sub('', outtext)\n",
    "##     print('2 sub')\n",
    "##     sys.stdout.flush()\n",
    "#    \n",
    "#    # templates\n",
    "#    outtext = re.sub(r'\\{\\{.*?(?:\\}\\}|$)','', outtext)\n",
    "##     outtext = re.sub(r'\\{\\{.*?(?:\\}\\}|$)','', outtext, flags=re.MULTILINE)\n",
    "##     outtext = temppat.sub('', outtext)\n",
    "##     print('3 sub')\n",
    "##     sys.stdout.flush()\n",
    "#    outtext = re.sub(r'^.*?\\}\\}','', outtext)\n",
    "##     outtext = re.sub(r'^.*?\\}\\}','', outtext, flags=re.MULTILINE)\n",
    "##     outtext = tempout.sub('', outtext)\n",
    "##     print('4 sub')\n",
    "##     sys.stdout.flush()\n",
    "#    \n",
    "#    # links\n",
    "#    outtext = re.sub(r'\\[\\[.*?\\|', '', outtext)\n",
    "##     outtext = re.sub(r'\\[\\[.*?\\|', '', outtext, flags=re.MULTILINE)\n",
    "##     outtext = linkpat.sub('', outtext)\n",
    "##     print('5 sub')\n",
    "##     sys.stdout.flush()\n",
    "    outtext = outtext.replace('[', '')\n",
    "    outtext = outtext.replace(']', '')\n",
    "    \n",
    "    outtext = outtext.replace('{', '')\n",
    "    outtext = outtext.replace('}', '')\n",
    "    \n",
    "    # <ref>\n",
    "    outtext = outtext.replace('<ref>', '')\n",
    "    outtext = outtext.replace('</ref>', '')\n",
    "    \n",
    "    # titles\n",
    "    outtext = outtext.replace('=', '')\n",
    "    \n",
    "    # categories\n",
    "    outtext = outtext.replace('Category:', '')\n",
    "    \n",
    "    return outtext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101_squadron_(israel)_citediff.csv\n",
      "2013–14_israeli–palestinian_peace_talks_citediff.csv\n"
     ]
    }
   ],
   "source": [
    "# Build editor dictionary with unigrams added\n",
    "edtext = {ed:'' for ed in talk_eds}\n",
    "\n",
    "cite_diffdir = '/home/michael/school/research/wp/wp_articles/citediffs/'\n",
    "# citedir = '/home/michael/school/research/wp/wp_articles/citations/'\n",
    "# revdir = '/home/michael/school/research/wp/wp_articles/extract/extract_articles/ipc_articles_raw/'\n",
    "\n",
    "for f in sorted(os.listdir(cite_diffdir))[:2]:\n",
    "    if not f.startswith('.'):\n",
    "        print(f)\n",
    "        with open(os.path.join(cite_diffdir,f)) as file:\n",
    "            r = csv.DictReader(file)\n",
    "            data = [row for row in r if row['editor'] in talk_eds and row['add/delete']=='add']\n",
    "#             diffdata = pd.read_csv()\n",
    "#             diffdata = diffdata[diffdata['editor'].isin(talk_eds)] # Restrict data to talk editors\n",
    "#             diffdata = diffdata[diffdata['add/delete']=='add'] # Just adds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 k\n",
      "0.1 k\n",
      "0.2 k\n",
      "18\n"
     ]
    }
   ],
   "source": [
    "# %%timeit -n 1 -r 1\n",
    "\n",
    "# Try ONE LINE\n",
    "# Without preprocess: 3.03 s per loop\n",
    "# With preprocess: 4min 41s without pre-compile, 4min 54s with pre-compile\n",
    "\n",
    "# Iterate thru citediffs, building editor dict\n",
    "wdwin = 50\n",
    "\n",
    "for i, row in enumerate(data):\n",
    "    if i % 100 == 0: print(i/1000, 'k')\n",
    "    before = word_tokenize(preprocess(row['before_citation']))\n",
    "#     if row['editor'] == 'Sepsis II':\n",
    "#    if '}' in before:\n",
    "#        debug_here()\n",
    "#     before = word_tokenize(row['before_citation'])\n",
    "#     before = preprocess(row['before_citation'])\n",
    "    edtext[row['editor']] += ' '.join(before[max(len(before)*-1, wdwin*-1):])\n",
    "    after = word_tokenize(preprocess(row['after_citation']))\n",
    "#     if '}' in after: debug_here()\n",
    "#     after = word_tokenize(row['after_citation'])\n",
    "#     after = preprocess(row['after_citation'])\n",
    "    edtext[row['editor']] += ' '.join(after[:min(len(after), wdwin)])\n",
    "\n",
    "print(len([ed for ed in edtext if edtext[ed] != '']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"in 1999 , but never did . Critics also worry that Israel will simply quietly re-arrest the potentially released Palestinians , and state that Israel is using the slow release to hold the negotiations hostage and that the main goal of the release is to bolster Israel 's image .The announcement came after Israel was accused of trying to sabotage the peace talks by approving nearly 1200 new settlement homes , which are on Palestinian land and illegal under international law . Israeli government spokesman Mark Regev dismissed the criticism . See also * Geneva II Middle East peace; he said that what he meant was no Israelis who were `` part of the occupation '' , but that he would n't have a problem with Jews or Israelis coming to Palestine for business or tourism reasons , as long as they were not an occupying force .See also *Peace process in the Israeli–Palestinian conflict * Geneva II Middle East peace conference on Syria References Israeli–Palestinian conflict; he said that what he meant was no Israelis who were `` part of the occupation '' , but that he would n't have a problem with Jews or Israelis coming to Palestine for business or tourism reasons , as long as they were not an occupying force .See also *Peace process in the Israeli–Palestinian conflict * Geneva II Middle East peace conference on Syria References Israeli–Palestinian conflictin 1999 , but never did . Critics also worry that Israel will simply quietly re-arrest the potentially released Palestinians , and state that Israel is using the slow release to hold the negotiations hostage and that the main goal of the release is to bolster Israel 's image .Israel and new settlements Israel was accused by Palestinian officials of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It is well known that these are an obstacle for the Palestinians as they are on Palestinian7 November John Kerry said that if the peace talks failed , there would likely be a third intifada .See also *Peace process in the Israeli–Palestinian conflict References External links * The History of the Peace Process in the Context of the 2013 John Kerry Peace Efforts Israeli–Palestinian conflict Israeli–Palestinian peace efforts7 November John Kerry said that if the peace talks failed , there would likely be a third intifada .See also *Peace process in the Israeli–Palestinian conflict References External links * The History of the Peace Process in the Context of the 2013 John Kerry Peace Efforts Israeli–Palestinian conflict Israeli–Palestinian peace efforts. It is well known that these are an obstacle for the Palestinians as they are on Palestinian land and are illegal under international law . Israeli government spokesman Mark Regev stated that these settlements in Palestine would `` remain part of Israel in any possible peace agreements . ''On 30 October , Israel stated it would go ahead with plans to build 3,500 more homes for Jewish Israelis in settlements in Palestine .On 30 October , Israel stated it would go ahead with plans to build 3,500 more homes for Jewish Israelis in settlements in Palestine .< ! -- one sentence section need merging -- > 29 July As Israeli and Palestinian negotiators met for a second day in Washington to discuss renewing peace talks , url Abbas foresees Palestinian state with no Israelis 13 August On the first day the Palestinian team leaders were Saebcompromises Before the peace talks began , both sides offered a gift . Palestine offered to put on hold international recognition as a state while Israel offered the release of 104 Palestinian prisoners , all of whom have been in Israeli jails since before the 1993 Oslo I Accord .|titleThe Sharm el Sheikh Memorandum but never did . Critics also worry that Israel will simply quietly re-arrest the potentially released Palestinians , and state that Israel is using the slow release to hold the negotiations hostage and that the main goal of the release is to bolster Israel 's|accessdate4 January 2014|newspaper2 External links Israeli–Palestinian peace efforts 2013 in international relations 2014 in international relations 2013 in Israel 2014 in Israel 2013 in the Palestinian territories 2014 in the Palestinian territories' '' Direct negotiations between Israel and the Palestinians '' ' began on July 29 , 2013 following an attempt by United States Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|dateJuly 29 2013 The negotiations were scheduled to lastStates Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|dateJuly 29 2013 The negotiations were scheduled to last up to nine months to reach a final status to the Palestinian-Israeli conflict by mid-2014 . They started in Washington , DC|titlePeace talks set to begin after Israel agrees to free 104 Palestinian prisoners|workThe Washington Post and will then move to the lastHarris|firstEmily|titleKerry Tries To Get Israeli-Palestinian Peace Talks Back On Track|url|titleThe Sharm el Sheikh Memorandum|workThe Knesset but never did .|accessdate4 January 2014|newspaperAl Jazeera|date10 August 2013 According to the Sharm el-Sheikh Fact-Finding Committee Report , Israel 's decision not to release the prisoners at the time was due to significantly increased violence against Israel by their partner in threlease from detention of terrorists ; the failure to control illegal weapons ; and the actual conduct of violent operations '' as a sign that `` the PLO has explicitly violated its renunciation of terrorism and other acts of violence , thereby significantly eroding trust between the parties . ''|workUS department of State|accessdate7 February 2014|date30 April 2001 Timeline On 29 July 2013 , as Israeli and Palestinian negotiators met for a second day in Washington to discuss renewing peace talks , titleMahmoud Abbas foresees Palestinian state with no Israelis|url January 2014|newspaperLos Angeles Times|date30 July 2013|titleThe Sharm el Sheikh Memorandum|workThe Knesset but never did .|accessdate4 January 2014|newspaperAl Jazeera|date10 August 2013 According to the Sharm el-Sheikh Fact-Finding Committee Report , Israel 's decision not to release the prisoners at the time was due to significantly increased violence against Israel by their partner in the memorandum , the PLto work for release of all inmates|workJpost Critics also worry that Israel will simply quietly re-arrest the potentially released Palestinians , and state that Israel is using the slow release to hold the negotiations hostage and that the main goal of the release is to bolster Israel 's image .|accessdate4 January 2014|newspaperAl Jazeera|date10 August 2013 According to the Sharm el-Sheikh Fact-Finding Committee Report , Israel 's decision not to release the prisoners at the time was due to significantly increased violence against Israel by their partner in the memorandum , the PLO , leading up to the Second Intifada, in all , 55 Israeli civilians , 15 Israeli security forces personnel , one French tourist and dozens of suspected Palestinian collaborators . Commenters have however pointed out that Israel had already promised to release these same 104 Palestinians , back in 1999 under the Sharm el-Sheikh Memorandum ,|accessdate4 January 2014|newspaperAl Jazeera|date10 August 2013 According to the Sharm el-She, in all , 55 Israeli civilians , 15 Israeli security forces personnel , one French tourist and dozens of suspected Palestinian collaborators . Commenters have however pointed out that Israel had already promised to release these same 104 Palestinians , back in 1999 under the Sharm el-Sheikh Memorandum ,|accessdate4 January 2014|newspaperAl Jazeera|date10 August 2013 According to the Sharm el-Sheikh Fact-Findingby Likud legislator url reject Regev proposal to annex West Bank settlements On 28th of March 2014 , Israel failed to release the fourth tranche of 26 Palestinian prisoners , as scheduled , in what Palestinian sources say was a violation of the original terms for the peace talks ,ael Wilner , Herb Keinon , 'US : No Palestinian prisoner release is violation of terms of talks , ' Jerusalem Post , 29 March 2014. which included a Palestinian undertaking not to sign up for international conventions . After Israel withheld the prisoners ' release , Mahmoud Abbas wentinternational conventions . After Israel withheld the prisoners ' release , Mahmoud Abbas went ahead and signed 15 conventions regarding adhesion to human and social rights . Israel then stated the prisoners ' release depended on a Palestinian commitment to continuing peace talks after the end of April deadline .Reactions US Secretary of State urlon its commitment to release the last 26 prisoners in late March 2014 . In response Palestinian President Mamhoud Abbas signed papers to join 15 of 63 international organizations and treaties . Several days later Israel approved 700 more Israeli settlements in Palestine followed by various sanctions against Palestinians .Reactions US Secretary of State url warns Israel of troubles ahead if peace talks fail|workReuters Despite all efforts of John Kerry , Palestinian President Mahmoud Abbas said that if `` there 's no progress , and it 's all Israel 's fault '' and blamed Israel , saying `` theon its commitment to release the last 26 prisoners in late March 2014 . In response Palestinian President Mamhoud Abbas signed papers to join 15 of 63 international organizations and treaties . Several days later Israel approved 700 more Israeli settlements in Palestine followed by various sanctions against Palestinians .Reactions US Secretary of State url warns Israel of troubles ahead if peace talks fail|workReuters Despite all efforts of John Kerry , Palestinian President Mahmoud Abbas said that if `` there 's no progress , and it 's all Israel 's fault '' and blamed Israel , saying `` thegetting somewhere , after all|date11 January 2014|workThe Economist On 10 January 2014 , Israel approved plans for 1,400 settler homes . Saeb Erekat responded by saying `` The recent announcement shows Israel 's clear commitment to the destruction of peace efforts and the imposition of an apartheid regime '' .titleKerry meeting with Netanyahu , Peres postponed due to snow|url January 2014|newspaperJpost On 14 January , Israel 's defense minister Moshe Ya'alon rejected the negotiations and insulted John Kerry , saying he was acting based upon `` messianic feeling '' , and that `` The only thing that can 'saveReactions US Secretary of State John Kerry said that if the peace talks failed , there would likely be a third intifada .Despite all efforts of John Kerry , Palestinian President Mahmoud Abbas said that if `` there 's no progress , and it 's all Israel 's fault '' and blamed Israel , saying `` the problem is with the Israeli side and not with us , '' EU Ambassador toat 85 in favor to 35 opposed . In addition to the Labor Party , American negotiators were also attempting to persuade Haredi parties Shas and United Torah Judaism , both of which are generally supportive of the peace process , to join the government to keep negotiations alive .On 10 January 2014 , Israel approved plans for 1,400 settler homes . Saeb Erekat responded by saying `` The recent announcement shows Israel 's clear commitment to the destruction of peace efforts and the imposition of an apartheid regime '' . titleKblamed for the break down . Yair Lapid , Israel 's finance minister said that the country could be targeted by an economically costly boycott if peace talks with the Palestinians fail , signalling that concerns about growing international isolation have moved centre stage in Israel 's public discourse .|accessdate7 February 2014|newspaperThe Independent|date31 January 2014|locationLondon|firstIan|lastDeitch Some critics believe that Israel is only trying to `` put on a show , '' claiming the Israelis do not seek a peace agreement , but are using these peace talks to further other goals , including improving their image , strengthening theirStates Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|date29 July 2013 The negotiations were scheduled to last up to nine months to reach a final status to the Palestinian-Israeli conflict by mid-2014 . They started in Washington , DC|titlePeace talks set to begin after Israel agrees to free 104 Palestinian prisoners|workThe Washington Post | firstWilliam|lastBooth|date29 July 2013 and will then move to the lastHarris|firstEmily|title, Mahmoud Abbas spoke in front of the UN Security Council , and welcomed the resumption of peace talks while at the same time criticizing Israel 's settlement building . The Israeli delegation was not present for Abbas ' speech , because they were observing the holiday of Sukkot .|titleAt U.N. , Palestinian leader Abbas criticizes Israel , but pledges good faith in peace talks|accessdate7 January 2014|workWashington post|firstAnne|lastGearan|date26 September 2013 Hamas and the Islamic Jihad called for a third intifada , and a spokesman for Hamas ' armed wing said that the current peace talks were `` futile ''the West Bank 's border with neighbouring Jordan . The new security arrangements would see a zone created with hi-tech fences equipped with sensors and drone surveillance planes flying overhead . Also the final peace treaty could also provide compensation for victims on both sides of the historic conflict .|accessdate7 February 2014|newspaperBloomberg|dateFebruary 4 , 2014 On 6 February , Israel reportedly seeks to annex 10 percent of the West Bank , but Palestinian negotiators insisted that they keep at leanot accept temporary solutions|accessdate7 February 2014|newspaperJpost On 8 September , Israel accused the Palestinians of leaking information about the talks , which are supposed to be kept secret , to the press . An Israeli official also stated that some of the information leaked by Palestinians was not true .|accessdate4 January 2014|newspaperYnet News On 26 September , Mahmoud Abbas spoke in front of the UN Security Council , and welcomed the resumption of peace talks while at the same time criticizing Israel 's settlement building . The Israeli delegation was not present for Abbas ' speech , because theyrenewing peace talks , Mahmoud Abbas said `` in a final resolution , we would not see the presence of a single Israeli – civilian or soldier – on our lands . '' His comments drew immediate condemnation from Israeli officials , who accused him for discriminating against Jews .As Obama Admin Pushes Peace Process , Abbas Vows No Israelis in Palestine : ‘Racist and Hateful Vision’ Abbas : 'Not a single Israeli ' in future Palestinian state On 13 August , the first day , the Palestinian team leaders were Saeb Erekat and Muhammed Shtayyeh while threlease the fourth tranche of 26 Palestinian prisoners , as scheduled , in what Palestinian sources say was a violation of the original terms for the peace talks , Michael Wilner , Herb Keinon , 'US : No Palestinian prisoner release is violation of terms of talks , ' E1Reactions US Secretary of State url warns Israel of troubles ahead if peace talks fail|worksettlements beyond the Green Line , in Gilo , followed by various sanctions against Palestinians in retaliation for their joining of international conventions.Nir Hasson , Barak Ravid , 'While Kerry tries to clinch deal , Israel issues 700 tenders beyond Green Line , ' Haaretz , 1 April 2014 .Despite all efforts of John Kerry , Palestinian President Mahmoud Abbas said that if `` there 's no progress , and it 's all Israel 's fault '' and blamed Israel , saying `` the problem is with the Israeli side and not with us , '''Palestinian UN moves designed to avoid U.S. retaliation , ' Reuters , 3 April 2003 Some days later , Israel approved tenders for 708 more Israeli settlements beyond the Green Line , in Gilo , followed by various sanctions against Palestinians in retaliation for their joining of international conventions .Despite all efforts of John Kerry , Palestinian Presi|accessdate4 January 2014|newspaper2 External links Israeli–Palestinian peace process 2013 in international relations 2014 in international relations 2013 in Israel 2014 in Israel 2013 in the Palestinian territories 2014 in the Palestinian territories' '' Direct negotiations between Israel and the Palestinians '' ' began on 29 July 2013 following an attempt by United States Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|date29 July 2013 The negotiations were scheduled to last up|newspaperYnet News|date30 July 2013|newspaperYnet News|date29 April 2014 Palestinian official titleIsraeli settlement construction doubles in 2013|url Press|date3 March 2014 For its part , Israeli spokesman titleIsrael blames terror surge on Palestinian incitement|url Times of Israel|date25 December 2013 < /ref|newspaperYnet News|date29 April 2014 Palestinian official titleIsraeli settlement construction doubles in 2013|url Press|date3 March 2014 For its part , Israeli spokesman titleIsrael blames terror surge on Palestinian incitement|url Times of Israel|date25 December 2013 According to B'Tselem , during this same period forty-five Palestinians and six Israelis were killed .Despite all efforts of John Kerry , Palestinian President Mahmoud Abbas said that if `` there 's no progress , and it 's all Israel 's fault '' and blamed Israel , saying `` the problem is with the Israeli side and not with us , '' In January ,titleIsraeli settlement construction doubles in 2013|url Press|date3 March 2014 For its part , Israeli spokesman Mark Regev condemned Palestinian incitement , saying `` the terrorist attacks against Israelis over the last few days are a direct result of the incitement and hatred propagated in Palestinian schools and media . ''According to url warns Israel of troubles ahead if peace talks fail|workReuters | date7 November 2013 Despite all efforts of John Kerry , Palestinian President Mahmoud Abbas said that if `` there 's no progress , and it 's all Israel 's fault '' and blamed Israel , saying ``|newspaperYnet News|date29 April 2014 Palestinian official Nabil Shaath condemned settlement construction , saying `` the settlement activities have made negotiations worthless . ''For its part , Israeli spokesman titleIsrael blames terror surge on Palestinian incitement|url Times of Israel|date25 December 2013 According to B'Tselem , during this same period forty-five Palestinians and six Israelis were killed . Fatalities after operation Cast Lead B'Tselem Reactions US Secretary of State John Kerry said that ifEU : Palestinians deceived Kerry Haaretz 7 May 2014 . Letter by Joseph Cohen 22 April 2014 . Conflict during peace talks According to Peace Now , during the nine months of peace talks Israel set a new record for settlement expansion at nearly 14,000 newly approved settler homes .|newspaperYnet News|date29 April 2014 Palestinian official titleIsraeli settlement construction doubles in 2013|url Press|date3 March 201Despite freezing settlements was not a precondition to restart peace talks ,|newspaperYnet News|date29 April 2014 Palestinian official titleIsraeli settlement construction doubles in 2013|url Press|date3 March 2014 For its part , Israeli spokesman titleIsrael blames|accessdate4 January 2014|newspaper2 External links Israeli–Palestinian peace process 2013 in international relations 2014 in international relations 2013 in Israel 2014 in Israel 2013 in the Palestinian territories 2014 in the Palestinian territories' '' Direct negotiations between Israel and the Palestinians '' ' began on 29 July 2013 following an attempt by United States Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|date29 July 2013 The negotiations were scheduled to last up|accessdate4 January 2014|newspaper2 External links Israeli–Palestinian peace process 2013 in international relations 2014 in international relations 2013 in Israel 2014 in Israel 2013 in the Palestinian territories 2014 in the Palestinian territories' '' Direct negotiations between Israel and the Palestinians '' ' began on 29 July 2013 following an attempt by United States Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|date29 July 2013 The negotiations were scheduled to last up|accessdate4 January 2014|newspaper2 External links Israeli–Palestinian peace process 2013 in international relations 2014 in international relations 2013 in Israel 2014 in Israel 2013 in the Palestinian territories 2014 in the Palestinian territories' '' Direct negotiations between Israel and the Palestinians '' ' began on 29 July 2013 following an attempt by United States Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|date29 July 2013 The negotiations were scheduled to last up|accessdate4 January 2014|newspaper2 External links Israeli–Palestinian peace process 2013 in international relations 2014 in international relations 2013 in Israel 2014 in Israel 2013 in the Palestinian territories 2014 in the Palestinian territories' '' Direct negotiations between Israel and the Palestinians '' ' began on 29 July 2013 following an attempt by United States Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|date29 July 2013 The negotiations were scheduled to last up|accessdate4 January 2014|newspaper2 External links Israeli–Palestinian peace process 2013 in international relations 2014 in international relations 2013 in Israel 2014 in Israel 2013 in the Palestinian territories 2014 in the Palestinian territories' '' Direct negotiations between Israel and the Palestinians '' ' began on 29 July 2013 following an attempt by United States Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|date29 July 2013 The negotiations were scheduled to last up|accessdate4 January 2014|newspaper2 External links Israeli–Palestinian peace process 2013 in international relations 2014 in international relations 2013 in Israel 2014 in Israel 2013 in the Palestinian territories 2014 in the Palestinian territories' '' Direct negotiations between Israel and the Palestinians '' ' began on 29 July 2013 following an attempt by United States Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|date29 July 2013 The negotiations were scheduled to last up|accessdate4 January 2014|newspaper2 External links Israeli–Palestinian peace process 2013 in international relations 2014 in international relations 2013 in Israel 2014 in Israel 2013 in the Palestinian territories 2014 in the Palestinian territories' '' Direct negotiations between Israel and the Palestinians '' ' began on 29 July 2013 following an attempt by United States Secretary of State lastRavid|firstBarak|titleObama welcomes renewal of Israeli-Palestinian talks , but says 'hard choices ' lie ahead|url February 2014|newspaperHaaretz|date29 July 2013 The negotiations were scheduled to last up\""
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edtext['Sepsis II']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 k\n",
      "0.1 k\n",
      "0.2 k\n",
      "5537\n",
      "1 loop, best of 1: 4min 54s per loop\n"
     ]
    }
   ],
   "source": [
    "%%timeit -n 1 -r 1\n",
    "\n",
    "# Without preprocess: 3.03 s per loop\n",
    "# With preprocess: 4min 41s without pre-compile, 4min 54s with pre-compile\n",
    "\n",
    "# Iterate thru citediffs, building editor dict\n",
    "wdwin = 50\n",
    "\n",
    "for i, row in enumerate(data):\n",
    "    if i % 100 == 0: print(i/1000, 'k')\n",
    "    before = word_tokenize(preprocess(row['before_citation']))\n",
    "#     before = word_tokenize(row['before_citation'])\n",
    "#     before = preprocess(row['before_citation'])\n",
    "    edtext[row['editor']] += ' '.join(before[max(len(before)*-1, wdwin*-1):])\n",
    "    after = word_tokenize(preprocess(row['after_citation']))\n",
    "#     after = word_tokenize(row['after_citation'])\n",
    "#     after = preprocess(row['after_citation'])\n",
    "    edtext[row['editor']] += ' '.join(after[:min(len(after), wdwin)])\n",
    "\n",
    "print(len(edtext))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 k\n",
      "0.1 k\n",
      "0.2 k\n",
      "5537\n",
      "1 loop, best of 1: 4min 54s per loop\n"
     ]
    }
   ],
   "source": [
    "%%timeit -n 1 -r 1\n",
    "\n",
    "# TRY AGAIN\n",
    "# Without preprocess: 3.03 s per loop\n",
    "# With preprocess: 4min 41s without pre-compile, 4min 54s with pre-compile\n",
    "\n",
    "# Iterate thru citediffs, building editor dict\n",
    "wdwin = 50\n",
    "\n",
    "for i, row in enumerate(data):\n",
    "    if i % 100 == 0: print(i/1000, 'k')\n",
    "    before = word_tokenize(preprocess(row['before_citation']))\n",
    "    matched = re.search(r'(https?:\\/\\/(?:www\\.)?[^ ]*)')\n",
    "    outtext = re.sub(r'https?:\\/\\/(?:www\\.)?[^ ]*', '', intext, flags=re.MULTILINE)\n",
    "#     outtext = urlpat.sub('', intext)\n",
    "#     print('1 sub')\n",
    "#     sys.stdout.flush()\n",
    "#     outtext = re.sub(r'[^ ].*?\\.html?', '', outtext)\n",
    "    outtext = re.sub(r'[^ ].*?\\.html?', '', outtext, flags=re.MULTILINE)\n",
    "#     outtext = urlhtml.sub('', outtext)\n",
    "#     print('2 sub')\n",
    "#     sys.stdout.flush()\n",
    "    \n",
    "    # templates\n",
    "#     outtext = re.sub(r'\\{\\{.*?(?:\\}\\}|$)','', outtext)\n",
    "    outtext = re.sub(r'\\{\\{.*?(?:\\}\\}|$)','', outtext, flags=re.MULTILINE)\n",
    "#     outtext = temppat.sub('', outtext)\n",
    "#     print('3 sub')\n",
    "#     sys.stdout.flush()\n",
    "#     outtext = re.sub(r'^.*?\\}\\}','', outtext)\n",
    "    outtext = re.sub(r'^.*?\\}\\}','', outtext, flags=re.MULTILINE)\n",
    "#     outtext = tempout.sub('', outtext)\n",
    "#     print('4 sub')\n",
    "#     sys.stdout.flush()\n",
    "    \n",
    "    # links\n",
    "#     outtext = re.sub(r'\\[\\[.*?\\|', '', outtext)\n",
    "    outtext = re.sub(r'\\[\\[.*?\\|', '', outtext, flags=re.MULTILINE)\n",
    "#     before = word_tokenize(row['before_citation'])\n",
    "#     before = preprocess(row['before_citation'])\n",
    "    edtext[row['editor']] += ' '.join(before[max(len(before)*-1, wdwin*-1):])\n",
    "    after = word_tokenize(preprocess(row['after_citation']))\n",
    "#     after = word_tokenize(row['after_citation'])\n",
    "#     after = preprocess(row['after_citation'])\n",
    "    edtext[row['editor']] += ' '.join(after[:min(len(after), wdwin)])\n",
    "\n",
    "print(len(edtext))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %%timeit -n 1 -r 1\n",
    "\n",
    "# Time-optimized\n",
    "# Iterate thru citediffs, building editor dict\n",
    "wdwin = 50\n",
    "\n",
    "# re sub preprocessing\n",
    "beforelen = len(data)\n",
    "#before_data = '\\n|||\\n'.join(row['before_citation'] for row in data)\n",
    "#after_data = '\\n|||\\n'.join(row['after_citation'] for row in data)\n",
    "before_data = '\\n'.join(row['before_citation'] for row in data)\n",
    "after_data = '\\n'.join(row['after_citation'] for row in data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Befores done\n",
      "Afters done\n"
     ]
    },
    {
     "ename": "AssertionError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m\u001b[0m",
      "\u001b[1;31mAssertionError\u001b[0mTraceback (most recent call last)",
      "\u001b[1;32m<ipython-input-31-7a36657c7e63>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Afters done'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstdout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[1;32massert\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbefore_pp\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mafter_pp\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mbeforelen\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrow\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAssertionError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "print('Preprocessing...')\n",
    "sys.stdout.flush()\n",
    "before_pp = preprocess(before_data).split('\\n')\n",
    "print('Befores done')\n",
    "sys.stdout.flush()\n",
    "after_pp = preprocess(before_data).split('\\n')\n",
    "print('Afters done')\n",
    "sys.stdout.flush()\n",
    "assert len(before_pp) == len(after_pp) == beforelen\n",
    "\n",
    "for i, row in enumerate(data):\n",
    "    if i % 100 == 0: print(i/1000, 'k')\n",
    "    before = word_tokenize(before_pp[i])\n",
    "    edtext[row['editor']] += ' '.join(before[max(len(before)*-1, wdwin*-1):])\n",
    "    \n",
    "    after = word_tokenize(after_pp[i])\n",
    "    edtext[row['editor']] += ' '.join(after[:min(len(after), wdwin)])\n",
    "\n",
    "print(len(edtext))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def delstring(line, m):\n",
    "    return line.replace(m.group(), '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Spesh531': '',\n",
       " 'Paulistano': '',\n",
       " 'Hecht': '',\n",
       " 'Bigshotnews': '',\n",
       " 'SheffieldSteel': '',\n",
       " 'Vladko': '',\n",
       " 'Dr. Manos': '',\n",
       " 'Karimi cae': '',\n",
       " 'Yohai.bs': '',\n",
       " 'Wwb06': '',\n",
       " 'EETech': '',\n",
       " 'Kleinecke': '',\n",
       " 'Cynwolfe': '',\n",
       " 'Drieakko': '',\n",
       " 'Light show': '',\n",
       " 'Yunis': '',\n",
       " 'Molestash': '',\n",
       " 'TurtleTurtle': '',\n",
       " 'Atefrat': '',\n",
       " 'Oszalał': '',\n",
       " 'Panyd': '',\n",
       " 'LemonLion': '',\n",
       " 'TimeOnHands': '',\n",
       " 'Leecharleswalker': '',\n",
       " 'Etothei': '',\n",
       " 'Eyrian': '',\n",
       " 'Positively': '',\n",
       " 'Screen stalker': '',\n",
       " 'RafaelG': '',\n",
       " 'Coolnewshere': '',\n",
       " 'Ashley kennedy3': '',\n",
       " 'Zginder': '',\n",
       " 'GRuban': '',\n",
       " 'Thehistorian10': '',\n",
       " 'StonedChipmunk': '',\n",
       " 'Andigisi': '',\n",
       " 'The Singing Badger': '',\n",
       " 'Marianian': '',\n",
       " 'Buspirtraz': '',\n",
       " 'Aaliyah Stevens': '',\n",
       " 'Juliancolton': '',\n",
       " 'Afroghost': '',\n",
       " 'Jdkag': '',\n",
       " 'ACEOREVIVED': '',\n",
       " 'Rodhullandemu': '',\n",
       " 'Muzher': '',\n",
       " 'Colipon': '',\n",
       " 'BozMo': '',\n",
       " 'E4024': '',\n",
       " 'Tulli': '',\n",
       " 'Constanz': '',\n",
       " 'BrendelSignature': '',\n",
       " 'Theo F': '',\n",
       " 'Ahmetyal': '',\n",
       " 'Lucretius6': '',\n",
       " 'Fram': '',\n",
       " '3bdulelah': '',\n",
       " 'Rgcrgcrgc': '',\n",
       " 'YaakovOfNY': '',\n",
       " 'Bagbesh': '',\n",
       " 'Stbalbach': '',\n",
       " 'Pathare Prabhu': '',\n",
       " 'Chagai': '',\n",
       " 'Ubikwit': '',\n",
       " 'Thumperward': '',\n",
       " 'Another name': '',\n",
       " 'Rcbutcher': '',\n",
       " 'Taivo': '',\n",
       " 'EvanHarper': '',\n",
       " 'JetJon': '',\n",
       " 'Mike4ty4': '',\n",
       " 'Cordless Larry': '',\n",
       " 'Vgy7ujm': '',\n",
       " 'DerekMBarnes': '',\n",
       " 'Adallaadalla': '',\n",
       " 'Arzkibar': '',\n",
       " 'J.christianson': '',\n",
       " 'Omerlives': '',\n",
       " 'Wavehunter': '',\n",
       " 'Leoboudv': '',\n",
       " 'Csi.southpark': '',\n",
       " 'Duhon': '',\n",
       " 'Therequiembellishere': '',\n",
       " 'Mary J. Marotta': '',\n",
       " 'Mtking': '',\n",
       " 'Benni seidel': '',\n",
       " 'Pax:Vobiscum': '',\n",
       " 'BMBTHC': '',\n",
       " 'Procrastinator supreme': '',\n",
       " 'WP Editor 2011': '',\n",
       " 'Dean Sayers': '',\n",
       " 'Skaye': '',\n",
       " 'YechezkelZilber': '',\n",
       " 'JokerXtreme': '',\n",
       " 'Mellum': '',\n",
       " 'Emijrp': '',\n",
       " 'Kadathdreamques': '',\n",
       " 'FeanorStar7': '',\n",
       " 'Peter Grey': '',\n",
       " 'Signaleer': '',\n",
       " 'Netanel h': '',\n",
       " 'Apeloverage': '',\n",
       " 'Tyomitch': '',\n",
       " 'Feliks': '',\n",
       " 'Emmanuelm': '',\n",
       " 'Andygx': '',\n",
       " 'Vecrumba': '',\n",
       " 'Agari': '',\n",
       " 'Deuar': '',\n",
       " 'Morningside Clio': '',\n",
       " 'Hydeparkblvd': '',\n",
       " 'Lindert': '',\n",
       " 'Worldedixor': '',\n",
       " 'Dove1950': '',\n",
       " 'Quietbritishjim': '',\n",
       " 'JimFarm': '',\n",
       " 'Ebralph': '',\n",
       " 'Magabund': '',\n",
       " 'OneGuy': '',\n",
       " 'Sicarii': '',\n",
       " 'Domichael': '',\n",
       " 'Rosaecruz': '',\n",
       " 'Dbrodbeck': '',\n",
       " 'HIZKIAH': '',\n",
       " 'Merhawie': '',\n",
       " 'Llosoc': '',\n",
       " 'Eternalbeans': '',\n",
       " 'Twobells': '',\n",
       " 'Sufisticated': '',\n",
       " 'Rcnet': '',\n",
       " 'Aronlip': '',\n",
       " 'Ringbark': '',\n",
       " 'Rickyrab': '',\n",
       " 'Peter Deer': '',\n",
       " 'Gal Kr': '',\n",
       " 'Cambalachero': '',\n",
       " 'Againme': '',\n",
       " 'Z y': '',\n",
       " 'Kwork2': '',\n",
       " 'ArnoldReinhold': '',\n",
       " 'Lucideer': '',\n",
       " 'Gzornenplatz': '',\n",
       " 'SEWilco': '',\n",
       " 'Nikola Štuban': '',\n",
       " 'Orartu': '',\n",
       " 'Old Pal': '',\n",
       " 'Mr. Stradivarius': '',\n",
       " 'Jkshapiro': '',\n",
       " 'Ynhockey': '',\n",
       " 'Mark3': '',\n",
       " 'CPWinter': '',\n",
       " 'Oncenawhile': '',\n",
       " 'Moriori': '',\n",
       " 'Ihmeidan': '',\n",
       " 'Danrolo': '',\n",
       " 'Saforrest': '',\n",
       " 'Georgianstar': '',\n",
       " 'Shint': '',\n",
       " 'Entwhiz': '',\n",
       " 'Abrnkak': '',\n",
       " 'Pejman47': '',\n",
       " 'Imladros': '',\n",
       " 'TheLongTone': '',\n",
       " 'CheskiChips': '',\n",
       " 'Fatal!ty': '',\n",
       " 'GaussianCopula': '',\n",
       " 'Bcwilmot': '',\n",
       " 'Escape Orbit': '',\n",
       " 'Joeboy': '',\n",
       " 'Jonathan Tweet': '',\n",
       " 'Blindman shady': '',\n",
       " 'Deinocheirus': '',\n",
       " 'Adrianke77': '',\n",
       " 'Ian Pitchford': '',\n",
       " 'Krommenaas': '',\n",
       " 'Chrono1084': '',\n",
       " 'Math.geek3.1415926': '',\n",
       " 'One last pharaoh': '',\n",
       " 'Mikael Häggström': '',\n",
       " 'Wgw2024': '',\n",
       " 'OneTopJob6': '',\n",
       " 'Solntsa90': '',\n",
       " 'Siafu': '',\n",
       " 'MiniMary12': '',\n",
       " \"D\\\\'Ranged 1\": '',\n",
       " 'Gidonb': '',\n",
       " 'GandalfDaGraay': '',\n",
       " 'Tachfin': '',\n",
       " 'Smkolins': '',\n",
       " 'RFoglio': '',\n",
       " 'DouglasCalvert': '',\n",
       " 'HagermanBot': '',\n",
       " 'Robofish': '',\n",
       " 'Ptrslv72': '',\n",
       " 'Stan Brin': '',\n",
       " 'Falafelis': '',\n",
       " 'Iosef aetos': '',\n",
       " 'Miranda': '',\n",
       " 'Aa2-2004': '',\n",
       " 'Davidelah': '',\n",
       " 'Jerzy': '',\n",
       " 'Nadav S': '',\n",
       " 'VsevolodKrolikov': '',\n",
       " 'Basejumper': '',\n",
       " 'Thom977': '',\n",
       " 'E Pluribus Anthony': '',\n",
       " 'Zalali': '',\n",
       " 'Andycjp': '',\n",
       " 'XavierGreen': '',\n",
       " 'John Bahrain': '',\n",
       " 'Faizan': '',\n",
       " 'Kitkat21': '',\n",
       " 'Attilios': '',\n",
       " 'Jarry1250': '',\n",
       " 'Pstanton': '',\n",
       " 'Tkenney2': '',\n",
       " 'MagnesianPhoenix': '',\n",
       " 'Invisible Flying Mangoes': '',\n",
       " 'Jros83': '',\n",
       " 'James Blond': '',\n",
       " 'Apteva': '',\n",
       " 'Tubesship': '',\n",
       " 'Richard BB': '',\n",
       " 'Olessi': '',\n",
       " 'Sca': '',\n",
       " 'Jadger': '',\n",
       " 'Mashkin': '',\n",
       " 'Nageeb': '',\n",
       " 'Mnbitar': '',\n",
       " 'Padres Hana': '',\n",
       " 'Renamed user 1253': '',\n",
       " 'Chedorlaomer': '',\n",
       " 'Gon4z': '',\n",
       " 'Rei': '',\n",
       " 'Rappatoni': '',\n",
       " 'KBurchfiel': '',\n",
       " 'Lagrange613': '',\n",
       " 'Mhazard9': '',\n",
       " 'Lifesatrain': '',\n",
       " 'Orthodox45': '',\n",
       " 'Jsmog': '',\n",
       " 'Abrooke': '',\n",
       " 'Zaid almasri': '',\n",
       " 'Kmweber': '',\n",
       " 'ACogloc': '',\n",
       " 'Illegitimate Barrister': '',\n",
       " 'Panairjdde': '',\n",
       " 'MJCdetroit': '',\n",
       " 'Nlu': '',\n",
       " 'Ahmadhusseini': '',\n",
       " 'Milnews.ca': '',\n",
       " 'Tomeasy': '',\n",
       " 'Guy Peters': '',\n",
       " 'HaterofIgnorance': '',\n",
       " 'Hermant patel': '',\n",
       " 'Blorg': '',\n",
       " 'Crushti': '',\n",
       " 'Smyth': '',\n",
       " 'This is not my last name': '',\n",
       " 'JudgeX': '',\n",
       " 'Jc3s5h': '',\n",
       " 'JK the unwise': '',\n",
       " 'BigNate37': '',\n",
       " 'Causteau': '',\n",
       " 'The Original Historygeek': '',\n",
       " 'Anomalocaris': '',\n",
       " 'Sharktopus': '',\n",
       " 'Aua': '',\n",
       " 'Mahmoud-Megahid': '',\n",
       " 'Aharon42': '',\n",
       " 'Yannismarou': '',\n",
       " 'Beeblebrox': '',\n",
       " 'Ohnoitsjamie': '',\n",
       " 'TimVickers': '',\n",
       " 'Patrick Berry': '',\n",
       " 'Charles-LF': '',\n",
       " 'CWenger': '',\n",
       " 'COGDEN': '',\n",
       " 'Muad': '',\n",
       " 'Buzzboy2000': '',\n",
       " 'Louboi': '',\n",
       " 'AniChai': '',\n",
       " 'EisenKnoechel': '',\n",
       " 'Srleffler': '',\n",
       " 'Peaceworld111': '',\n",
       " 'Deodar': '',\n",
       " 'Zbvhs': '',\n",
       " 'George45646': '',\n",
       " 'Firefly322': '',\n",
       " 'RCS': '',\n",
       " 'Spawn Man': '',\n",
       " 'Ycarcomed': '',\n",
       " 'Arctic.gnome': '',\n",
       " 'PinchasC': '',\n",
       " 'Historyisapov': '',\n",
       " 'Abdull': '',\n",
       " 'Redxiv': '',\n",
       " 'CTIPryor': '',\n",
       " 'Qwyrxian': '',\n",
       " 'Andrew Gray': '',\n",
       " 'Typhoon9410': '',\n",
       " 'Dbleyou': '',\n",
       " 'BrooklynHabiru': '',\n",
       " 'Stefan': '',\n",
       " 'Beland': '',\n",
       " 'N10452': '',\n",
       " 'Cezveci': '',\n",
       " 'Clairemont': '',\n",
       " 'SpaceFalcon2001': '',\n",
       " 'Tu-49': '',\n",
       " 'Arjayay': '',\n",
       " '157.228.x.x': '',\n",
       " 'Mike Doughney': '',\n",
       " 'Yonatan': '',\n",
       " 'Xt828': '',\n",
       " 'Alinor': '',\n",
       " 'Schwarzschachtel': '',\n",
       " 'Gentoo-Michael': '',\n",
       " 'AndyTheGrump': '',\n",
       " 'Striver': '',\n",
       " 'Adamrce': '',\n",
       " 'Noleander': '',\n",
       " 'DavidD4scnrt': '',\n",
       " 'Zath42': '',\n",
       " 'Mor2': '',\n",
       " 'Goon Noot': '',\n",
       " 'Ghione': '',\n",
       " 'Woodah': '',\n",
       " 'Will Lakeman': '',\n",
       " 'Rmsharpe': '',\n",
       " 'Shiftadot': '',\n",
       " 'Stat-ist-ikk': '',\n",
       " 'Basejumper2': '',\n",
       " 'Bk0': '',\n",
       " 'Basesurge': '',\n",
       " 'Ann arbor street': '',\n",
       " 'Samuel B52': '',\n",
       " 'Polozooza': '',\n",
       " 'Christopher Thomas': '',\n",
       " 'Daimanta': '',\n",
       " 'Bastun': '',\n",
       " 'Zadil': '',\n",
       " 'Nescio vos': '',\n",
       " 'Hamdan2': '',\n",
       " 'Albinomonkey': '',\n",
       " 'Highfields': '',\n",
       " 'Sam Spade': '',\n",
       " 'Joe407': '',\n",
       " 'DAVIDY': '',\n",
       " \"Bearian\\\\'sBooties\": '',\n",
       " 'Mourabitoun': '',\n",
       " 'Awiseman': '',\n",
       " 'Cymbelmineer': '',\n",
       " 'Basket of Puppies': '',\n",
       " 'No more bongos': '',\n",
       " 'MYLO': '',\n",
       " 'Throwaway85': '',\n",
       " 'Narson': '',\n",
       " 'HiLo48': '',\n",
       " 'Creidieki': '',\n",
       " 'Gideonaa': '',\n",
       " 'Samw': '',\n",
       " 'Tewfik': '',\n",
       " 'Kevlar67': '',\n",
       " 'Mavromatis': '',\n",
       " 'Trackteur': '',\n",
       " 'WLRoss': '',\n",
       " 'Jtrainor': '',\n",
       " 'GegenIsrael': '',\n",
       " 'Tumadoireacht': '',\n",
       " 'Wbm1058': '',\n",
       " 'Cosainsé': '',\n",
       " 'RudolfRed': '',\n",
       " 'Goodoldpolonius2': '',\n",
       " 'Antonrojo': '',\n",
       " 'Jonathunder': '',\n",
       " 'Edinborgarstefan': '',\n",
       " 'Maverick821': '',\n",
       " 'Xx236': '',\n",
       " 'Steve Hart': '',\n",
       " 'Lance6968': '',\n",
       " 'Twilightsojourn': '',\n",
       " 'Aaron Brenneman': '',\n",
       " 'Tpbradbury': '',\n",
       " 'GDonato': '',\n",
       " 'Emmette Hernandez Coleman': '',\n",
       " 'Feezo': '',\n",
       " 'Matt Crypto': '',\n",
       " 'Charlesg1': '',\n",
       " 'JdeJ': '',\n",
       " 'Green Giant': '',\n",
       " 'AbdulHornochsmannn': '',\n",
       " 'Lestrade': '',\n",
       " 'Zkharya': '',\n",
       " 'Offbeam': '',\n",
       " 'Spookfish': '',\n",
       " 'Virtua67': '',\n",
       " 'Takenitora': '',\n",
       " 'Ian.thomson': '',\n",
       " 'FormerIP': '',\n",
       " 'Groupuscule': '',\n",
       " 'Dr. Persi': '',\n",
       " 'Physchim62': '',\n",
       " 'Raphaelhui': '',\n",
       " 'NewsWatcher99': '',\n",
       " 'Indigenius': '',\n",
       " 'Truth 06': '',\n",
       " 'Werldwayd': '',\n",
       " 'Audancer1993': '',\n",
       " 'Colourinthemeaning': '',\n",
       " 'KLN2000': '',\n",
       " 'WeedWhacker': '',\n",
       " 'Kaelus': '',\n",
       " 'Tarek': '',\n",
       " 'Ruthfulbarbarity': '',\n",
       " 'Sasquatch': '',\n",
       " 'Reithy': '',\n",
       " 'LyTe': '',\n",
       " 'Dijxtra': '',\n",
       " 'DMacks': '',\n",
       " 'F22 Raptor': '',\n",
       " 'Lamoonia': '',\n",
       " 'Umalee': '',\n",
       " 'Allard': '',\n",
       " 'Cadwallader': '',\n",
       " 'Richardjames444': '',\n",
       " 'L1A1 FAL': '',\n",
       " 'ClockworkLunch': '',\n",
       " '-- -- --': '',\n",
       " 'Wwoods': '',\n",
       " 'Nafpaktitism': '',\n",
       " 'Fishhead64': '',\n",
       " 'Ayinyud': '',\n",
       " 'Lexington50': '',\n",
       " 'SamuelTheGhost': '',\n",
       " 'Biruitorul': '',\n",
       " 'Stumink': '',\n",
       " 'HJV': '',\n",
       " 'Epoch era': '',\n",
       " 'K012957': '',\n",
       " 'Mr. Billion': '',\n",
       " 'Beit Or': '',\n",
       " 'GJ': '',\n",
       " 'Novidmarana': '',\n",
       " 'XLR8TION': '',\n",
       " 'Ruslik0': '',\n",
       " 'Wolfe': '',\n",
       " 'Doom777': '',\n",
       " 'DrDagless': '',\n",
       " 'Ponyo': '',\n",
       " 'MuZemike': '',\n",
       " 'No More Mr Nice Guy': '',\n",
       " 'Lemmiwinks2': '',\n",
       " 'Oneworld25': '',\n",
       " 'RichardMathews': '',\n",
       " 'MiShogun': '',\n",
       " 'JaapBoBo': '',\n",
       " 'PetesGuide': '',\n",
       " 'Brutannica': '',\n",
       " 'Thomasmeeks': '',\n",
       " 'Carwil': '',\n",
       " 'Camundongo': '',\n",
       " 'Faro0485': '',\n",
       " 'Strausszek': '',\n",
       " 'Skington': '',\n",
       " 'Fintelia': '',\n",
       " 'Wallie': '',\n",
       " 'Avihu': '',\n",
       " 'Peter G Werner': '',\n",
       " 'Rich Farmbrough': '',\n",
       " 'Dougweller': '',\n",
       " 'Mmlj4': '',\n",
       " 'Petrarchan47': '',\n",
       " 'Dendirrek': '',\n",
       " 'Totorotroll': '',\n",
       " 'LeFrog': '',\n",
       " 'EclecticGeek': '',\n",
       " 'Paxse': '',\n",
       " 'Capricorn42': '',\n",
       " 'Sam metal': '',\n",
       " 'Weregerbil': '',\n",
       " 'Midnightblueowl': '',\n",
       " 'Super Nintendo Chalmers': '',\n",
       " '109XXX': '',\n",
       " 'Yaakov Birthright Franklin': '',\n",
       " 'Addoula': '',\n",
       " 'Lordrichie': '',\n",
       " 'Zafiroblue05': '',\n",
       " 'Ben Ammi': '',\n",
       " 'Secondat of Orange': '',\n",
       " 'Leflyman': '',\n",
       " 'NotAWeasel': '',\n",
       " 'Flamgirlant': '',\n",
       " 'Mark Schierbecker': '',\n",
       " 'Sp0': '',\n",
       " 'Greyshark09': \"'save ' us is that John Kerry will get a Nobel Peace Prize and leave us alone . '' Yuval Steinitz expressed general agreement with Ya'alon 's views , but disagreed with the personal insult . Ya'alon and Steinitz are both members of the Likud party . < ref namelon '' > Israeli settlements Israel was accused by Palestinian officials of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It is well known that these are an obstacle for the Palestinians as they are onthat John Kerry will get a Nobel Peace Prize and leave us alone . '' Yuval Steinitz expressed general agreement with Ya'alon 's views , but disagreed with the personal insult . Ya'alon and Steinitz are both members of the Likud party . < ref name '' yaalon '' >Israeli settlements Israel was accused by Palestinian officials of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It is well known that these are an obstacle for the Palestinians as they are on Palestinian land and'save ' us is that John Kerry will get a Nobel Peace Prize and leave us alone . '' Yuval Steinitz expressed general agreement with Ya'alon 's views , but disagreed with the personal insult . Ya'alon and Steinitz are both members of the Likud party . < ref namelon '' > Israeli settlements Israel was accused by Palestinian officials of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It is well known that these are an obstacle for the Palestinians as they are onthat John Kerry will get a Nobel Peace Prize and leave us alone . '' Yuval Steinitz expressed general agreement with Ya'alon 's views , but disagreed with the personal insult . Ya'alon and Steinitz are both members of the Likud party . < ref name '' yaalon '' >Israeli settlements Israel was accused by Palestinian officials of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It is well known that these are an obstacle for the Palestinians as they are on Palestinian land and'save ' us is that John Kerry will get a Nobel Peace Prize and leave us alone . '' Yuval Steinitz expressed general agreement with Ya'alon 's views , but disagreed with the personal insult . Ya'alon and Steinitz are both members of the Likud party . < ref namelon '' > Israeli settlements Israel was accused by Palestinian officials of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It is well known that these are an obstacle for the Palestinians as they are onthat John Kerry will get a Nobel Peace Prize and leave us alone . '' Yuval Steinitz expressed general agreement with Ya'alon 's views , but disagreed with the personal insult . Ya'alon and Steinitz are both members of the Likud party . < ref name '' yaalon '' >Israeli settlements Israel was accused by Palestinian officials of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It is well known that these are an obstacle for the Palestinians as they are on Palestinian land and\",\n",
       " 'Omrim': '',\n",
       " 'Rabbi-m': '',\n",
       " 'Aleph4': '',\n",
       " 'Trashbag': '',\n",
       " 'Ekem': '',\n",
       " 'EhavEliyahu': '',\n",
       " 'Java7837': '',\n",
       " 'Beorhtwulf': '',\n",
       " 'Ryulong': '',\n",
       " 'AndySimpson': '',\n",
       " 'Lustright': '',\n",
       " 'Bladesmulti': '',\n",
       " 'ALUOPline2': '',\n",
       " 'Wahoofive': '',\n",
       " 'AnotherBDA': '',\n",
       " 'Jabotito48': '',\n",
       " 'Josh Parris': '',\n",
       " 'ErikWarmelink': '',\n",
       " 'Zer0fighta': '',\n",
       " 'Notaneditorjustaneditor': '',\n",
       " 'Tibetan Prayer': '',\n",
       " 'Deleteyourself16': '',\n",
       " 'NoCal100': '',\n",
       " 'CPS': '',\n",
       " 'Hbomberman': '',\n",
       " 'Pir': '',\n",
       " 'Lionelt': '',\n",
       " 'Factchecker atyourservice': '',\n",
       " 'Komaknacon': '',\n",
       " 'Kirill Lokshin': '',\n",
       " 'Ww2censor': '',\n",
       " 'Mnnlaxer': '',\n",
       " 'Hassanfarooqi': '',\n",
       " 'Mzyxptlk': '',\n",
       " 'Epson291': '',\n",
       " 'Cumbria4': '',\n",
       " 'Harvestman': '',\n",
       " 'Sal73x': '',\n",
       " 'Spoil29': '',\n",
       " 'Indisciplined': '',\n",
       " 'Greyhood': '',\n",
       " 'Luke85uk': '',\n",
       " 'Aucitypops': '',\n",
       " 'Doron': '',\n",
       " 'Bbb23': '',\n",
       " 'House3434': '',\n",
       " 'Black Falcon': '',\n",
       " \"CasualObserver\\\\'48\": '',\n",
       " 'Atypicaloracle': '',\n",
       " 'Gamaliel': '',\n",
       " 'HG': '',\n",
       " 'MnJWalker': '',\n",
       " 'CWatchman': '',\n",
       " 'Zizouzenati': '',\n",
       " 'GRBerry': '',\n",
       " 'Taifarious1': '',\n",
       " 'Hayabusa future': '',\n",
       " 'Barakooda': '',\n",
       " 'Amorice': '',\n",
       " 'Mitrebox': '',\n",
       " 'Stuck in SD with Yaming': '',\n",
       " 'Textor': '',\n",
       " 'Phase1': '',\n",
       " 'Asarelah': '',\n",
       " 'Rym torch': '',\n",
       " 'Livna-Maor': '',\n",
       " 'Freestylefrappe': '',\n",
       " 'Gheronihmo': '',\n",
       " 'Brightgalrs': '',\n",
       " 'Pare Mo': '',\n",
       " 'Rsynnott': '',\n",
       " 'Jaro7788': '',\n",
       " 'Sikon': '',\n",
       " 'Rueben lys': '',\n",
       " 'TPaineTX': '',\n",
       " 'Dennis Brown': '',\n",
       " 'Wndl42': '',\n",
       " 'Snowded': '',\n",
       " 'Aldux': '',\n",
       " 'Down In The Air': '',\n",
       " 'Zorroz': '',\n",
       " 'Jeffpw': '',\n",
       " 'Raylin': '',\n",
       " 'Ocaasi': '',\n",
       " 'Scoobydunk': '',\n",
       " 'Ellaivarios': '',\n",
       " 'Emt147': '',\n",
       " 'Jackmcbarn': '',\n",
       " 'The Lesser Merlin': '',\n",
       " 'Kdammers': '',\n",
       " 'CsDix': '',\n",
       " 'Esn': '',\n",
       " 'Ziggysdaydream': '',\n",
       " 'Ikar.us': '',\n",
       " \"Mark O\\\\'Sullivan\": '',\n",
       " 'Epimetreus': '',\n",
       " 'Herostratus': '',\n",
       " 'Reuvenk': '',\n",
       " 'Jachin': '',\n",
       " 'Wittym': '',\n",
       " 'Cloptonson': '',\n",
       " 'Zhochaka': '',\n",
       " 'Jackehammond': '',\n",
       " 'Herunar': '',\n",
       " 'Apus': '',\n",
       " 'Historygypsy': '',\n",
       " 'DimaY2K': '',\n",
       " 'Mohamed CJ': '',\n",
       " 'Parvazbato59': '',\n",
       " \"Kassab\\\\'s\": '',\n",
       " 'Mflcs': '',\n",
       " 'JIP': '',\n",
       " 'Midrashah': '',\n",
       " 'Roniblr': '',\n",
       " 'Filll': '',\n",
       " 'Galloglass': '',\n",
       " 'Colossus': '',\n",
       " 'Akubhai': '',\n",
       " 'Bigweeboy': '',\n",
       " 'Jtdirl': '',\n",
       " 'Eriann': '',\n",
       " 'Eheitner': '',\n",
       " 'Bantman': '',\n",
       " 'Bukvoed': '',\n",
       " 'Cam': '',\n",
       " 'Birdman9': '',\n",
       " 'Pharos': '',\n",
       " 'Suhajdab': '',\n",
       " 'SimplesC': '',\n",
       " 'Megapixie': '',\n",
       " 'DoctorKubla': '',\n",
       " 'Terrancommander': '',\n",
       " 'Kotniski': '',\n",
       " 'Darkwind': '',\n",
       " 'Cbarbry': '',\n",
       " 'Moth1701': '',\n",
       " 'Jim Sukwutput': '',\n",
       " 'Zyido': '',\n",
       " 'Gilabrand': '',\n",
       " 'Robert Lindsay': '',\n",
       " 'Discospinster': '',\n",
       " 'P3Y229': '',\n",
       " 'Pb8bije6a7b6a3w': '',\n",
       " 'Duk': '',\n",
       " 'Spasemunki': '',\n",
       " 'Wayward': '',\n",
       " 'Trachys': '',\n",
       " 'Tombseye': '',\n",
       " 'Murftown': '',\n",
       " 'Shtove': '',\n",
       " 'Hope&amp;Act3!': '',\n",
       " 'Slgrandson': '',\n",
       " 'EBY3221': '',\n",
       " 'Sonntagsbraten': '',\n",
       " 'Fjmustak': '',\n",
       " 'Rm uk': '',\n",
       " 'Slashme': '',\n",
       " 'Phydeaux': '',\n",
       " 'Acjelen': '',\n",
       " 'Gingermint': '',\n",
       " 'Rjensen': '',\n",
       " 'Christian Askeland': '',\n",
       " 'Rathersilly': '',\n",
       " 'Jcc': '',\n",
       " 'George Kaplan': '',\n",
       " 'Tasc': '',\n",
       " 'GrundyCamellia': '',\n",
       " 'Chuborno': '',\n",
       " 'Sandstein': '',\n",
       " 'Chhe': '',\n",
       " 'Kumioko (renamed)': '',\n",
       " 'Emurphy42': '',\n",
       " 'Charvex': '',\n",
       " 'WerWil': '',\n",
       " 'UDSS': '',\n",
       " 'Mingus ah um': '',\n",
       " 'Alton': '',\n",
       " 'Brooks Lindsay': '',\n",
       " 'Odedee': '',\n",
       " 'MA1988': '',\n",
       " 'Nimur': '',\n",
       " 'MustangAficionado': '',\n",
       " 'MPerel': '',\n",
       " 'Ptah, the El Daoud': '',\n",
       " 'Andre Engels': '',\n",
       " 'TresÁrboles': '',\n",
       " 'Mustafa Mustamann': '',\n",
       " 'Yogi Bearstein': '',\n",
       " 'Okedem': '',\n",
       " 'Ricardo monteiro': '',\n",
       " 'ApolloCreed': '',\n",
       " 'Wapondaponda': '',\n",
       " 'Dean Armond': '',\n",
       " 'Exoucia1': '',\n",
       " 'Vulpesvulpes201': '',\n",
       " 'Djampa': '',\n",
       " 'Canutethegreat': '',\n",
       " 'Otterathome': '',\n",
       " 'IANVS': '',\n",
       " \"Vrubel\\\\'s Demons\": '',\n",
       " 'Abyssal': '',\n",
       " 'Mikicook': '',\n",
       " 'Aaron north': '',\n",
       " 'Oiqbal': '',\n",
       " 'Dwegowy': '',\n",
       " 'PaulHammond': '',\n",
       " 'EWikist': '',\n",
       " 'Jarhed': '',\n",
       " 'MrZaf': '',\n",
       " 'Graham1973': '',\n",
       " 'Armbrust': '',\n",
       " 'Taiwan boi': '',\n",
       " 'Staszek Lem': '',\n",
       " 'Bonkalicious': '',\n",
       " 'Pygmypony': '',\n",
       " 'Romis': '',\n",
       " 'Noor Aalam': '',\n",
       " 'Aslaningozu': '',\n",
       " 'Yulia Romero': '',\n",
       " 'ThuranX': '',\n",
       " 'Roderick E': '',\n",
       " 'Mathew5000': '',\n",
       " 'Fashionfreakforever': '',\n",
       " 'Kempler video': '',\n",
       " 'Jrobin08': '',\n",
       " '8een4Tfor': '',\n",
       " 'Petrous Fire': '',\n",
       " 'YeshuaDavid': '',\n",
       " 'OuroborosCobra': '',\n",
       " 'Raistlin': '',\n",
       " 'Tinlv7': '',\n",
       " 'Jsolinsky': '',\n",
       " 'Soewinhan': '',\n",
       " 'Saflid': '',\n",
       " 'Jake95': '',\n",
       " 'KillerChihuahua': '',\n",
       " 'Phoniel': '',\n",
       " 'BigEyedFish': '',\n",
       " 'Herobill': '',\n",
       " 'Autismal': '',\n",
       " 'Sweetfreek': '',\n",
       " 'Ido50': '',\n",
       " 'Tiv': '',\n",
       " 'Jonney2000': '',\n",
       " 'Tritomex': '',\n",
       " 'TopGun': '',\n",
       " 'Mrg3105': '',\n",
       " 'Guy Montag': '',\n",
       " 'Känsterle': '',\n",
       " 'JonathanCook': '',\n",
       " 'Krou': '',\n",
       " 'Pustelnik': '',\n",
       " 'ApuNahasaminajustApu': '',\n",
       " 'GabrielF': '',\n",
       " 'Spike monotheist': '',\n",
       " 'EvilSuggestions': '',\n",
       " 'Bee Cliff River Slob': '',\n",
       " 'Widmerpool': '',\n",
       " 'RamuelMRaagas': '',\n",
       " 'Banaticus': '',\n",
       " 'Anandks007': '',\n",
       " 'B4Ctom1': '',\n",
       " 'Steven X': '',\n",
       " 'Irishman76m': '',\n",
       " 'Ghostreveries': '',\n",
       " 'AbrahamIsaac': '',\n",
       " 'Derwig': '',\n",
       " 'J21': '',\n",
       " 'Gabe2000000': '',\n",
       " 'Fsem10': '',\n",
       " 'Hithisishal': '',\n",
       " 'Richigi': '',\n",
       " 'Der letzte Konsul': '',\n",
       " 'ExoditeTyr': '',\n",
       " 'JackhammerSwirl': '',\n",
       " 'Saddhiyama': '',\n",
       " 'Other Choices': '',\n",
       " 'קולנואני': '',\n",
       " 'Capitalistroadster': '',\n",
       " 'Casliber': '',\n",
       " 'Intelligent Mr Toad': '',\n",
       " 'Pollinosisss': '',\n",
       " 'Andrew Scott Selkirk': '',\n",
       " 'SimonTrew': '',\n",
       " 'Hammerquill': '',\n",
       " 'JHawk88': '',\n",
       " 'Philopp': '',\n",
       " 'Asad112': '',\n",
       " '3210': '',\n",
       " 'Bevo74': '',\n",
       " 'Leobaumgart': '',\n",
       " 'Balcer': '',\n",
       " 'Good Olfactory': \"28 October Netanyahu categorically rejected the Palestinian right of return and said that Jerusalem must remain undivided. < ref name '' pointofnoreturn '' > 6 November Israeli negotiators said there will not be a state based on the 1967 borders and that the Separation Wall will be a boundary .14 November On 14 November , the Palestinian team quit the negotiations blaming the `` escalation of settlement-building . ''compromises Before the peace talks began , both sides offered a gift . Palestine offered to put on hold international recognition as a state while Israel offered the release of 104 Palestinian prisoners , all of whom have been in Israeli jails since before the 1993 Oslo I Accord .Commenters have however pointed out that Israel had already promised to release these same 104 Palestinians , back in 1999 under the Sharm el-Sheikh Memorandum , < ref name '' knesset-sharm '' > but never did. < ref name '' jpost-PA-welcomes '' > Critics also worry that Israel will simplythereby significantly eroding trust between the parties . ''As Obama Admin Pushes Peace Process , Abbas Vows No Israelis in Palestine : ‘Racist and Hateful Vision’ Abbas : 'Not a single Israeli ' in future Palestinian state 13 August On the first day the Palestinian team leaders were Saeb Erekat and Muhammed Shtayyeh while their Israeli counterparts wererelease from detention of terrorists ; the failure to control illegal weapons ; and the actual conduct of violent operations '' as a sign that `` the PLO has explicitly violated its renunciation of terrorism and other acts of violence , thereby significantly eroding trust between the parties . ''te news|url Timeline 29 July As Israeli and Palestinian negotiators met for a second day in Washington to discuss renewing peace talks , Mahmoud Abbas said `` in a final resolution , we would not see the presence of a single Israeli – civilian or soldier – on our landsin the West Bank would have the option of remaining if they renounced their Israeli citizenship and applied to be citizens of Palestine . A poll has shown that 4.5 % of Jewish settlers would consider becoming Palestinian citizens under such an arrangement. < ref name '' settlersstay '' >31 January According to Martin Indyk , the framework for the US-backed Middle East peace deal will allow up to 80 per cent of Jewish settlers to remain in the West Bank . The deal would redraw borders so that some 80 per cent of settlers’ homes would be redesignated28 October Netanyahu categorically rejected the Palestinian right of return and said that Jerusalem must remain undivided. < ref name '' pointofnoreturn '' > 6 November Israeli negotiators said there will not be a state based on the 1967 borders and that the Separation Wall will be a boundary .14 November On 14 November , the Palestinian team quit the negotiations blaming the `` escalation of settlement-building . ''compromises Before the peace talks began , both sides offered a gift . Palestine offered to put on hold international recognition as a state while Israel offered the release of 104 Palestinian prisoners , all of whom have been in Israeli jails since before the 1993 Oslo I Accord .Commenters have however pointed out that Israel had already promised to release these same 104 Palestinians , back in 1999 under the Sharm el-Sheikh Memorandum , < ref name '' knesset-sharm '' > but never did. < ref name '' jpost-PA-welcomes '' > Critics also worry that Israel will simplythereby significantly eroding trust between the parties . ''As Obama Admin Pushes Peace Process , Abbas Vows No Israelis in Palestine : ‘Racist and Hateful Vision’ Abbas : 'Not a single Israeli ' in future Palestinian state 13 August On the first day the Palestinian team leaders were Saeb Erekat and Muhammed Shtayyeh while their Israeli counterparts wererelease from detention of terrorists ; the failure to control illegal weapons ; and the actual conduct of violent operations '' as a sign that `` the PLO has explicitly violated its renunciation of terrorism and other acts of violence , thereby significantly eroding trust between the parties . ''te news|url Timeline 29 July As Israeli and Palestinian negotiators met for a second day in Washington to discuss renewing peace talks , Mahmoud Abbas said `` in a final resolution , we would not see the presence of a single Israeli – civilian or soldier – on our landsin the West Bank would have the option of remaining if they renounced their Israeli citizenship and applied to be citizens of Palestine . A poll has shown that 4.5 % of Jewish settlers would consider becoming Palestinian citizens under such an arrangement. < ref name '' settlersstay '' >31 January According to Martin Indyk , the framework for the US-backed Middle East peace deal will allow up to 80 per cent of Jewish settlers to remain in the West Bank . The deal would redraw borders so that some 80 per cent of settlers’ homes would be redesignated28 October Netanyahu categorically rejected the Palestinian right of return and said that Jerusalem must remain undivided. < ref name '' pointofnoreturn '' > 6 November Israeli negotiators said there will not be a state based on the 1967 borders and that the Separation Wall will be a boundary .14 November On 14 November , the Palestinian team quit the negotiations blaming the `` escalation of settlement-building . ''compromises Before the peace talks began , both sides offered a gift . Palestine offered to put on hold international recognition as a state while Israel offered the release of 104 Palestinian prisoners , all of whom have been in Israeli jails since before the 1993 Oslo I Accord .Commenters have however pointed out that Israel had already promised to release these same 104 Palestinians , back in 1999 under the Sharm el-Sheikh Memorandum , < ref name '' knesset-sharm '' > but never did. < ref name '' jpost-PA-welcomes '' > Critics also worry that Israel will simplythereby significantly eroding trust between the parties . ''As Obama Admin Pushes Peace Process , Abbas Vows No Israelis in Palestine : ‘Racist and Hateful Vision’ Abbas : 'Not a single Israeli ' in future Palestinian state 13 August On the first day the Palestinian team leaders were Saeb Erekat and Muhammed Shtayyeh while their Israeli counterparts wererelease from detention of terrorists ; the failure to control illegal weapons ; and the actual conduct of violent operations '' as a sign that `` the PLO has explicitly violated its renunciation of terrorism and other acts of violence , thereby significantly eroding trust between the parties . ''te news|url Timeline 29 July As Israeli and Palestinian negotiators met for a second day in Washington to discuss renewing peace talks , Mahmoud Abbas said `` in a final resolution , we would not see the presence of a single Israeli – civilian or soldier – on our landsin the West Bank would have the option of remaining if they renounced their Israeli citizenship and applied to be citizens of Palestine . A poll has shown that 4.5 % of Jewish settlers would consider becoming Palestinian citizens under such an arrangement. < ref name '' settlersstay '' >31 January According to Martin Indyk , the framework for the US-backed Middle East peace deal will allow up to 80 per cent of Jewish settlers to remain in the West Bank . The deal would redraw borders so that some 80 per cent of settlers’ homes would be redesignated\",\n",
       " 'Noble Story': '',\n",
       " 'Jamrifis': '',\n",
       " 'Piotrus': '',\n",
       " 'Najdazy': '',\n",
       " 'Vlmastra': '',\n",
       " 'Richard001': '',\n",
       " 'Neodymium60': '',\n",
       " 'Domsta333': '',\n",
       " 'Lisa': '',\n",
       " 'Mr. Hicks The III': '',\n",
       " 'Silverchemist': '',\n",
       " 'Hamamelis': '',\n",
       " 'Vanished User 1004': '',\n",
       " 'AreaMan': '',\n",
       " 'Avaya1': '',\n",
       " 'Misty MH': '',\n",
       " 'Ocdncntx': '',\n",
       " 'Queenrani': '',\n",
       " 'It is me i think': '',\n",
       " 'Ha-y Gavra': '',\n",
       " 'Alexey Topol': '',\n",
       " 'Danezra': '',\n",
       " 'Gobbleblotchit': '',\n",
       " 'El Sordo1': '',\n",
       " 'Ulkomaalainen': '',\n",
       " 'Micahbrwn': '',\n",
       " '3dnatureguy': '',\n",
       " 'Tyrenius': '',\n",
       " 'Dumaka': '',\n",
       " 'Ceedjee': '',\n",
       " 'Verdatum': '',\n",
       " 'Xenonius': '',\n",
       " 'Jb1944': \"in 1999 , but never did . Critics also worry that Israel will simply quietly re-arrest the potentially released Palestinians , and state that Israel is using the slow release to hold the negotiations hostage and that the main goal of the release is to bolster Israel 's image .Palestinians worry about prisoner release - Features . Al Jazeera English . Retrieved on 2013-08-14 . Of the first 26 detainees , eight ( the Israeli newspaper ''Ha'aretz '' put the number at nine ) are near the end of their sentences - six would have finished their sentence withinRetrieved on 2013-08-14 . Of the first 26 detainees , eight ( the Israeli newspaper ''Ha'aretz '' put the number at nine ) are near the end of their sentences - six would have finished their sentence within the next three years , and two others within six months .Israel to release 'low-risk ' prisoners - Middle East . Al Jazeera English . Retrieved on 2013-08-14 . Israel and new settlements Israel was accused of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It isnegotiations were scheduled to last up to nine months to reach a final status to the Palestinian-Israeli conflict by mid-2014 . They started in Washington , DC < ref name '' wapo '' > and will then move to the King David Hotel in Jerusalem and then to Hebron .an , Lazar . ( 2013-08-07 ) US peace envoy to come to Israel next week . The Times of Israel . Retrieved on 2013-08-14 . Prisoner release In July 2013 , Israel announced as a `` good will gesture '' that they would release 104 Palestinian prisoners , allin 1999 , but never did . Critics also worry that Israel will simply quietly re-arrest the potentially released Palestinians , and state that Israel is using the slow release to hold the negotiations hostage and that the main goal of the release is to bolster Israel 's image .Palestinians worry about prisoner release - Features . Al Jazeera English . Retrieved on 2013-08-14 . Of the first 26 detainees , eight ( the Israeli newspaper ''Ha'aretz '' put the number at nine ) are near the end of their sentences - six would have finished their sentence withinRetrieved on 2013-08-14 . Of the first 26 detainees , eight ( the Israeli newspaper ''Ha'aretz '' put the number at nine ) are near the end of their sentences - six would have finished their sentence within the next three years , and two others within six months .Israel to release 'low-risk ' prisoners - Middle East . Al Jazeera English . Retrieved on 2013-08-14 . Israel and new settlements Israel was accused of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It isnegotiations were scheduled to last up to nine months to reach a final status to the Palestinian-Israeli conflict by mid-2014 . They started in Washington , DC < ref name '' wapo '' > and will then move to the King David Hotel in Jerusalem and then to Hebron .an , Lazar . ( 2013-08-07 ) US peace envoy to come to Israel next week . The Times of Israel . Retrieved on 2013-08-14 . Prisoner release In July 2013 , Israel announced as a `` good will gesture '' that they would release 104 Palestinian prisoners , allin 1999 , but never did . Critics also worry that Israel will simply quietly re-arrest the potentially released Palestinians , and state that Israel is using the slow release to hold the negotiations hostage and that the main goal of the release is to bolster Israel 's image .Palestinians worry about prisoner release - Features . Al Jazeera English . Retrieved on 2013-08-14 . Of the first 26 detainees , eight ( the Israeli newspaper ''Ha'aretz '' put the number at nine ) are near the end of their sentences - six would have finished their sentence withinRetrieved on 2013-08-14 . Of the first 26 detainees , eight ( the Israeli newspaper ''Ha'aretz '' put the number at nine ) are near the end of their sentences - six would have finished their sentence within the next three years , and two others within six months .Israel to release 'low-risk ' prisoners - Middle East . Al Jazeera English . Retrieved on 2013-08-14 . Israel and new settlements Israel was accused of trying to sabotage the peace talks by approving nearly 1200 new settlement homes shortly before the negotiations were due to start . It isnegotiations were scheduled to last up to nine months to reach a final status to the Palestinian-Israeli conflict by mid-2014 . They started in Washington , DC < ref name '' wapo '' > and will then move to the King David Hotel in Jerusalem and then to Hebron .an , Lazar . ( 2013-08-07 ) US peace envoy to come to Israel next week . The Times of Israel . Retrieved on 2013-08-14 . Prisoner release In July 2013 , Israel announced as a `` good will gesture '' that they would release 104 Palestinian prisoners , all\",\n",
       " 'WhatamIdoing': '',\n",
       " 'Mohamed Magdy': '',\n",
       " 'Jamesday': '',\n",
       " 'Zero0000': '',\n",
       " 'Crisco 1492': '',\n",
       " 'GateKeeper': '',\n",
       " 'Clinkophonist': '',\n",
       " 'ElrodeoLover': '',\n",
       " 'Cybbe': '',\n",
       " 'Trackinfo': '',\n",
       " 'Elockid': '',\n",
       " 'Wachholder': '',\n",
       " 'Pedantrician': '',\n",
       " 'Redaktor': '',\n",
       " 'Future777': '',\n",
       " 'Joseph E. Saad': '',\n",
       " 'Immeëmosol': '',\n",
       " 'Renner8592': '',\n",
       " 'Menchi': '',\n",
       " 'Pollinator': '',\n",
       " 'MaZiltHona': '',\n",
       " 'Federalist51': '',\n",
       " 'NZUlysses': '',\n",
       " 'Tabarez': '',\n",
       " 'Joefromrandb': '',\n",
       " 'Mrman2': '',\n",
       " 'Takabeg': '',\n",
       " 'Childe Roland of Gilead': '',\n",
       " 'Skydude176': '',\n",
       " 'Rt66lt': '',\n",
       " 'Cristiano Tomás': '',\n",
       " '172': '',\n",
       " 'Strongbrow': '',\n",
       " 'JonathanGo': '',\n",
       " 'Shlomif': '',\n",
       " 'Sire22': '',\n",
       " 'Theophilus Wallace': '',\n",
       " 'K260260': '',\n",
       " 'Mtsmallwood': '',\n",
       " 'Markinship': '',\n",
       " 'Adam Bishop': '',\n",
       " 'Becksguy': '',\n",
       " 'Maarten Hermans': '',\n",
       " 'SebastianGS': '',\n",
       " 'Eastcote': '',\n",
       " 'FeldBum': '',\n",
       " 'Dendoi': '',\n",
       " 'Mdwh': '',\n",
       " 'Haploidavey': '',\n",
       " 'Astanhope': '',\n",
       " 'JohnChrysostom': '',\n",
       " 'Philomath2004': '',\n",
       " 'John D. Croft': '',\n",
       " 'Ajnem': '',\n",
       " 'GwydionM': '',\n",
       " 'Bob Burkhardt': '',\n",
       " 'Sunray': '',\n",
       " 'Solatic': '',\n",
       " 'Quinacrine': '',\n",
       " 'Snakeswithfeet': '',\n",
       " 'PlasticFork': '',\n",
       " 'Sernager': '',\n",
       " 'Jan CZ': '',\n",
       " 'Expo1892': '',\n",
       " 'Bellerophon': '',\n",
       " 'J1.grammar natz': '',\n",
       " 'Scaligera': '',\n",
       " 'DrFrench': '',\n",
       " 'Gram123': '',\n",
       " 'Modinyr': '',\n",
       " 'Luqmancharsobis': '',\n",
       " 'AgadaUrbanit': '',\n",
       " 'John': '',\n",
       " 'Mcattell': '',\n",
       " 'Ianblair23': '',\n",
       " 'Miniapolis': '',\n",
       " 'Polmandc': '',\n",
       " 'Sharavanabhava': '',\n",
       " 'MiguelMunoz': '',\n",
       " 'Zimriel': '',\n",
       " 'Kudzu1': '',\n",
       " 'Asbl': '',\n",
       " 'Woogie10w': '',\n",
       " 'Shapeshyfter': '',\n",
       " 'Rabagasst': '',\n",
       " 'One Night In Hackney': '',\n",
       " 'Gene s': '',\n",
       " 'BernardZ': '',\n",
       " 'Meelar': '',\n",
       " 'Ef80': '',\n",
       " 'Peter jackson': '',\n",
       " 'Jessicapierce': '',\n",
       " 'Fipplet': '',\n",
       " 'MuffledThud': '',\n",
       " 'Tintin1107': '',\n",
       " 'AcidSnow': '',\n",
       " 'HueSatLum': '',\n",
       " 'Gentgeen': '',\n",
       " 'Knowitall369': '',\n",
       " 'John J. Bulten': '',\n",
       " 'Slash': '',\n",
       " 'Omdo': '',\n",
       " 'Seerus': '',\n",
       " 'AchtungAchtung': '',\n",
       " 'Oichiro': '',\n",
       " 'Azalea pomp': '',\n",
       " 'רסטיניאק': '',\n",
       " 'Johncandrew': '',\n",
       " 'Male1979': '',\n",
       " 'Poliocretes': '',\n",
       " 'PeterWaldo': '',\n",
       " 'Alphaboi867': '',\n",
       " 'Escoville': '',\n",
       " 'Tnxman307': '',\n",
       " 'Asnac': '',\n",
       " 'AVR2012': '',\n",
       " 'AnaHona': '',\n",
       " 'Kmac1986': '',\n",
       " 'Kontrolleur Cro': '',\n",
       " 'Tchadienne': '',\n",
       " 'MPLX': '',\n",
       " 'Wb678': '',\n",
       " 'Callanecc': '',\n",
       " 'BringItOn TheAteam': '',\n",
       " 'Father Goose': '',\n",
       " 'Osl97': '',\n",
       " 'Anonymous Dissident': '',\n",
       " 'Lopakhin': '',\n",
       " 'Omeganian': '',\n",
       " 'Wikipeterproject': '',\n",
       " 'Maxman': '',\n",
       " 'Vetrisimino0': '',\n",
       " 'Arxiloxos': '',\n",
       " 'Glenntwo': '',\n",
       " 'Pedant': '',\n",
       " '84user': '',\n",
       " 'KingHiggins': '',\n",
       " 'Syrian77': '',\n",
       " 'Demiurge1000': '',\n",
       " 'Sambot': '',\n",
       " 'Arre': '',\n",
       " 'Maerk': '',\n",
       " 'Stevenwhig': '',\n",
       " 'Miona152': '',\n",
       " 'Relata refero': '',\n",
       " 'Epictatus': '',\n",
       " 'Colombo Man': '',\n",
       " 'Rhetth': '',\n",
       " 'Wile E. Heresiarch': '',\n",
       " 'Bob drobbs': '',\n",
       " 'Betzalelf': '',\n",
       " 'CatcherWry': '',\n",
       " 'Dinkytown': '',\n",
       " 'Mangojuice': '',\n",
       " 'Nicolas1981': '',\n",
       " '48Lugur': '',\n",
       " 'SpidErxD': '',\n",
       " 'JamesMLane': '',\n",
       " 'Halgin': '',\n",
       " 'Tangerine': '',\n",
       " 'Steel1943': '',\n",
       " 'Sixtyideas': '',\n",
       " 'MickMacNee': '',\n",
       " 'RW Marloe': '',\n",
       " 'Naghmeh et': '',\n",
       " 'Maglanist': '',\n",
       " 'EhJJ': '',\n",
       " 'Rolf acker': '',\n",
       " 'Jennavecia': '',\n",
       " 'StuffOfInterest': '',\n",
       " 'Minguo': '',\n",
       " 'Dumarest': '',\n",
       " 'Ellenois': '',\n",
       " 'JodyB': '',\n",
       " 'Ory Amitay': '',\n",
       " 'Diren Yardimli': '',\n",
       " 'Alaina3': '',\n",
       " 'Dimadick': '',\n",
       " ...}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edtext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index 0 is out of bounds for axis 0 with size 0",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0mTraceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-fff32c2cfcd8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;31m#         i, row in diffdata.iterrows():\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0med\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtalk_eds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m         \u001b[0mbefore_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m' '\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mpreprocess\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtext\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdiffdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdiffdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'editor'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m==\u001b[0m\u001b[0med\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'before_citation'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m         \u001b[0mafter_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m' '\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mpreprocess\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtext\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdiffdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdiffdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'editor'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m==\u001b[0m\u001b[0med\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'after_citation'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m         \u001b[0medtext\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0med\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbefore_text\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m' '\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mafter_text\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: index 0 is out of bounds for axis 0 with size 0"
     ]
    }
   ],
   "source": [
    "outpath = '/home/michael/school/research/wp/wp_articles/editor_citation_window.csv'\n",
    "with open(outpath, 'w') as f:\n",
    "    w = csv.writer(f)\n",
    "    w.writerow(['article_name', 'timestamp', 'editor', 'citation', 'before', 'after',])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      " set to begin after Israel agrees to free 104 Palestinian prisoners}}</ref>  and will then move to the [[King David Hotel bombing|King David Hotel in Jerusalem]][http://www.timesofisrael.com/us-peace-envoy-to-come-to-israel-next-week/] and then to Hebron.  ==Prisoner release== In July 2013, Israel announced as a \"good will jesture\" that it would be releasing 104 Palestinian prisoners, all of whom have been in Israeli jails since before the 1993 Oslo Accords. Israel then stated that it would be releasing the prisoners slowing throughout the negotiations rather than before the negotiations.   Commenters have however pointed out that Israel had already promised to release these same 104 Palestinians, back in 1999, but never did. Critics also worry that Israel will simply quietly re-arrest the potentially released Palestinians, and state that Israel is using the slow release to hold the negotiations hostage and that the main goal of the release is to bolster Israel's image.<ref>http://www.\n",
      "\n",
      "  and will then move to the King David Hotel in Jerusalem and then to Hebron.  Prisoner release In July 2013, Israel announced as a \"good will jesture\" that it would be releasing 104 Palestinian prisoners, all of whom have been in Israeli jails since before the 1993 Oslo Accords. Israel then stated that it would be releasing the prisoners slowing throughout the negotiations rather than before the negotiations.   Commenters have however pointed out that Israel had already promised to release these same 104 Palestinians, back in 1999, but never did. Critics also worry that Israel will simply quietly re-arrest the potentially released Palestinians, and state that Israel is using the slow release to hold the negotiations hostage and that the main goal of the release is to bolster Israel's image.\n",
      "143\n",
      "\n",
      "indepth/features/2013/08/201389101325337889.html</ref>   The announcement came after Israel was accused of trying to sabotage the peace talks by approving nearly 1200 new settlement homes, which are on Palestinian land and illegal under international law. Israeli government spokesman Mark Regev dismissed the criticism.[http://www.bbc.co.uk/news/world-middle-east-23662160]  ==See also== * [[Geneva II Middle East peace conference]]  ==References== {{reflist}}  [[Category:Israeli–Palestinian conflict]]\n",
      "\n",
      "   The announcement came after Israel was accused of trying to sabotage the peace talks by approving nearly 1200 new settlement homes, which are on Palestinian land and illegal under international law. Israeli government spokesman Mark Regev dismissed the criticism.  See also * Geneva II Middle East peace conference  References   Israeli–Palestinian conflict\n",
      "54\n"
     ]
    }
   ],
   "source": [
    "# Iterate thru citediffs, finding citation windows in text\n",
    "charwin = 1000\n",
    "\n",
    "outpath = '/home/michael/school/research/wp/wp_articles/citation_windows/{:s}'.format(f)\n",
    "\n",
    "with open(outpath, 'w') as f:\n",
    "    w = csv.writer(f)\n",
    "    w.writerow(['article_name', 'timestamp', 'editor', 'citation', 'before', 'after',])\n",
    "    \n",
    "    for i, row in diffdata.iterrows():\n",
    "        ts = row['timestamp']\n",
    "        c = citedata[citedata['timestamp'==ts]['citation']\n",
    "        ctype = row['citation_type']\n",
    "\n",
    "        text = revdata[revdata['timestamp']==ts]['text'].values[0]\n",
    "        ccount = text.count(c)\n",
    "        print(ccount)\n",
    "        cidx = text.rfind(c) \n",
    "        after_idx = cidx+len(c)+1\n",
    "\n",
    "        before = text[max(cidx-charwin,0):cidx]\n",
    "        after = text[after_idx:min(after_idx+charwin,len(text))]\n",
    "    #     print(before)\n",
    "    #     print()\n",
    "    #     print(preprocess(before))\n",
    "    # #     print(len(word_tokenize(preprocess(before))))\n",
    "    #     print()\n",
    "    #     print(after)\n",
    "    #     print()\n",
    "    #     print(preprocess(after))\n",
    "    # #     print(len(word_tokenize(preprocess(after))))\n",
    "\n",
    "        if i > 100: break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}