Mixed Model Eval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# mixed model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pickle the model files and load here to test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dannyyang/Library/Python/3.6/lib/python/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import spacy\n",
    "import networkx as nx\n",
    "import score as sc\n",
    "import time\n",
    "from sklearn.metrics import confusion_matrix, pairwise, f1_score, precision_score\n",
    "from scipy.spatial import distance\n",
    "from preprocessing.utils import plot_confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "from collections import Counter, defaultdict\n",
    "import itertools\n",
    "import importlib\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import en_coref_md\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "coref = en_coref_md.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# In[21]:\n",
    "\n",
    "\n",
    "negating_words = set([\n",
    "    \"n't\", \"not\", \"no\", \n",
    "    \"never\", \"nobody\", \"non\", \"nope\"])\n",
    "doubting_words = set([\n",
    "    'fake','fraud', 'hoax', \n",
    "    'false', 'deny', 'denies', \n",
    "    'despite', 'doubt', \n",
    "    'bogus', 'debunk', 'prank', \n",
    "    'retract', 'scam', \"withdrawn\",\n",
    "    \"misinformation\"])\n",
    "hedging_words = set([\n",
    "    'allege', 'allegedly','apparently',\n",
    "    'appear','claim','could',\n",
    "    'evidently','largely','likely',\n",
    "    'mainly','may', 'maybe', 'might',\n",
    "    'mostly','perhaps','presumably',\n",
    "    'probably','purport', 'purportedly',\n",
    "    'reported', 'reportedly',\n",
    "    'rumor', 'rumour', 'rumored', 'rumoured',\n",
    "    'says','seem','somewhat',\n",
    "    'unconfirmed'])\n",
    "sus_words = doubting_words.union(hedging_words)\n",
    "\n",
    "\n",
    "# In[92]:\n",
    "\n",
    "\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "vader = SentimentIntensityAnalyzer()\n",
    "\n",
    "def get_sentiment(sentence):\n",
    "    sent =  vader.polarity_scores(sentence)\n",
    "    return [sent[\"pos\"],sent[\"neg\"],sent[\"neu\"],sent[\"compound\"]]\n",
    "\n",
    "def get_avg_sentiment(lst):\n",
    "    sents = np.array([get_sentiment(s) for s in lst])\n",
    "    return list(np.mean(sents, axis = 0))\n",
    "\n",
    "def get_diff_sentiment(a,b):\n",
    "    return list(np.array(a) - np.array(b))\n",
    "\n",
    "\n",
    "# In[23]:\n",
    "\n",
    "\n",
    "def train_test_split(bodies, stances, split=0.8):\n",
    "    idx = np.random.permutation(np.arange(len(bodies)))\n",
    "    bodies = bodies.values[idx]\n",
    "    train = int(len(bodies)*0.8)\n",
    "    bodies_tr = set([i[0] for i in bodies[:train]])\n",
    "    bodies_val = set([i[0] for i in bodies[train:]])\n",
    "    stances_tr = stances.loc[stances[\"Body ID\"].isin(bodies_tr), :]\n",
    "    stances_val = stances.loc[stances[\"Body ID\"].isin(bodies_val), :]\n",
    "    return stances_tr, stances_val\n",
    "\n",
    "# In[26]:\n",
    "\n",
    "\n",
    "def get_body(n):\n",
    "    return test_bodies.loc[lambda x: x[\"Body ID\"] == n, \"articleBody\"].item()\n",
    "\n",
    "\n",
    "# In[27]:\n",
    "\n",
    "\n",
    "def preprocess(text):\n",
    "    text = text.replace(\"' \",' ')\n",
    "    text = text.replace(\"'\\n\",'\\n')\n",
    "    text = text.replace(\" '\",' ')\n",
    "    text = text.replace('\"',' ')\n",
    "    text = text.replace('“',' ')\n",
    "    text = text.replace('”', ' ')\n",
    "    text = text.replace(\":\", \". \")\n",
    "    text = text.replace(\";\", \". \")\n",
    "    text = text.replace(\"...\", \" \")\n",
    "    return text\n",
    "\n",
    "\n",
    "# In[28]:\n",
    "\n",
    "\n",
    "def cosine_similarity(x,y):\n",
    "    if all([a == 0 for a in x]) or all([a == 0 for a in y]):\n",
    "        return 0\n",
    "    return 1 - np.nan_to_num(distance.cosine(x,y))\n",
    "\n",
    "\n",
    "# In[29]:\n",
    "\n",
    "\n",
    "def get_topics(doc):\n",
    "    \"\"\"\n",
    "    get topics of a sentence\n",
    "    input: spacy doc\n",
    "    output: dictionary with nouns as the key, and the set of noun chunks that contain the noun as the value\n",
    "    special entry _vocab has the set of all tokens in the dict\n",
    "    \"\"\"\n",
    "    subjs = {}\n",
    "    for chunk in doc.noun_chunks:\n",
    "        if len(chunk.root.text) > 2 and chunk.root.pos_ not in [\"NUM\", \"SYM\",\"PUNCT\"]:\n",
    "            txt = chunk.root.lemma_.lower()\n",
    "            if txt not in subjs:\n",
    "                subjs[txt] = set([txt])\n",
    "            subjs[txt].add(chunk.text.lower())\n",
    "    subjects_= []\n",
    "    for word in subjs:\n",
    "        for phrase in subjs[word]:\n",
    "            subjects_ += phrase.split(\" \")\n",
    "    subjs[\"_vocab\"] = set(subjects_)\n",
    "    return subjs\n",
    "\n",
    "\n",
    "# In[30]:\n",
    "\n",
    "\n",
    "def get_svos(sent):\n",
    "    \"\"\"\n",
    "    input: Spacy processed sentence\n",
    "    output: dict of subj, dict of v, dict of obj (each word is lemmatized and lowercased)\n",
    "    each entry in dict has key of lemmatized token, value is actual token (to do traversals with later if needed)\n",
    "    \"\"\"\n",
    "    s = {}\n",
    "    v = {}\n",
    "    o = {}\n",
    "    for token in sent:\n",
    "        if token.dep_ == 'ROOT':\n",
    "            v[token.lemma_.lower()] = token\n",
    "        elif token.dep_ in [\"nsubj\", \"nsubjpass\", \"csubj\",\"csubjpass\", \"agent\",\"compound\"]:\n",
    "            s[token.lemma_.lower()] = token\n",
    "        elif token.dep_ in [\"dobj\", \"dative\", \"attr\", \"oprd\", \"pobj\"]:\n",
    "            o[token.lemma_.lower()] = token\n",
    "    # https://github.com/clir/clearnlp-guidelines/blob/master/md/specifications/dependency_labels.md\n",
    "    return (s,v,o)\n",
    "\n",
    "\n",
    "# In[31]:\n",
    "\n",
    "\n",
    "def build_graph(doc):\n",
    "    \"\"\"\n",
    "    build a NetworkX graph of the dependency tree\n",
    "    input: spacy Doc\n",
    "    output: networkx graph\n",
    "    \"\"\"\n",
    "    edges = set()\n",
    "    for token in doc:\n",
    "        if token.pos_ not in ['SPACE']:\n",
    "            for child in token.children:\n",
    "                if child.pos_ not in ['SPACE']:\n",
    "                    edges.add((token.lemma_.lower(),child.lemma_.lower()))\n",
    "    graph = nx.DiGraph(list(edges))\n",
    "    return graph\n",
    "\n",
    "\n",
    "# In[32]:\n",
    "\n",
    "\n",
    "def get_edges(doc):\n",
    "    \"\"\"\n",
    "    return list of edges\n",
    "    \"\"\"\n",
    "    edges = []\n",
    "    for token in doc:\n",
    "        if token.pos_ not in ['SPACE', 'PUNCT', 'SYM']:\n",
    "            for child in token.children:\n",
    "                if child.pos_ not in ['SPACE', 'PUNCT', 'SYM']:\n",
    "                    edges.append((\n",
    "                        {\"token\":token.lemma_.lower(), \"dep\":token.dep_ , \"pos\":token.pos_},\n",
    "                        {\"token\":child.lemma_.lower(), \"dep\":child.dep_ , \"pos\":child.pos_}\n",
    "                    ))\n",
    "    return edges\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# In[33]:\n",
    "\n",
    "\n",
    "def get_summary(doc, subjects, n = 5):\n",
    "    \"\"\"\n",
    "    get summary of n sentences in document\n",
    "    first meaningful sentence will always be returned\n",
    "    \"\"\"\n",
    "    subjects_ = subjects\n",
    "    def score_sentence(sent):\n",
    "        # not very robust right now\n",
    "        score = 0\n",
    "        word_count = 0\n",
    "        for token in sent:\n",
    "            word_count += 1\n",
    "            t = token.lemma_.lower()\n",
    "            if t in subjects_:\n",
    "                score += 1\n",
    "            elif t in negating_words or t in doubting_words or t in hedging_words:\n",
    "                score += 1\n",
    "        return score/word_count if word_count > 4 else 0\n",
    "    sentences = [s for s in doc.sents]\n",
    "    scored_sentences = [[idx, sent, score_sentence(sent)] for idx, sent in enumerate(sentences)]\n",
    "    scored_sentences = [s for s in scored_sentences if s[2] > 0 and s[0] > 0] #filter out non-scoring sentences\n",
    "    scored_sentences.sort(key = lambda x: x[2], reverse = True)\n",
    "    top = scored_sentences[:n]\n",
    "    top.sort(key = lambda x: x[0])\n",
    "    scored_sentences.sort(key = lambda x: x[0])\n",
    "    result = None\n",
    "    if len(scored_sentences) == 0:\n",
    "        result = [sentences[0]]\n",
    "    else:\n",
    "        result = [scored_sentences[0][1]] + [s[1] for s in top]\n",
    "    return result\n",
    "\n",
    "\n",
    "# In[34]:\n",
    "\n",
    "\n",
    "def get_shortest_path_to_negating(graph, subjects):\n",
    "    \"\"\"\n",
    "    get the shortest path from each subject to any negating or doubting/hedging word\n",
    "    returns: dictionary with subject as key, and 2-element list of path lengths [negating, doubting]\n",
    "    - if a subject does not exist in graph or have a path to any negating word, then the value will be [None, None]\n",
    "    \"\"\"\n",
    "    results = {}\n",
    "    for s in subjects:\n",
    "        results[s] = [None, None, None]\n",
    "        if graph.has_node(s):\n",
    "            for word in negating_words:\n",
    "                if word in graph:\n",
    "                    try:\n",
    "                        path = nx.shortest_path(graph, source = s, target = word)\n",
    "                        if results[s][0] == None or len(path) < results[s][0]:\n",
    "                            results[s][0] = len(path)\n",
    "                    except:\n",
    "                        continue\n",
    "            for word in hedging_words:\n",
    "                if word in graph:\n",
    "                    try:\n",
    "                        path = nx.shortest_path(graph, source = s, target = word)\n",
    "                        if results[s][1] == None or len(path) < results[s][1]:\n",
    "                            results[s][1] = len(path)\n",
    "                    except:\n",
    "                        continue\n",
    "            for word in doubting_words:\n",
    "                if word in graph:\n",
    "                    try:\n",
    "                        path = nx.shortest_path(graph, source = s, target = word)\n",
    "                        if results[s][2] == None or len(path) < results[s][2]:\n",
    "                            results[s][2] = len(path)\n",
    "                    except:\n",
    "                        continue\n",
    "    return results\n",
    "\n",
    "\n",
    "# In[35]:\n",
    "\n",
    "\n",
    "def root_distance(graph, root):\n",
    "    \"\"\"\n",
    "    as implemented in the Emergent paper - return the shortest distance between the given root and any \n",
    "    doubting or hedging words in the graph, or None if no such path exists\n",
    "    \"\"\"\n",
    "    if root == None:\n",
    "        return None\n",
    "    min_dist = None\n",
    "    for word in sus_words:\n",
    "        if word in graph:\n",
    "            try:\n",
    "                path = nx.shortest_path(graph, source = root, target = word)\n",
    "                if min_dist == None or len(path) < min_dist:\n",
    "                    min_dist = len(path)\n",
    "            except:\n",
    "                continue\n",
    "    return min_dist\n",
    "\n",
    "\n",
    "# In[36]:\n",
    "\n",
    "\n",
    "def get_neg_ancestors(doc):\n",
    "    \"\"\"\n",
    "    get the ancestors of every negating word\n",
    "    input: spacy Doc\n",
    "    returns: tuple  - set of words that were in the ancestor list of negating words, \n",
    "    set of words that were in ancestor list of refuting words, # negating words, # refuting words\n",
    "    \"\"\"\n",
    "    results = [set(), set(), set(), 0, 0, 0]\n",
    "    for token in doc:\n",
    "        if token.lemma_.lower() in negating_words:\n",
    "            results[0] = results[0].union(\n",
    "                set([ancestor.lemma_.lower() for ancestor in token.ancestors if len(ancestor) > 2]).union(\n",
    "                    set([child.lemma_.lower() for child in token.head.children if child.text != token.text and len(child) > 2])\n",
    "                )\n",
    "            )\n",
    "            results[3] += 1\n",
    "        elif token.lemma_.lower() in doubting_words:\n",
    "            results[1] = results[1].union(\n",
    "                set([ancestor.lemma_.lower() for ancestor in token.ancestors if len(ancestor) > 2]).union(\n",
    "                    set([child.lemma_.lower() for child in token.head.children if child.text != token.text and len(child) > 2])\n",
    "                )\n",
    "            )\n",
    "            results[4] += 1\n",
    "        elif token.lemma_.lower() in hedging_words:\n",
    "            results[2] = results[1].union(\n",
    "                set([ancestor.lemma_.lower() for ancestor in token.ancestors if len(ancestor) > 2]).union(\n",
    "                    set([child.lemma_.lower() for child in token.head.children if child.text != token.text and len(child) > 2])\n",
    "                )\n",
    "            )\n",
    "            results[5] += 1\n",
    "    return tuple(results)\n",
    "\n",
    "\n",
    "# In[49]:\n",
    "\n",
    "\n",
    "from sentic import SenticPhrase\n",
    "text = \"Hello, World!\"\n",
    "sp = SenticPhrase(text)\n",
    "\n",
    "def get_sentics(sent):\n",
    "    \"\"\"\n",
    "        input: Spacy processed sentence\n",
    "        output: a tuple containing the polarity score and a list of sentic values \n",
    "            (pleasantness, attention, sensitiviy, aptitude )\n",
    "    \"\"\"\n",
    "    info = sp.info(sent)\n",
    "          \n",
    "    # Sometimes sentic doesn't returns any sentics values, seems to be only when purely neutral. \n",
    "    # Some sort of tag to make sure this is true could help with classiciation! (if all 0's not enough)\n",
    "    sentics = {\"pleasantness\":0, \"attention\":0, \"sensitivity\":0, \"aptitude\":0}\n",
    "    sentics.update(info[\"sentics\"])\n",
    "    return [info['polarity'], sentics['aptitude'], sentics['attention'], sentics['sensitivity'], sentics['pleasantness']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25413, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Headline</th>\n",
       "      <th>Body ID</th>\n",
       "      <th>Stance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ferguson riots: Pregnant woman loses eye after...</td>\n",
       "      <td>2008</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Crazy Conservatives Are Sure a Gitmo Detainee ...</td>\n",
       "      <td>1550</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A Russian Guy Says His Justin Bieber Ringtone ...</td>\n",
       "      <td>2</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Zombie Cat: Buried Kitty Believed Dead, Meows ...</td>\n",
       "      <td>1793</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Argentina's President Adopts Boy to End Werewo...</td>\n",
       "      <td>37</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Headline  Body ID     Stance\n",
       "0  Ferguson riots: Pregnant woman loses eye after...     2008  unrelated\n",
       "1  Crazy Conservatives Are Sure a Gitmo Detainee ...     1550  unrelated\n",
       "2  A Russian Guy Says His Justin Bieber Ringtone ...        2  unrelated\n",
       "3  Zombie Cat: Buried Kitty Believed Dead, Meows ...     1793  unrelated\n",
       "4  Argentina's President Adopts Boy to End Werewo...       37  unrelated"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#load test data\n",
    "test_stances = pd.read_csv(\"fn_data/competition_test_stances.csv\")\n",
    "print(test_stances.shape)\n",
    "test_stances.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(904, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Body ID</th>\n",
       "      <th>articleBody</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Al-Sisi has denied Israeli reports stating tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>A bereaved Afghan mother took revenge on the T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>CNBC is reporting Tesla has chosen Nevada as t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>A 4-inch version of the iPhone 6 is said to be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19</td>\n",
       "      <td>GR editor’s Note\\n\\nThere are no reports in th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Body ID                                        articleBody\n",
       "0        1  Al-Sisi has denied Israeli reports stating tha...\n",
       "1        2  A bereaved Afghan mother took revenge on the T...\n",
       "2        3  CNBC is reporting Tesla has chosen Nevada as t...\n",
       "3       12  A 4-inch version of the iPhone 6 is said to be...\n",
       "4       19  GR editor’s Note\\n\\nThere are no reports in th..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_bodies = pd.read_csv(\"fn_data/competition_test_bodies.csv\")\n",
    "print(test_bodies.shape)\n",
    "test_bodies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 0\n",
      "Processed 2500\n",
      "Processed 5000\n",
      "Processed 7500\n",
      "Processed 10000\n",
      "Processed 12500\n",
      "Processed 15000\n",
      "Processed 17500\n",
      "Processed 20000\n",
      "Processed 22500\n",
      "Processed 25000\n",
      "Done!\n",
      "Processed 0\n",
      "Processed 100\n",
      "Processed 200\n",
      "Processed 300\n",
      "Processed 400\n",
      "Processed 500\n",
      "Processed 600\n",
      "Processed 700\n",
      "Processed 800\n",
      "Processed 900\n",
      "Done!\n",
      "382\n"
     ]
    }
   ],
   "source": [
    "headline_info = {}\n",
    "body_info = {}\n",
    "start = time.time()\n",
    "stance_data = list(test_stances.values)\n",
    "body_data = list(test_bodies.values)\n",
    "for headline in range(len(stance_data)):\n",
    "    if headline % 2500 == 0:\n",
    "        print(\"Processed \"+str(headline))\n",
    "    h, b_id, s = tuple(stance_data[headline])\n",
    "    if h not in headline_info:\n",
    "        nlp_h = nlp(preprocess(h))\n",
    "        headline_edges = get_edges(nlp_h)\n",
    "        headline_graph = nx.DiGraph(list(set([(e[0]['token'], e[1]['token']) for e in headline_edges])))\n",
    "        headline_subj = get_topics(nlp_h)\n",
    "        headline_svo = get_svos(nlp_h)\n",
    "        headline_root_dist = root_distance(headline_graph, list(headline_svo[1].keys())[0])\n",
    "        headline_neg_ancestors = get_neg_ancestors(nlp_h)\n",
    "        headline_info[h] = (nlp_h, headline_graph, headline_subj, headline_svo, headline_root_dist, headline_neg_ancestors, headline_edges)\n",
    "print(\"Done!\")\n",
    "for body in range(len(body_data)):\n",
    "    if body % 100 == 0:\n",
    "        print(\"Processed \"+str(body))\n",
    "    b_id, txt = tuple(body_data[body])\n",
    "    nlp_a = coref(preprocess(txt))\n",
    "    nlp_b = nlp(nlp_a._.coref_resolved.lower())\n",
    "    body_graph = build_graph(nlp_b)\n",
    "    body_info[b_id] = (nlp_b, body_graph)\n",
    "print(\"Done!\")\n",
    "end = time.time()\n",
    "print(int(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 0\n",
      "Processed 100\n",
      "Processed 200\n",
      "Processed 300\n",
      "Processed 400\n",
      "Processed 500\n",
      "Processed 600\n",
      "Processed 700\n",
      "Processed 800\n",
      "Processed 900\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "def build_idf(body_data):\n",
    "    idf = {}\n",
    "    for body in range(len(body_data)):\n",
    "        if body % 100 == 0:\n",
    "            print(\"Processed \"+str(body))\n",
    "        b_id, txt = tuple(body_data[body])\n",
    "        nlp_b = nlp(preprocess(txt))\n",
    "        tokens = [t for t in nlp_b if not t.is_stop and t.pos_ not in ['PUNCT','NUM','SYM','SPACE','PART']]\n",
    "        lemmatized = set([token.lemma_.lower() for token in tokens])\n",
    "        for tok in lemmatized:\n",
    "            if tok not in idf:\n",
    "                idf[tok] = 0\n",
    "            idf[tok] += 1\n",
    "    avg = float(sum(idf.values())) / len(idf)\n",
    "    for i in idf:\n",
    "        idf[i] = math.log(len(body_data)/idf[i])\n",
    "    idf[\"_avg\"] = math.log(len(body_data)/avg)\n",
    "    return idf\n",
    "idf = build_idf(list(test_bodies.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_sentence(sentence):\n",
    "    svo = get_svos(sentence)\n",
    "\n",
    "    # list of words that belong to that part of speech\n",
    "    nouns = []\n",
    "    verbs = []\n",
    "    adjectives = []\n",
    "    adverbs = []\n",
    "    tokens = []\n",
    "    for token in sentence:\n",
    "        if not token.is_stop and token.pos_ not in ['PUNCT', 'NUM', 'SYM','SPACE','PART']:\n",
    "            if token.pos_ == \"NOUN\":\n",
    "                nouns.append(token.lemma_.lower())\n",
    "            elif token.pos_ == \"VERB\":\n",
    "                verbs.append(token.lemma_.lower())\n",
    "            elif token.pos_ == \"ADJ\":\n",
    "                adjectives.append(token.lemma_.lower())\n",
    "            elif token.pos_ == \"ADV\":\n",
    "                adverbs.append(token.lemma_.lower())\n",
    "            tokens.append(token.lemma_.lower())   \n",
    "    \n",
    "    bigram = list(nltk.bigrams(tokens))\n",
    "    bigram_str = [x[0]+' '+x[1] for x in bigram]\n",
    "\n",
    "    return {\n",
    "        \"raw\": sentence.text,\n",
    "        \"tokens\": tokens,\n",
    "        \"bigrams\": bigram_str,\n",
    "        \"nouns\": nouns,\n",
    "        \"verbs\": verbs,\n",
    "        \"adjectives\": adjectives,\n",
    "        \"adverbs\": adverbs,\n",
    "        \"svo\": [list(item) for item in svo]\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def score_sentence_idf(sent, idf):\n",
    "    # not very robust right now\n",
    "    score = 0\n",
    "    word_count = 0\n",
    "    for token in sent:\n",
    "        word_count += 1\n",
    "        t = token.lemma_.lower()\n",
    "        if t in idf:\n",
    "            score += idf[t]\n",
    "    return score/word_count if word_count > 4 else 0\n",
    "\n",
    "def process_body(body, idf):\n",
    "    sentences = [s for s in body.sents]\n",
    "    if len(sentences) == 0:\n",
    "        sentences = [body]\n",
    "\n",
    "    # first sentence of article\n",
    "    first_sentence_data = process_sentence(sentences[0])\n",
    "\n",
    "    nouns = []\n",
    "    verbs = []\n",
    "    adjectives = []\n",
    "    adverbs = []\n",
    "    tokens = []\n",
    "    for token in body:\n",
    "        if not token.is_stop and token.pos_ not in ['PUNCT', 'NUM', 'SYM','SPACE','PART']:\n",
    "            if token.pos_ == \"NOUN\":\n",
    "                nouns.append(token.lemma_.lower())\n",
    "            elif token.pos_ == \"VERB\":\n",
    "                verbs.append(token.lemma_.lower())\n",
    "            elif token.pos_ == \"ADJ\":\n",
    "                adjectives.append(token.lemma_.lower())\n",
    "            elif token.pos_ == \"ADV\":\n",
    "                adverbs.append(token.lemma_.lower())\n",
    "            tokens.append(token.lemma_.lower())   \n",
    "    \n",
    "    bigram = list(nltk.bigrams(tokens))\n",
    "    bigram_str = [x[0]+' '+x[1] for x in bigram]\n",
    "\n",
    "    doc_len = len(tokens)\n",
    "    n_counter = Counter(nouns)\n",
    "    v_counter = Counter(verbs)\n",
    "    b_counter = Counter(bigram)\n",
    "    t_counter = Counter(tokens)\n",
    "\n",
    "    avg_idf = idf[\"_avg\"]\n",
    "    n_tfidf, v_tfidf, t_tfidf = {}, {}, {}\n",
    "    for n in n_counter:\n",
    "        n_tfidf[n] = (n_counter[n]/doc_len) *             (idf[n] if n in idf else avg_idf)\n",
    "    for v in v_counter:\n",
    "        v_tfidf[v] = (v_counter[v]/doc_len) *             (idf[v] if v in idf else avg_idf)\n",
    "    for t in t_counter:\n",
    "        t_tfidf[t] = (t_counter[t]/doc_len) *             (idf[t] if t in idf else avg_idf)\n",
    "    \n",
    "    common_nouns = sorted(n_tfidf, key=n_tfidf.get, reverse=True)[:5]\n",
    "    common_verbs = sorted(v_tfidf, key=v_tfidf.get, reverse=True)[:5]\n",
    "    common_tokens = sorted(t_tfidf, key=t_tfidf.get, reverse=True)[:5]\n",
    "\n",
    "    # no idf for bigrams increase \"common\" count to 10\n",
    "    common_bigrams = [x[0] for x in b_counter.most_common(10)]\n",
    "    \n",
    "    scored_sentences = [[idx, sent, score_sentence_idf(sent, idf)] for idx, sent in enumerate(sentences)]\n",
    "    scored_sentences = [s for s in scored_sentences] #filter out non-scoring sentences\n",
    "    scored_sentences.sort(key = lambda x: x[2], reverse = True)\n",
    "    most_significant_sentence_data = process_sentence(scored_sentences[0][1])\n",
    "\n",
    "    return {\n",
    "        \"raw\" : body.text,\n",
    "        \"tokens\": tokens,\n",
    "        \"bigrams\": bigram_str,\n",
    "        \"nouns\": nouns,\n",
    "        \"verbs\": verbs,\n",
    "        \"first_sentence\": first_sentence_data,\n",
    "        \"significant_sentence\": most_significant_sentence_data,\n",
    "        \"vocabulary\": list(set(tokens)),\n",
    "        \"common_tokens\": common_tokens,\n",
    "        \"common_nouns\": common_nouns,\n",
    "        \"common_verbs\": common_verbs,\n",
    "        \"common_bigrams\": common_bigrams,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 0\n",
      "Processed 2500\n",
      "Processed 5000\n",
      "Processed 7500\n",
      "Processed 10000\n",
      "Processed 12500\n",
      "Processed 15000\n",
      "Processed 17500\n",
      "Processed 20000\n",
      "Processed 22500\n",
      "Processed 25000\n",
      "Done!\n",
      "Processed 0\n",
      "Processed 100\n",
      "Processed 200\n",
      "Processed 300\n",
      "Processed 400\n",
      "Processed 500\n",
      "Processed 600\n",
      "Processed 700\n",
      "Processed 800\n",
      "Processed 900\n",
      "Done!\n",
      "706\n"
     ]
    }
   ],
   "source": [
    "headline_info_rel = {}\n",
    "body_info_rel = {}\n",
    "start = time.time()\n",
    "stance_data = list(test_stances.values)\n",
    "body_data = list(test_bodies.values)\n",
    "for headline in range(len(stance_data)):\n",
    "    if headline % 2500 == 0:\n",
    "        print(\"Processed \"+str(headline))\n",
    "    h, b_id, s = tuple(stance_data[headline])\n",
    "    if h not in headline_info_rel:\n",
    "        nlp_h = nlp(preprocess(h))\n",
    "        headline_processed = process_sentence(nlp_h)\n",
    "        headline_info_rel[h] = headline_processed\n",
    "print(\"Done!\")\n",
    "for body in range(len(body_data)):\n",
    "    if body % 100 == 0:\n",
    "        print(\"Processed \"+str(body))\n",
    "    b_id, txt = tuple(body_data[body])\n",
    "    nlp_a = coref(preprocess(txt))\n",
    "    nlp_b = nlp(nlp_a._.coref_resolved.lower())\n",
    "    body_processed = process_body(nlp_b, idf)\n",
    "    body_info_rel[b_id] = body_processed\n",
    "print(\"Done!\")\n",
    "end = time.time()\n",
    "print(int(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    }
   ],
   "source": [
    "def get_features_rel(stance_df):\n",
    "    start = time.time()\n",
    "    data = list(stance_df.values)\n",
    "    features = []\n",
    "    actual = []\n",
    "    for item in data:\n",
    "        h, b, s = tuple(item)\n",
    "        fts = get_feats_rel(h, b)\n",
    "        features.append(fts)\n",
    "        actual.append(s)\n",
    "    end = time.time()\n",
    "    print(int(end-start))\n",
    "    return features, actual\n",
    "\n",
    "\n",
    "# In[59]:\n",
    "\n",
    "\n",
    "def bow_cos_similarity(a, b):\n",
    "    vocab = list(set(a).union(set(b)))\n",
    "    a_bow, b_bow = set(a), set(b)\n",
    "    if len(a) == 0 or len(b) == 0:\n",
    "        return -1\n",
    "    a_vec = [(1 if i in a_bow else 0) for i in vocab]\n",
    "    b_vec = [(1 if i in b_bow else 0) for i in vocab]\n",
    "    return 1 - distance.cosine(a_vec, b_vec)\n",
    "\n",
    "def get_feats_rel(headline, body_id):\n",
    "    headline_data = headline_info_rel[headline]\n",
    "    body_data = body_info_rel[body_id]\n",
    "\n",
    "    shared_common_nouns = len(set(headline_data['nouns']).intersection(\n",
    "        set(body_data['common_nouns'])))\n",
    "    shared_common_verbs = len(set(headline_data['verbs']).intersection(\n",
    "        set(body_data['common_verbs'])))\n",
    "    shared_common_tokens = len(set(headline_data['tokens']).intersection(\n",
    "        set(body_data['common_tokens'])))\n",
    "    shared_bigrams = len(set(headline_data['bigrams']).intersection(\n",
    "        set(body_data['common_bigrams'])))\n",
    "\n",
    "    shared_nouns_first = len(set(headline_data['nouns']).intersection(\n",
    "        set(body_data['first_sentence']['nouns'])))\n",
    "    shared_verbs_first = len(set(headline_data['verbs']).intersection(\n",
    "        set(body_data['first_sentence']['verbs'])))\n",
    "    shared_bigrams_first = len(set(headline_data['bigrams']).intersection(\n",
    "        set(body_data['first_sentence']['bigrams'])))\n",
    "    shared_tokens_first = len(set(headline_data['tokens']).intersection(\n",
    "        set(body_data['first_sentence']['tokens'])))\n",
    "\n",
    "    shared_nouns_sig = len(set(headline_data['nouns']).intersection(\n",
    "        set(body_data['significant_sentence']['nouns'])))\n",
    "    shared_verbs_sig = len(set(headline_data['verbs']).intersection(\n",
    "        set(body_data['significant_sentence']['verbs'])))\n",
    "    shared_bigrams_sig = len(set(headline_data['bigrams']).intersection(\n",
    "        set(body_data['significant_sentence']['bigrams'])))\n",
    "    shared_tokens_sig = len(set(headline_data['tokens']).intersection(\n",
    "        set(body_data['significant_sentence']['tokens'])))\n",
    "\n",
    "    headline_svo = headline_data['svo']\n",
    "    body_fst_svo = body_data['first_sentence']['svo']\n",
    "    body_sig_svo = body_data['significant_sentence']['svo']\n",
    "\n",
    "    # cosine similarity - no verbs because relatively few per sentence\n",
    "    cos_nouns_first = bow_cos_similarity(\n",
    "        headline_data['nouns'], body_data['first_sentence']['nouns'])\n",
    "    cos_bigrams_first = bow_cos_similarity(\n",
    "        headline_data['bigrams'], body_data['first_sentence']['bigrams'])\n",
    "    cos_tokens_first = bow_cos_similarity(\n",
    "        headline_data['tokens'], body_data['first_sentence']['tokens'])\n",
    "\n",
    "    cos_nouns_sig = bow_cos_similarity(\n",
    "        headline_data['nouns'], body_data['significant_sentence']['nouns'])\n",
    "    cos_bigrams_sig = bow_cos_similarity(\n",
    "        headline_data['bigrams'], body_data['significant_sentence']['bigrams'])\n",
    "    cos_tokens_sig = bow_cos_similarity(\n",
    "        headline_data['tokens'], body_data['significant_sentence']['tokens'])\n",
    "    \n",
    "    svo_cos_sim_fst = bow_cos_similarity(\n",
    "        body_fst_svo[0]+body_fst_svo[1]+body_fst_svo[2], \n",
    "        headline_svo[0]+headline_svo[1]+headline_svo[2])\n",
    "\n",
    "    svo_cos_sim_sig = bow_cos_similarity(\n",
    "        body_sig_svo[0]+body_sig_svo[1]+body_sig_svo[2], \n",
    "        headline_svo[0]+headline_svo[1]+headline_svo[2])\n",
    "    \n",
    "    svo_s_fst = len(set(body_fst_svo[0]).intersection(set(headline_svo[0]))) \n",
    "    svo_v_fst = len(set(body_fst_svo[1]).intersection(set(headline_svo[1])))\n",
    "    svo_o_fst = len(set(body_fst_svo[2]).intersection(set(headline_svo[2])))\n",
    "    svo_s_sig = len(set(body_sig_svo[0]).intersection(set(headline_svo[0])))\n",
    "    svo_v_sig = len(set(body_sig_svo[1]).intersection(set(headline_svo[1])))\n",
    "    svo_o_sig = len(set(body_sig_svo[2]).intersection(set(headline_svo[2])))\n",
    "    \n",
    "    return {\n",
    "        'shared_nouns': shared_common_nouns,\n",
    "        'shared_verbs': shared_common_verbs,\n",
    "        'shared_bigrams': shared_bigrams,\n",
    "        'shared_tokens': shared_common_tokens,\n",
    "\n",
    "        'shared_nouns_fst': shared_nouns_first,\n",
    "        'shared_verbs_fst': shared_verbs_first,\n",
    "        'shared_bigrams_fst': shared_bigrams_first,\n",
    "        'shared_tokens_fst': shared_tokens_first,\n",
    "\n",
    "        'shared_nouns_sig': shared_nouns_sig,\n",
    "        'shared_verbs_sig': shared_verbs_sig,\n",
    "        'shared_bigrams_sig': shared_bigrams_sig,\n",
    "        'shared_tokens_sig': shared_tokens_sig,\n",
    "\n",
    "        'cos_nouns_sig': cos_nouns_sig,\n",
    "        'cos_bigrams_sig': cos_bigrams_sig,\n",
    "        'cos_tokens_sig': cos_tokens_sig,\n",
    "\n",
    "        'cos_nouns_fst': cos_nouns_first,\n",
    "        'cos_bigrams_fst': cos_bigrams_first,\n",
    "        'cos_tokens_fst': cos_tokens_first,\n",
    "\n",
    "        'svo_cos_sim_fst' : svo_cos_sim_fst,\n",
    "        'svo_cos_sim_sig' : svo_cos_sim_sig,\n",
    "        \n",
    "        'svo_s_fst': svo_s_fst,\n",
    "        'svo_v_fst': svo_v_fst,\n",
    "        'svo_o_fst': svo_o_fst,\n",
    "\n",
    "        'svo_s_sig': svo_s_sig,\n",
    "        'svo_v_sig': svo_v_sig,\n",
    "        'svo_o_sig': svo_o_sig,\n",
    "    }\n",
    "\n",
    "\n",
    "# In[56]:\n",
    "\n",
    "\n",
    "stance_info_rel = get_features_rel(test_stances)\n",
    "stance_dict_rel = {}\n",
    "for idx, d in enumerate(list(test_stances.values)):\n",
    "    h, b, s = d\n",
    "    stance_dict_rel[(h, b)] = stance_info_rel[0][idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/numpy/core/fromnumeric.py:83: RuntimeWarning: overflow encountered in reduce\n",
      "  return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/ipykernel_launcher.py:120: RuntimeWarning: overflow encountered in double_scalars\n",
      "/Users/dannyyang/Library/Python/3.6/lib/python/site-packages/scipy/spatial/distance.py:505: RuntimeWarning: invalid value encountered in true_divide\n",
      "  dist = 1.0 - np.dot(u, v) / (norm(u) * norm(v))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "545\n"
     ]
    }
   ],
   "source": [
    "def get_sentence_vec(s):\n",
    "    vecs = [token.vector for token in s]\n",
    "    return np.nan_to_num(np.product(vecs, axis = 0))\n",
    "\n",
    "def get_features_stance(stance_df, n_sent = 5):\n",
    "    start = time.time()\n",
    "    data = list(stance_df.values)\n",
    "    features = []\n",
    "    actual = []\n",
    "    for item in data:\n",
    "        h, b, s = tuple(item)\n",
    "        headline, headline_graph, headline_subjs, headline_svo, headline_root_dist, headline_neg_ancestors, headline_edges  = headline_info[h]\n",
    "        body, body_graph = body_info[b]\n",
    "        \n",
    "        h_important_words = set(headline_subjs[\"_vocab\"]).union(set(headline_svo[0])).union(set(headline_svo[1])).union(set(headline_svo[2]))\n",
    "        \n",
    "        #sometimes the coref deletes bodies that are one sentence\n",
    "        if len(body) == 0:\n",
    "            body = nlp(preprocess(get_body(b)))\n",
    "            body_graph = build_graph(body)\n",
    "\n",
    "        #return the shortest path to negating word for each subject in headline_subjs, if one exists\n",
    "        neg_h = get_shortest_path_to_negating(headline_graph, h_important_words)\n",
    "        neg_b = get_shortest_path_to_negating(body_graph, h_important_words)\n",
    "\n",
    "        #body summary\n",
    "        summary = get_summary(body, h_important_words, n_sent)\n",
    "        first_summ_sentence = summary[0]\n",
    "        \n",
    "        summary_svos = [get_svos(s) for s in summary]\n",
    "        summary_root_dist = [root_distance(body_graph, list(s[1].keys())[0]) for s in summary_svos]\n",
    "        summary_neg_ancestors = [get_neg_ancestors(s) for s in summary]\n",
    "        summary_neg_counts = [s[3:] for s in summary_neg_ancestors]\n",
    "        \n",
    "        summary_neg_ancestors_superset = [set(), set(), set()]\n",
    "        for a in summary_neg_ancestors:\n",
    "            summary_neg_ancestors_superset[0] = summary_neg_ancestors_superset[0].union(a[0])\n",
    "            summary_neg_ancestors_superset[1] = summary_neg_ancestors_superset[1].union(a[1])\n",
    "            summary_neg_ancestors_superset[2] = summary_neg_ancestors_superset[2].union(a[2])\n",
    "            \n",
    "        #ancestors\n",
    "        h_anc = [[1 if w in headline_neg_ancestors[0] else -1 for w in h_important_words],\n",
    "                [1 if w in headline_neg_ancestors[1] else -1 for w in h_important_words],\n",
    "                [1 if w in headline_neg_ancestors[2] else -1 for w in h_important_words]]\n",
    "        b_anc = [[1 if w in summary_neg_ancestors_superset[0] else -1 for w in h_important_words],\n",
    "                [1 if w in summary_neg_ancestors_superset[1] else -1 for w in h_important_words],\n",
    "                [1 if w in summary_neg_ancestors_superset[2] else -1 for w in h_important_words]]    \n",
    "        neg_anc_sim = cosine_similarity(h_anc[0], b_anc[0])\n",
    "        doubt_anc_sim = cosine_similarity(h_anc[1], b_anc[1])\n",
    "        hedge_anc_sim = cosine_similarity(h_anc[2], b_anc[2])\n",
    "        neg_anc_overlap = len(headline_neg_ancestors[0].union(summary_neg_ancestors_superset[0]))\n",
    "        doubt_anc_overlap = len(headline_neg_ancestors[1].union(summary_neg_ancestors_superset[1]))\n",
    "        hedge_anc_overlap = len(headline_neg_ancestors[2].union(summary_neg_ancestors_superset[2]))\n",
    "        #svo\n",
    "        body_s, body_v, body_o = {}, {}, {}\n",
    "        headline_s, headline_v, headline_o = headline_svo\n",
    "        for svo in summary_svos:\n",
    "            body_s.update(svo[0])\n",
    "            body_v.update(svo[1])\n",
    "            body_o.update(svo[2])\n",
    "        body_s_vec = list(np.sum([body_s[s].vector for s in body_s], axis = 0)) if len(body_s) > 0 else np.zeros(384)\n",
    "        body_v_vec = list(np.sum([body_v[s].vector for s in body_v], axis = 0)) if len(body_v) > 0 else np.zeros(384)\n",
    "        body_o_vec = list(np.sum([body_o[s].vector for s in body_o], axis = 0)) if len(body_o) > 0 else np.zeros(384)\n",
    "    \n",
    "        headline_s_vec = list(np.sum([headline_s[s].vector for s in headline_s], axis = 0)) if len(headline_s) > 0 else np.zeros(384)\n",
    "        headline_v_vec = list(np.sum([headline_v[s].vector for s in headline_v], axis = 0)) if len(headline_v) > 0 else np.zeros(384)\n",
    "        headline_o_vec = list(np.sum([headline_o[s].vector for s in headline_o], axis = 0)) if len(headline_o) > 0 else np.zeros(384)\n",
    "        \n",
    "        cos_sim_s = cosine_similarity(body_s_vec, headline_s_vec)\n",
    "        cos_sim_v = cosine_similarity(body_v_vec, headline_v_vec)\n",
    "        cos_sim_o = cosine_similarity(body_o_vec, headline_o_vec)\n",
    "        \n",
    "        #negating paths\n",
    "        headline_paths = [neg_h[x] for x in neg_h]\n",
    "        headline_neg_paths = [1 if x[0] != None else -1 for x in headline_paths]\n",
    "        headline_doubt_paths = [1 if x[1] != None else -1 for x in headline_paths]\n",
    "        headline_hedge_paths = [1 if x[2] != None else -1 for x in headline_paths]\n",
    "        body_paths = [neg_b[x] for x in neg_h]\n",
    "        body_neg_paths = [1 if x[0] != None else -1 for x in body_paths]\n",
    "        body_doubt_paths = [1 if x[1] != None else -1 for x in body_paths]\n",
    "        body_hedge_paths = [1 if x[2] != None else -1 for x in body_paths]\n",
    "        \n",
    "        neg_path_cos_sim = cosine_similarity(headline_neg_paths, body_neg_paths)\n",
    "        hedge_path_cos_sim = cosine_similarity(headline_hedge_paths, body_hedge_paths)\n",
    "        doubt_path_cos_sim = cosine_similarity(headline_doubt_paths, body_doubt_paths)\n",
    "        \n",
    "        #root distance\n",
    "        summary_root_dists = [x if x != None else 15 for x in summary_root_dist]\n",
    "        avg_summary_root_dist = sum(summary_root_dists)/len(summary_root_dists)\n",
    "        root_dist_feats = [headline_root_dist, avg_summary_root_dist]\n",
    "        root_dist_feats = [x/15 if x != None else 1 for x in root_dist_feats]\n",
    "        root_dist_feats = root_dist_feats + [int(headline_root_dist == None), len([x for x in summary_root_dist if x != None])]\n",
    "    \n",
    "        #sentiment\n",
    "        headline_sent = get_sentiment(headline.text)\n",
    "        body_sents = [get_sentiment(s.text) for s in summary]\n",
    "        avg_body_sent = list(np.mean(body_sents, axis = 0))\n",
    "        diff_avg_sents = list(np.array(headline_sent) - avg_body_sent)\n",
    "        diff_sents = list(np.sum([get_diff_sentiment(headline_sent, s) for s in body_sents], axis = 0))\n",
    "        sent_cos_sim = cosine_similarity(headline_sent, avg_body_sent)\n",
    "\n",
    "        headline_sentics = get_sentics(headline.text)\n",
    "        body_sentics = [get_sentics(s.text) for s in summary]\n",
    "        avg_body_sentics = list(np.mean(body_sentics, axis = 0))\n",
    "        diff_avg_sentics = list(np.array(headline_sentics) - avg_body_sentics)\n",
    "        diff_sentics = list(np.sum([get_diff_sentiment(headline_sentics, s) for s in body_sentics], axis = 0))\n",
    "        sentics_cos_sim = cosine_similarity(headline_sentics, avg_body_sentics)\n",
    "        \n",
    "        #bow\n",
    "        headline_vocab = set([tok.lemma_.lower() for tok in headline])\n",
    "        fst_summ_vocab = set([tok.lemma_.lower() for tok in first_summ_sentence])\n",
    "        total_vocab = list(headline_vocab.union(fst_summ_vocab))\n",
    "        headline_embedding = [1 if tok in headline_vocab else -1 for tok in total_vocab]\n",
    "        fst_summ_embedding = [1 if tok in fst_summ_vocab else -1 for tok in total_vocab]\n",
    "        bow_cos_sim = cosine_similarity(headline_embedding, fst_summ_embedding)\n",
    "        \n",
    "        #word vecs\n",
    "        cos_sims = [cosine_similarity(get_sentence_vec(s), headline.vector) for s in summary]\n",
    "        fst_cos_sim = cos_sims[0]\n",
    "        avg_cos_sim = sum(cos_sims)/len(cos_sims)\n",
    "        \n",
    "        #neg_hedge_doubt distributions\n",
    "        hd_dist = list(headline_neg_ancestors[3:])\n",
    "        body_dist = list(np.sum(summary_neg_counts, axis = 0))\n",
    "        dist_sim = cosine_similarity(hd_dist, body_dist)\n",
    "        #build final features list\n",
    "        fts = (\n",
    "            [fst_cos_sim, avg_cos_sim, bow_cos_sim, \n",
    "                neg_path_cos_sim, hedge_path_cos_sim, doubt_path_cos_sim,\n",
    "                neg_anc_sim, hedge_anc_sim, doubt_anc_sim,\n",
    "                neg_anc_overlap, hedge_anc_overlap, doubt_anc_overlap,\n",
    "                cos_sim_s, cos_sim_v, cos_sim_o,\n",
    "                dist_sim, sent_cos_sim, sentics_cos_sim] + \n",
    "            diff_avg_sents + diff_sents + diff_avg_sentics + diff_sentics + \n",
    "            root_dist_feats + hd_dist + body_dist +\n",
    "            headline_sent + avg_body_sent + headline_sentics + avg_body_sentics\n",
    "        )\n",
    "        features.append(fts)\n",
    "        actual.append(s)\n",
    "    end = time.time()\n",
    "    print(int(end-start))\n",
    "    return features, actual\n",
    "\n",
    "\n",
    "# # Generate Features from headline/body pairs\n",
    "\n",
    "# In[96]:\n",
    "\n",
    "\n",
    "stance_data_stance = get_features_stance(test_stances, 5)\n",
    "stance_dict_stance = {}\n",
    "for idx, d in enumerate(list(test_stances.values)):\n",
    "    h, b, s = d\n",
    "    stance_dict_stance[(h, b)] = stance_data_stance[0][idx]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from joblib import dump, load\n",
    "rel_model = load('saved_models/relevance_detection_trained.joblib')\n",
    "stance_model = load('saved_models/stance_detection_trained.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "testing_data = [[],[],[]]\n",
    "for h,b,s in list(test_stances.values):\n",
    "    testing_data[0].append(list(stance_dict_rel[(h,b)].values()))\n",
    "    testing_data[1].append(stance_dict_stance[(h,b)])\n",
    "    testing_data[2].append(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/numpy/core/_methods.py:36: RuntimeWarning: overflow encountered in reduce\n",
      "  return umr_sum(a, axis, dtype, out, keepdims, initial)\n"
     ]
    }
   ],
   "source": [
    "rel_predicted = rel_model.predict(testing_data[0])\n",
    "stance_predicted = stance_model.predict(np.nan_to_num(np.array(testing_data[1]).astype(\"float32\")))\n",
    "predicted = []\n",
    "for i in range(len(rel_predicted)):\n",
    "    r = rel_predicted[i]\n",
    "    s = stance_predicted[i]\n",
    "    if r == \"unrelated\":\n",
    "        predicted.append(r)\n",
    "    else:\n",
    "        predicted.append(s)\n",
    "actual = testing_data[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------------------\n",
      "|           |   agree   | disagree  |  discuss  | unrelated |\n",
      "-------------------------------------------------------------\n",
      "|   agree   |    425    |    69     |   1144    |    265    |\n",
      "-------------------------------------------------------------\n",
      "| disagree  |    67     |    91     |    332    |    207    |\n",
      "-------------------------------------------------------------\n",
      "|  discuss  |    328    |    148    |   3222    |    766    |\n",
      "-------------------------------------------------------------\n",
      "| unrelated |    37     |    25     |    400    |   17887   |\n",
      "-------------------------------------------------------------\n",
      "Score: 8731.75 out of 11651.25\t(74.9426027250295%)\n",
      "F1 Score\n",
      "[0.30797101 0.17669903 0.67391759 0.95463521]\n",
      "Avg Precision Score\n",
      "[0.49591599 0.27327327 0.63201255 0.93526797]\n",
      "Normalized confusion matrix\n",
      "[[0.22333158 0.03625854 0.60115607 0.13925381]\n",
      " [0.09612626 0.13055954 0.47632712 0.29698709]\n",
      " [0.0734767  0.03315412 0.72177419 0.17159498]\n",
      " [0.00201646 0.00136247 0.02179955 0.97482152]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEYCAYAAAApuP8NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FOX2wPHvSULASA8lhRo6oRqaSJMmHUGaIsrFyk+R\na0OliRRFRAWveC2oqKhcBAWkCwIKihAUAkE6QZIAUoPS0s7vj5msuyEkAXazCXk/z7NPdmbemTmz\n2T37lplZUVUMwzDyMx9vB2AYhuFtJhEahpHvmURoGEa+ZxKhYRj5nkmEhmHkeyYRGoaR75lEmE+I\nyDgRmW0/ryAif4uIr5v3ESMi7d25zWzsc6iIHLOPJ/A6tvO3iIS5MzZvEZFoEWnj7TjyEpMI3cRO\nAn+KyM1O8x4UkbVeDCtDqvqHqhZW1RRvx3I9RKQA8AbQ0T6ek9e6LXv9A+6Lzv1EZJaITMyqnKqG\nq+raHAjphmESoXv5AsOvdyNiMf+brJUFCgHR3g4kNxARP2/HkFeZD5t7vQY8IyLFM1ooIs1FZLOI\nJNh/mzstWysik0RkA3AeCLPnTRSRn+ym27ciEigin4vIWXsblZy2MV1EDtvLtohIyyvEUUlEVET8\nRORWe9tpj4siEmOX8xGR50Vkv4icFJG5IlLSaTuDROSQvWxUZi+MiNwkIq/b5RNEZL2I3GQv62E3\n587Yx1zLab0YEXlGRKLs9f4nIoVEpDqw2y52RkS+dz6udK/rg/bzqiKyzt7OCRH5n1M5FZGq9vNi\nIvKpiBy34x2d9sUkIoPt2KeKyGkROSginTM57hgRedaO/5yIfCgiZUVkmYj8JSKrRKSEU/mvROSo\nHeMPIhJuz38YGAiMSHsvOG3/ORGJAs7Z/1NHF4WILBWR1522P0dEPsrsf5Uvqap5uOEBxADtga+B\nifa8B4G19vOSwGlgEOAH3G1PB9rL1wJ/AOH28gL2vH1AFaAYsBPYY+/HD/gU+NgphnuBQHvZ08BR\noJC9bBww235eCVDAL90xFADWAa/Y08OBjUA5oCDwHvClvaw28DfQyl72BpAMtL/C6zPDPp5QrJpz\nc3u96sA5oIO9/xH2Mfs7va6bgBD7NfwdeDSj48jouOx9Pmg//xIYhVUBKAS0cCqnQFX7+afAQqCI\nvc09wAP2ssFAEvCQfRxDgXhAMnlfbMSqvYYCfwK/Ag3tGL4HXnQqP8Teb0FgGrDVadks7PdWuu1v\nBcoDNzm/F+3nQfY+22Il0gNAEW9/XnLbw+sB3CgP/kmEdYAEoDSuiXAQsCndOj8Dg+3na4Hx6Zav\nBUY5Tb8OLHOa7u78QckgptNAffv5OLJOhP8FFgM+9vTvQDun5cF2EvADxgJznJbdDCSSQSK0E8+F\ntFjSLRsDzE1XNg5o4/S63uu0fArwbkbHkdFx4ZoIPwXeB8plEIcCVbGSWyJQ22nZI07/x8HAPqdl\nAfa6QZm8LwY6Tc8H/us0PQxYcIV1i9vbLmZPzyLjRDgko/ei0/RdwGHgBE7J3zz+eZimsZup6g6s\nZPJ8ukUhwKF08w5h1RLSHM5gk8ecnl/IYLpw2oTdhPzdbladwapFlspO3CLyCNAGuEdVU+3ZFYFv\n7CbrGazEmIJVuwlxjldVzwFXGqwohVX72Z/BMpfXxd73YVxfl6NOz8/jdMxXaQQgwCa7KT7kCrEW\nwPV/lf7/5IhHVc/bTzOLKVv/QxHxFZHJdlfEWayElhZTZjJ63zj7FivB71bV9VmUzZdMIvSMF7Ga\nTs4fnnisxOKsAlbtJ8013wrI7g8cAfQDSqhqcayaqWRz3QlAT1U967ToMNBZVYs7PQqpahxwBKs5\nlraNAKxmeUZOABexmvjpubwuIiL2duMyKJuVc/bfAKd5QWlPVPWoqj6kqiFYtbx30voF08WahOv/\nKv3/yVPuAXpitSyKYdVw4Z//4ZXeH1m9byZhfYkFi8jd1xnjDckkQg9Q1X3A/4AnnGYvBaqLyD12\nh3Z/rH62xW7abRGsPrrjgJ+IjAWKZrWSiJQH5gL3qeqedIvfBSaJSEW7bGkR6Wkvmwd0E5EWIuIP\njOcK7ye7lvcR8IaIhNg1n1tFpKC9764i0k6s02GeBi4BP13V0Vv7OY6VsO619zEEp+QrIn1FpJw9\neRorgaSm20aKHdMkESliH/tTwOyrjecaFME69pNYyfzldMuPAVd1rqOItAL+BdwH3A/8R0RCM18r\n/zGJ0HPGY/WbAaDWOW7dsD7oJ7Fqb91U9YSb9rcCWI7VsX8IqwaWVZMJoB1WU3ee/DNynHY6ynRg\nEbBSRP7C6vRvah9PNPAY8AVW7fA0EJvJfp4BtgObgVPAq1h9kbuxBnn+g1Ub6w50V9XEbB53eg8B\nz2K9xuG4JtTGwC8i8rd9XMM143MHh2HVLg8A6+1jzImR1k+x/ndxWANjG9Mt/xCobXdVLMhqYyJS\n1N7m46oap6o/2tv42K55GzaxO1MNwzDyLVMjNAwj3zOJ0DCMfM8kQsMw8j2TCA3DyPfMRdpZKF68\nuAYHB3s7jGt20003eTuE63L4cHYGvnOvoKCgrAvlUnFxcZw6dcpto8s+Pj7ZGplV1RWq2sld+80O\nkwizEBwczEcf5d1r1Bs0aODtEK7Lv//9b2+HcF1GjBjh7RCu2Z133un2bRYoUCDLMomJidm6Gsqd\nTCI0DCPH+Pjkzt44kwgNw8gxufU8bpMIDcPIESJiEqFhGIZpGhuGke+ZGqFhGPmeSYSGYeRrImKa\nxoZhGKZGaBhGvmcSoWEY+ZppGhuGYWBqhIZhGCYRGoZhmKaxYRj5mrnEzjAMA9M0zhc2btzItGnT\nSElJoXv37tx3330uy7/88ku+/fZbfH19KV68OCNHjiQ4OJg9e/bw2muvcf78eXx8fLj//vtp3749\nAC+//DK7du1CVSlfvjyjR48mICAgo91ft5UrV/Lss8+SkpLC4MGDeeaZZ1yWX7p0iQcffJDffvuN\nkiVL8tlnn1GxYkU2b97M448/7ig3cuRIeva0fv74zJkz/N///R87d+5ERHj33Xdp2rSpR+IPDw+n\nX79++Pj4sH79elasWHFZmYiICLp16wZAbGwsH374IQDNmjWjS5cuACxdupSNG61f0uzZsyfNmjUj\nICCA4cOHeyTuNOvWrWPixImkpKTQr18/Hn30UZflmzZtYuLEiezevZtp06bRuXNnwLqB6tChQ1FV\nkpKSuO+++7jnnnsAWLJkCe+88w4pKSm0bdvW6/dHNE3jG1xKSgpTp05l+vTplClThgceeICWLVtS\nuXJlR5nq1avz0UcfUahQIb7++mveeecdJkyYQKFChRg7dizly5fn+PHjDBkyhKZNm1KkSBGGDx/O\nzTdbP488ffp05s2bd1mCdVf8Tz75JIsXLyY0NJSWLVvStWtXatWq5Sgza9Ysihcvzo4dO/jqq68Y\nPXo0n332GeHh4WzYsAE/Pz+OHDlCs2bN6Nq1K35+fjz77LN06NCBL774gsTERM6fP+/22MGqadx9\n991MmzaN06dP88ILLxAVFcWRI0ccZcqUKUOnTp0cXzpFihQBICAggG7duvHyy9bvqY8cOZKoqCjO\nnz9PVFQUa9asYcKECR6JO01KSgrjxo3jk08+ISgoiN69e9OuXTuqVavmKBMSEsKUKVOYOXOmy7ql\nS5fmq6++omDBgpw7d44uXbrQrl07/P39mTx5MgsWLCAwMJBnn32Wn376iebNm3v0WDKTW2uEuTM9\n50E7d+6kXLlyhIaGUqBAAdq3b8+PP/7oUiYiIoJChQoBVu3lzz//BKBChQqUL18esN7UJUqU4MyZ\nMwCOJKiqJCYmeuyNFBkZSZUqVahcuTL+/v706dOHxYsXu5RZsmQJ9957LwC9evVi7dq1qCoBAQH4\n+VnfqZcuXXLEmJCQwPr16xk8eDAA/v7+FC9e3CPxV65cmT///JMTJ06QkpJCZGQk9evXdynTokUL\n1q5d60jGf/31F2D9L37//XfOnz/P+fPn+f333wkPDwfg4MGDnD171iMxO9u2bRsVK1akQoUK+Pv7\n07VrV1atWuVSply5ctSsWfOyWpW/vz8FCxYEIDExkdTUVMD6mYNKlSoRGBgIQPPmzVm+fLnHj+VK\n0voIs3p4g0mEbnL8+HHKli3rmC5dujTHjx+/YvnFixfTrFmzy+bv3LmTpKQkQkNDHfMmTpxIt27d\nOHToEH379nVv4Lb4+HiXfYaGhhIfH3/FMn5+fhQtWpSTJ08CVrMtIiKCxo0bM336dPz8/IiJiaFU\nqVI88sgjNGvWjKFDh3Lu3DmPxF+8eHFOnz7tmD59+vRlSbds2bKULVuWZ599lueee86R7LKzrqcd\nO3YM59/GCQoK4tixY9lePz4+nq5du9KyZUsefvhhypYtS8WKFTlw4ACxsbEkJyezatUqlxqyN/j4\n+GT58EpcXtlrPrd8+XJ27drFwIEDXeafOHGC8ePHM2rUKJc3xOjRo1m0aBEVK1a8rJaQWzRp0oQt\nW7bw448/MnXqVC5evEhycjJbt27lwQcfZOPGjdx8881MnTrVazH6+PhQpkwZXn/9dWbOnMm9996b\n53/cKk1ISAhLlixh9erVfPPNN5w4cYJixYoxfvx4hg8fzoABAwgNDcXX19ercZoaYQ4QEa/1eZYu\nXdrlG/z48eOULl36snKbN2/mk08+4dVXX8Xf398x/9y5czzzzDM8/PDD1KlT57L1fH19ad++PWvX\nrvVI/CEhIcTFxTmm4+LiCAkJuWKZ5ORkzp4962h2palZsyaFCxcmOjqa0NBQQkNDadKkCWA1p7du\n3eqR+M+cOUOJEiUc087dC2lOnz5NVFQUqampnDx5kj///JMyZcpka11PK1u2rEtt7ejRoy4tjKvZ\nTvXq1dm8eTMA7dq1Y/78+cybN4+wsDCXPmtvMInwKojIAhHZIiLRIvKwPe8BEdkjIptE5AMRedue\nP0tE3hWRX4ApInKziHxkl/tNRHra5XxF5DUR2SwiUSLyiDtjrlWrFrGxscTHx5OUlMSqVato0aKF\nS5ndu3fz6quvMmXKFEqWLOmYn5SUxPPPP0/nzp1p27atY76qEhsb63i+fv16Klas6M6wHSIiIti3\nbx8xMTEkJiYyb948unbt6lKmS5cuzJ49G4BvvvmG1q1bIyLExMSQnJwMwB9//MHu3bupWLEiQUFB\nlCtXjj179gCwZs0al8EXd4qJiaFMmTIEBgbi6+tLo0aN2LZtm0uZbdu2Ub16dcDqey1TpgwnTpwg\nOjqa2rVrExAQQEBAALVr1yY6OtojcV5JvXr1OHToEIcPHyYxMZElS5bQrl27bK175MgRLl68CFj9\nspGRkYSFhQE4ui4SEhL4/PPP6devn2cOIBvSrjXOjU3j3DpqPERVT4nITcBmEVkCjAFuAf4Cvgec\n3+XlgOaqmiIiLwPfq+oQESkObBKRVcBAIEFVG4tIQWCDiKxU1YPpd24n34eBbH8r+/n58dRTT/Hk\nk0+SkpJCt27dCAsL44MPPqBmzZq0bNmSGTNmcOHCBUaPHk3atqdMmcLq1avZunUrZ8+eZenSpQCM\nGjWKqlWrMmHCBM6dO4eqUq1aNZ599tlreDmzF/8bb7xBjx49SElJ4b777qN27dqMHz+eW265hW7d\nujF48GAeeOAB6tSpQ4kSJfj0008B+Omnn3j99dfx8/PDx8eHadOmUaqU9YuMr7/+Ov/6179ISkqi\nUqVKvPfeex6JPzU1lTlz5jB8+HB8fHzYsGEDR44coXv37hw6dIioqChHwnvxxRdRVebPn+/os1yy\nZAkvvPCC43nagErv3r1p0qSJYwR2/fr1lw0iuYOfnx8vvvgi//rXv0hJSaFv375Ur16dadOmUadO\nHdq3b09UVBRDhw7l7NmzfP/990yfPp3ly5ezf/9+XnnlFUQEVeXBBx+kRo0aAEyYMIHff/8dgGHD\nhuWKGmFuJKrZ+s3lHCUi44Be9mQl4BWglqreby9/Aqiuqo+LyCxgjap+Yi+LBAoByfb6JYE7gAlA\nPSDt/I1iwCOqujKzWGrVqqXmd429x/yusffceeedbN++3W2Zq1ChQpqdFs2ePXu2qGojd+03O3Jd\njVBE2gDtgVtV9byIrAV2AZm1qZyHIgW4S1V3p9uuAMNU9fKzbA3D8LjcfBuu3BhVMeC0nQRrAs2A\nm4HWIlLCHhC5K5P1VwDD7MSHiDR0mj9URArY86uLyM0eOwrDMC6TWwdLcl2NEFgOPCoivwO7gY1A\nHPAysAk4hVVDTLjC+hOAaUCUiPgAB4FuwEysZvavdpI8DtzpucMwDCOvyHWJUFUvAZ3TzxeRSFV9\n364RfgMssMsPTrf+BeCyEWFVTQVG2g/DMLwgtzaNc10izMQ4EWmPNRCyEjsRGoaRd+TWUeM8kwhV\n9ZmsSxmGkVvl5vsR5s56qmEYNyR3nFAtIp1EZLeI7BOR5zNYXkFE1tgXVESJSJcs47rG4zEMw7hq\n1ztqLCK+wAyscYTawN0iUjtdsdHAXFVtCAwA3skqLpMIDcPIMW44faYJsE9VD6hqIjAH6JmujAJF\n7efFgHiykGf6CA3DyNuu4oTqUvYVYmneV9X37eehwGGnZbFA+luejwNWisgwrHOQ22e1Q5MIDcPI\nMdkcLDlxnZfY3Q3MUtXXReRW4DMRqWOfQpchkwgNw8gxbhg1jgPKO02Xs+c5ewDoBKCqP4tIIaAU\n8OeVNmr6CA3DyBFuug3XZqCaiFQWEX+swZBF6cr8AbSz91kL69zjK98uHlMjNAwjB11vjVBVk0Xk\ncax7B/gCH6lqtIiMByJVdRHwNPCBiDyJNXAyWLO4zZZJhIZh5Bh3nFCtqkuBpenmjXV6vhO47Wq2\naRKhYRg5xlxrbBhGvpabL7EzidAwjBxjEqFhGPmeaRobhpHvmRqhYRj5mukjNAzDwDSN8yx/f3+v\n/xbs9Th69Ki3Q7gulSpV8nYI1yXtN4XzorQfjXcnUyM0DCPfM4nQMIx8LTf/rrFJhIZh5BhTIzQM\nI98zidAwjHzNNI0NwzAwNULDMAyTCA3DMEwiNAwj3zOJ0DCMfM1ca2wYhoG51tgwDMPUCA3DyN9M\n09gwDAPTNDYMwzA1QsMwDJMI84Hvv/+esWPHkpKSwj333MOwYcNcll+6dIknnniCqKgoSpQowXvv\nvUf58uVJTExkxIgRbNu2DR8fHyZMmEDz5s0BSExMZOTIkfz888+ICM8//zzdunXzSPzr1q1j/Pjx\npKam0q9fP4YOHeqyfNOmTUyYMIFdu3Yxffp0unTpAkBcXByPPvooqampJCcnc9999zFw4EAApk6d\nyjfffENCQgI7duzwSNxp9uzZw+LFi0lNTaVx48a0bt06w3I7duzgiy++4P/+7/8oV64cW7du5ccf\nf3QsP3r0KI899hghISEkJyfz7bffcuDAAUSEjh07UqdOHY/Ev2XLFj744ANSU1Pp0KEDffv2dVm+\nYMECVq5cia+vL0WLFmX48OGUKVMGgFmzZrF582YABgwYQMuWLQF47rnnuHDhAgAJCQlUq1aN0aNH\neyT+rJhrjfOBlJQURo4cyf/+9z+Cg4Pp3LkzHTt2pEaNGo4yX375JcWKFePnn39mwYIFTJw4kffe\ne4/PP/8cgDVr1nDixAnuueceli9fjo+PD9OnT6dUqVJs2LCB1NRUTp8+7bH4X3zxRT799FOCgoK4\n8847ad++PdWqVXOUCQkJYcqUKcycOdNl3dKlSzNv3jwKFizIuXPn6NSpE+3bt6ds2bK0a9eO++67\nj7Zt23ok7jSpqaksWrSIIUOGULRoUd555x1q1qxJ2bJlXcpdunSJn376ifLlyzvmNWjQgAYNGgBW\nEpw9ezYhISEArF27lptvvpmnn36a1NRUR1Jxt5SUFN59910mTJhAYGAgTz31FE2bNqVChQqOMmFh\nYbzxxhsUKlSIpUuX8vHHH/Pcc8+xefNm9u/fz1tvvUVSUhIvvPACERERBAQE8OqrrzrWf/nll2nW\nrJlH4s+u3FojzJ3pOQ/67bffqFSpEhUrVsTf35+ePXuyYsUKlzLLly+nX79+AHTr1o0ff/wRVWXP\nnj3cdtttAJQqVYpixYqxbds2AObMmcMTTzwBWB3NgYGBHol/27ZtVKxYkQoVKuDv70+3bt347rvv\nXMqUK1eOWrVqXfat7u/vT8GCBQGrBpuamupY1rBhQ0etxZNiY2MJDAykZMmS+Pn5Ua9evQxvk//d\nd9/RqlUr/PwyrgNs27aNevXqOaa3bNlCmzZtAOv1v/nmmz0S/969ewkODiYoKIgCBQrQqlUrfvnl\nF5cy9erVo1ChQgDUqFGDkydPAnD48GHCw8Px9fWlUKFCVK5cmS1btrise/78eaKionJFIszq4Q0m\nEbrJ0aNHCQ0NdUwHBwdf9nshR48eddQ0/Pz8KFq0KKdOnaJ27dqsXLmS5ORk/vjjD6KiooiLiyMh\nIQGAV199lQ4dOvDQQw9x/Phxj8UfHBzsEv+xY8eyvX58fDydO3fmtttu45FHHrmsJuZpCQkJFCtW\nzDFdrFgxzp4961Im7TWtWbPmFbezfft2RyJMq/199913vP3223zxxRf89ddfHogeTp48SalSpRzT\ngYGBjkSXke+++46IiAjA+l2XX3/9lYsXL5KQkEBUVBQnTpxwKb9x40bq169PQECAR+LPjrSmcVYP\nb8ixvYrIOBF5RkTGi0j7nNpvXnD33XcTHBxMp06dGDt2LI0aNcLX15fk5GTi4+Np3Lix443/0ksv\neTvcDIWEhLBs2TLWrFnD119/7bGEfa1SU1NZunSpo18zI4cPH6ZAgQIEBQU51klISKBChQo8/vjj\nVKhQgWXLluVUyFe0Zs0a9u3bR+/evQG45ZZbiIiIYMSIEUydOpWaNWtellDWrVtHq1atvBGuC1Mj\ntKnqWFVd5Ylti8UrXylBQUHExcU5po8cOeL4QDmXiY+PByA5OZmzZ886mnLjx49n1apVzJo1i7Nn\nzxIWFkbJkiW56aabHB/e7t27s337do/Ff+TIEZf4r6VWV7ZsWapXr+7ouM8pxYoVc9SgwaohFi1a\n1DGdmJjIsWPH+OCDD5gyZQqHDx/ms88+IzY21lEmKiqK+vXrO6YDAgIoUKAA4eHhANSpU8fx/3O3\nwMBAl1rcyZMnM+wG2bp1K3PnzmX06NEUKFDAMb9///689dZbTJgwAVV1aZ0kJCSwd+9eGjdu7JHY\nr0a+TIQiMkpE9ojIeqCGPW+WiPSxn08WkZ0iEiUiU+153UXkFxH5TURWiUhZe35pEflORKJFZKaI\nHBKRUiJSSUR2i8inwA6gvIh0FJGfReRXEflKRArb24gQkXUiskVEVohIcIaBX4MGDRpw8OBB/vjj\nDxITE1m4cCF33HGHS5k77riDuXPnArB48WJatGiBiHD+/HnOnz8PWN/cvr6+1KhRwzFK+dNPPwGw\nfv16qlev7q6QXdSrV4+YmBgOHz5MYmIiixcvpn377FXcjxw54vjpx4SEBCIjIwkLC/NInFcSGhrK\niRMnOHXqFMnJyURFRVGrVi3H8kKFCjF69GhGjBjBiBEjKF++PIMGDaJcuXKAVftzbhaD9aGtWbMm\nBw8eBGD//v0e6++sVq0a8fHxHD16lKSkJH744QeaNGniUmb//v3MmDGDMWPGULx4ccf8lJQURzfA\nwYMHiYmJoWHDho7lP/30E40bN8bf398jsV8NdzSNRaST/ZnfJyLPX6FMPzu3RIvIF1lt02OjxiIS\nAQwAGtj7+RXY4rQ8EOgF1FRVFZG0/+x6oJk970FgBPA08CLwvaq+IiKdgAecdlcNuF9VN4pIKWA0\n0F5Vz4nIc8BTIvIK8B+gp6oeF5H+wCRgSAaxPww8DLh8s2bGz8+Pl19+mbvvvpuUlBQGDBhAjRo1\nmDJlCvXr1+eOO+7g7rvvZtiwYdx6660UL16cd999F7C+/e+++25EhODgYP7zn/84tjtq1CiGDRvG\n2LFjCQwM5M0338xWPFfLz8+PcePGcf/995Oamkrfvn2pXr06b775JnXr1qV9+/Zs27aNoUOHkpCQ\nwOrVq5k+fTorVqxg3759vPzyy4gIqspDDz3k6IebPHkyixYt4sKFCzRv3px+/frx73//2+3x+/r6\n0qNHDz7++GNUlYiICMqWLct3333nGOTJTExMDMWKFaNkyZIu8zt16sRXX33FkiVLCAgIoE+fPm6P\nPS3+Rx99lBdffJHU1FTat29PxYoVmT17NtWqVaNp06Z8/PHHXLx4kcmTJwPWaP2YMWNISUnh+eet\nfBAQEMDTTz+Nr6+vY9s//PCDx+K+Gu6o8YmILzAD6ADEAptFZJGq7nQqUw14AbhNVU+LSJbfXqKq\nV9ph0QwX2FT1bGbLReTfQElVHWtPvwHEA3WAxcACrMS4xZ5erKqJIlIXeB0IBvyBg6raSUS2Ar1U\n9aC9vVNAdaAwsEZVK9vzuwGzsF4k7G38DLwJ/AQcsOf7AkdUtWNmx1G/fn1NP/qbl3jqdI+cMmfO\nHG+HcF3q1q3r7RCu2ZNPPsnevXvd1lYtXbq03nnnnVmWmzlz5hZVbZTRMhG5FRinqnfY0y8AqOor\nTmWmAHtUdWZG28hIZjXCaEAB5xcibVqBChmtlF2qmiwiTYB2QB/gcaAtVq3tDVVdJCJtgHHZ2Nw5\np+cCfKeqdzsXsBNstKreej1xG4Zx7bI5KlxKRCKdpt9X1fft56HAYadlsUDTdOtXBxCRDVgVnnGq\nujyzHV4xEapq+Ssty6YfgFl2k9QP6A68l7bQ7rcLUNWldsBpNbViQNqow/1O29sA9ANeFZGOQIkr\n7HcjMENEqqrqPhG5GevF2w2UFpFbVfVnESkAVFfV6Os8TsMwsimbTeMTV6oRZpMfVndZG6Ac8IOI\n1FXVM1daIVvpWUQGiMhI+3k5u/8vU6r6K/A/YBuwDEg/jFgEWCwiUVj9gk/Z88cBX4nIFsD5ZKiX\ngI4isgPoCxwFLjupS1WPA4OBL+1t/4zVD5mIVfN8VUS2AVuB5lkfvWEY7pCdEeNsJMo4wLmSVo5/\nKk5pYoEJ1SE7AAAgAElEQVRFqppkd6XtwUqMV5TlYImIvA0UAFoBLwPngXeBLMfiVXUS1oDElTRJ\nP0NVFwILMyibANxhN6lvBRqr6iUgBqvf0Xkb32cUn6putY/DMAwvcMMJ05uBaiJSGSsBDgDuSVdm\nAXA38LE9eFqdf1qcGcrOqHFzVb1FRH4DUNVTIuKNcfgKwFz7PMFE4CEvxGAYxnW43lFjuyL0OLAC\nq//vI1WNFpHxQKSqLrKXdRSRnUAK8KyqXvkyHbKXCJPs5KPgOO0lNfNV3E9V9wINsyxoGEau5Y4T\nplV1KbA03byxTs8Vq6vtKbIpO4lwBjAfa6DhJawBi9x5nZdhGLmW5OXbcKnqp/bARdplBn1V1bM3\nljMM44bkrUvospLdK0t8gSSs5nHuTOmGYeR6uTURZpnURGQU8CUQgjVU/UXa2dyGYRjZldY0zo23\n4cpOjfA+oKGqngcQkUnAb8Arma5lGIaRR2QnER5JV87PnmcYhnFV8txgiYi8idUneAqIFpEV9nRH\nLr9KxDAMI0u5tY8wsxph2shwNLDEaf5Gz4VjGMaNLM8lQlX9MCcDMQzjxpanzyMUkSpY1wvXBgql\nzVdVz9wq2TCMG1ZurRFmJz3PAj7Gus9fZ2Au1l1lDMMwroob7j7jEdlJhAGqugJAVfer6mishGgY\nhpFtef08wkv2TRf2i8ijWLe+KeLZsAzDuBHl1qZxdhLhk8DNwBNYfYXFyOAHjwzDMLKSZxOhqv5i\nP/0LGOTZcAzDuJHluVFjEfkG+x6EGVHV3h6JyDCMG5I3B0OyklmN8O0ciyIXE5Fc8cPY18r5h8Dz\nonXr1nk7hOvSvXt3b4dwzTzxvs9ziVBVV+dkIIZh3PjyXNPYMAzD3fJcjdAwDMOd8mofoQsRKWj/\nfKZhGMY1ya1N4+zcobqJiGwH9trT9UXkPx6PzDCMG05evsTuLaAbcBJAVbcBt3syKMMwbky5NRFm\np2nso6qH0gWY4qF4DMO4QeXp23ABh0WkCaAi4gsMA/Z4NizDMG5EeXmwZChW87gCcAxYZc8zDMO4\nKnk2Earqn8CAHIjFMIwbWJ5uGovIB2RwzbGqPuyRiAzDuGHl2RohVlM4TSGgF3DYM+EYhnEjy7OJ\nUFVdbssvIp8B6z0WkWEYN6zc2jS+lqgqA2XdHYhhGDe27JxDmJ0ao4h0EpHdIrJPRJ7PpNxdIqIi\n0iirbWanj/A0//QR+mD94PsVd24YhnEl19s0tk/hmwF0AGKBzSKySFV3pitXBBgO/HL5Vi6XaSIU\nK+r6WL9TApCqqle8WathGEZm3NBH2ATYp6oH7O3NAXoCO9OVmwC8CjybnY1m2jS2k95SVU2xHyYJ\nZmL16tU0adKERo0aMW3atMuWX7p0iQceeIBGjRrRoUMH/vjjDwC++uorWrdu7XiUKlWK7du3A9C3\nb19atWpF8+bNefrpp0lJ8dxFPStXrqRevXqEh4fz2muvZRj/vffeS3h4OC1btuTQoUMAbN68maZN\nm9K0aVOaNGnCwoULAbh48SItWrSgSZMm3HLLLUyYMMFjsQNEREQwc+ZMPvroI/r163fZ8ocffpgZ\nM2YwY8YMZs6cybx58wAICwvjzTff5L333uO///0vrVq1cqwzYsQIZs6cybvvvsuTTz6Jr6+vx+Jf\nv3493bt3p0uXLsycOfOy5ZGRkfTr148GDRqwcuVKx/xNmzbRp08fxyMiIoLVq63biT733HN0796d\nXr16MWbMGJKSkjwWf3a4oWkciutgbaw9z3kftwDlVXVJduPKzqjxVhFpqKq/ZXej+VFKSgojRoxg\n/vz5hISE0L59ezp16kTNmjUdZWbPnk3x4sWJjIzk66+/5qWXXuLDDz+kb9++9O3bF4CdO3cyaNAg\n6tatC8CHH35I0aJFUVUGDx7MwoUL6d3b/b+SkJKSwr///W+WLFlCaGgoLVq0oFu3btSqVctRZtas\nWZQoUYLo6Gjmzp3LqFGjmD17NuHh4WzYsAE/Pz+OHDlC06ZN6dq1KwULFmT58uUULlyYpKQk2rZt\nS8eOHWnatKnb4/fx8eGxxx5j5MiRnDhxgrfeeouNGzc6vmwA3n//fcfzHj16UKVKFcBK8K+99hrx\n8fGULFmSt99+my1btnDu3DnWrFnDlClTAHj++efp1KkTS5Zk+/OVbSkpKUyaNIn333+foKAgBgwY\nwO233+6IESA4OJgJEybwySefuKzbpEkTR1JPSEigS5cuNG/eHICuXbsyefJkwEqKX3/9Nf3793d7\n/NmVzRphKRGJdJp+X1Xfv2Jp1+37AG8Ag68mrivWCEUkLUk2xGqH7xaRX0XkNxH59Wp2kh/8+uuv\nVK5cmUqVKuHv70+vXr1YtmyZS5lly5YxYIB1bnqPHj344YcfSF/Jnj9/Pr169XJMFy1aFIDk5GSS\nkpI8dvrB5s2bqVKlCpUrV8bf35++ffuyePFilzKLFy9m4MCBAPTu3Zu1a9eiqgQEBODnZ71dLl26\n5IhRRChcuDAASUlJJCcneyz+GjVqcOTIEY4ePUpycjLr1q3j1ltvvWL5Nm3asHbtWgDi4uKIj48H\n4NSpU5w5c4ZixYoB1uuSZvfu3ZQqVcoj8W/fvp0KFSpQvnx5ChQoQOfOnVmzZo1LmdDQUGrUqJHp\na7hy5UpatGjBTTfdBECrVq0cNa06depw7Ngxj8SfHVfxu8YnVLWR08M5CcYB5Z2my/FP1x1YPzVc\nB1grIjFAM2BRVgMmmTWNN9l/ewA1gC5AX6CP/ddwcuTIEUJD/6mhh4SEcOTIkcvKhISEAODn50fR\nokU5deqUS5kFCxZw1113uczr06cPNWrUoHDhwvTo0cMj8cfHx1OuXDnHdGhoKHFxcVcskxb/yZMn\nAat5dsstt9CoUSPeeustR2JMSUmhadOmVKhQgbZt29KkSROPxB8YGMjx48cd0ydOnCAwMDDDsmXK\nlCEoKIht27Zdtqx69eqOmq0zX19f2rVrR2Rk5GXruMOff/5JUFCQY7ps2bLXlLSWL19Oly5dLpuf\nlJTE4sWLue22264rzuvlhqbxZqCaiFQWEX+sq94WpS1U1QRVLaWqlVS1ErAR6KGqmf7jMkuEYm94\nf0aPbBxztojIOBF5RkTGi0h7d203L4qMjOSmm25yaY4CzJs3j507d3Lp0iV++OEHL0WXuSZNmvDr\nr7+yfv16XnvtNS5evAhYCeSXX35h3759REZGEh0d7eVIoXXr1vz444+kpqa6zC9ZsiQjRozgjTfe\nuKym/vjjj7N9+/ZcEf+VHD9+nL179zqaxc4mTZpEREQEERERXojsH9ebCFU1GXgcWAH8DsxV1Wg7\nf1xzLSGzPsLSIvJUJgG9ca07vcL2xrpzezktODjYpQYVHx9PcHDwZWXi4+MJDQ0lOTmZs2fPUrJk\nScfyb7755or9f4UKFaJz584sW7aM2293/+0gQ0JCiI2NdUzHxcW51HCdy5QrV84Rf/paV82aNSlc\nuDDR0dEuH7rixYvTunVrVq5cSXh4uNvjP3nyJKVLl3ZMlypVylFbTa9169bMmDHDZV5AQADjx49n\n1qxZ7Nq1y2XZwIEDKVasGG+99Zbb405TpkwZjh496pg+duwYZcte3em6K1asoG3bthQoUMBl/n//\n+19OnTqV4QBeTnLXtcaquhRYmm5ehvlDVdtkZ5uZReULFMZqc2f0uGYiMkpE9ojIeqxmNyIyS0T6\n2M8ni8hOEYkSkan2vLIi8o2IbLMfzUWkkojscNruMyIyzn7+hNM25tjzWovIVvvxm32ukVs0bNiQ\nAwcOcOjQIRITE/nmm2/o3LmzS5lOnToxZ84cABYtWkTLli0d34CpqaksWLDAJRH+/fffjg9HcnIy\n3333HdWqVXNXyC4aNWrEvn37iImJITExka+++oquXbu6lOnatSuff/45AF9//TWtW7dGRIiJiSE5\nORmAQ4cOsXv3bipWrMjx48c5c+YMABcuXGD16tXUqFHDI/Hv3r2bkJAQypYti5+fH61bt2bjxo2X\nlStXrhxFihTh999/d8zz8/NjzJgxrFq1ivXrXS+a6tSpExEREUyePPmyWqI71alTh0OHDhEbG0tS\nUhLLli2jTZs2V7WNZcuWXdYsnj9/Phs2bGDKlCm54qoOd5xQ7QmZ1QiPqOp4d+9QRCKw2vUN7P3/\nCmxxWh6IdT1zTVVVEUn7Yd63gHWq2ss+qbIwUCKTXT0PVFbVS07beAZ4TFU3iEhh4OIVYnwYeBhw\n6TfLjJ+fH6+++ip9+/YlJSWFe+65h5o1a/LKK6/QoEEDOnfuzL333svQoUNp1KgRxYsXdzlF4qef\nfiI0NJRKlSo55p0/f56BAweSmJhIamoqLVq04F//+le24rlafn5+vPnmm3Tv3p2UlBTuv/9+ateu\nzfjx47nlllvo1q0bgwcPZsiQIYSHh1OiRAk+++wzR+xTp06lQIEC+Pj4MH36dMcpQA899BApKSmk\npqZy1113Zdh/5Q6pqam88847TJo0CR8fH1auXMmhQ4cYNGgQe/fudSRF50GSNK1ataJu3boULVqU\nDh06APD6669z4MABhg0bxrFjx3jzzTcB2LBhA1988YXb4/fz82PkyJE8+uijpKSk0KtXL6pWrcrb\nb79NeHg4t99+Ozt27GD48OH89ddfrFu3jnfeeYcFCxYAVg3+6NGjNGrkOiYwYcIEgoODuffeewFo\n164dQ4d67y563kp0WZErfcuJyG+q2tDtOxT5N1AyrSorIm8A8VgjPYuBBViJcYs9vVhVE0XkOFDO\n+QekRKSSvbyOPf0MUFhVx4nIcuBve3sLVPVv+3KcXsDnwNeq+k9b8AoaNGig33//vXsO3gsCAgK8\nHcJ1ufPOO70dwnWZOnWqt0O4Zv379yc6OtptmSssLEzHj8+6bjVo0KAtqprlZXHulFlduV2OReHE\n7gxtAszD+q2U5ZkUT8b1GAo5Pe+KdSnOLVin//ip6mTgQeAmYIOI1MQwjByTW5vGV0yEqnrqSsuu\n0w/AnSJyk91H1915od1kLWZ3iD6JdYkfwGrsO2OLiK+IFMO6Y3YZEQkUkYJYiTPtpMryqroGeA4o\nBhQWkSqqul1VX8UahjeJ0DByUG5NhDn+A++q+quI/A/YBvyJlZCcFQEWikghrFN40kauhwPvi8gD\nWD8eNVRVfxaR8VjnPMYBacN9vsBsO1kK8JaqnhGRCSJyO5AKRAOuZzwbhuFRuWHAJiM5nggBVHUS\nMCmTIpeddauqx7Aurk4//y2sgZT0WmRQdthVhGkYhht5s8aXFa8kQsMw8ieTCA3DyPdM09gwjHzP\n1AgNw8jXTB+hYRgGpmlsGIZhaoSGYRgmERqGka+56zZcnmASoWEYOcbUCA3DyPdMIjQMI18zTWPD\nMAxMjdAwDMMkQsMwjNzaNM6dURmGYeQgUyM0DCNHmGuNDcMwyL1NY5MIDcPIMaZGmEf5+flRsmRJ\nb4eRb3377bfeDuG6FC5c2NshXLOkpCS3b9MkQsMw8jVzQrVhGAamRmgYhmESoWEY+ZtpGhuGYZB7\na4S5Mz0bhnFDSjupOrNHNrbRSUR2i8g+EXk+g+VPichOEYkSkdUiUjGrbZpEaBhGjvHx8cnykRkR\n8QVmAJ2B2sDdIlI7XbHfgEaqWg+YB0zJMq5rOhrDMIyrlJ3aYDZqhE2Afap6QFUTgTlAT+cCqrpG\nVc/bkxuBcllt1PQRGoaRY7LZR1hKRCKdpt9X1fft56HAYadlsUDTTLb1ALAsqx2aRGgYRo7J5qjx\nCVVtdL37EpF7gUZA66zKmkRoGEaOccOocRxQ3mm6nD0v/X7aA6OA1qp6KauNmj5CwzByhJv6CDcD\n1USksoj4AwOARen20xB4D+ihqn9mJzZTIzQMI8dc7wnVqposIo8DKwBf4CNVjRaR8UCkqi4CXgMK\nA1/ZifUPVe2R2XZNIjQMI8e444RqVV0KLE03b6zT8/ZXu02TCA3DyDG59coSkwgNw8gRufla49wZ\nVR61fPlyatSoQdWqVZk8efJlyy9dukT//v2pWrUqTZs2JSYmxrHslVdeoWrVqtSoUYMVK1YAcPjw\nYW6//XZq165NeHg406dPz1PxAwwZMoQyZcpQp04dj8YOsGLFCsLDw6lVqxZTplx+McGlS5e45557\nqFWrFrfddpsj/lWrVtG0aVMaNmxI06ZNWbNmDQDnz5+nZ8+e1KlTh/r16zNy5EiPxt+xY0e2b9/O\nzp07eeaZZy5bXqFCBZYvX05kZCQrV64kNDQUgNatW7Np0ybHIyEhgR49rC6xDz74gN27dzuW1atX\nz6PHkBV3XGLnEapqHpk8IiIiNDuSk5M1LCxM9+/fr5cuXdJ69eppdHS0S5kZM2boI488oqqqX375\npfbr109VVaOjo7VevXp68eJFPXDggIaFhWlycrLGx8frli1bVFX17NmzWq1atcu26S6eiF9Vdd26\ndbplyxYNDw+/prgSExOz9bhw4YKGhYXprl279O+//9a6devq1q1bXcq89dZb+tBDD2liYqJ+9tln\n2qdPH01MTNRffvlFY2JiNDExUX/99VcNCQnRxMREPXPmjK5cuVITExP177//1ttuu00XLVqU7ZgS\nExPV398/W49ChQrp/v37tUaNGnrzzTfrtm3btF69ei5l5s2bp0OGDFF/f3/t2LGjzp49+7LtlC1b\nVk+ePKnFihVTf39//eSTT7R///7ZjsP5ISKqbvwshYeH665du7J8YA165Ojn3NQI3WTTpk1UrVqV\nsLAw/P39GTBgAAsXLnQps3DhQu6//34A+vTpw+rVq1FVFi5cyIABAyhYsCCVK1ematWqbNq0ieDg\nYG655RYAihQpQq1atYiLu+yUqVwbP0CrVq1y5KcONm/eTJUqVRzx9+vX77Lb/H/77bcMGjQIgLvu\nuos1a9agqjRs2JCQkBAAwsPDuXDhApcuXSIgIIA2bdoA4O/vT8OGDT32+jdu3Jj9+/dz8OBBkpKS\nmDt3Lt27d3cpU6tWLdauXQvA2rVrL1sO0Lt3b1asWMGFCxc8Euf1SGsaX8+1xp5iEqGbxMXFUb78\nP+d5litX7rIPjXMZPz8/ihUrxsmTJ7O1bkxMDL/99htNm2Z2NVHujd/T4uLiKFfun0tKQ0NDiY+P\nv2IZ5/idff311zRs2JCCBQu6zD9z5gxLlizh9ttv90j8ISEhHD78z5VjcXFxjqZvmqioKO68804A\nevbsSdGiRS/7kunbty9z5851mTd+/HgiIyN57bXX8Pf390j82ZVbm8YmEeYBf//9N3fddRfTpk2j\naNGi3g7nhhUdHc2oUaOYMWOGy/zk5GQGDRrEY489RlhYmJeig+eff56WLVvyyy+/0KpVK2JjY0lJ\nSXEsDwoKok6dOqxcudIxb8yYMdStW5fmzZtTokSJDPsec5JJhDe40NBQl2/02NjYy77RncskJyeT\nkJBAYGBgpusmJSVx1113MXDgQHr37p3n4s8poaGhxMbGOqbj4uIczd2MyjjHD1bMffv25aOPPqJK\nlSou6w0dOpSqVavyxBNPeCz++Ph4l1p1aGjoZbXqI0eO0L9/f5o2bcrYsdZpcwkJCY7lffr0YdGi\nRSQnJzvmHT16FIDExEQ+/fRTGjdu7LFjyA7TNL7BNW7cmL1793Lw4EESExOZM2eOY+QuTY8ePfjk\nk08AmDdvHm3btkVE6NGjB3PmzOHSpUscPHiQvXv30qRJE1SVBx54gFq1avHUU0/lufhzUqNGjdi3\nb58j/rlz59KtWzeXMt26deOzzz4DYP78+bRp0wYR4cyZM/Ts2ZNJkybRvHlzl3XGjh1LQkICr7/+\nukfjj4yMpGrVqlSqVIkCBQrQr18/Fi9e7FImMDDQUWMaMWKE43+Rpl+/fvzvf/9zmRcUFOR43qNH\nD6Kjoz10BFlz0yV2npHTozN57ZHdUWNV1SVLlmi1atU0LCxMJ06cqKqqY8aM0YULF6qq6oULF7RP\nnz5apUoVbdy4se7fv9+x7sSJEzUsLEyrV6+uS5cuVVXVH3/8UQGtW7eu1q9fX+vXr69LlizJdjxX\ny93xq6oOGDBAg4KC1M/PT0NDQ3XmzJlXFdPVjNAuXLhQq1atqmFhYfrSSy9pYmKijhw5UufPn6+J\niYl69uxZ7d27t1apUkUbNWqku3bt0sTERB03bpwGBARovXr1HI/Y2Fg9cOCAAlqjRg3H/Hfffdcj\no8b+/v7ao0cP3bNnj+7fv1/HjBmj/v7+OnHiRO3du7f6+/tr//79de/evbpnzx798MMPtXDhwo51\nq1WrprGxsVqwYEGXbX7//fe6fft23bFjh37++edaokQJr40a161bV2NiYrJ84IVRY1FV72TgPKJR\no0YaGRmZdUHDIzzxI+M5Ka//wHtqaqrbqmj16tXT9LXcjFSsWHGLuuE2XFfDXFliGEaOMZfYGYaR\n75lEaBhGvpabrzU2idAwjBxjaoSGYeR7JhEahpGvmaaxYRgGpkZoGIZhEqFhGIZpGhuGka959Vri\nLJhEaBhGjjGJ0DCMfM80jQ3DyNdM09gwDAPTNDYMwzBNY8MwDFMjNAwjXzN9hIZhGJimsWEYhqkR\nGoZhmERoGEa+Zm7DZRiGQe6tEZqf88yCiBwHDnlwF6WAEx7cvifl5dgh78fvaRVVtbS7NiYiy7Fe\n86ycUNVO7tpvdphE6GUiEpnTv+HqLnk5dsj78Rvukzsb7IZhGDnIJELDMPI9kwi9731vB3Ad8nLs\nkPfjN9zE9BEahpHvmRqhYRj5nkmEhmHkeyYRGoaR75lEmItJbj0N3zBuMCYR5kIiUhBAb6CRLOek\nnhcSfPoY80LMxrUziTCXEZGuwGsi8qqI1BGRPH09uFMCCRIRfxEpoKqamxOLiEjal5CI3CsilW+k\nLyXjciYR5iIi0hJ4BXgb6AI8QR7/H9lJrzPwNTAGmCUi/rk5sTglwWHACKCgdyMyPC1Pf8huFE61\no2ZYyaIMcB6YqKqJInKT14K7TiLSECu5/wtIBCoChZyW58qaoYiEA/cCHVR1l4i0F5FWIlLC27EZ\n7penm103Cqfa0UHgIaAs0EdV/xCRgUBV4CVvxXctnJqXArwJhADdgQGqelZEmgKbVTXVm3GmSdcc\n9gNOAbuAJ0SkNFAZuBmYBCzxWqCGR5gaoZeJSDMR6SkitwDRwE3ATOC8Pe85YLM3Y7waaTU8p+Se\nALyKdUytVfWAiLQBhpG9WzJ5XLok+DAwVlWPAFuAS8B7qtoBWAk0916khqeYS+y8SETuAKYBbwDv\nAu2AEsAdWE1If2C6qi5y/rDmVmkxikh7oDewAyuJVwaeBMZjtUImYiWbhV4LNgMi8iTQH3hQVXek\nW3YP8DzQV1V3eyM+w3NM09hLRKQwMBToBZQEfgd+V9VjIrIYqxkWoKrxeSEJgmNg5A5gClZf58NA\nfeAZIBVr8CceeEFVl+am4xKRm4FbgQ5AUREZBPTDqrkWwUqQ95gkeGMyNUIvEJEGQBxwD1Z/YFtg\noKruF5HBwBZV3e7FEK+ZiDwNLAaCgNeBnqoalz7p5bIkWENVd4vIp0BDrL7BbUAVoLCq9hWRoqp6\n1quBGh5jaoQ5TESaA69h1ZZCgb5YI5P7RaQ+8CzwiBdDvCYi0hY4CvgCc4CLQHdVPSIiXbDOI5yt\nqomQe04WF5FawLMislxV7xORnliDOPH2MT0qIgVNEryxmUSYg0SkHnA/8LmqRovIOCAcGGWPMTTA\najau916UV09EIrBGtR8HPgPaAFvtJNgCa9T48bQkmMvEAhuBViJSAPhSVVNF5AWspvH9qnrJqxEa\nHmcSYQ6xR1ODsRKfikiIXevoDbTG+l+8rapbclOzMSsiEgp8BcxV1W0iUgSYDgwVkdVAMeBpVf3O\nm3GmJyIDgJOq+p2IfIF1jmNzIEVEvsI6o2JQ+kET48Zk+ghzgN38uher1nQb1rmC3wKrVfVPb8Z2\nLZxGh/3tE74nAP+HdXrMDqflFYDk3DDgk+4UGQEewO6GUNW1dgJ/A6gJvKmqX3srViPnmRqhh4lI\nB+A+IAKr3+w1rHMFBwAFReRbVT3pxRCvilOSuxWYJCL3qeoYETkHfGpP7wBQ1T/S1stFSfA+4E/g\nI6xa4DQReUpVvxeRzUASsMFbsRreYRKhB4lII6wTiQcBkUAlYDRWzbAg1ikZK70V37Wwk2BHoA/W\n+YGrRKStqk4WkURgvojclZualOlOlh6GNZKdipW4FSvmL4H2QFdVPea9aA1vME1jDxKRO4Fuqvqg\n3RxrC4wC1gAvA0VV9bQ3Y7xaIlIdWIp1us8vIjIDaAV0VtVYEXkeWJ+bBnxExAcIBD4HnrX7Mn1V\nNcVefitQA9igqnu9GKrhJeYSOw8QkSp2wvgZqC8iHdSyGojBOj+tL3DG/pDmJQnAT1jn2qGqjwF/\nAMtFpIiqTs4NSTDtUr80qnocOA78nVbELldXVX9W1VkmCeZfee1DmOuJSHesW069Zj/mA71EZJB9\n7XBt4ATQ1E6OueKmA1eSllBExNeedQHr7jidnYq9h/Ve+tapnNek6xO8F2twCqyrW2YAqGqyiNwN\nTLEHSox8zPQRupGINAPGYl2m1QHrNJLzwA9Y59idxDqROhgYItbttS7m1lNlnAZGOgEDRWQ71p1X\nXgA+EZGqWDcl6IM1IPQIUBTwanPfKQk+hdUPO8SeP0hEvhCRtcAB4P/bO/cYu6oqjP++QbCV1o7B\nF+WRUqA+WqGhLSpE0jSkESikgChExcZKoRJDhZIYwQQTH0Q0IcQAAibUVE15SFJ8hIhGhKZom6El\ntcK0tNUMYjLEQKAtAfHzj7WmuY6ddqZluHc465fczJ1z9z17ncnky1p7r73WR4DFtl9ql61FZ1BC\n+MbSR6SRzASWAR8FbgOOBa4mjm19gkgwvtj27jbZOSxSBM8i1jO/RiSDn08UUFiY76cSHte7gdlk\nyBYbnBIAAAYqSURBVNkOJJ0IHJbJ6pMJr/UMYJyki4jjc5cSAjgJ2GF7R7vsLTqHCo3fQGz32V5H\nJEivtP0MsJIQiecJ72kmcL7tJ9tn6fCQNA6YQ3hVXURdxJXA94HjbP8g1wi7ifD487b/1SZbjyNy\nNbdKmmT7H8ALwGNEhZ/TiQIQdxGnXv5QIlgMULvGo0CeWricCCMvIHYq1+RnXZ28LtgSDk8lNna6\niVSfVcQzbSF2vQ8lvMJ+Io3mP7a3tcnmowlPvA94ggjVVwBbgS8BD9reniH+AuCqgR3jooAKjUeL\nXxPicR7w7RYRVIeLYFees11A5NtdkydFJhPrm31EeLkN+I7tf+ZXt7bH4j08C/QC04iSWUcAFwL3\n2r4FQNIyIixeVCJYDKaEcBTISiUrJP00dyeVO8Qd6X5LGmf7lRTB2cRu96dTBA/PI3J9hFc4HVjm\nDqnL1+LBdgGnEGuUDxF2XqioM7iFSGb/gsdoebNidKnQeBRp9/na4SDpSOAc4D7bL0j6TP5+HVF9\n5SxgArHGdizQ5SgZ1jHPpujrspxoELWY6DeykwjZXyIq4mwuT7AYitosGUU6RSiGQtIRxBpmD1ER\n52QirD+ayH/cRYhLL1FQYXtuAHXas30A+JntDcA1RPrOaUQoPx54rkSw2BclhA0lE6XPBk4i1v1u\nJs5EH2V7HjDP9m1ED5XZxBphp9IDnC5puu1Xbd9MFL01cIPt59trXtHpVGjccBSl9d9DVGSZlj9X\nAxuIPssriDXBB9tm5H6Q1E2U1AL4PeEFfhW41PazbTOsGDOUEDYYRaOlrxORQT/wJ0IMtwMP589j\nbK/vpDXBvZE72xfk69/A8rGQq1l0BiWEDUXSe4kz0Utsb5Z0JXH0r58IhXcA3xtrx89yl1i2X97v\n4KJIao2wubxGpE8NNFm/gwiRzwU2Ezl4Y0oEAWzvLBEsRkoJYUPJOoj3AHMlzbD9GuEh7gJWVVhZ\nNIkKjRtMHk27AjgVWEccTbvS9sNtNawo3mRKCBtO1uL7ODCDaCz/SJtNKoo3nRLCoigaT60RFkXR\neEoIi6JoPCWERVE0nhLCoigaTwlhURSNp4Sw2IOk1yVtkLRJ0r2S3nEQ95or6Zf5/rxs/D7U2G5J\nXz6AOW6QtHy41weNuVvSp0Yw1xRJm0ZqYzE2KCEsWtlte6btGcCrRLL1HhSM+H/G9mrbN+5jSDfR\nc6Qo2kIJYTEUjwInpCf0tKSfAJuAYyTNl7RWUk96jhMAJH1S0lOSeogqMOT1RZJ+mO/fJ+kBSRvz\ndRpwI3B8eqM35bhrJa2T9KSkb7bc6zpJvZIeIwqy7hNJl+V9Nkq6f5CXe6ak9Xm/BTn+EEk3tcx9\n+cH+IYvOp4Sw+D8kvY0o0T/Q3+NE4Fbb04kS+NcDZ9o+BVgPXJ2tP+8kijbMAt4/xO1vAR6xfTLR\nY+QvRM/kZ9IbvVbS/JzzVKL96SxJZ0iaBVyc184mWo3uj1/YnpPz/ZUo5T/AlJzjHOD2fIbFwIu2\n5+T9L8tWocVbmGreVLQyXtKGfP8o8GNgMvA324/n9Y8BHwbWRJFrDgPWAh8EttveAiBpJbBkL3PM\nI7rJkeXzX5T0rkFj5ufrifx9AiGME4EHbO/KOVYP45lmSPoWEX5PIBo7DXBPdhXcImlbPsN84KSW\n9cNJOXfvMOYqxiglhEUru23PbL2QYrez9RLwW9uXDBr3P987SAR81/aPBs2x7ADudTew0PZGSYuA\nuS2fDT5f6pz7K7ZbBRNJUw5g7mKMUKFxMVIeJ/qDnABRCFXSNOApYIqk43PcJUN8/3fA0vzuIZIm\nEZ3mJraMeQj4Ysva41FZSPaPwEJJ47NYxLnDsHci8JykQ4HPDvrsIkldafNU4Omce2mOR9K0LPZa\nvIUpj7AYEbb707P6uaS35+XrbfdKWgL8StIuIrSeuJdbXAXcIWkx8Dqw1PZaSWsyPeU3uU74IWBt\neqQvA5+z3SNpFbCR6K2ybhgmf4NoQTDQiqDVpr8DfwbeCVxh+xVJdxFrhz3Z4KofWDi8v04xVqnq\nM0VRNJ4KjYuiaDwlhEVRNJ4SwqIoGk8JYVEUjaeEsCiKxlNCWBRF4ykhLIqi8fwXjAi6BUkqx1sA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x102176710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sc.report_score(actual, predicted)\n",
    "print(\"F1 Score\")\n",
    "print(f1_score(actual, predicted, average = None))\n",
    "print(\"Avg Precision Score\")\n",
    "print(precision_score(actual, predicted, average = None))\n",
    "matrix = confusion_matrix(actual,predicted)\n",
    "plot_confusion_matrix(matrix, classes=[\"agree\",\"disagree\", \"discuss\"],\n",
    "                      title='Normalized confusion matrix')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}