Q&A.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:09:38.204501Z",
     "start_time": "2019-04-16T15:09:38.198742Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:09:33.006153Z",
     "start_time": "2019-04-16T15:09:32.996298Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def path_to_url(nid, data_type, paths):\n",
    "    \n",
    "    try:\n",
    "\n",
    "        path_type = (paths['type'] == data_type)\n",
    "        path_nid = (paths['nid'] == nid)\n",
    "\n",
    "        path = paths.loc[path_type & path_nid]['path'].values[0]\n",
    "\n",
    "        return 'https://publiclab.org' + path\n",
    "    \n",
    "    except:\n",
    "        \n",
    "        return False\n",
    "    \n",
    "# path_to_url(18446, 'qstn', paths), path_to_url(13356, 'qstn', paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:09:33.676459Z",
     "start_time": "2019-04-16T15:09:33.664474Z"
    },
    "code_folding": [],
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def create_csv_paths(data_types, base='', exp='', full=True):\n",
    "    \n",
    "    csv_paths = {}\n",
    "    \n",
    "    for data_type in data_types:\n",
    "\n",
    "        if not full:\n",
    "            csv_dir = '../data/2016_2019/samples/'\n",
    "            e = '_' + str(base) + 'e' + str(exp)\n",
    "            csv_path = csv_dir + data_type + e + '.csv'\n",
    "        else:\n",
    "            csv_dir = '../data/2016_2019/full/'\n",
    "            e = ''\n",
    "            csv_path = csv_dir + data_type + e + '.csv'\n",
    "            \n",
    "        csv_paths[data_type] = csv_path\n",
    "    \n",
    "    return csv_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T16:50:28.089766Z",
     "start_time": "2019-04-16T16:50:28.082353Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def set_edge_direction(row):\n",
    "    \n",
    "    if row['q_uid'] == row['a_uid']:\n",
    "        \n",
    "        direction = -1\n",
    "        \n",
    "    elif row['q_uid'] != row['a_uid']:\n",
    "        \n",
    "        direction = 1\n",
    "        \n",
    "    return direction\n",
    "\n",
    "# set_edge_direction(q_and_a.iloc[3254]), set_edge_direction(q_and_a.iloc[3255]) # -1 , 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### LOAD PATHS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### CSV DIRECTORY PATHS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:28:04.647058Z",
     "start_time": "2019-04-16T15:28:04.641502Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data_types = ['answers', 'comments', 'notes', 'questions', 'users', 'wikis', 'paths', 'q_and_a']\n",
    "csv_paths = create_csv_paths(data_types)\n",
    "\n",
    "# print csv_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### WEBSITE PATHS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:09:43.599030Z",
     "start_time": "2019-04-16T15:09:43.572108Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "paths = pd.read_csv(csv_paths['paths'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:09:44.157662Z",
     "start_time": "2019-04-16T15:09:44.125266Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>nid</th>\n",
       "      <th>date</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2026</th>\n",
       "      <td>note</td>\n",
       "      <td>13969</td>\n",
       "      <td>1488297079</td>\n",
       "      <td>/notes/nicholas/02-28-2017/foldable-spectromet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>wiki</td>\n",
       "      <td>15087</td>\n",
       "      <td>1508797068</td>\n",
       "      <td>/wiki/infragram-filters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2166</th>\n",
       "      <td>note</td>\n",
       "      <td>14206</td>\n",
       "      <td>1495047283</td>\n",
       "      <td>/notes/stevie/05-17-2017/last-annual-barnraisi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1036</th>\n",
       "      <td>note</td>\n",
       "      <td>12375</td>\n",
       "      <td>1446810564</td>\n",
       "      <td>/notes/Holger/11-06-2015/raspberry-pi-based-mi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>wiki</td>\n",
       "      <td>18412</td>\n",
       "      <td>1550945796</td>\n",
       "      <td>/wiki/filters</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      type    nid        date  \\\n",
       "2026  note  13969  1488297079   \n",
       "349   wiki  15087  1508797068   \n",
       "2166  note  14206  1495047283   \n",
       "1036  note  12375  1446810564   \n",
       "487   wiki  18412  1550945796   \n",
       "\n",
       "                                                   path  \n",
       "2026  /notes/nicholas/02-28-2017/foldable-spectromet...  \n",
       "349                             /wiki/infragram-filters  \n",
       "2166  /notes/stevie/05-17-2017/last-annual-barnraisi...  \n",
       "1036  /notes/Holger/11-06-2015/raspberry-pi-based-mi...  \n",
       "487                                       /wiki/filters  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### WIKI PAGES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ],
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# # ORIGINAL WIKI COLS\n",
    "# [u'nid', u'vid', u'type', u'language', u'title', u'uid', u'status', u'created', u'changed', u'comment', u'promote', \n",
    "#  u'moderate', u'sticky', u'tnid', u'translate', u'cached_likes', u'comments_count', u'drupal_node_revisions_count', \n",
    "#  u'path', u'main_image_id', u'slug', u'legacy_views', u'views', u'latitude', u'longitude', u'precision']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:26.208933Z",
     "start_time": "2019-04-16T15:20:26.103621Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>date</th>\n",
       "      <th>comments</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10388</td>\n",
       "      <td>7</td>\n",
       "      <td>1525745225</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>wiki</td>\n",
       "      <td>OpenHour</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11502</td>\n",
       "      <td>1</td>\n",
       "      <td>1420149623</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>wiki</td>\n",
       "      <td>MapMill Help</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11508</td>\n",
       "      <td>7</td>\n",
       "      <td>1420405663</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>wiki</td>\n",
       "      <td>choosing-how-to-track-progress</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11509</td>\n",
       "      <td>7</td>\n",
       "      <td>1420476955</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>wiki</td>\n",
       "      <td>infrared-garden-experiment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11511</td>\n",
       "      <td>7</td>\n",
       "      <td>1420666454</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>wiki</td>\n",
       "      <td>pole-mapping-guide</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid  uid        date  comments  likes  type  \\\n",
       "0  10388    7  1525745225         0     27  wiki   \n",
       "1  11502    1  1420149623         0      0  wiki   \n",
       "2  11508    7  1420405663         0      0  wiki   \n",
       "3  11509    7  1420476955         0      1  wiki   \n",
       "4  11511    7  1420666454         0      3  wiki   \n",
       "\n",
       "                            title  \n",
       "0                        OpenHour  \n",
       "1                    MapMill Help  \n",
       "2  choosing-how-to-track-progress  \n",
       "3      infrared-garden-experiment  \n",
       "4              pole-mapping-guide  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wiki_cols = [u'nid',  u'uid', u'created', u'comments_count', u'cached_likes', u'type', u'title']\n",
    "renamed_wiki_cols = {'created': 'date', 'comments_count': 'comments', 'cached_likes': 'likes'}\n",
    "\n",
    "wikis = pd.read_csv(csv_paths['wikis'])\n",
    "wikis['type'] = 'wiki'\n",
    "\n",
    "wikis = wikis[wiki_cols] # questions.columns = cols\n",
    "wikis.rename(index=str, columns=renamed_wiki_cols, inplace=True)\n",
    "\n",
    "# wikis.columns\n",
    "wikis.head()\n",
    "# print wikis.shape # (514, 6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### RESEARCH NOTES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:27.774388Z",
     "start_time": "2019-04-16T15:20:27.769361Z"
    },
    "code_folding": [
     0
    ],
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# # ORIGINAL NOTES COLUMNS\n",
    "# ['nid', 'vid', 'type', 'language', 'title', 'uid', 'status', 'created', 'changed', 'comment', 'promote', \n",
    "#  'moderate', 'sticky', 'tnid', 'translate', 'cached_likes', 'comments_count', 'drupal_node_revisions_count', \n",
    "#  'path', 'main_image_id', 'slug', 'legacy_views', 'views', 'latitude', 'longitude', 'precision']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:28.568269Z",
     "start_time": "2019-04-16T15:20:28.457338Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>date</th>\n",
       "      <th>comments</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11503</td>\n",
       "      <td>554</td>\n",
       "      <td>1420328192</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>note</td>\n",
       "      <td>Public Lab at Science for Action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11504</td>\n",
       "      <td>554</td>\n",
       "      <td>1420328228</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>note</td>\n",
       "      <td>Public Lab Barnraising 2014, day 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11505</td>\n",
       "      <td>554</td>\n",
       "      <td>1420328254</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>note</td>\n",
       "      <td>Public Lab Barnraising 2014, day 2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11506</td>\n",
       "      <td>554</td>\n",
       "      <td>1420328276</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>note</td>\n",
       "      <td>Public Lab Barnraising 2014, day 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11507</td>\n",
       "      <td>554</td>\n",
       "      <td>1420328303</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>note</td>\n",
       "      <td>Public Lab Organizers Summit, 2014</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid  uid        date  comments  likes  type  \\\n",
       "0  11503  554  1420328192         0      3  note   \n",
       "1  11504  554  1420328228         0      0  note   \n",
       "2  11505  554  1420328254         0      0  note   \n",
       "3  11506  554  1420328276         1      0  note   \n",
       "4  11507  554  1420328303         2      4  note   \n",
       "\n",
       "                                title  \n",
       "0   Public Lab at Science for Action   \n",
       "1  Public Lab Barnraising 2014, day 3  \n",
       "2  Public Lab Barnraising 2014, day 2  \n",
       "3  Public Lab Barnraising 2014, day 1  \n",
       "4  Public Lab Organizers Summit, 2014  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes_cols = [u'nid',  u'uid', u'created', u'comments_count', u'cached_likes', u'type', u'title']\n",
    "renamed_notes_cols = {'created': 'date', 'comments_count': 'comments', 'cached_likes': 'likes'}\n",
    "\n",
    "notes = pd.read_csv(csv_paths['notes'])\n",
    "notes['type'] = 'note'\n",
    "notes = notes[notes_cols]\n",
    "notes.rename(index=str, columns=renamed_notes_cols, inplace=True)\n",
    "\n",
    "# notes.columns\n",
    "notes.head() # (3155, 6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### COMMENTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:32.054479Z",
     "start_time": "2019-04-16T15:20:32.050865Z"
    },
    "code_folding": [
     0
    ],
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# # ORIGINAL COMMENTS COLUMNS\n",
    "# [u'cid', u'pid', u'nid', u'uid', u'subject', u'comment', u'hostname', u'timestamp', u'status', u'format', \n",
    "#  u'thread', u'name', u'mail', u'homepage', u'aid', u'comment_via', u'message_id', u'tweet_id', u'reply_to']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:33.102543Z",
     "start_time": "2019-04-16T15:20:32.849787Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>cid</th>\n",
       "      <th>aid</th>\n",
       "      <th>date</th>\n",
       "      <th>type</th>\n",
       "      <th>thread</th>\n",
       "      <th>comment</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11500</td>\n",
       "      <td>433330</td>\n",
       "      <td>11010</td>\n",
       "      <td>0</td>\n",
       "      <td>1420182880</td>\n",
       "      <td>cmnt</td>\n",
       "      <td>02/</td>\n",
       "      <td>fyi, I just noticed that the local public libr...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9509</td>\n",
       "      <td>433359</td>\n",
       "      <td>11011</td>\n",
       "      <td>0</td>\n",
       "      <td>1420238273</td>\n",
       "      <td>cmnt</td>\n",
       "      <td>02/</td>\n",
       "      <td>Thank you very much Ned. This information will...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11493</td>\n",
       "      <td>51</td>\n",
       "      <td>11012</td>\n",
       "      <td>0</td>\n",
       "      <td>1420483545</td>\n",
       "      <td>cmnt</td>\n",
       "      <td>01/</td>\n",
       "      <td>I will be attending!</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11493</td>\n",
       "      <td>51</td>\n",
       "      <td>11013</td>\n",
       "      <td>0</td>\n",
       "      <td>1420483578</td>\n",
       "      <td>cmnt</td>\n",
       "      <td>02/</td>\n",
       "      <td>Not certain but perhaps.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11178</td>\n",
       "      <td>7</td>\n",
       "      <td>11014</td>\n",
       "      <td>0</td>\n",
       "      <td>1420487544</td>\n",
       "      <td>cmnt</td>\n",
       "      <td>02/</td>\n",
       "      <td>Hey @Tbtouaki , how did this turn out?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid     uid    cid  aid        date  type thread  \\\n",
       "0  11500  433330  11010    0  1420182880  cmnt    02/   \n",
       "1   9509  433359  11011    0  1420238273  cmnt    02/   \n",
       "2  11493      51  11012    0  1420483545  cmnt    01/   \n",
       "3  11493      51  11013    0  1420483578  cmnt    02/   \n",
       "4  11178       7  11014    0  1420487544  cmnt    02/   \n",
       "\n",
       "                                             comment  reply_to  \n",
       "0  fyi, I just noticed that the local public libr...         0  \n",
       "1  Thank you very much Ned. This information will...         0  \n",
       "2                               I will be attending!         0  \n",
       "3                           Not certain but perhaps.         0  \n",
       "4             Hey @Tbtouaki , how did this turn out?         0  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmnt_cols = ['nid', 'uid', 'cid', 'aid', 'timestamp', 'type', 'thread', 'comment', 'reply_to']\n",
    "\n",
    "comments = pd.read_csv(csv_paths['comments'])\n",
    "comments['type'] = 'cmnt'\n",
    "comments['reply_to'].fillna(0, inplace=True)\n",
    "comments['reply_to'] = comments['reply_to'].astype('int64')\n",
    "comments = comments[cmnt_cols]\n",
    "comments.rename(index=str, columns={'timestamp': 'date'}, inplace=True)\n",
    "\n",
    "# comments.columns\n",
    "comments.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### QUESTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:36.006701Z",
     "start_time": "2019-04-16T15:20:36.002762Z"
    },
    "code_folding": [
     0
    ],
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# # ORIGINAL QUESTIONS COLUMNS\n",
    "# ['nid', 'vid', 'type', 'language', 'title', 'uid', 'status', 'created', 'changed', 'comment', 'promote', \n",
    "#  'moderate', 'sticky', 'tnid', 'translate', 'cached_likes', 'comments_count', 'drupal_node_revisions_count', \n",
    "#  'path', 'main_image_id', 'slug', 'legacy_views', 'views', 'latitude', 'longitude', 'precision']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:36.930372Z",
     "start_time": "2019-04-16T15:20:36.817816Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>qid</th>\n",
       "      <th>date</th>\n",
       "      <th>user_cmnts</th>\n",
       "      <th>thread_cmnts</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11516</td>\n",
       "      <td>433692</td>\n",
       "      <td>1</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11524</td>\n",
       "      <td>433790</td>\n",
       "      <td>2</td>\n",
       "      <td>1421278243</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Is the dustduino (with Shinyei sensor) appropr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11526</td>\n",
       "      <td>433849</td>\n",
       "      <td>3</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11545</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>qstn</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11610</td>\n",
       "      <td>435436</td>\n",
       "      <td>5</td>\n",
       "      <td>1424457388</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Is the project still active?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid     uid  qid        date  user_cmnts  thread_cmnts  likes  type  \\\n",
       "0  11516  433692    1  1421053078           2             3      1  qstn   \n",
       "1  11524  433790    2  1421278243           2             1      1  qstn   \n",
       "2  11526  433849    3  1421429978           2             2      0  qstn   \n",
       "3  11545       4    4  1422557787           2             6      5  qstn   \n",
       "4  11610  435436    5  1424457388           2             2      0  qstn   \n",
       "\n",
       "                                               title  \n",
       "0  Is it possible to log the data from a dustduin...  \n",
       "1  Is the dustduino (with Shinyei sensor) appropr...  \n",
       "2                How can I test my tap water for BPA  \n",
       "3  dealing with patents: Black & Decker and the T...  \n",
       "4                       Is the project still active?  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quest_cols = [u'nid', u'uid', u'qid', u'created', u'comment', u'comments_count', u'cached_likes', u'type', u'title']\n",
    "\n",
    "renamed_quest_cols = {'created': 'date', 'comments_count': 'thread_cmnts', \n",
    "                      'comment': 'user_cmnts', 'cached_likes': 'likes'}\n",
    "\n",
    "questions = pd.read_csv(csv_paths['questions'])\n",
    "questions['type'] = 'qstn'\n",
    "questions['qid'] = range(1, questions.shape[0] + 1)\n",
    "\n",
    "questions = questions[quest_cols] # questions.columns = cols\n",
    "questions.rename(index=str, columns=renamed_quest_cols, inplace=True)\n",
    "\n",
    "# questions.columns\n",
    "questions.head() # (902, 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### ANSWERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:39.559168Z",
     "start_time": "2019-04-16T15:20:39.553355Z"
    },
    "code_folding": [
     0
    ],
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# # ORIGINAL ANSWERS COLUMNS\n",
    "# [u'id', u'uid', u'nid', u'content', u'cached_likes', u'created_at', u'updated_at', u'accepted']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:40.050107Z",
     "start_time": "2019-04-16T15:20:39.928370Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>aid</th>\n",
       "      <th>created_at</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>content</th>\n",
       "      <th>accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13190</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-17 15:42:47 UTC</td>\n",
       "      <td>0</td>\n",
       "      <td>ans</td>\n",
       "      <td>Yes, it there should be an edit link (with pen...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13312</td>\n",
       "      <td>468506</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-08-02 09:16:09 UTC</td>\n",
       "      <td>1</td>\n",
       "      <td>ans</td>\n",
       "      <td>Sorry to bother you. I was just testing the ne...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13356</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2016-08-16 23:04:23 UTC</td>\n",
       "      <td>0</td>\n",
       "      <td>ans</td>\n",
       "      <td>Looks like it does indeed work... now to see i...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13356</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>2016-08-17 12:35:13 UTC</td>\n",
       "      <td>0</td>\n",
       "      <td>ans</td>\n",
       "      <td>Here's an answer for you: 42!</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13358</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2016-08-17 14:27:28 UTC</td>\n",
       "      <td>0</td>\n",
       "      <td>ans</td>\n",
       "      <td>Hi, CCDs do not have a linear response curve -...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid     uid  aid               created_at  likes type  \\\n",
       "0  13190       1    1  2016-06-17 15:42:47 UTC      0  ans   \n",
       "1  13312  468506    2  2016-08-02 09:16:09 UTC      1  ans   \n",
       "2  13356       1    3  2016-08-16 23:04:23 UTC      0  ans   \n",
       "3  13356       7    4  2016-08-17 12:35:13 UTC      0  ans   \n",
       "4  13358       1    5  2016-08-17 14:27:28 UTC      0  ans   \n",
       "\n",
       "                                             content  accepted  \n",
       "0  Yes, it there should be an edit link (with pen...      True  \n",
       "1  Sorry to bother you. I was just testing the ne...      True  \n",
       "2  Looks like it does indeed work... now to see i...      True  \n",
       "3                      Here's an answer for you: 42!     False  \n",
       "4  Hi, CCDs do not have a linear response curve -...      True  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans_cols = [u'nid', u'uid', u'id', u'created_at', u'cached_likes', u'type', u'content', u'accepted']\n",
    "\n",
    "answers = pd.read_csv(csv_paths['answers'])\n",
    "answers['type'] = 'ans'\n",
    "answers = answers[ans_cols]\n",
    "\n",
    "renamed_ans_cols = {'id': 'aid', 'created': 'date', 'cached_likes': 'likes'}\n",
    "answers.rename(index=str, columns=renamed_ans_cols, inplace=True)\n",
    "\n",
    "# answers.columns\n",
    "answers.head()  #(1081, 8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T19:03:05.811019Z",
     "start_time": "2019-04-15T19:03:05.798600Z"
    },
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### TOTALS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:20:44.438704Z",
     "start_time": "2019-04-16T15:20:44.430685Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WIKI PAGES: 514\n",
      "RESEARCH NOTES: 3155\n",
      "COMMENTS: 11636\n",
      "QUESTIONS: 902\n",
      "ANSWERS: 1081\n"
     ]
    }
   ],
   "source": [
    "print 'WIKI PAGES:', wikis.shape[0]\n",
    "print 'RESEARCH NOTES:', notes.shape[0]\n",
    "print 'COMMENTS:', comments.shape[0]\n",
    "print 'QUESTIONS:', questions.shape[0]\n",
    "print 'ANSWERS:', answers.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### DATA RELATIONSIHPS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### QUESTIONS VS. NOTES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 521,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T19:07:05.886651Z",
     "start_time": "2019-04-15T19:07:05.859361Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>qid</th>\n",
       "      <th>date</th>\n",
       "      <th>user_cmnts</th>\n",
       "      <th>thread_cmnts</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>13745</td>\n",
       "      <td>498969</td>\n",
       "      <td>153</td>\n",
       "      <td>1480917522</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Is anyone doing any work with fungi? or biorem...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       nid     uid  qid        date  user_cmnts  thread_cmnts  likes  type  \\\n",
       "152  13745  498969  153  1480917522           2             3      0  qstn   \n",
       "\n",
       "                                                 title  \n",
       "152  Is anyone doing any work with fungi? or biorem...  "
      ]
     },
     "execution_count": 521,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions.loc[questions['nid'] == 13745]\n",
    "# questions.loc[questions['nid'] == 18446]\n",
    "# questions.loc[questions['nid'] == 13356] # NO QUESTION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "A QUESTION IS A NOTE WITH A POWER TAG OF QUESTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 522,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T19:07:08.916205Z",
     "start_time": "2019-04-15T19:07:08.891245Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>date</th>\n",
       "      <th>comments</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1370</th>\n",
       "      <td>13745</td>\n",
       "      <td>498969</td>\n",
       "      <td>1480917522</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>note</td>\n",
       "      <td>Is anyone doing any work with fungi? or biorem...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        nid     uid        date  comments  likes  type  \\\n",
       "1370  13745  498969  1480917522         3      0  note   \n",
       "\n",
       "                                                  title  \n",
       "1370  Is anyone doing any work with fungi? or biorem...  "
      ]
     },
     "execution_count": 522,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes.loc[notes['nid'] == 13745]\n",
    "# notes.loc[notes['nid'] == 18446]\n",
    "# notes.loc[notes['nid'] == 13356] # NO NOTE (E.G., NO QUESTION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### ANSWERS VS. COMMENTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 523,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T19:07:16.992195Z",
     "start_time": "2019-04-15T19:07:16.965139Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>aid</th>\n",
       "      <th>created_at</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>content</th>\n",
       "      <th>accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>13745</td>\n",
       "      <td>499993</td>\n",
       "      <td>149</td>\n",
       "      <td>2016-12-15 00:59:26 UTC</td>\n",
       "      <td>1</td>\n",
       "      <td>ans</td>\n",
       "      <td>Hey Mushroomman! \\r\\n\\r\\nFellow mycology fanat...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>13745</td>\n",
       "      <td>237313</td>\n",
       "      <td>251</td>\n",
       "      <td>2017-06-06 19:08:04 UTC</td>\n",
       "      <td>0</td>\n",
       "      <td>ans</td>\n",
       "      <td>I'm late to the party, but a very cool topic! ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       nid     uid  aid               created_at  likes type  \\\n",
       "138  13745  499993  149  2016-12-15 00:59:26 UTC      1  ans   \n",
       "237  13745  237313  251  2017-06-06 19:08:04 UTC      0  ans   \n",
       "\n",
       "                                               content  accepted  \n",
       "138  Hey Mushroomman! \\r\\n\\r\\nFellow mycology fanat...     False  \n",
       "237  I'm late to the party, but a very cool topic! ...     False  "
      ]
     },
     "execution_count": 523,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers.loc[answers['nid'] == 13745]\n",
    "# answers.loc[answers['nid'] == 18446]\n",
    "# answers.loc[answers['nid'] == 13356] # DOES NOT HAVE A QUESTION/NOTE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "ALL DISCUSSION ABOUT A QUESTION, INCLUDING ANSWERS AND REPLIES IN ALL LEVELS OF HIERARCHIES ARE COMMENTS.\n",
    "\n",
    "HOWEVER, AN ANSWER IS *TECHNICALLY* A COMMENT OF THE HIGHEST LEVEL THREAD.\n",
    "\n",
    "**NOTICE:** *AID 149 AND 251 ABOVE* ARE THE SAME AS *CID 22382 and 22481 BELOW*.  HOWEVER, *CID 22223 BELOW* IS IN REPLY TO *AID 149 (ABOVE AND BELOW) AND CID 22382 (BELOW)* WHICH ARE ALL THE SAME.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 542,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T19:13:57.704017Z",
     "start_time": "2019-04-15T19:13:57.664524Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>cid</th>\n",
       "      <th>aid</th>\n",
       "      <th>date</th>\n",
       "      <th>type</th>\n",
       "      <th>thread</th>\n",
       "      <th>comment</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10042</th>\n",
       "      <td>13745</td>\n",
       "      <td>579767</td>\n",
       "      <td>22223</td>\n",
       "      <td>149</td>\n",
       "      <td>1552994882</td>\n",
       "      <td>cmnt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hello Jlmaybach.\\r\\n\\r\\nI am actually doing a...</td>\n",
       "      <td>22382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10195</th>\n",
       "      <td>13745</td>\n",
       "      <td>499993</td>\n",
       "      <td>22382</td>\n",
       "      <td>0</td>\n",
       "      <td>1481763566</td>\n",
       "      <td>cmnt</td>\n",
       "      <td>/01</td>\n",
       "      <td>Hey Mushroomman! \\r\\n\\r\\nFellow mycology fanat...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10294</th>\n",
       "      <td>13745</td>\n",
       "      <td>237313</td>\n",
       "      <td>22481</td>\n",
       "      <td>0</td>\n",
       "      <td>1496776084</td>\n",
       "      <td>cmnt</td>\n",
       "      <td>/01</td>\n",
       "      <td>I'm late to the party, but a very cool topic! ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         nid     uid    cid  aid        date  type thread  \\\n",
       "10042  13745  579767  22223  149  1552994882  cmnt    NaN   \n",
       "10195  13745  499993  22382    0  1481763566  cmnt    /01   \n",
       "10294  13745  237313  22481    0  1496776084  cmnt    /01   \n",
       "\n",
       "                                                 comment  reply_to  \n",
       "10042   Hello Jlmaybach.\\r\\n\\r\\nI am actually doing a...     22382  \n",
       "10195  Hey Mushroomman! \\r\\n\\r\\nFellow mycology fanat...         0  \n",
       "10294  I'm late to the party, but a very cool topic! ...         0  "
      ]
     },
     "execution_count": 542,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.loc[comments['nid'] == 13745]\n",
    "# comments.loc[comments['nid'] == 18446]\n",
    "# comments.loc[comments['nid'] == 13356] # DOES NOT HAVE A QUESTION/NOTE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### WIKI PAGES"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "WIKI PAGES DO NOT APPEAR TO HAVE ANY SUBTYPES SUCH AS WITH QUESTIONS & NOTES AND ANSWERS & COMMENTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T19:14:24.892036Z",
     "start_time": "2019-04-15T19:14:24.875799Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>date</th>\n",
       "      <th>comments</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [nid, uid, date, comments, likes, type, title]\n",
       "Index: []"
      ]
     },
     "execution_count": 545,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikis.loc[wikis['nid'] == 13745]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### SUMMARY OF Q&A RELATIONSIPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "DATES -> WIKI_DATES + NOTES_DATES + COMMENT_DATES\n",
    "USERS -> USERS_COUNT\n",
    "WIKI PAGES -> WIKIS\n",
    "RESEARCH NOTES -> NOTES - QUESTIONS\n",
    "COMMENTS -> WIKI_COMMENTS + NOTE_COMMENTS + ANSWERS\n",
    "QUESTIONS -> NOTES WHERE QUESTION_NID EQUALS NOTES_NID\n",
    "ANSWERS -> COMMENTS WHERE QUESTIONS_NID EQUALS COMMENTS_NID\n",
    "\n",
    "USER_WIKI_PAGES -> USERS_UID == WIKI_PAGES_UID\n",
    "USER_NOTES -> USERS_UID == NOTES_UID\n",
    "USER_COMMENTS -> USERS_UID == WIKI_PAGES_UID\n",
    "USER_QUESTIONS -> USERS_UID == QUESTIONS_UID\n",
    "USER_ANSWERS -> USERS_UID == ANSWERS_UID\n",
    "\n",
    "WIKI PAGE COMMENTS -> COMMENTS WHERE WIKI_NID EQUALS COMMENTS_NID\n",
    "RESEARCH NOTE COMMENTS -> COMMENTS WHERE NOTE_NID EQUALS COMMENTS_NID"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### MERGING QUESTIONS AND ANSWERS  \n",
    "SEE \"SUMMARY STATISTICS\" TO SEE WHY THIS MERGE WAS CHOSEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:43:07.749630Z",
     "start_time": "2019-04-16T15:43:07.740252Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nid                                                         11516\n",
       "uid                                                        433692\n",
       "qid                                                             1\n",
       "date                                                   1421053078\n",
       "user_cmnts                                                      2\n",
       "thread_cmnts                                                    3\n",
       "likes                                                           1\n",
       "type                                                         qstn\n",
       "title           Is it possible to log the data from a dustduin...\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:43:22.124036Z",
     "start_time": "2019-04-16T15:43:22.114760Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nid                                                     11500\n",
       "uid                                                    433330\n",
       "cid                                                     11010\n",
       "aid                                                         0\n",
       "date                                               1420182880\n",
       "type                                                     cmnt\n",
       "thread                                                    02/\n",
       "comment     fyi, I just noticed that the local public libr...\n",
       "reply_to                                                    0\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:46:50.873865Z",
     "start_time": "2019-04-16T15:46:50.790543Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>q_id</th>\n",
       "      <th>q_uid</th>\n",
       "      <th>q_date</th>\n",
       "      <th>q_text</th>\n",
       "      <th>a_id</th>\n",
       "      <th>a_uid</th>\n",
       "      <th>a_cid</th>\n",
       "      <th>a_date</th>\n",
       "      <th>a_text</th>\n",
       "      <th>merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11516</td>\n",
       "      <td>1.0</td>\n",
       "      <td>433692.0</td>\n",
       "      <td>1.421053e+09</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>304279.0</td>\n",
       "      <td>11037.0</td>\n",
       "      <td>1.421161e+09</td>\n",
       "      <td>Hi KP55,  \\r\\nI think that the first thing you...</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11516</td>\n",
       "      <td>1.0</td>\n",
       "      <td>433692.0</td>\n",
       "      <td>1.421053e+09</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11391.0</td>\n",
       "      <td>1.427129e+09</td>\n",
       "      <td>I agree that a non-wifi DustDuino variant woul...</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11516</td>\n",
       "      <td>1.0</td>\n",
       "      <td>433692.0</td>\n",
       "      <td>1.421053e+09</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22929.0</td>\n",
       "      <td>1.518818e+09</td>\n",
       "      <td>There's now a way to collect data from a #dust...</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11524</td>\n",
       "      <td>2.0</td>\n",
       "      <td>433790.0</td>\n",
       "      <td>1.421278e+09</td>\n",
       "      <td>Is the dustduino (with Shinyei sensor) appropr...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>11043.0</td>\n",
       "      <td>1.421343e+09</td>\n",
       "      <td>currently the dust counts for the dustduino ar...</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11526</td>\n",
       "      <td>3.0</td>\n",
       "      <td>433849.0</td>\n",
       "      <td>1.421430e+09</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>11064.0</td>\n",
       "      <td>1.421795e+09</td>\n",
       "      <td>That is a good question.  I'm not sure what as...</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid  q_id     q_uid        q_date  \\\n",
       "0  11516   1.0  433692.0  1.421053e+09   \n",
       "1  11516   1.0  433692.0  1.421053e+09   \n",
       "2  11516   1.0  433692.0  1.421053e+09   \n",
       "3  11524   2.0  433790.0  1.421278e+09   \n",
       "4  11526   3.0  433849.0  1.421430e+09   \n",
       "\n",
       "                                              q_text  a_id     a_uid    a_cid  \\\n",
       "0  Is it possible to log the data from a dustduin...   0.0  304279.0  11037.0   \n",
       "1  Is it possible to log the data from a dustduin...   0.0       1.0  11391.0   \n",
       "2  Is it possible to log the data from a dustduin...   0.0       1.0  22929.0   \n",
       "3  Is the dustduino (with Shinyei sensor) appropr...   0.0       4.0  11043.0   \n",
       "4                How can I test my tap water for BPA   0.0       4.0  11064.0   \n",
       "\n",
       "         a_date                                             a_text merge  \n",
       "0  1.421161e+09  Hi KP55,  \\r\\nI think that the first thing you...  both  \n",
       "1  1.427129e+09  I agree that a non-wifi DustDuino variant woul...  both  \n",
       "2  1.518818e+09  There's now a way to collect data from a #dust...  both  \n",
       "3  1.421343e+09  currently the dust counts for the dustduino ar...  both  \n",
       "4  1.421795e+09  That is a good question.  I'm not sure what as...  both  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_q_cols = ['nid', 'q_id', 'q_uid','q_date', 'q_text']\n",
    "merge_a_cols = ['a_id', 'a_uid',  'a_cid', 'a_date', 'a_text']\n",
    "\n",
    "merge_cols = merge_q_cols + merge_a_cols\n",
    "\n",
    "q_cols = ['nid', 'qid', 'uid','date', 'title']\n",
    "a_cols = ['nid', 'aid', 'uid',  'cid', 'date', 'comment']\n",
    "\n",
    "# qac for question and answers from comments\n",
    "qac = pd.merge(questions[q_cols], comments[a_cols], on='nid', how='outer', indicator=True)\n",
    "qac.columns = merge_cols + ['merge']\n",
    "qac.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T16:06:56.180163Z",
     "start_time": "2019-04-16T16:06:56.159306Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "QAC MERGE - NID, OUTER\n",
      "LEFT ONLY: 104   RIGHT ONLY: 8280   BOTH: 3356\n"
     ]
    }
   ],
   "source": [
    "print 'QAC MERGE - NID, OUTER'\n",
    "print 'LEFT ONLY:', qac.loc[qac['merge'] == 'left_only'].shape[0], \\\n",
    "      '  RIGHT ONLY:', qac.loc[qac['merge'] == 'right_only'].shape[0], \\\n",
    "      '  BOTH:', qac.loc[qac['merge'] == 'both'].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T16:13:13.158260Z",
     "start_time": "2019-04-16T16:13:13.123403Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>q_id</th>\n",
       "      <th>q_uid</th>\n",
       "      <th>q_date</th>\n",
       "      <th>q_text</th>\n",
       "      <th>a_id</th>\n",
       "      <th>a_uid</th>\n",
       "      <th>a_cid</th>\n",
       "      <th>a_date</th>\n",
       "      <th>a_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0</td>\n",
       "      <td>304279</td>\n",
       "      <td>11037</td>\n",
       "      <td>1421161394</td>\n",
       "      <td>Hi KP55,  \\r\\nI think that the first thing you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11391</td>\n",
       "      <td>1427129027</td>\n",
       "      <td>I agree that a non-wifi DustDuino variant woul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>22929</td>\n",
       "      <td>1518817538</td>\n",
       "      <td>There's now a way to collect data from a #dust...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11524</td>\n",
       "      <td>2</td>\n",
       "      <td>433790</td>\n",
       "      <td>1421278243</td>\n",
       "      <td>Is the dustduino (with Shinyei sensor) appropr...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>11043</td>\n",
       "      <td>1421342622</td>\n",
       "      <td>currently the dust counts for the dustduino ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11526</td>\n",
       "      <td>3</td>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>11064</td>\n",
       "      <td>1421795312</td>\n",
       "      <td>That is a good question.  I'm not sure what as...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid  q_id   q_uid      q_date  \\\n",
       "0  11516     1  433692  1421053078   \n",
       "1  11516     1  433692  1421053078   \n",
       "2  11516     1  433692  1421053078   \n",
       "3  11524     2  433790  1421278243   \n",
       "4  11526     3  433849  1421429978   \n",
       "\n",
       "                                              q_text  a_id   a_uid  a_cid  \\\n",
       "0  Is it possible to log the data from a dustduin...     0  304279  11037   \n",
       "1  Is it possible to log the data from a dustduin...     0       1  11391   \n",
       "2  Is it possible to log the data from a dustduin...     0       1  22929   \n",
       "3  Is the dustduino (with Shinyei sensor) appropr...     0       4  11043   \n",
       "4                How can I test my tap water for BPA     0       4  11064   \n",
       "\n",
       "       a_date                                             a_text  \n",
       "0  1421161394  Hi KP55,  \\r\\nI think that the first thing you...  \n",
       "1  1427129027  I agree that a non-wifi DustDuino variant woul...  \n",
       "2  1518817538  There's now a way to collect data from a #dust...  \n",
       "3  1421342622  currently the dust counts for the dustduino ar...  \n",
       "4  1421795312  That is a good question.  I'm not sure what as...  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# # CREATE DF WITH MERGE EQUALS BOTH ONLY\n",
    "\n",
    "qac_both = qac.loc[(~qac.isnull()).all(axis=1)]\n",
    "\n",
    "cols_to_type = ['nid', 'q_id', 'q_uid','q_date', 'a_id', 'a_uid',  'a_cid', 'a_date']\n",
    "\n",
    "# GIVES .loc ALERT, NOT SURE WHY, TRIED IT W/ .loc AND THEN THERE WAS A KEY ERROR\n",
    "for col_to_type in cols_to_type:\n",
    "    qac_both[col_to_type] = qac_both[col_to_type].astype('int64')\n",
    "    \n",
    "qac_both.drop(['merge'], inplace=True, axis=1)\n",
    "qac_both.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T16:14:55.852204Z",
     "start_time": "2019-04-16T16:14:55.750038Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "qac_both.to_csv('../data/2016_2019/full/q_and_a.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T13:57:31.535635Z",
     "start_time": "2019-04-19T13:57:31.482039Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOTAL ROWS: 3356\n"
     ]
    }
   ],
   "source": [
    "q_and_a = pd.read_csv(csv_paths['q_and_a'])\n",
    "q_and_a.head()\n",
    "print 'TOTAL ROWS:', q_and_a.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-17T20:08:12.628548Z",
     "start_time": "2019-04-17T20:08:12.620032Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30, 9)"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions.loc[questions['uid'] == 7].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T16:19:10.897483Z",
     "start_time": "2019-04-16T16:19:10.862760Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>q_id</th>\n",
       "      <th>q_uid</th>\n",
       "      <th>q_date</th>\n",
       "      <th>q_text</th>\n",
       "      <th>a_id</th>\n",
       "      <th>a_uid</th>\n",
       "      <th>a_cid</th>\n",
       "      <th>a_date</th>\n",
       "      <th>a_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3254</th>\n",
       "      <td>18446</td>\n",
       "      <td>869</td>\n",
       "      <td>217812</td>\n",
       "      <td>1551498746</td>\n",
       "      <td>How do I interpret the relationship between th...</td>\n",
       "      <td>1346</td>\n",
       "      <td>217812</td>\n",
       "      <td>22034</td>\n",
       "      <td>1551731181</td>\n",
       "      <td>Yes!  Perfect.  Thank you!  Speaking of a nod...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3255</th>\n",
       "      <td>18446</td>\n",
       "      <td>869</td>\n",
       "      <td>217812</td>\n",
       "      <td>1551498746</td>\n",
       "      <td>How do I interpret the relationship between th...</td>\n",
       "      <td>1346</td>\n",
       "      <td>1</td>\n",
       "      <td>22035</td>\n",
       "      <td>1551732030</td>\n",
       "      <td>that's right!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3256</th>\n",
       "      <td>18446</td>\n",
       "      <td>869</td>\n",
       "      <td>217812</td>\n",
       "      <td>1551498746</td>\n",
       "      <td>How do I interpret the relationship between th...</td>\n",
       "      <td>1346</td>\n",
       "      <td>217812</td>\n",
       "      <td>22036</td>\n",
       "      <td>1551732144</td>\n",
       "      <td>Very likable indeed.  Thanks, @warren!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3257</th>\n",
       "      <td>18446</td>\n",
       "      <td>869</td>\n",
       "      <td>217812</td>\n",
       "      <td>1551498746</td>\n",
       "      <td>How do I interpret the relationship between th...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>23322</td>\n",
       "      <td>1551730472</td>\n",
       "      <td>Hi @bsugar!\\r\\n\\r\\n 1) While there are answer ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        nid  q_id   q_uid      q_date  \\\n",
       "3254  18446   869  217812  1551498746   \n",
       "3255  18446   869  217812  1551498746   \n",
       "3256  18446   869  217812  1551498746   \n",
       "3257  18446   869  217812  1551498746   \n",
       "\n",
       "                                                 q_text  a_id   a_uid  a_cid  \\\n",
       "3254  How do I interpret the relationship between th...  1346  217812  22034   \n",
       "3255  How do I interpret the relationship between th...  1346       1  22035   \n",
       "3256  How do I interpret the relationship between th...  1346  217812  22036   \n",
       "3257  How do I interpret the relationship between th...     0       1  23322   \n",
       "\n",
       "          a_date                                             a_text  \n",
       "3254  1551731181   Yes!  Perfect.  Thank you!  Speaking of a nod...  \n",
       "3255  1551732030                                      that's right!  \n",
       "3256  1551732144             Very likable indeed.  Thanks, @warren!  \n",
       "3257  1551730472  Hi @bsugar!\\r\\n\\r\\n 1) While there are answer ...  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_and_a.loc[q_and_a['nid'] == 18446]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### CREATE EDGELIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### ADD EDGE DIRECTIONALITY FOR IN/OUT DEGREE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 478,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T14:19:40.707061Z",
     "start_time": "2019-04-19T14:19:40.380207Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>q_id</th>\n",
       "      <th>q_uid</th>\n",
       "      <th>q_date</th>\n",
       "      <th>q_text</th>\n",
       "      <th>a_id</th>\n",
       "      <th>a_uid</th>\n",
       "      <th>a_cid</th>\n",
       "      <th>a_date</th>\n",
       "      <th>a_text</th>\n",
       "      <th>edge_dir</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0</td>\n",
       "      <td>304279</td>\n",
       "      <td>11037</td>\n",
       "      <td>1421161394</td>\n",
       "      <td>Hi KP55,  \\r\\nI think that the first thing you...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11391</td>\n",
       "      <td>1427129027</td>\n",
       "      <td>I agree that a non-wifi DustDuino variant woul...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>22929</td>\n",
       "      <td>1518817538</td>\n",
       "      <td>There's now a way to collect data from a #dust...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11524</td>\n",
       "      <td>2</td>\n",
       "      <td>433790</td>\n",
       "      <td>1421278243</td>\n",
       "      <td>Is the dustduino (with Shinyei sensor) appropr...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>11043</td>\n",
       "      <td>1421342622</td>\n",
       "      <td>currently the dust counts for the dustduino ar...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11526</td>\n",
       "      <td>3</td>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>11064</td>\n",
       "      <td>1421795312</td>\n",
       "      <td>That is a good question.  I'm not sure what as...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid  q_id   q_uid      q_date  \\\n",
       "0  11516     1  433692  1421053078   \n",
       "1  11516     1  433692  1421053078   \n",
       "2  11516     1  433692  1421053078   \n",
       "3  11524     2  433790  1421278243   \n",
       "4  11526     3  433849  1421429978   \n",
       "\n",
       "                                              q_text  a_id   a_uid  a_cid  \\\n",
       "0  Is it possible to log the data from a dustduin...     0  304279  11037   \n",
       "1  Is it possible to log the data from a dustduin...     0       1  11391   \n",
       "2  Is it possible to log the data from a dustduin...     0       1  22929   \n",
       "3  Is the dustduino (with Shinyei sensor) appropr...     0       4  11043   \n",
       "4                How can I test my tap water for BPA     0       4  11064   \n",
       "\n",
       "       a_date                                             a_text  edge_dir  \n",
       "0  1421161394  Hi KP55,  \\r\\nI think that the first thing you...         1  \n",
       "1  1427129027  I agree that a non-wifi DustDuino variant woul...         1  \n",
       "2  1518817538  There's now a way to collect data from a #dust...         1  \n",
       "3  1421342622  currently the dust counts for the dustduino ar...         1  \n",
       "4  1421795312  That is a good question.  I'm not sure what as...         1  "
      ]
     },
     "execution_count": 478,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_and_a['edge_dir'] = q_and_a.apply(set_edge_direction, axis=1)\n",
    "q_and_a.head()\n",
    "# q_and_a.loc[q_and_a['nid'] == 18446]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 578,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T17:10:39.372593Z",
     "start_time": "2019-04-19T17:10:39.333653Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>s_date</th>\n",
       "      <th>s_text</th>\n",
       "      <th>target</th>\n",
       "      <th>t_date</th>\n",
       "      <th>t_text</th>\n",
       "      <th>edge_dir</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>4</td>\n",
       "      <td>1421795312</td>\n",
       "      <td>That is a good question.  I'm not sure what as...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>524546</td>\n",
       "      <td>1517961102</td>\n",
       "      <td>Might want to check for answers here? https://...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>8</td>\n",
       "      <td>1422654308</td>\n",
       "      <td>Interesting Matt, how did you find out about t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>4</td>\n",
       "      <td>1422654623</td>\n",
       "      <td>Well, I saw the device and did a patent search...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>43651</td>\n",
       "      <td>1422716715</td>\n",
       "      <td>I myself am really curious about this issue in...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>1</td>\n",
       "      <td>1422719975</td>\n",
       "      <td>Here are some potentially helpful notes from O...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>53164</td>\n",
       "      <td>1422724009</td>\n",
       "      <td>I'm also very curious to see how this progresses.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>1</td>\n",
       "      <td>1424729885</td>\n",
       "      <td>So, does priority date allow them to back-date...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>435436</td>\n",
       "      <td>1424457388</td>\n",
       "      <td>Is the project still active?</td>\n",
       "      <td>4</td>\n",
       "      <td>1424903115</td>\n",
       "      <td>I don't believe this project is still active, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>435436</td>\n",
       "      <td>1424457388</td>\n",
       "      <td>Is the project still active?</td>\n",
       "      <td>443858</td>\n",
       "      <td>1434435160</td>\n",
       "      <td>Great project. The project fruit will benefit ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>328260</td>\n",
       "      <td>1424795839</td>\n",
       "      <td>NDVI in realtime video</td>\n",
       "      <td>1</td>\n",
       "      <td>1424796093</td>\n",
       "      <td>One way to do this could involve the python ve...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    source      s_date                                             s_text  \\\n",
       "4   433849  1421429978                How can I test my tap water for BPA   \n",
       "5   433849  1421429978                How can I test my tap water for BPA   \n",
       "6        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "7        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "8        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "9        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "10       4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "11       4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "12  435436  1424457388                       Is the project still active?   \n",
       "13  435436  1424457388                       Is the project still active?   \n",
       "14  328260  1424795839                             NDVI in realtime video   \n",
       "\n",
       "    target      t_date                                             t_text  \\\n",
       "4        4  1421795312  That is a good question.  I'm not sure what as...   \n",
       "5   524546  1517961102  Might want to check for answers here? https://...   \n",
       "6        8  1422654308  Interesting Matt, how did you find out about t...   \n",
       "7        4  1422654623  Well, I saw the device and did a patent search...   \n",
       "8    43651  1422716715  I myself am really curious about this issue in...   \n",
       "9        1  1422719975  Here are some potentially helpful notes from O...   \n",
       "10   53164  1422724009  I'm also very curious to see how this progresses.   \n",
       "11       1  1424729885  So, does priority date allow them to back-date...   \n",
       "12       4  1424903115  I don't believe this project is still active, ...   \n",
       "13  443858  1434435160  Great project. The project fruit will benefit ...   \n",
       "14       1  1424796093  One way to do this could involve the python ve...   \n",
       "\n",
       "    edge_dir  \n",
       "4          1  \n",
       "5          1  \n",
       "6          1  \n",
       "7         -1  \n",
       "8          1  \n",
       "9          1  \n",
       "10         1  \n",
       "11         1  \n",
       "12         1  \n",
       "13         1  \n",
       "14         1  "
      ]
     },
     "execution_count": 578,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_edges = q_and_a[['q_uid', 'q_date', 'q_text', 'a_uid', 'a_date', 'a_text', 'edge_dir']].copy()\n",
    "q_edges.columns = ['source', 's_date', 's_text', 'target', 't_date', 't_text', 'edge_dir']\n",
    "q_edges[4:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 579,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T17:10:50.868841Z",
     "start_time": "2019-04-19T17:10:50.840996Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>s_date</th>\n",
       "      <th>s_text</th>\n",
       "      <th>target</th>\n",
       "      <th>t_date</th>\n",
       "      <th>t_text</th>\n",
       "      <th>edge_dir</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>4</td>\n",
       "      <td>1422654623</td>\n",
       "      <td>Well, I saw the device and did a patent search...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>328260</td>\n",
       "      <td>1424795839</td>\n",
       "      <td>NDVI in realtime video</td>\n",
       "      <td>328260</td>\n",
       "      <td>1424796168</td>\n",
       "      <td>Another thing that confuses me a little bit mo...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>328260</td>\n",
       "      <td>1424795839</td>\n",
       "      <td>NDVI in realtime video</td>\n",
       "      <td>328260</td>\n",
       "      <td>1424805532</td>\n",
       "      <td>cfastie Great, in that case the formula that I...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>432132</td>\n",
       "      <td>1426707115</td>\n",
       "      <td>Question: HELP NEEDED</td>\n",
       "      <td>432132</td>\n",
       "      <td>1426713736</td>\n",
       "      <td>Thanks Chris for all the material that you cre...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>435648</td>\n",
       "      <td>1427020365</td>\n",
       "      <td>Question: purchase sensor</td>\n",
       "      <td>435648</td>\n",
       "      <td>1427189401</td>\n",
       "      <td>Thanks for replying...well by my house there i...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    source      s_date                                             s_text  \\\n",
       "7        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "15  328260  1424795839                             NDVI in realtime video   \n",
       "17  328260  1424795839                             NDVI in realtime video   \n",
       "24  432132  1426707115                              Question: HELP NEEDED   \n",
       "32  435648  1427020365                          Question: purchase sensor   \n",
       "\n",
       "    target      t_date                                             t_text  \\\n",
       "7        4  1422654623  Well, I saw the device and did a patent search...   \n",
       "15  328260  1424796168  Another thing that confuses me a little bit mo...   \n",
       "17  328260  1424805532  cfastie Great, in that case the formula that I...   \n",
       "24  432132  1426713736  Thanks Chris for all the material that you cre...   \n",
       "32  435648  1427189401  Thanks for replying...well by my house there i...   \n",
       "\n",
       "    edge_dir  \n",
       "7         -1  \n",
       "15        -1  \n",
       "17        -1  \n",
       "24        -1  \n",
       "32        -1  "
      ]
     },
     "execution_count": 579,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg_one = edges.loc[q_edges['edge_dir'] == -1].copy()\n",
    "neg_one.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 580,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T17:11:04.117253Z",
     "start_time": "2019-04-19T17:11:04.084590Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>t_date</th>\n",
       "      <th>t_text</th>\n",
       "      <th>source</th>\n",
       "      <th>s_date</th>\n",
       "      <th>s_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "      <td>1422654623</td>\n",
       "      <td>Well, I saw the device and did a patent search...</td>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>328260</td>\n",
       "      <td>1424796168</td>\n",
       "      <td>Another thing that confuses me a little bit mo...</td>\n",
       "      <td>328260</td>\n",
       "      <td>1424795839</td>\n",
       "      <td>NDVI in realtime video</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>328260</td>\n",
       "      <td>1424805532</td>\n",
       "      <td>cfastie Great, in that case the formula that I...</td>\n",
       "      <td>328260</td>\n",
       "      <td>1424795839</td>\n",
       "      <td>NDVI in realtime video</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>432132</td>\n",
       "      <td>1426713736</td>\n",
       "      <td>Thanks Chris for all the material that you cre...</td>\n",
       "      <td>432132</td>\n",
       "      <td>1426707115</td>\n",
       "      <td>Question: HELP NEEDED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>435648</td>\n",
       "      <td>1427189401</td>\n",
       "      <td>Thanks for replying...well by my house there i...</td>\n",
       "      <td>435648</td>\n",
       "      <td>1427020365</td>\n",
       "      <td>Question: purchase sensor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    target      t_date                                             t_text  \\\n",
       "7        4  1422654623  Well, I saw the device and did a patent search...   \n",
       "15  328260  1424796168  Another thing that confuses me a little bit mo...   \n",
       "17  328260  1424805532  cfastie Great, in that case the formula that I...   \n",
       "24  432132  1426713736  Thanks Chris for all the material that you cre...   \n",
       "32  435648  1427189401  Thanks for replying...well by my house there i...   \n",
       "\n",
       "    source      s_date                                             s_text  \n",
       "7        4  1422557787  dealing with patents: Black & Decker and the T...  \n",
       "15  328260  1424795839                             NDVI in realtime video  \n",
       "17  328260  1424795839                             NDVI in realtime video  \n",
       "24  432132  1426707115                              Question: HELP NEEDED  \n",
       "32  435648  1427020365                          Question: purchase sensor  "
      ]
     },
     "execution_count": 580,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg_one = neg_one[[u'target', u't_date', u't_text', u'source', u's_date', u's_text']]\n",
    "neg_one.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 581,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T17:11:10.477149Z",
     "start_time": "2019-04-19T17:11:10.449454Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>s_date</th>\n",
       "      <th>s_text</th>\n",
       "      <th>target</th>\n",
       "      <th>t_date</th>\n",
       "      <th>t_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "      <td>1422654623</td>\n",
       "      <td>Well, I saw the device and did a patent search...</td>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>328260</td>\n",
       "      <td>1424796168</td>\n",
       "      <td>Another thing that confuses me a little bit mo...</td>\n",
       "      <td>328260</td>\n",
       "      <td>1424795839</td>\n",
       "      <td>NDVI in realtime video</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>328260</td>\n",
       "      <td>1424805532</td>\n",
       "      <td>cfastie Great, in that case the formula that I...</td>\n",
       "      <td>328260</td>\n",
       "      <td>1424795839</td>\n",
       "      <td>NDVI in realtime video</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>432132</td>\n",
       "      <td>1426713736</td>\n",
       "      <td>Thanks Chris for all the material that you cre...</td>\n",
       "      <td>432132</td>\n",
       "      <td>1426707115</td>\n",
       "      <td>Question: HELP NEEDED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>435648</td>\n",
       "      <td>1427189401</td>\n",
       "      <td>Thanks for replying...well by my house there i...</td>\n",
       "      <td>435648</td>\n",
       "      <td>1427020365</td>\n",
       "      <td>Question: purchase sensor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    source      s_date                                             s_text  \\\n",
       "7        4  1422654623  Well, I saw the device and did a patent search...   \n",
       "15  328260  1424796168  Another thing that confuses me a little bit mo...   \n",
       "17  328260  1424805532  cfastie Great, in that case the formula that I...   \n",
       "24  432132  1426713736  Thanks Chris for all the material that you cre...   \n",
       "32  435648  1427189401  Thanks for replying...well by my house there i...   \n",
       "\n",
       "    target      t_date                                             t_text  \n",
       "7        4  1422557787  dealing with patents: Black & Decker and the T...  \n",
       "15  328260  1424795839                             NDVI in realtime video  \n",
       "17  328260  1424795839                             NDVI in realtime video  \n",
       "24  432132  1426707115                              Question: HELP NEEDED  \n",
       "32  435648  1427020365                          Question: purchase sensor  "
      ]
     },
     "execution_count": 581,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg_one.columns = [u'source', u's_date', u's_text', u'target', u't_date', u't_text']\n",
    "neg_one.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 582,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T17:11:19.614017Z",
     "start_time": "2019-04-19T17:11:19.584772Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>s_date</th>\n",
       "      <th>s_text</th>\n",
       "      <th>target</th>\n",
       "      <th>t_date</th>\n",
       "      <th>t_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>304279</td>\n",
       "      <td>1421161394</td>\n",
       "      <td>Hi KP55,  \\r\\nI think that the first thing you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>1</td>\n",
       "      <td>1427129027</td>\n",
       "      <td>I agree that a non-wifi DustDuino variant woul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>1</td>\n",
       "      <td>1518817538</td>\n",
       "      <td>There's now a way to collect data from a #dust...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>433790</td>\n",
       "      <td>1421278243</td>\n",
       "      <td>Is the dustduino (with Shinyei sensor) appropr...</td>\n",
       "      <td>4</td>\n",
       "      <td>1421342622</td>\n",
       "      <td>currently the dust counts for the dustduino ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>4</td>\n",
       "      <td>1421795312</td>\n",
       "      <td>That is a good question.  I'm not sure what as...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source      s_date                                             s_text  \\\n",
       "0  433692  1421053078  Is it possible to log the data from a dustduin...   \n",
       "1  433692  1421053078  Is it possible to log the data from a dustduin...   \n",
       "2  433692  1421053078  Is it possible to log the data from a dustduin...   \n",
       "3  433790  1421278243  Is the dustduino (with Shinyei sensor) appropr...   \n",
       "4  433849  1421429978                How can I test my tap water for BPA   \n",
       "\n",
       "   target      t_date                                             t_text  \n",
       "0  304279  1421161394  Hi KP55,  \\r\\nI think that the first thing you...  \n",
       "1       1  1427129027  I agree that a non-wifi DustDuino variant woul...  \n",
       "2       1  1518817538  There's now a way to collect data from a #dust...  \n",
       "3       4  1421342622  currently the dust counts for the dustduino ar...  \n",
       "4       4  1421795312  That is a good question.  I'm not sure what as...  "
      ]
     },
     "execution_count": 582,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_one = q_edges.loc[edges['edge_dir'] == 1].copy()\n",
    "pos_one.drop('edge_dir', axis=1, inplace=True)\n",
    "pos_one.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 583,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T17:11:30.748488Z",
     "start_time": "2019-04-19T17:11:30.714055Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>s_date</th>\n",
       "      <th>s_text</th>\n",
       "      <th>target</th>\n",
       "      <th>t_date</th>\n",
       "      <th>t_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>4</td>\n",
       "      <td>1421795312</td>\n",
       "      <td>That is a good question.  I'm not sure what as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>524546</td>\n",
       "      <td>1517961102</td>\n",
       "      <td>Might want to check for answers here? https://...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>8</td>\n",
       "      <td>1422654308</td>\n",
       "      <td>Interesting Matt, how did you find out about t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "      <td>1422654623</td>\n",
       "      <td>Well, I saw the device and did a patent search...</td>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>43651</td>\n",
       "      <td>1422716715</td>\n",
       "      <td>I myself am really curious about this issue in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>1</td>\n",
       "      <td>1422719975</td>\n",
       "      <td>Here are some potentially helpful notes from O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>53164</td>\n",
       "      <td>1422724009</td>\n",
       "      <td>I'm also very curious to see how this progresses.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>1</td>\n",
       "      <td>1424729885</td>\n",
       "      <td>So, does priority date allow them to back-date...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>435436</td>\n",
       "      <td>1424457388</td>\n",
       "      <td>Is the project still active?</td>\n",
       "      <td>4</td>\n",
       "      <td>1424903115</td>\n",
       "      <td>I don't believe this project is still active, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>435436</td>\n",
       "      <td>1424457388</td>\n",
       "      <td>Is the project still active?</td>\n",
       "      <td>443858</td>\n",
       "      <td>1434435160</td>\n",
       "      <td>Great project. The project fruit will benefit ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>328260</td>\n",
       "      <td>1424795839</td>\n",
       "      <td>NDVI in realtime video</td>\n",
       "      <td>1</td>\n",
       "      <td>1424796093</td>\n",
       "      <td>One way to do this could involve the python ve...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    source      s_date                                             s_text  \\\n",
       "4   433849  1421429978                How can I test my tap water for BPA   \n",
       "5   433849  1421429978                How can I test my tap water for BPA   \n",
       "6        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "7        4  1422654623  Well, I saw the device and did a patent search...   \n",
       "8        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "9        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "10       4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "11       4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "12  435436  1424457388                       Is the project still active?   \n",
       "13  435436  1424457388                       Is the project still active?   \n",
       "14  328260  1424795839                             NDVI in realtime video   \n",
       "\n",
       "    target      t_date                                             t_text  \n",
       "4        4  1421795312  That is a good question.  I'm not sure what as...  \n",
       "5   524546  1517961102  Might want to check for answers here? https://...  \n",
       "6        8  1422654308  Interesting Matt, how did you find out about t...  \n",
       "7        4  1422557787  dealing with patents: Black & Decker and the T...  \n",
       "8    43651  1422716715  I myself am really curious about this issue in...  \n",
       "9        1  1422719975  Here are some potentially helpful notes from O...  \n",
       "10   53164  1422724009  I'm also very curious to see how this progresses.  \n",
       "11       1  1424729885  So, does priority date allow them to back-date...  \n",
       "12       4  1424903115  I don't believe this project is still active, ...  \n",
       "13  443858  1434435160  Great project. The project fruit will benefit ...  \n",
       "14       1  1424796093  One way to do this could involve the python ve...  "
      ]
     },
     "execution_count": 583,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edges = pd.concat([pos_one, neg_one])\n",
    "edges.sort_index(inplace=True)\n",
    "edges[4:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 584,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T17:11:36.427202Z",
     "start_time": "2019-04-19T17:11:36.391117Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>s_date</th>\n",
       "      <th>s_text</th>\n",
       "      <th>target</th>\n",
       "      <th>t_date</th>\n",
       "      <th>t_text</th>\n",
       "      <th>edge_dir</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>4</td>\n",
       "      <td>1421795312</td>\n",
       "      <td>That is a good question.  I'm not sure what as...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>524546</td>\n",
       "      <td>1517961102</td>\n",
       "      <td>Might want to check for answers here? https://...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>8</td>\n",
       "      <td>1422654308</td>\n",
       "      <td>Interesting Matt, how did you find out about t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>4</td>\n",
       "      <td>1422654623</td>\n",
       "      <td>Well, I saw the device and did a patent search...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>43651</td>\n",
       "      <td>1422716715</td>\n",
       "      <td>I myself am really curious about this issue in...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>1</td>\n",
       "      <td>1422719975</td>\n",
       "      <td>Here are some potentially helpful notes from O...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>53164</td>\n",
       "      <td>1422724009</td>\n",
       "      <td>I'm also very curious to see how this progresses.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "      <td>1</td>\n",
       "      <td>1424729885</td>\n",
       "      <td>So, does priority date allow them to back-date...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>435436</td>\n",
       "      <td>1424457388</td>\n",
       "      <td>Is the project still active?</td>\n",
       "      <td>4</td>\n",
       "      <td>1424903115</td>\n",
       "      <td>I don't believe this project is still active, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>435436</td>\n",
       "      <td>1424457388</td>\n",
       "      <td>Is the project still active?</td>\n",
       "      <td>443858</td>\n",
       "      <td>1434435160</td>\n",
       "      <td>Great project. The project fruit will benefit ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>328260</td>\n",
       "      <td>1424795839</td>\n",
       "      <td>NDVI in realtime video</td>\n",
       "      <td>1</td>\n",
       "      <td>1424796093</td>\n",
       "      <td>One way to do this could involve the python ve...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    source      s_date                                             s_text  \\\n",
       "4   433849  1421429978                How can I test my tap water for BPA   \n",
       "5   433849  1421429978                How can I test my tap water for BPA   \n",
       "6        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "7        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "8        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "9        4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "10       4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "11       4  1422557787  dealing with patents: Black & Decker and the T...   \n",
       "12  435436  1424457388                       Is the project still active?   \n",
       "13  435436  1424457388                       Is the project still active?   \n",
       "14  328260  1424795839                             NDVI in realtime video   \n",
       "\n",
       "    target      t_date                                             t_text  \\\n",
       "4        4  1421795312  That is a good question.  I'm not sure what as...   \n",
       "5   524546  1517961102  Might want to check for answers here? https://...   \n",
       "6        8  1422654308  Interesting Matt, how did you find out about t...   \n",
       "7        4  1422654623  Well, I saw the device and did a patent search...   \n",
       "8    43651  1422716715  I myself am really curious about this issue in...   \n",
       "9        1  1422719975  Here are some potentially helpful notes from O...   \n",
       "10   53164  1422724009  I'm also very curious to see how this progresses.   \n",
       "11       1  1424729885  So, does priority date allow them to back-date...   \n",
       "12       4  1424903115  I don't believe this project is still active, ...   \n",
       "13  443858  1434435160  Great project. The project fruit will benefit ...   \n",
       "14       1  1424796093  One way to do this could involve the python ve...   \n",
       "\n",
       "    edge_dir  \n",
       "4          1  \n",
       "5          1  \n",
       "6          1  \n",
       "7         -1  \n",
       "8          1  \n",
       "9          1  \n",
       "10         1  \n",
       "11         1  \n",
       "12         1  \n",
       "13         1  \n",
       "14         1  "
      ]
     },
     "execution_count": 584,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_edges[4:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 593,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T17:31:16.643860Z",
     "start_time": "2019-04-19T17:31:16.555458Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>s_date</th>\n",
       "      <th>s_text</th>\n",
       "      <th>target</th>\n",
       "      <th>t_date</th>\n",
       "      <th>t_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>433692</td>\n",
       "      <td>2015-01-12</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>304279</td>\n",
       "      <td>2015-01-13</td>\n",
       "      <td>Hi KP55,  \\r\\nI think that the first thing you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>433692</td>\n",
       "      <td>2015-01-12</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>1</td>\n",
       "      <td>2015-03-23</td>\n",
       "      <td>I agree that a non-wifi DustDuino variant woul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>433692</td>\n",
       "      <td>2015-01-12</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-02-16</td>\n",
       "      <td>There's now a way to collect data from a #dust...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>433790</td>\n",
       "      <td>2015-01-14</td>\n",
       "      <td>Is the dustduino (with Shinyei sensor) appropr...</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-01-15</td>\n",
       "      <td>currently the dust counts for the dustduino ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>433849</td>\n",
       "      <td>2015-01-16</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-01-20</td>\n",
       "      <td>That is a good question.  I'm not sure what as...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source      s_date                                             s_text  \\\n",
       "0  433692  2015-01-12  Is it possible to log the data from a dustduin...   \n",
       "1  433692  2015-01-12  Is it possible to log the data from a dustduin...   \n",
       "2  433692  2015-01-12  Is it possible to log the data from a dustduin...   \n",
       "3  433790  2015-01-14  Is the dustduino (with Shinyei sensor) appropr...   \n",
       "4  433849  2015-01-16                How can I test my tap water for BPA   \n",
       "\n",
       "   target      t_date                                             t_text  \n",
       "0  304279  2015-01-13  Hi KP55,  \\r\\nI think that the first thing you...  \n",
       "1       1  2015-03-23  I agree that a non-wifi DustDuino variant woul...  \n",
       "2       1  2018-02-16  There's now a way to collect data from a #dust...  \n",
       "3       4  2015-01-15  currently the dust counts for the dustduino ar...  \n",
       "4       4  2015-01-20  That is a good question.  I'm not sure what as...  "
      ]
     },
     "execution_count": 593,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# edges['s_date'] = pd.to_datetime(edges['s_date'], unit='s').dt.strftime('%Y-%m-%d')\n",
    "# edges['t_date'] = pd.to_datetime(edges['t_date'], unit='s').dt.strftime('%Y-%m-%d')\n",
    "\n",
    "edges.to_csv('../data/2016_2019/full/qa_edges.csv', index=False)\n",
    "edges.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### NODELIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# NAKE A NODE LIST WHERE -1 IS ADDED FOR MAKING THE QUESTION.  SO IN THE ABOVE, I'VE ACTUALLY PARTICIPATED 5 TIMES \n",
    "# IF I INCLUDE THE FIRST QUESTION, WHICH I SHOULD!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-17T18:07:42.673593Z",
     "start_time": "2019-04-17T18:07:42.652950Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>q_id</th>\n",
       "      <th>q_uid</th>\n",
       "      <th>q_date</th>\n",
       "      <th>q_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11524</td>\n",
       "      <td>2</td>\n",
       "      <td>433790</td>\n",
       "      <td>1421278243</td>\n",
       "      <td>Is the dustduino (with Shinyei sensor) appropr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11526</td>\n",
       "      <td>3</td>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid  q_id   q_uid      q_date  \\\n",
       "0  11516     1  433692  1421053078   \n",
       "1  11516     1  433692  1421053078   \n",
       "2  11516     1  433692  1421053078   \n",
       "3  11524     2  433790  1421278243   \n",
       "4  11526     3  433849  1421429978   \n",
       "\n",
       "                                              q_text  \n",
       "0  Is it possible to log the data from a dustduin...  \n",
       "1  Is it possible to log the data from a dustduin...  \n",
       "2  Is it possible to log the data from a dustduin...  \n",
       "3  Is the dustduino (with Shinyei sensor) appropr...  \n",
       "4                How can I test my tap water for BPA  "
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_nodes = q_and_a[['nid', 'q_id', 'q_uid','q_date', 'q_text']].copy()\n",
    "q_nodes.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### TOTAL QUESTIONS ASKED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-17T20:35:25.502117Z",
     "start_time": "2019-04-17T20:35:25.490674Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "902"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions.loc[~questions['nid'].duplicated()].shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### TOTAL RESPONSES INCLUDING OP'S"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T18:04:35.764543Z",
     "start_time": "2019-04-18T18:04:35.758919Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3356"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_and_a.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### COUNT OF QUESTIONS WITH RESPONSES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T12:49:37.648690Z",
     "start_time": "2019-04-18T12:49:37.618543Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "798\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>q_id</th>\n",
       "      <th>q_uid</th>\n",
       "      <th>q_date</th>\n",
       "      <th>q_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11524</td>\n",
       "      <td>2</td>\n",
       "      <td>433790</td>\n",
       "      <td>1421278243</td>\n",
       "      <td>Is the dustduino (with Shinyei sensor) appropr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11526</td>\n",
       "      <td>3</td>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11545</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1422557787</td>\n",
       "      <td>dealing with patents: Black &amp; Decker and the T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>11610</td>\n",
       "      <td>5</td>\n",
       "      <td>435436</td>\n",
       "      <td>1424457388</td>\n",
       "      <td>Is the project still active?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      nid  q_id   q_uid      q_date  \\\n",
       "0   11516     1  433692  1421053078   \n",
       "3   11524     2  433790  1421278243   \n",
       "4   11526     3  433849  1421429978   \n",
       "6   11545     4       4  1422557787   \n",
       "12  11610     5  435436  1424457388   \n",
       "\n",
       "                                               q_text  \n",
       "0   Is it possible to log the data from a dustduin...  \n",
       "3   Is the dustduino (with Shinyei sensor) appropr...  \n",
       "4                 How can I test my tap water for BPA  \n",
       "6   dealing with patents: Black & Decker and the T...  \n",
       "12                       Is the project still active?  "
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_unq = q_nodes.loc[~q_nodes['nid'].duplicated()]\n",
    "print q_unq.shape[0]\n",
    "q_unq.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### COUNT OF QUESTIONS W/O RESPONSES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T20:39:06.731804Z",
     "start_time": "2019-04-18T20:39:06.699784Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>qid</th>\n",
       "      <th>date</th>\n",
       "      <th>user_cmnts</th>\n",
       "      <th>thread_cmnts</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>11946</td>\n",
       "      <td>443091</td>\n",
       "      <td>17</td>\n",
       "      <td>1433350344</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Question: Colorimetry using the desktop spectr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>11953</td>\n",
       "      <td>443196</td>\n",
       "      <td>18</td>\n",
       "      <td>1433708737</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Question: Stray light correction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>12135</td>\n",
       "      <td>420458</td>\n",
       "      <td>26</td>\n",
       "      <td>1439246576</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Question:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>12767</td>\n",
       "      <td>468156</td>\n",
       "      <td>49</td>\n",
       "      <td>1456950440</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Question: RE: Desktop Spectrometry Kit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>13636</td>\n",
       "      <td>451398</td>\n",
       "      <td>121</td>\n",
       "      <td>1477067715</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>qstn</td>\n",
       "      <td>How does bio-waste move within the waste stream?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       nid     uid  qid        date  user_cmnts  thread_cmnts  likes  type  \\\n",
       "16   11946  443091   17  1433350344           2             0      0  qstn   \n",
       "17   11953  443196   18  1433708737           2             0      0  qstn   \n",
       "25   12135  420458   26  1439246576           2             0      0  qstn   \n",
       "48   12767  468156   49  1456950440           2             0      0  qstn   \n",
       "120  13636  451398  121  1477067715           2             0      1  qstn   \n",
       "\n",
       "                                                 title  \n",
       "16   Question: Colorimetry using the desktop spectr...  \n",
       "17                    Question: Stray light correction  \n",
       "25                                          Question:   \n",
       "48              Question: RE: Desktop Spectrometry Kit  \n",
       "120   How does bio-waste move within the waste stream?  "
      ]
     },
     "execution_count": 339,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_without_resp = questions.loc[~questions['nid'].isin(q_unq['nid'])]\n",
    "print q_without_resp.shape[0]\n",
    "q_without_resp.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### COUNT OF UNIQUE USERS WHO ASKED QUESTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T20:40:22.549608Z",
     "start_time": "2019-04-18T20:40:22.528786Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>q_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>160</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  q_count\n",
       "0    1      114\n",
       "1    4        7\n",
       "2    7       23\n",
       "3    8        1\n",
       "4  160        1"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_ask_unq_user = pd.DataFrame(q_unq.groupby('q_uid').size())\n",
    "q_ask_unq_user.reset_index(inplace=True)#.head()\n",
    "q_ask_unq_user.columns = ['uid', 'q_count']\n",
    "q_ask_unq_user.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### COUNTS OF UNIQUE USERS WHO ASKED QUESTIONS W/O A RESPONSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T20:40:41.209685Z",
     "start_time": "2019-04-18T20:40:41.177849Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>qid</th>\n",
       "      <th>date</th>\n",
       "      <th>user_cmnts</th>\n",
       "      <th>thread_cmnts</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>11946</td>\n",
       "      <td>443091</td>\n",
       "      <td>17</td>\n",
       "      <td>1433350344</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Question: Colorimetry using the desktop spectr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>11953</td>\n",
       "      <td>443196</td>\n",
       "      <td>18</td>\n",
       "      <td>1433708737</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Question: Stray light correction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>12135</td>\n",
       "      <td>420458</td>\n",
       "      <td>26</td>\n",
       "      <td>1439246576</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Question:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>12767</td>\n",
       "      <td>468156</td>\n",
       "      <td>49</td>\n",
       "      <td>1456950440</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Question: RE: Desktop Spectrometry Kit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>13636</td>\n",
       "      <td>451398</td>\n",
       "      <td>121</td>\n",
       "      <td>1477067715</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>qstn</td>\n",
       "      <td>How does bio-waste move within the waste stream?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       nid     uid  qid        date  user_cmnts  thread_cmnts  likes  type  \\\n",
       "16   11946  443091   17  1433350344           2             0      0  qstn   \n",
       "17   11953  443196   18  1433708737           2             0      0  qstn   \n",
       "25   12135  420458   26  1439246576           2             0      0  qstn   \n",
       "48   12767  468156   49  1456950440           2             0      0  qstn   \n",
       "120  13636  451398  121  1477067715           2             0      1  qstn   \n",
       "\n",
       "                                                 title  \n",
       "16   Question: Colorimetry using the desktop spectr...  \n",
       "17                    Question: Stray light correction  \n",
       "25                                          Question:   \n",
       "48              Question: RE: Desktop Spectrometry Kit  \n",
       "120   How does bio-waste move within the waste stream?  "
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_without_resp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T20:41:09.905883Z",
     "start_time": "2019-04-18T20:41:09.882808Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>q_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>45586</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>237313</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>380298</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      uid  q_count\n",
       "0       1        9\n",
       "1       7        7\n",
       "2   45586        1\n",
       "3  237313        1\n",
       "4  380298        1"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_without_resp = pd.DataFrame(q_without_resp.groupby('uid').size())\n",
    "q_without_resp.reset_index(inplace=True)#.head()\n",
    "q_without_resp.columns = ['uid', 'q_count']\n",
    "print q_without_resp.shape[0]\n",
    "q_without_resp.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### COUNT OF \"ANSWERS\" BY USERS *EXCLUDING* THE ORIGINAL QUESTIONER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T20:36:42.840245Z",
     "start_time": "2019-04-18T20:36:42.811969Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "307\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>a_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  a_count\n",
       "0    1      687\n",
       "1    4       28\n",
       "2    7      144\n",
       "3    8        1\n",
       "4    9        2"
      ]
     },
     "execution_count": 330,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a_nodes = q_and_a[['nid', 'a_id', 'a_uid','a_date', 'a_text', 'edge_dir']].copy()\n",
    "a_resp_no_q = a_nodes.loc[a_nodes['edge_dir'] == 1]\n",
    "a_resp_no_q = pd.DataFrame(a_resp_no_q.groupby('a_uid').size())\n",
    "a_resp_no_q.reset_index(inplace=True)#.head()\n",
    "\n",
    "# a_resp_no_q.head()\n",
    "a_resp_no_q.columns = ['uid', 'a_count']\n",
    "print a_resp_no_q.shape[0]\n",
    "a_resp_no_q.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### COUNT OF ALL RESPONSES FROM QUESTIONER *EXCLUDING* ORIGINAL QUESTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T20:36:46.962259Z",
     "start_time": "2019-04-18T20:36:46.937923Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "219\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>q_disc_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>160</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>554</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  q_disc_count\n",
       "0    1           290\n",
       "1    4             7\n",
       "2    7            13\n",
       "3  160             5\n",
       "4  554            12"
      ]
     },
     "execution_count": 331,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a_resp_from_q = a_nodes.loc[a_nodes['edge_dir'] == -1]\n",
    "a_resp_from_q = pd.DataFrame(a_resp_from_q.groupby('a_uid').size())\n",
    "# a_resp_from_q.head()\n",
    "a_resp_from_q.reset_index(inplace=True)\n",
    "a_resp_from_q.columns = ['uid', 'q_disc_count']\n",
    "print a_resp_from_q.shape[0]\n",
    "a_resp_from_q.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T21:32:45.283871Z",
     "start_time": "2019-04-18T21:32:45.252747Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>q_id</th>\n",
       "      <th>q_uid</th>\n",
       "      <th>q_date</th>\n",
       "      <th>q_text</th>\n",
       "      <th>a_id</th>\n",
       "      <th>a_uid</th>\n",
       "      <th>a_cid</th>\n",
       "      <th>a_date</th>\n",
       "      <th>a_text</th>\n",
       "      <th>edge_dir</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0</td>\n",
       "      <td>304279</td>\n",
       "      <td>11037</td>\n",
       "      <td>1421161394</td>\n",
       "      <td>Hi KP55,  \\r\\nI think that the first thing you...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11391</td>\n",
       "      <td>1427129027</td>\n",
       "      <td>I agree that a non-wifi DustDuino variant woul...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11516</td>\n",
       "      <td>1</td>\n",
       "      <td>433692</td>\n",
       "      <td>1421053078</td>\n",
       "      <td>Is it possible to log the data from a dustduin...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>22929</td>\n",
       "      <td>1518817538</td>\n",
       "      <td>There's now a way to collect data from a #dust...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11524</td>\n",
       "      <td>2</td>\n",
       "      <td>433790</td>\n",
       "      <td>1421278243</td>\n",
       "      <td>Is the dustduino (with Shinyei sensor) appropr...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>11043</td>\n",
       "      <td>1421342622</td>\n",
       "      <td>currently the dust counts for the dustduino ar...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11526</td>\n",
       "      <td>3</td>\n",
       "      <td>433849</td>\n",
       "      <td>1421429978</td>\n",
       "      <td>How can I test my tap water for BPA</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>11064</td>\n",
       "      <td>1421795312</td>\n",
       "      <td>That is a good question.  I'm not sure what as...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     nid  q_id   q_uid      q_date  \\\n",
       "0  11516     1  433692  1421053078   \n",
       "1  11516     1  433692  1421053078   \n",
       "2  11516     1  433692  1421053078   \n",
       "3  11524     2  433790  1421278243   \n",
       "4  11526     3  433849  1421429978   \n",
       "\n",
       "                                              q_text  a_id   a_uid  a_cid  \\\n",
       "0  Is it possible to log the data from a dustduin...     0  304279  11037   \n",
       "1  Is it possible to log the data from a dustduin...     0       1  11391   \n",
       "2  Is it possible to log the data from a dustduin...     0       1  22929   \n",
       "3  Is the dustduino (with Shinyei sensor) appropr...     0       4  11043   \n",
       "4                How can I test my tap water for BPA     0       4  11064   \n",
       "\n",
       "       a_date                                             a_text  edge_dir  \n",
       "0  1421161394  Hi KP55,  \\r\\nI think that the first thing you...         1  \n",
       "1  1427129027  I agree that a non-wifi DustDuino variant woul...         1  \n",
       "2  1518817538  There's now a way to collect data from a #dust...         1  \n",
       "3  1421342622  currently the dust counts for the dustduino ar...         1  \n",
       "4  1421795312  That is a good question.  I'm not sure what as...         1  "
      ]
     },
     "execution_count": 388,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_and_a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T17:46:26.059460Z",
     "start_time": "2019-04-16T17:46:26.050530Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "run_control": {
     "marked": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2553"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# NUMBER OF FIRST RESPONSES\n",
    "q_and_a.loc[q_and_a['a_id'] == 0].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T20:42:29.646131Z",
     "start_time": "2019-04-18T20:42:29.622805Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "405\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>160</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>554</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1083</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>45586</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>46742</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>46795</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0    1\n",
       "0      1  114\n",
       "1      4    7\n",
       "2      7   23\n",
       "3      8    1\n",
       "4    160    1\n",
       "5    554    4\n",
       "6   1083    1\n",
       "7  45586    1\n",
       "8  46742    1\n",
       "9  46795    1"
      ]
     },
     "execution_count": 346,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_dfs = [q_ask_unq_users, q_without_resp, a_resp_no_q, a_resp_from_q, desc_count_unq_user]\n",
    "col_names = ['9_7', '9_8',  '9_9', '9_10', '9_11']\n",
    "nodes = pd.concat(node_dfs, axis=1, names=col_names, join='outer', ignore_index=True)\n",
    "print nodes.shape[0]\n",
    "nodes.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 431,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T12:50:22.019356Z",
     "start_time": "2019-04-19T12:50:21.976132Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "run_control": {
     "marked": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "436\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>q_count_x</th>\n",
       "      <th>q_count_y</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>114.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>160</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  q_count_x  q_count_y     _merge\n",
       "0    1      114.0        9.0       both\n",
       "1    4        7.0        NaN  left_only\n",
       "2    7       23.0        7.0       both\n",
       "3    8        1.0        NaN  left_only\n",
       "4  160        1.0        NaN  left_only"
      ]
     },
     "execution_count": 431,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_9_7_9_8 = pd.merge(q_ask_unq_users, q_without_resp, on='uid', how='outer', indicator=True)\n",
    "print nodes_9_7_9_8.shape[0]\n",
    "nodes_9_7_9_8.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T12:50:23.728078Z",
     "start_time": "2019-04-19T12:50:23.685820Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "463\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>q_disc_count</th>\n",
       "      <th>a_count</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>290.0</td>\n",
       "      <td>687.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>7.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>13.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>160</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>554</td>\n",
       "      <td>12.0</td>\n",
       "      <td>239.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  q_disc_count  a_count _merge\n",
       "0    1         290.0    687.0   both\n",
       "1    4           7.0     28.0   both\n",
       "2    7          13.0    144.0   both\n",
       "3  160           5.0      1.0   both\n",
       "4  554          12.0    239.0   both"
      ]
     },
     "execution_count": 432,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_9_9_9_10 = pd.merge(a_resp_from_q, a_resp_no_q, on='uid', how='outer', indicator=True)\n",
    "print nodes_9_9_9_10.shape[0]\n",
    "nodes_9_9_9_10.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T13:09:26.874930Z",
     "start_time": "2019-04-19T13:09:26.747951Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>q_count_x</th>\n",
       "      <th>q_count_y</th>\n",
       "      <th>q_disc_count</th>\n",
       "      <th>a_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>114.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>687.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>144.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>160</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  q_count_x  q_count_y  q_disc_count  a_count\n",
       "0    1      114.0        9.0         290.0    687.0\n",
       "1    4        7.0        0.0           7.0     28.0\n",
       "2    7       23.0        7.0          13.0    144.0\n",
       "3    8        1.0        0.0           0.0      1.0\n",
       "4  160        1.0        0.0           5.0      1.0"
      ]
     },
     "execution_count": 453,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_9_7_9_8 = pd.merge(q_ask_unq_users, q_without_resp, on='uid', how='outer', indicator=True)\n",
    "nodes_9_7_9_8.drop(['_merge'], axis=1, inplace=True)\n",
    "\n",
    "nodes_9_9_9_10 = pd.merge(a_resp_from_q, a_resp_no_q, on='uid', how='outer', indicator=True)\n",
    "nodes_9_9_9_10.drop(['_merge'], axis=1, inplace=True)\n",
    "\n",
    "nodes_97_910 = pd.merge(nodes_9_7_9_8, nodes_9_9_9_10, on='uid', how='outer', indicator=True)\n",
    "nodes_97_910.drop(['_merge'], axis=1, inplace=True)\n",
    "\n",
    "nodes = pd.merge(nodes_97_910, desc_count_unq_user, on='uid', how='outer', indicator=True)\n",
    "nodes.drop(['_merge'], axis=1, inplace=True)\n",
    "nodes.fillna(0, axis=1, inplace=True)\n",
    "\n",
    "nodes.drop('disc_count', axis=1, inplace=True)\n",
    "nodes.reset_index(drop=True, inplace=True)\n",
    "nodes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T13:10:11.039232Z",
     "start_time": "2019-04-19T13:10:10.990885Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>q_w_r</th>\n",
       "      <th>q_wo_r</th>\n",
       "      <th>r_not_op</th>\n",
       "      <th>r_as_op</th>\n",
       "      <th>total_q</th>\n",
       "      <th>total_r</th>\n",
       "      <th>total_activity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>114.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>687.0</td>\n",
       "      <td>123.0</td>\n",
       "      <td>977.0</td>\n",
       "      <td>1100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>42.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>187.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>160</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  q_w_r  q_wo_r  r_not_op  r_as_op  total_q  total_r  total_activity\n",
       "0    1  114.0     9.0     290.0    687.0    123.0    977.0          1100.0\n",
       "1    4    7.0     0.0       7.0     28.0      7.0     35.0            42.0\n",
       "2    7   23.0     7.0      13.0    144.0     30.0    157.0           187.0\n",
       "3    8    1.0     0.0       0.0      1.0      1.0      1.0             2.0\n",
       "4  160    1.0     0.0       5.0      1.0      1.0      6.0             7.0"
      ]
     },
     "execution_count": 454,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes.columns = ['uid', 'q_w_r', 'q_wo_r', 'r_not_op', 'r_as_op']\n",
    "\n",
    "# BECAUSE ALL_R_NO_Q is wrong.\n",
    "nodes['total_q'] = nodes['q_w_r'] + nodes['q_wo_r']\n",
    "nodes['total_r'] = nodes['r_not_op'] + nodes['r_as_op']\n",
    "nodes['total_activity'] = nodes['total_q'] + nodes['total_r']\n",
    "\n",
    "nodes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T13:10:15.504320Z",
     "start_time": "2019-04-19T13:10:15.465891Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>q_w_r</th>\n",
       "      <th>q_wo_r</th>\n",
       "      <th>r_not_op</th>\n",
       "      <th>r_as_op</th>\n",
       "      <th>total_q</th>\n",
       "      <th>total_r</th>\n",
       "      <th>total_activity</th>\n",
       "      <th>ask_ratio</th>\n",
       "      <th>ans_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>114.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>687.0</td>\n",
       "      <td>123.0</td>\n",
       "      <td>977.0</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>0.111818</td>\n",
       "      <td>0.888182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>0.160428</td>\n",
       "      <td>0.839572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>160</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.857143</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  q_w_r  q_wo_r  r_not_op  r_as_op  total_q  total_r  total_activity  \\\n",
       "0    1  114.0     9.0     290.0    687.0    123.0    977.0          1100.0   \n",
       "1    4    7.0     0.0       7.0     28.0      7.0     35.0            42.0   \n",
       "2    7   23.0     7.0      13.0    144.0     30.0    157.0           187.0   \n",
       "3    8    1.0     0.0       0.0      1.0      1.0      1.0             2.0   \n",
       "4  160    1.0     0.0       5.0      1.0      1.0      6.0             7.0   \n",
       "\n",
       "   ask_ratio  ans_ratio  \n",
       "0   0.111818   0.888182  \n",
       "1   0.166667   0.833333  \n",
       "2   0.160428   0.839572  \n",
       "3   0.500000   0.500000  \n",
       "4   0.142857   0.857143  "
      ]
     },
     "execution_count": 455,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes['ask_ratio'] = nodes['total_q']/(nodes['total_q'] + nodes['total_r'])\n",
    "nodes['ans_ratio'] = nodes['total_r']/(nodes['total_q'] + nodes['total_r'])\n",
    "\n",
    "nodes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 459,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T13:12:04.067852Z",
     "start_time": "2019-04-19T13:12:04.060903Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "q_w_r\n",
      "q_wo_r\n",
      "r_not_op\n",
      "r_as_op\n",
      "total_q\n",
      "total_r\n",
      "total_activity\n"
     ]
    }
   ],
   "source": [
    "for col in nodes.columns[1:8]:\n",
    "    print col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 460,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T13:12:18.931439Z",
     "start_time": "2019-04-19T13:12:18.884228Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>q_w_r</th>\n",
       "      <th>q_wo_r</th>\n",
       "      <th>r_not_op</th>\n",
       "      <th>r_as_op</th>\n",
       "      <th>total_q</th>\n",
       "      <th>total_r</th>\n",
       "      <th>total_activity</th>\n",
       "      <th>ask_ratio</th>\n",
       "      <th>ans_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>114</td>\n",
       "      <td>9</td>\n",
       "      <td>290</td>\n",
       "      <td>687</td>\n",
       "      <td>123</td>\n",
       "      <td>977</td>\n",
       "      <td>1100</td>\n",
       "      <td>0.111818</td>\n",
       "      <td>0.888182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>28</td>\n",
       "      <td>7</td>\n",
       "      <td>35</td>\n",
       "      <td>42</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>23</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>144</td>\n",
       "      <td>30</td>\n",
       "      <td>157</td>\n",
       "      <td>187</td>\n",
       "      <td>0.160428</td>\n",
       "      <td>0.839572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>160</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.857143</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid  q_w_r  q_wo_r  r_not_op  r_as_op  total_q  total_r  total_activity  \\\n",
       "0    1    114       9       290      687      123      977            1100   \n",
       "1    4      7       0         7       28        7       35              42   \n",
       "2    7     23       7        13      144       30      157             187   \n",
       "3    8      1       0         0        1        1        1               2   \n",
       "4  160      1       0         5        1        1        6               7   \n",
       "\n",
       "   ask_ratio  ans_ratio  \n",
       "0   0.111818   0.888182  \n",
       "1   0.166667   0.833333  \n",
       "2   0.160428   0.839572  \n",
       "3   0.500000   0.500000  \n",
       "4   0.142857   0.857143  "
      ]
     },
     "execution_count": 460,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for col in nodes.columns[1:8]:\n",
    "    nodes[col] = nodes[col].astype('int64')\n",
    "\n",
    "nodes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T13:27:27.779377Z",
     "start_time": "2019-04-19T13:27:27.673698Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "nodes.to_csv('../data/2016_2019/full/qa_nodes.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 427,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T22:12:12.290865Z",
     "start_time": "2019-04-18T22:12:12.265040Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9943931688991793"
      ]
     },
     "execution_count": 427,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def ask_or_ans(row):\n",
    "    \n",
    "    total_questions = float(row['total_q'])\n",
    "    responses = float(row['r_not_op'])\n",
    "    \n",
    "    if total_questions and responses:\n",
    "        \n",
    "        ratio = float(total_questions)/responses\n",
    "        \n",
    "    elif row['total_q'] and not row['r_not_op']:\n",
    "        \n",
    "        ratio = 0.0\n",
    "        \n",
    "    elif not row['total_q'] and row['r_not_op']:\n",
    "        \n",
    "        ratio = 1.0\n",
    "        \n",
    "    return ratio\n",
    "\n",
    "d = {'total_q': 651, 'r_not_op' : 0 }\n",
    "\n",
    "ask_or_ans(nodes.loc[nodes['uid'] == 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T19:57:13.060576Z",
     "start_time": "2019-04-18T19:57:13.054499Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'q_uid', u'q_count'], dtype='object')"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "COUNT OF UNIQUE USERS ASKING QUESTIONS W/ RESPONSES: 405\n",
    "DESCRIPTION OF NUMBER OF QUESTIONS EACH: MEAN: 1.97037037037 MEDIAN: 1.0 STD: 6.98932078678\n",
    "\n",
    "COUNT OF UNIQUE USERS ASKING QUESTIONS W/O RESPONSES: 65\n",
    "DESCRIPTION OF NUMBER OF QUESTIONS EACH: MEAN: 1.6 MEDIAN: 1.0 STD: 1.71208936683\n",
    "\n",
    "COUNT OF UNIQUE USERS RESPONDING AND NOT OP: 307\n",
    "DESCRIPTION OF NUMBER OF RESPONSES EACH: MEAN: 7.45928338762 MEDIAN: 1.0 STD: 43.1315149874\n",
    "\n",
    "RESPONSES FROM OP: 219\n",
    "DESCRIPTION OF NUMBER OF RESPONSES EACH: MEAN: 4.86757990868 MEDIAN: 2.0 STD: 20.4276705459\n",
    "\n",
    "COUNT OF UNIQUE RESPONDERS INCLUDING OP BUT W/O ORIGINAL QUESTION: 627\n",
    "DESCRIPTION OF NUMBER OF RESPONSES EACH: MEAN: 10.7049441786 MEDIAN: 3.0 STD: 67.0944498387\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### SUMMARY STATSTICS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T19:44:33.264798Z",
     "start_time": "2019-04-18T19:44:33.219572Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOTAL QUESTIONS ASKED: 902\n",
      "TOTAL RESPONSES: 3356\n",
      "\n",
      "QUESTIONS WITH RESPONSES: 798\n",
      "QUESTIONS W/O RESPONSES: 65\n",
      "\n",
      "COUNT OF UNIQUE USERS ASKING QUESTIONS W/ RESPONSES: 405\n",
      "DESCRIPTION OF NUMBER OF QUESTIONS EACH: MEAN: 1.97037037037 MEDIAN: 1.0 STD: 6.98932078678\n",
      "\n",
      "COUNT OF UNIQUE USERS ASKING QUESTIONS W/O RESPONSES: 65\n",
      "DESCRIPTION OF NUMBER OF QUESTIONS EACH: MEAN: 1.6 MEDIAN: 1.0 STD: 1.71208936683\n",
      "\n",
      "COUNT OF UNIQUE USERS RESPONDING AND NOT OP: 307\n",
      "DESCRIPTION OF NUMBER OF RESPONSES EACH: MEAN: 7.45928338762 MEDIAN: 1.0 STD: 43.1315149874\n",
      "\n",
      "RESPONSES FROM OP: 219\n",
      "DESCRIPTION OF NUMBER OF RESPONSES EACH: MEAN: 4.86757990868 MEDIAN: 2.0 STD: 20.4276705459\n",
      "\n",
      "COUNT OF UNIQUE RESPONDERS INCLUDING OP: 627\n",
      "DESCRIPTION OF NUMBER OF RESPONSES EACH: MEAN: 10.7049441786 MEDIAN: 3.0 STD: 67.0944498387\n"
     ]
    }
   ],
   "source": [
    "print 'TOTAL QUESTIONS ASKED:', questions.shape[0] # 9.3\n",
    "print 'TOTAL RESPONSES:', q_and_a.shape[0] # 9.4\n",
    "print ''\n",
    "print 'QUESTIONS WITH RESPONSES:', q_nodes['nid'].unique().shape[0] # 9.5\n",
    "print 'QUESTIONS W/O RESPONSES:', q_without_resp.shape[0] # 9.6\n",
    "print ''\n",
    "print 'COUNT OF UNIQUE USERS ASKING QUESTIONS W/ RESPONSES:',  q_ask_unq_users.shape[0]\n",
    "print 'DESCRIPTION OF NUMBER OF QUESTIONS EACH:', 'MEAN:', q_ask_unq_users['q_count'].mean(), \\\n",
    "                                                  'MEDIAN:', q_ask_unq_users['q_count'].median(), \\\n",
    "                                                  'STD:', q_ask_unq_users['q_count'].std()\n",
    "print ''\n",
    "print 'COUNT OF UNIQUE USERS ASKING QUESTIONS W/O RESPONSES:',  q_without_resp.shape[0]\n",
    "print 'DESCRIPTION OF NUMBER OF QUESTIONS EACH:', 'MEAN:', q_without_resp['q_count'].mean(), \\\n",
    "                                                  'MEDIAN:', q_without_resp['q_count'].median(), \\\n",
    "                                                  'STD:', q_without_resp['q_count'].std()\n",
    "print ''\n",
    "        \n",
    "print 'COUNT OF UNIQUE USERS RESPONDING AND NOT OP:', a_resp_no_q.shape[0]\n",
    "print 'DESCRIPTION OF NUMBER OF RESPONSES EACH:', 'MEAN:', a_resp_no_q['a_count'].mean(), \\\n",
    "                                                  'MEDIAN:', a_resp_no_q['a_count'].median(), \\\n",
    "                                                  'STD:', a_resp_no_q['a_count'].std()\n",
    "print ''\n",
    "\n",
    "print 'RESPONSES FROM OP:', a_resp_from_q.shape[0] #9.8 \n",
    "print 'DESCRIPTION OF NUMBER OF RESPONSES EACH:', 'MEAN:', a_resp_from_q['q_disc_count'].mean(), \\\n",
    "                                                  'MEDIAN:', a_resp_from_q['q_disc_count'].median(), \\\n",
    "                                                  'STD:', a_resp_from_q['q_disc_count'].std()\n",
    "\n",
    "        \n",
    "print ''\n",
    "\n",
    "print 'COUNT OF UNIQUE RESPONDERS INCLUDING OP:', desc_count_unq_user.shape[0] #9.9\n",
    "print 'DESCRIPTION OF NUMBER OF RESPONSES EACH:', 'MEAN:', desc_count_unq_user['disc_count'].mean(), \\\n",
    "                                                  'MEDIAN:', desc_count_unq_user['disc_count'].median(), \\\n",
    "                                                  'STD:', desc_count_unq_user['disc_count'].std()\n",
    "        \n",
    "NUMBER OF POSTS QUESTIONS USER HAS ASKED/TOTAL NO OP ENGAGEMENT POSTS + COMMENTS \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:35:11.653892Z",
     "start_time": "2019-04-16T15:35:11.647112Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "QUESTIONS MERGED WITH ANSWERS: q_and_a\n",
    "TOTAL Q: 902, TOTAL A: 1081\n",
    "LQ: 346, RA: 29, BOTH: 1052\n",
    "\n",
    "NOTES MERGED WITH ANSWERS: q_and_a_from_n\n",
    "TOTAL NOTES: 3155, TOTAL ANS: 1081 \n",
    "LNQ: 2597 RA: 21 BOTH: 1060\n",
    "\n",
    "QUESTIONS MERGED WITH COMMENTS (ANSWERS) q_and_a_from_c\n",
    "TOTAL QUESTS: 902 TOTAL CMTS: 11636\n",
    "LQ: 104 RAC: 8280 BOTH: 3356\n",
    "\n",
    "NOTES (QUESTIONS) MERGED WITH COMMENTS (ANSWERS): qn_and_ac\n",
    "TOTAL NOTES: 3155 TOTAL CMTS: 11636 \n",
    "LNQ: 1032 RAC: 1101 BOTH: 10535"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-16T15:32:47.279225Z",
     "start_time": "2019-04-16T15:32:47.254785Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOTAL COMMENTS: 11636\n",
      "TOTAL COMMENTS RELATED TO A QUESTION: 3356\n",
      "TOTAL COMMENTS RELATED TO A QUESTION AND IN TOP THREAD: 2520\n",
      "TOTAL COMMENTS RELATED TO A QUESTION AND IN A SUB THREAD: 836\n",
      "TOTAL ANSWERS WHEN QUETIONS ARE MERGED WITH COMMENTS: 3356\n"
     ]
    }
   ],
   "source": [
    "qnids = list(questions['nid'])\n",
    "q_resp = comments.loc[(comments['nid'].isin(qnids))]\n",
    "q_resp_top_thread = comments.loc[(comments['nid'].isin(qnids)) & (comments['reply_to'] == 0)]\n",
    "q_resp_sub_threads = comments.loc[(comments['nid'].isin(qnids)) & (comments['reply_to'] != 0)]\n",
    "print 'TOTAL COMMENTS:', comments.shape[0]\n",
    "print 'TOTAL COMMENTS RELATED TO A QUESTION:', q_resp.shape[0]\n",
    "print 'TOTAL COMMENTS RELATED TO A QUESTION AND IN TOP THREAD:', q_resp_top_thread.shape[0]\n",
    "print 'TOTAL COMMENTS RELATED TO A QUESTION AND IN A SUB THREAD:', q_resp_sub_threads.shape[0]\n",
    "print 'TOTAL ANSWERS WHEN QUETIONS ARE MERGED WITH COMMENTS:', qac_both.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### EXAMPLE MISSING DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-17T20:19:43.884220Z",
     "start_time": "2019-04-17T20:19:43.862485Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>date</th>\n",
       "      <th>comments</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1417</th>\n",
       "      <td>13836</td>\n",
       "      <td>7</td>\n",
       "      <td>1483554758</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>note</td>\n",
       "      <td>Is using iodine a good way to determine if ric...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        nid  uid        date  comments  likes  type  \\\n",
       "1417  13836    7  1483554758         0      0  note   \n",
       "\n",
       "                                                  title  \n",
       "1417  Is using iodine a good way to determine if ric...  "
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes[notes['nid'] == 13836]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-17T20:19:59.100067Z",
     "start_time": "2019-04-17T20:19:59.073610Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>qid</th>\n",
       "      <th>date</th>\n",
       "      <th>user_cmnts</th>\n",
       "      <th>thread_cmnts</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>13836</td>\n",
       "      <td>7</td>\n",
       "      <td>170</td>\n",
       "      <td>1483554758</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>qstn</td>\n",
       "      <td>Is using iodine a good way to determine if ric...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       nid  uid  qid        date  user_cmnts  thread_cmnts  likes  type  \\\n",
       "169  13836    7  170  1483554758           2             0      0  qstn   \n",
       "\n",
       "                                                 title  \n",
       "169  Is using iodine a good way to determine if ric...  "
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions[questions['nid'] == 13836]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-17T20:20:53.394887Z",
     "start_time": "2019-04-17T20:20:53.379008Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>aid</th>\n",
       "      <th>created_at</th>\n",
       "      <th>likes</th>\n",
       "      <th>type</th>\n",
       "      <th>content</th>\n",
       "      <th>accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [nid, uid, aid, created_at, likes, type, content, accepted]\n",
       "Index: []"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers[answers['nid'] == 13836]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-17T20:21:18.097726Z",
     "start_time": "2019-04-17T20:21:18.079434Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>cid</th>\n",
       "      <th>aid</th>\n",
       "      <th>date</th>\n",
       "      <th>type</th>\n",
       "      <th>thread</th>\n",
       "      <th>comment</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [nid, uid, cid, aid, date, type, thread, comment, reply_to]\n",
       "Index: []"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments[comments['nid'] == 13836]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "291px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}