03merge_hand_truth.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/albarron/anaconda/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import re\n",
    "import time\n",
    "import datetime\n",
    "# Requires for ipython to pick up on twitter?\n",
    "sys.path.append('/Library/Python/2.7/site-packages/')\n",
    "import twitter\n",
    "import pandas as pd\n",
    "import func\n",
    "# import pyowm # Historical API is paid\n",
    "\n",
    "# inline plot\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#%load 'data/raw-twt2016-01-26-14/21/09.csv'\n",
    "truth = pd.read_csv(\"data/truth_tweets.csv\",sep=',',error_bad_lines=False)\n",
    "twts = pd.read_csv(\"data/formated_twts.csv\",sep=',',error_bad_lines=False)\n",
    "weather = pd.read_csv(\"data/weather-add-twt2016-03-06-21:26:39.csv\",sep='\\t',error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>train_id</th>\n",
       "      <th>is_delay</th>\n",
       "      <th>delay_minor</th>\n",
       "      <th>delay_med</th>\n",
       "      <th>delay_major</th>\n",
       "      <th>delay_catastrophic</th>\n",
       "      <th>is_backlog</th>\n",
       "      <th>is_canceled</th>\n",
       "      <th>is_passing</th>\n",
       "      <th>is_accident</th>\n",
       "      <th>is_medical</th>\n",
       "      <th>is_mechanical</th>\n",
       "      <th>is_customer</th>\n",
       "      <th>is_event</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>691768068385718275</td>\n",
       "      <td>#NB155 is 22 mins late. #Caltrain</td>\n",
       "      <td>155</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>690667494906814464</td>\n",
       "      <td>#NB 151 is running 10 mins behind #Caltrain</td>\n",
       "      <td>151</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>689936168893329408</td>\n",
       "      <td>#NB151 is 10 mins down at San Mateo. #Caltrain</td>\n",
       "      <td>151</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>689509733640732672</td>\n",
       "      <td>#SB138 will board on the northbound platform a...</td>\n",
       "      <td>138</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>689485484838416388</td>\n",
       "      <td>#SB216 delayed 13 minutes at SAT\\n#NB225 delay...</td>\n",
       "      <td>216</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id                                               text  \\\n",
       "0  691768068385718275                  #NB155 is 22 mins late. #Caltrain   \n",
       "1  690667494906814464        #NB 151 is running 10 mins behind #Caltrain   \n",
       "2  689936168893329408     #NB151 is 10 mins down at San Mateo. #Caltrain   \n",
       "3  689509733640732672  #SB138 will board on the northbound platform a...   \n",
       "4  689485484838416388  #SB216 delayed 13 minutes at SAT\\n#NB225 delay...   \n",
       "\n",
       "   train_id is_delay  delay_minor  delay_med  delay_major  delay_catastrophic  \\\n",
       "0       155        1            0          0            1                   0   \n",
       "1       151        1            1          0            0                   0   \n",
       "2       151        1            1          0            0                   0   \n",
       "3       138        0            0          0            0                   0   \n",
       "4       216        1            0          1            0                   0   \n",
       "\n",
       "   is_backlog  is_canceled  is_passing  is_accident  is_medical  \\\n",
       "0           0            0           0            0           0   \n",
       "1           0            0           0            0           0   \n",
       "2           0            0           0            0           0   \n",
       "3           0            0           0            0           0   \n",
       "4           0            0           0            0           0   \n",
       "\n",
       "   is_mechanical  is_customer  is_event  \n",
       "0              0            0         0  \n",
       "1              0            0         0  \n",
       "2              0            0         0  \n",
       "3              0            0         0  \n",
       "4              0            0         0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del truth['Unnamed: 0']\n",
    "truth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>month</th>\n",
       "      <th>time_of_day</th>\n",
       "      <th>topic_train</th>\n",
       "      <th>t_bullet</th>\n",
       "      <th>t_limited</th>\n",
       "      <th>t_northbound</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>train_id</th>\n",
       "      <th>tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-01-25 23:42:14</td>\n",
       "      <td>1</td>\n",
       "      <td>NB155, Caltrain</td>\n",
       "      <td>691768068385718275</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>#NB155 is 22 mins late. #Caltrain</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>error</td>\n",
       "      <td>'155'</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-01-25 23:42:14</td>\n",
       "      <td>155</td>\n",
       "      <td>691768068385718275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-01-22 22:48:57</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NB, Caltrain</td>\n",
       "      <td>690667494906814464</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>#NB 151 is running 10 mins behind #Caltrain</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>error</td>\n",
       "      <td>'151'</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-01-22 22:48:57</td>\n",
       "      <td>151</td>\n",
       "      <td>690667494906814464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-01-20 22:22:55</td>\n",
       "      <td>1</td>\n",
       "      <td>NB151, Caltrain</td>\n",
       "      <td>689936168893329408</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>#NB151 is 10 mins down at San Mateo. #Caltrain</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>error</td>\n",
       "      <td>'151'</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-01-20 22:22:55</td>\n",
       "      <td>151</td>\n",
       "      <td>689936168893329408</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            created_at  favorite_count         hashtags                  id  \\\n",
       "0  2016-01-25 23:42:14               1  NB155, Caltrain  691768068385718275   \n",
       "1  2016-01-22 22:48:57             NaN     NB, Caltrain  690667494906814464   \n",
       "2  2016-01-20 22:22:55               1  NB151, Caltrain  689936168893329408   \n",
       "\n",
       "   in_reply_to_screen_name  retweet_count  \\\n",
       "0                      NaN              2   \n",
       "1                      NaN              2   \n",
       "2                      NaN              2   \n",
       "\n",
       "                                             text  day_of_week  day_of_month  \\\n",
       "0               #NB155 is 22 mins late. #Caltrain            0            25   \n",
       "1     #NB 151 is running 10 mins behind #Caltrain            4            22   \n",
       "2  #NB151 is 10 mins down at San Mateo. #Caltrain            2            20   \n",
       "\n",
       "   month time_of_day topic_train  t_bullet  t_limited  t_northbound  \\\n",
       "0      1       error       '155'         0          0             1   \n",
       "1      1       error       '151'         0          0             1   \n",
       "2      1       error       '151'         0          0             1   \n",
       "\n",
       "             timestamp  train_id            tweet_id  \n",
       "0  2016-01-25 23:42:14       155  691768068385718275  \n",
       "1  2016-01-22 22:48:57       151  690667494906814464  \n",
       "2  2016-01-20 22:22:55       151  689936168893329408  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del twts['Unnamed: 0']\n",
    "twts.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "truth['uid'] = truth.apply(lambda x: int(str(x.train_id)+str(x.id)),axis=1)\n",
    "twts['uid'] = truth.apply(lambda x: int(str(x.train_id)+str(x.id)),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(584, 35)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df = twts.merge(truth,on=['id','train_id'],how='inner') # now have uid\n",
    "df = twts.merge(truth,on='uid',how='inner')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# wt = weather[['id','temp','precipiation','visability','windspeed','humidity','cloudcover']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# wt.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# df = df.merge(wt,left_on='tweet_id',right_on='id',how='outer')\n",
    "# df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id_x</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text_x</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>month</th>\n",
       "      <th>...</th>\n",
       "      <th>delay_major</th>\n",
       "      <th>delay_catastrophic</th>\n",
       "      <th>is_backlog</th>\n",
       "      <th>is_canceled</th>\n",
       "      <th>is_passing</th>\n",
       "      <th>is_accident</th>\n",
       "      <th>is_medical</th>\n",
       "      <th>is_mechanical</th>\n",
       "      <th>is_customer</th>\n",
       "      <th>is_event</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-01-25 23:42:14</td>\n",
       "      <td>1</td>\n",
       "      <td>NB155, Caltrain</td>\n",
       "      <td>691768068385718275</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>#NB155 is 22 mins late. #Caltrain</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-01-22 22:48:57</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NB, Caltrain</td>\n",
       "      <td>690667494906814464</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>#NB 151 is running 10 mins behind #Caltrain</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-01-20 22:22:55</td>\n",
       "      <td>1</td>\n",
       "      <td>NB151, Caltrain</td>\n",
       "      <td>689936168893329408</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>#NB151 is 10 mins down at San Mateo. #Caltrain</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-01-19 18:08:25</td>\n",
       "      <td>1</td>\n",
       "      <td>SB138, Caltrain</td>\n",
       "      <td>689509733640732672</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>#SB138 will board on the northbound platform a...</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-01-19 16:32:04</td>\n",
       "      <td>2</td>\n",
       "      <td>SB216, NB225, NB329, Caltrain</td>\n",
       "      <td>689485484838416388</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>#SB216 delayed 13 minutes at SAT\\n#NB225 delay...</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            created_at  favorite_count                       hashtags  \\\n",
       "0  2016-01-25 23:42:14               1                NB155, Caltrain   \n",
       "1  2016-01-22 22:48:57             NaN                   NB, Caltrain   \n",
       "2  2016-01-20 22:22:55               1                NB151, Caltrain   \n",
       "3  2016-01-19 18:08:25               1                SB138, Caltrain   \n",
       "4  2016-01-19 16:32:04               2  SB216, NB225, NB329, Caltrain   \n",
       "\n",
       "                 id_x  in_reply_to_screen_name  retweet_count  \\\n",
       "0  691768068385718275                      NaN              2   \n",
       "1  690667494906814464                      NaN              2   \n",
       "2  689936168893329408                      NaN              2   \n",
       "3  689509733640732672                      NaN              2   \n",
       "4  689485484838416388                      NaN              4   \n",
       "\n",
       "                                              text_x  day_of_week  \\\n",
       "0                  #NB155 is 22 mins late. #Caltrain            0   \n",
       "1        #NB 151 is running 10 mins behind #Caltrain            4   \n",
       "2     #NB151 is 10 mins down at San Mateo. #Caltrain            2   \n",
       "3  #SB138 will board on the northbound platform a...            1   \n",
       "4  #SB216 delayed 13 minutes at SAT\\n#NB225 delay...            1   \n",
       "\n",
       "   day_of_month  month    ...    delay_major delay_catastrophic  is_backlog  \\\n",
       "0            25      1    ...              1                  0           0   \n",
       "1            22      1    ...              0                  0           0   \n",
       "2            20      1    ...              0                  0           0   \n",
       "3            19      1    ...              0                  0           0   \n",
       "4            19      1    ...              0                  0           0   \n",
       "\n",
       "   is_canceled  is_passing is_accident  is_medical  is_mechanical is_customer  \\\n",
       "0            0           0           0           0              0           0   \n",
       "1            0           0           0           0              0           0   \n",
       "2            0           0           0           0              0           0   \n",
       "3            0           0           0           0              0           0   \n",
       "4            0           0           0           0              0           0   \n",
       "\n",
       "   is_event  \n",
       "0         0  \n",
       "1         0  \n",
       "2         0  \n",
       "3         0  \n",
       "4         0  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['created_at', 'favorite_count', 'hashtags', 'id_x',\n",
       "       'in_reply_to_screen_name', 'retweet_count', 'text_x', 'day_of_week',\n",
       "       'day_of_month', 'month', 'time_of_day', 'topic_train', 't_bullet',\n",
       "       't_limited', 't_northbound', 'timestamp', 'train_id_x', 'tweet_id',\n",
       "       'uid', 'id_y', 'text_y', 'train_id_y', 'is_delay', 'delay_minor',\n",
       "       'delay_med', 'delay_major', 'delay_catastrophic', 'is_backlog',\n",
       "       'is_canceled', 'is_passing', 'is_accident', 'is_medical',\n",
       "       'is_mechanical', 'is_customer', 'is_event'], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "del df['created_at']\n",
    "del df['favorite_count']\n",
    "del df['hashtags']\n",
    "del df['in_reply_to_screen_name']\n",
    "del df['retweet_count']\n",
    "del df['text_y']\n",
    "del df['train_id_y']\n",
    "del df['id_y']\n",
    "del df['id_x']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>month</th>\n",
       "      <th>t_bullet</th>\n",
       "      <th>t_limited</th>\n",
       "      <th>t_northbound</th>\n",
       "      <th>train_id_x</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>delay_minor</th>\n",
       "      <th>delay_med</th>\n",
       "      <th>delay_major</th>\n",
       "      <th>delay_catastrophic</th>\n",
       "      <th>is_backlog</th>\n",
       "      <th>is_canceled</th>\n",
       "      <th>is_passing</th>\n",
       "      <th>is_accident</th>\n",
       "      <th>is_medical</th>\n",
       "      <th>is_mechanical</th>\n",
       "      <th>is_customer</th>\n",
       "      <th>is_event</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>day_of_week</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.064527</td>\n",
       "      <td>0.032273</td>\n",
       "      <td>0.038234</td>\n",
       "      <td>-0.015425</td>\n",
       "      <td>-0.060972</td>\n",
       "      <td>0.119804</td>\n",
       "      <td>-0.074660</td>\n",
       "      <td>-0.050183</td>\n",
       "      <td>-0.041209</td>\n",
       "      <td>0.017500</td>\n",
       "      <td>0.035520</td>\n",
       "      <td>0.026806</td>\n",
       "      <td>-0.031875</td>\n",
       "      <td>0.043513</td>\n",
       "      <td>0.031715</td>\n",
       "      <td>0.050550</td>\n",
       "      <td>-0.041497</td>\n",
       "      <td>-0.044278</td>\n",
       "      <td>-0.030502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>day_of_month</th>\n",
       "      <td>-0.064527</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.167183</td>\n",
       "      <td>0.030566</td>\n",
       "      <td>0.050568</td>\n",
       "      <td>0.053409</td>\n",
       "      <td>0.023401</td>\n",
       "      <td>-0.226543</td>\n",
       "      <td>0.097248</td>\n",
       "      <td>0.077398</td>\n",
       "      <td>0.079371</td>\n",
       "      <td>-0.074655</td>\n",
       "      <td>-0.182253</td>\n",
       "      <td>-0.012005</td>\n",
       "      <td>0.016955</td>\n",
       "      <td>0.025270</td>\n",
       "      <td>-0.101009</td>\n",
       "      <td>-0.087499</td>\n",
       "      <td>0.020955</td>\n",
       "      <td>0.009100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <td>0.032273</td>\n",
       "      <td>-0.167183</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.104911</td>\n",
       "      <td>0.057563</td>\n",
       "      <td>0.073191</td>\n",
       "      <td>0.005346</td>\n",
       "      <td>0.000845</td>\n",
       "      <td>-0.011825</td>\n",
       "      <td>0.028640</td>\n",
       "      <td>-0.038361</td>\n",
       "      <td>0.012834</td>\n",
       "      <td>-0.049368</td>\n",
       "      <td>-0.000624</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>0.041778</td>\n",
       "      <td>-0.233609</td>\n",
       "      <td>0.039058</td>\n",
       "      <td>0.048913</td>\n",
       "      <td>0.000024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t_bullet</th>\n",
       "      <td>0.038234</td>\n",
       "      <td>0.030566</td>\n",
       "      <td>-0.104911</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.406237</td>\n",
       "      <td>-0.003214</td>\n",
       "      <td>0.657885</td>\n",
       "      <td>-0.066599</td>\n",
       "      <td>-0.021947</td>\n",
       "      <td>0.033449</td>\n",
       "      <td>-0.057969</td>\n",
       "      <td>0.095674</td>\n",
       "      <td>0.006079</td>\n",
       "      <td>0.056771</td>\n",
       "      <td>-0.005847</td>\n",
       "      <td>0.042806</td>\n",
       "      <td>-0.089497</td>\n",
       "      <td>0.049790</td>\n",
       "      <td>0.005040</td>\n",
       "      <td>-0.035980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t_limited</th>\n",
       "      <td>-0.015425</td>\n",
       "      <td>0.050568</td>\n",
       "      <td>0.057563</td>\n",
       "      <td>-0.406237</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.049954</td>\n",
       "      <td>0.133330</td>\n",
       "      <td>-0.094702</td>\n",
       "      <td>-0.036353</td>\n",
       "      <td>-0.018982</td>\n",
       "      <td>0.005696</td>\n",
       "      <td>-0.039966</td>\n",
       "      <td>0.101631</td>\n",
       "      <td>-0.034239</td>\n",
       "      <td>0.073836</td>\n",
       "      <td>0.005562</td>\n",
       "      <td>-0.038704</td>\n",
       "      <td>-0.008069</td>\n",
       "      <td>-0.066965</td>\n",
       "      <td>-0.012378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t_northbound</th>\n",
       "      <td>-0.060972</td>\n",
       "      <td>0.053409</td>\n",
       "      <td>0.073191</td>\n",
       "      <td>-0.003214</td>\n",
       "      <td>-0.049954</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.006201</td>\n",
       "      <td>0.045560</td>\n",
       "      <td>0.008760</td>\n",
       "      <td>-0.058795</td>\n",
       "      <td>0.022023</td>\n",
       "      <td>0.018741</td>\n",
       "      <td>0.080351</td>\n",
       "      <td>-0.002916</td>\n",
       "      <td>0.076562</td>\n",
       "      <td>0.066943</td>\n",
       "      <td>-0.103233</td>\n",
       "      <td>-0.026955</td>\n",
       "      <td>-0.045545</td>\n",
       "      <td>0.003113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train_id_x</th>\n",
       "      <td>0.119804</td>\n",
       "      <td>0.023401</td>\n",
       "      <td>0.005346</td>\n",
       "      <td>0.657885</td>\n",
       "      <td>0.133330</td>\n",
       "      <td>0.006201</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.111896</td>\n",
       "      <td>-0.089655</td>\n",
       "      <td>-0.009991</td>\n",
       "      <td>-0.008451</td>\n",
       "      <td>0.065518</td>\n",
       "      <td>0.015809</td>\n",
       "      <td>0.031720</td>\n",
       "      <td>0.015133</td>\n",
       "      <td>0.047335</td>\n",
       "      <td>-0.054273</td>\n",
       "      <td>0.037056</td>\n",
       "      <td>-0.051603</td>\n",
       "      <td>-0.009753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tweet_id</th>\n",
       "      <td>-0.074660</td>\n",
       "      <td>-0.226543</td>\n",
       "      <td>0.000845</td>\n",
       "      <td>-0.066599</td>\n",
       "      <td>-0.094702</td>\n",
       "      <td>0.045560</td>\n",
       "      <td>-0.111896</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.067052</td>\n",
       "      <td>-0.130514</td>\n",
       "      <td>-0.017715</td>\n",
       "      <td>0.047553</td>\n",
       "      <td>0.083153</td>\n",
       "      <td>-0.036222</td>\n",
       "      <td>0.132490</td>\n",
       "      <td>-0.051096</td>\n",
       "      <td>0.296229</td>\n",
       "      <td>-0.129205</td>\n",
       "      <td>-0.005532</td>\n",
       "      <td>-0.070528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delay_minor</th>\n",
       "      <td>-0.050183</td>\n",
       "      <td>0.097248</td>\n",
       "      <td>-0.011825</td>\n",
       "      <td>-0.021947</td>\n",
       "      <td>-0.036353</td>\n",
       "      <td>0.008760</td>\n",
       "      <td>-0.089655</td>\n",
       "      <td>-0.067052</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.294072</td>\n",
       "      <td>-0.109563</td>\n",
       "      <td>-0.080127</td>\n",
       "      <td>-0.129274</td>\n",
       "      <td>-0.059536</td>\n",
       "      <td>-0.091557</td>\n",
       "      <td>-0.089406</td>\n",
       "      <td>-0.118606</td>\n",
       "      <td>-0.062823</td>\n",
       "      <td>0.002914</td>\n",
       "      <td>0.101125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delay_med</th>\n",
       "      <td>-0.041209</td>\n",
       "      <td>0.077398</td>\n",
       "      <td>0.028640</td>\n",
       "      <td>0.033449</td>\n",
       "      <td>-0.018982</td>\n",
       "      <td>-0.058795</td>\n",
       "      <td>-0.009991</td>\n",
       "      <td>-0.130514</td>\n",
       "      <td>-0.294072</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.150462</td>\n",
       "      <td>-0.110038</td>\n",
       "      <td>-0.106410</td>\n",
       "      <td>-0.116536</td>\n",
       "      <td>-0.086000</td>\n",
       "      <td>-0.162627</td>\n",
       "      <td>-0.005602</td>\n",
       "      <td>0.206398</td>\n",
       "      <td>0.224643</td>\n",
       "      <td>0.078902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delay_major</th>\n",
       "      <td>0.017500</td>\n",
       "      <td>0.079371</td>\n",
       "      <td>-0.038361</td>\n",
       "      <td>-0.057969</td>\n",
       "      <td>0.005696</td>\n",
       "      <td>0.022023</td>\n",
       "      <td>-0.008451</td>\n",
       "      <td>-0.017715</td>\n",
       "      <td>-0.109563</td>\n",
       "      <td>-0.150462</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.040997</td>\n",
       "      <td>0.030199</td>\n",
       "      <td>-0.043418</td>\n",
       "      <td>-0.046845</td>\n",
       "      <td>0.040378</td>\n",
       "      <td>-0.028929</td>\n",
       "      <td>0.109105</td>\n",
       "      <td>0.036739</td>\n",
       "      <td>-0.032805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delay_catastrophic</th>\n",
       "      <td>0.035520</td>\n",
       "      <td>-0.074655</td>\n",
       "      <td>0.012834</td>\n",
       "      <td>0.095674</td>\n",
       "      <td>-0.039966</td>\n",
       "      <td>0.018741</td>\n",
       "      <td>0.065518</td>\n",
       "      <td>0.047553</td>\n",
       "      <td>-0.080127</td>\n",
       "      <td>-0.110038</td>\n",
       "      <td>-0.040997</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.030178</td>\n",
       "      <td>-0.031753</td>\n",
       "      <td>-0.034259</td>\n",
       "      <td>0.170691</td>\n",
       "      <td>-0.044381</td>\n",
       "      <td>-0.038042</td>\n",
       "      <td>-0.016581</td>\n",
       "      <td>-0.023991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_backlog</th>\n",
       "      <td>0.026806</td>\n",
       "      <td>-0.182253</td>\n",
       "      <td>-0.049368</td>\n",
       "      <td>0.006079</td>\n",
       "      <td>0.101631</td>\n",
       "      <td>0.080351</td>\n",
       "      <td>0.015809</td>\n",
       "      <td>0.083153</td>\n",
       "      <td>-0.129274</td>\n",
       "      <td>-0.106410</td>\n",
       "      <td>0.030199</td>\n",
       "      <td>0.030178</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.067674</td>\n",
       "      <td>0.093118</td>\n",
       "      <td>-0.072046</td>\n",
       "      <td>-0.094586</td>\n",
       "      <td>-0.020911</td>\n",
       "      <td>-0.037555</td>\n",
       "      <td>-0.051131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_canceled</th>\n",
       "      <td>-0.031875</td>\n",
       "      <td>-0.012005</td>\n",
       "      <td>-0.000624</td>\n",
       "      <td>0.056771</td>\n",
       "      <td>-0.034239</td>\n",
       "      <td>-0.002916</td>\n",
       "      <td>0.031720</td>\n",
       "      <td>-0.036222</td>\n",
       "      <td>-0.059536</td>\n",
       "      <td>-0.116536</td>\n",
       "      <td>-0.043418</td>\n",
       "      <td>-0.031753</td>\n",
       "      <td>-0.067674</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.036282</td>\n",
       "      <td>0.015404</td>\n",
       "      <td>-0.047002</td>\n",
       "      <td>0.069861</td>\n",
       "      <td>-0.056113</td>\n",
       "      <td>-0.025408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_passing</th>\n",
       "      <td>0.043513</td>\n",
       "      <td>0.016955</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>-0.005847</td>\n",
       "      <td>0.073836</td>\n",
       "      <td>0.076562</td>\n",
       "      <td>0.015133</td>\n",
       "      <td>0.132490</td>\n",
       "      <td>-0.091557</td>\n",
       "      <td>-0.086000</td>\n",
       "      <td>-0.046845</td>\n",
       "      <td>-0.034259</td>\n",
       "      <td>0.093118</td>\n",
       "      <td>-0.036282</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.059208</td>\n",
       "      <td>-0.050711</td>\n",
       "      <td>0.001630</td>\n",
       "      <td>-0.060542</td>\n",
       "      <td>-0.027413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_accident</th>\n",
       "      <td>0.031715</td>\n",
       "      <td>0.025270</td>\n",
       "      <td>0.041778</td>\n",
       "      <td>0.042806</td>\n",
       "      <td>0.005562</td>\n",
       "      <td>0.066943</td>\n",
       "      <td>0.047335</td>\n",
       "      <td>-0.051096</td>\n",
       "      <td>-0.089406</td>\n",
       "      <td>-0.162627</td>\n",
       "      <td>0.040378</td>\n",
       "      <td>0.170691</td>\n",
       "      <td>-0.072046</td>\n",
       "      <td>0.015404</td>\n",
       "      <td>-0.059208</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.001065</td>\n",
       "      <td>-0.080839</td>\n",
       "      <td>-0.091570</td>\n",
       "      <td>0.004398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_medical</th>\n",
       "      <td>0.050550</td>\n",
       "      <td>-0.101009</td>\n",
       "      <td>-0.233609</td>\n",
       "      <td>-0.089497</td>\n",
       "      <td>-0.038704</td>\n",
       "      <td>-0.103233</td>\n",
       "      <td>-0.054273</td>\n",
       "      <td>0.296229</td>\n",
       "      <td>-0.118606</td>\n",
       "      <td>-0.005602</td>\n",
       "      <td>-0.028929</td>\n",
       "      <td>-0.044381</td>\n",
       "      <td>-0.094586</td>\n",
       "      <td>-0.047002</td>\n",
       "      <td>-0.050711</td>\n",
       "      <td>0.001065</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.100631</td>\n",
       "      <td>-0.052983</td>\n",
       "      <td>0.016859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_mechanical</th>\n",
       "      <td>-0.041497</td>\n",
       "      <td>-0.087499</td>\n",
       "      <td>0.039058</td>\n",
       "      <td>0.049790</td>\n",
       "      <td>-0.008069</td>\n",
       "      <td>-0.026955</td>\n",
       "      <td>0.037056</td>\n",
       "      <td>-0.129205</td>\n",
       "      <td>-0.062823</td>\n",
       "      <td>0.206398</td>\n",
       "      <td>0.109105</td>\n",
       "      <td>-0.038042</td>\n",
       "      <td>-0.020911</td>\n",
       "      <td>0.069861</td>\n",
       "      <td>0.001630</td>\n",
       "      <td>-0.080839</td>\n",
       "      <td>-0.100631</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.185693</td>\n",
       "      <td>-0.054399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_customer</th>\n",
       "      <td>-0.044278</td>\n",
       "      <td>0.020955</td>\n",
       "      <td>0.048913</td>\n",
       "      <td>0.005040</td>\n",
       "      <td>-0.066965</td>\n",
       "      <td>-0.045545</td>\n",
       "      <td>-0.051603</td>\n",
       "      <td>-0.005532</td>\n",
       "      <td>0.002914</td>\n",
       "      <td>0.224643</td>\n",
       "      <td>0.036739</td>\n",
       "      <td>-0.016581</td>\n",
       "      <td>-0.037555</td>\n",
       "      <td>-0.056113</td>\n",
       "      <td>-0.060542</td>\n",
       "      <td>-0.091570</td>\n",
       "      <td>-0.052983</td>\n",
       "      <td>0.185693</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.002621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_event</th>\n",
       "      <td>-0.030502</td>\n",
       "      <td>0.009100</td>\n",
       "      <td>0.000024</td>\n",
       "      <td>-0.035980</td>\n",
       "      <td>-0.012378</td>\n",
       "      <td>0.003113</td>\n",
       "      <td>-0.009753</td>\n",
       "      <td>-0.070528</td>\n",
       "      <td>0.101125</td>\n",
       "      <td>0.078902</td>\n",
       "      <td>-0.032805</td>\n",
       "      <td>-0.023991</td>\n",
       "      <td>-0.051131</td>\n",
       "      <td>-0.025408</td>\n",
       "      <td>-0.027413</td>\n",
       "      <td>0.004398</td>\n",
       "      <td>0.016859</td>\n",
       "      <td>-0.054399</td>\n",
       "      <td>0.002621</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    day_of_week  day_of_month     month  t_bullet  t_limited  \\\n",
       "day_of_week            1.000000     -0.064527  0.032273  0.038234  -0.015425   \n",
       "day_of_month          -0.064527      1.000000 -0.167183  0.030566   0.050568   \n",
       "month                  0.032273     -0.167183  1.000000 -0.104911   0.057563   \n",
       "t_bullet               0.038234      0.030566 -0.104911  1.000000  -0.406237   \n",
       "t_limited             -0.015425      0.050568  0.057563 -0.406237   1.000000   \n",
       "t_northbound          -0.060972      0.053409  0.073191 -0.003214  -0.049954   \n",
       "train_id_x             0.119804      0.023401  0.005346  0.657885   0.133330   \n",
       "tweet_id              -0.074660     -0.226543  0.000845 -0.066599  -0.094702   \n",
       "delay_minor           -0.050183      0.097248 -0.011825 -0.021947  -0.036353   \n",
       "delay_med             -0.041209      0.077398  0.028640  0.033449  -0.018982   \n",
       "delay_major            0.017500      0.079371 -0.038361 -0.057969   0.005696   \n",
       "delay_catastrophic     0.035520     -0.074655  0.012834  0.095674  -0.039966   \n",
       "is_backlog             0.026806     -0.182253 -0.049368  0.006079   0.101631   \n",
       "is_canceled           -0.031875     -0.012005 -0.000624  0.056771  -0.034239   \n",
       "is_passing             0.043513      0.016955  0.000035 -0.005847   0.073836   \n",
       "is_accident            0.031715      0.025270  0.041778  0.042806   0.005562   \n",
       "is_medical             0.050550     -0.101009 -0.233609 -0.089497  -0.038704   \n",
       "is_mechanical         -0.041497     -0.087499  0.039058  0.049790  -0.008069   \n",
       "is_customer           -0.044278      0.020955  0.048913  0.005040  -0.066965   \n",
       "is_event              -0.030502      0.009100  0.000024 -0.035980  -0.012378   \n",
       "\n",
       "                    t_northbound  train_id_x  tweet_id  delay_minor  \\\n",
       "day_of_week            -0.060972    0.119804 -0.074660    -0.050183   \n",
       "day_of_month            0.053409    0.023401 -0.226543     0.097248   \n",
       "month                   0.073191    0.005346  0.000845    -0.011825   \n",
       "t_bullet               -0.003214    0.657885 -0.066599    -0.021947   \n",
       "t_limited              -0.049954    0.133330 -0.094702    -0.036353   \n",
       "t_northbound            1.000000    0.006201  0.045560     0.008760   \n",
       "train_id_x              0.006201    1.000000 -0.111896    -0.089655   \n",
       "tweet_id                0.045560   -0.111896  1.000000    -0.067052   \n",
       "delay_minor             0.008760   -0.089655 -0.067052     1.000000   \n",
       "delay_med              -0.058795   -0.009991 -0.130514    -0.294072   \n",
       "delay_major             0.022023   -0.008451 -0.017715    -0.109563   \n",
       "delay_catastrophic      0.018741    0.065518  0.047553    -0.080127   \n",
       "is_backlog              0.080351    0.015809  0.083153    -0.129274   \n",
       "is_canceled            -0.002916    0.031720 -0.036222    -0.059536   \n",
       "is_passing              0.076562    0.015133  0.132490    -0.091557   \n",
       "is_accident             0.066943    0.047335 -0.051096    -0.089406   \n",
       "is_medical             -0.103233   -0.054273  0.296229    -0.118606   \n",
       "is_mechanical          -0.026955    0.037056 -0.129205    -0.062823   \n",
       "is_customer            -0.045545   -0.051603 -0.005532     0.002914   \n",
       "is_event                0.003113   -0.009753 -0.070528     0.101125   \n",
       "\n",
       "                    delay_med  delay_major  delay_catastrophic  is_backlog  \\\n",
       "day_of_week         -0.041209     0.017500            0.035520    0.026806   \n",
       "day_of_month         0.077398     0.079371           -0.074655   -0.182253   \n",
       "month                0.028640    -0.038361            0.012834   -0.049368   \n",
       "t_bullet             0.033449    -0.057969            0.095674    0.006079   \n",
       "t_limited           -0.018982     0.005696           -0.039966    0.101631   \n",
       "t_northbound        -0.058795     0.022023            0.018741    0.080351   \n",
       "train_id_x          -0.009991    -0.008451            0.065518    0.015809   \n",
       "tweet_id            -0.130514    -0.017715            0.047553    0.083153   \n",
       "delay_minor         -0.294072    -0.109563           -0.080127   -0.129274   \n",
       "delay_med            1.000000    -0.150462           -0.110038   -0.106410   \n",
       "delay_major         -0.150462     1.000000           -0.040997    0.030199   \n",
       "delay_catastrophic  -0.110038    -0.040997            1.000000    0.030178   \n",
       "is_backlog          -0.106410     0.030199            0.030178    1.000000   \n",
       "is_canceled         -0.116536    -0.043418           -0.031753   -0.067674   \n",
       "is_passing          -0.086000    -0.046845           -0.034259    0.093118   \n",
       "is_accident         -0.162627     0.040378            0.170691   -0.072046   \n",
       "is_medical          -0.005602    -0.028929           -0.044381   -0.094586   \n",
       "is_mechanical        0.206398     0.109105           -0.038042   -0.020911   \n",
       "is_customer          0.224643     0.036739           -0.016581   -0.037555   \n",
       "is_event             0.078902    -0.032805           -0.023991   -0.051131   \n",
       "\n",
       "                    is_canceled  is_passing  is_accident  is_medical  \\\n",
       "day_of_week           -0.031875    0.043513     0.031715    0.050550   \n",
       "day_of_month          -0.012005    0.016955     0.025270   -0.101009   \n",
       "month                 -0.000624    0.000035     0.041778   -0.233609   \n",
       "t_bullet               0.056771   -0.005847     0.042806   -0.089497   \n",
       "t_limited             -0.034239    0.073836     0.005562   -0.038704   \n",
       "t_northbound          -0.002916    0.076562     0.066943   -0.103233   \n",
       "train_id_x             0.031720    0.015133     0.047335   -0.054273   \n",
       "tweet_id              -0.036222    0.132490    -0.051096    0.296229   \n",
       "delay_minor           -0.059536   -0.091557    -0.089406   -0.118606   \n",
       "delay_med             -0.116536   -0.086000    -0.162627   -0.005602   \n",
       "delay_major           -0.043418   -0.046845     0.040378   -0.028929   \n",
       "delay_catastrophic    -0.031753   -0.034259     0.170691   -0.044381   \n",
       "is_backlog            -0.067674    0.093118    -0.072046   -0.094586   \n",
       "is_canceled            1.000000   -0.036282     0.015404   -0.047002   \n",
       "is_passing            -0.036282    1.000000    -0.059208   -0.050711   \n",
       "is_accident            0.015404   -0.059208     1.000000    0.001065   \n",
       "is_medical            -0.047002   -0.050711     0.001065    1.000000   \n",
       "is_mechanical          0.069861    0.001630    -0.080839   -0.100631   \n",
       "is_customer           -0.056113   -0.060542    -0.091570   -0.052983   \n",
       "is_event              -0.025408   -0.027413     0.004398    0.016859   \n",
       "\n",
       "                    is_mechanical  is_customer  is_event  \n",
       "day_of_week             -0.041497    -0.044278 -0.030502  \n",
       "day_of_month            -0.087499     0.020955  0.009100  \n",
       "month                    0.039058     0.048913  0.000024  \n",
       "t_bullet                 0.049790     0.005040 -0.035980  \n",
       "t_limited               -0.008069    -0.066965 -0.012378  \n",
       "t_northbound            -0.026955    -0.045545  0.003113  \n",
       "train_id_x               0.037056    -0.051603 -0.009753  \n",
       "tweet_id                -0.129205    -0.005532 -0.070528  \n",
       "delay_minor             -0.062823     0.002914  0.101125  \n",
       "delay_med                0.206398     0.224643  0.078902  \n",
       "delay_major              0.109105     0.036739 -0.032805  \n",
       "delay_catastrophic      -0.038042    -0.016581 -0.023991  \n",
       "is_backlog              -0.020911    -0.037555 -0.051131  \n",
       "is_canceled              0.069861    -0.056113 -0.025408  \n",
       "is_passing               0.001630    -0.060542 -0.027413  \n",
       "is_accident             -0.080839    -0.091570  0.004398  \n",
       "is_medical              -0.100631    -0.052983  0.016859  \n",
       "is_mechanical            1.000000     0.185693 -0.054399  \n",
       "is_customer              0.185693     1.000000  0.002621  \n",
       "is_event                -0.054399     0.002621  1.000000  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "del df['text_x']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "filename = \"./data/merged_delay.csv\".format(date=datetime.datetime.now().strftime(\"%Y-%m-%d-%H:%M:%S\"))\n",
    "df.to_csv(filename, sep='\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}