APMA_DRP_Fire_Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RlZUXLce5qH5"
   },
   "source": [
    "# Predicting Fires in Charlottesville\n",
    "Jackson Barkstrom, under the guidance of Michael Lee"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "phy9ec8A488x"
   },
   "source": [
    "Initial authors: Jackson Barkstrom, Habib Karaky, Josh Schuck, Garrett Vercoe. The data we took from GitHub was worked on by many, including the four of us, during Charlottesville Civic Innovation Day 2018 (special shoutouts to Stephen and Katharine). Edited again by Jackson as of 11/9/2018 to properly add fires to housing data and to (if I have time) give a proper train test split for time. The Machine Learning section is all for the Applied Math Directed Reading Program at Brown University--work by Jackson Barkstrom under the guidance of grad student Michael Lee."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     }
    },
    "colab_type": "code",
    "id": "kR35uSaU1ust"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jwbarkstrom/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import numpy as nd\n",
    "import urllib.request, json\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, Imputer, OneHotEncoder\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.model_selection import train_test_split\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import matplotlib.pyplot as plt\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jYmqzJkSe0tE"
   },
   "source": [
    "## Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     },
     "base_uri": "https://localhost:8080/",
     "height": 884.0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1820.0,
     "status": "error",
     "timestamp": 1.529857539957E12,
     "user": {
      "displayName": "Jackson Barkstrom",
      "photoUrl": "//lh5.googleusercontent.com/-vVdnq6QxZKc/AAAAAAAAAAI/AAAAAAAAAC8/hz-Hu2YlLus/s50-c-k-no/photo.jpg",
      "userId": "100905809459765639952"
     },
     "user_tz": 240.0
    },
    "id": "j5mlDYwkr43z",
    "outputId": "2f78f167-cd1f-493a-9f70-324840107013"
   },
   "outputs": [],
   "source": [
    "# Import data from Civic Innovation Day, and get them all in DataFrame format\n",
    "fire_late = pd.read_excel('https://github.com/Smart-Cville/CID-2018-CFD-Challenge/blob/master/data/CFD_Fires_12_16to9_19_18.xlsx?raw=true')\n",
    "commercial_original = pd.read_csv('https://raw.githubusercontent.com/Smart-Cville/CID-2018-CFD-Challenge/master/data/addresses-joined-to-details-commercial-with-lat-lon.csv')\n",
    "residential_original = pd.read_csv('https://raw.githubusercontent.com/Smart-Cville/CID-2018-CFD-Challenge/master/data/addresses-joined-to-details-residential-with-lat-lon.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reading the JSON data using json.load() for the early fire data\n",
    "with urllib.request.urlopen('https://github.com/Smart-Cville/CID-2018-CFD-Challenge/blob/master/data/HISTORICAL_FIRE.json?raw=true') as url:\n",
    "    file = json.loads(url.read().decode())\n",
    "    \n",
    "datetime = []\n",
    "for i in range(len(file[\"features\"])):\n",
    "    datetime.append(file['features'][i]['properties']['IN_AlarmDa'])\n",
    "address = []\n",
    "for i in range(len(file[\"features\"])):\n",
    "    address.append(file['features'][i]['properties']['Match_addr'])\n",
    "cause = []\n",
    "for i in range(len(file[\"features\"])):\n",
    "    cause.append(file['features'][i]['properties']['FR_CauseOf'])\n",
    "\n",
    "fire_early = pd.DataFrame({'datetime':datetime, 'address':address, 'cause':cause})\n",
    "del file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "commercial = pd.DataFrame(commercial_original)\n",
    "residential = pd.DataFrame(residential_original)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3FW1ZdYOjIL7"
   },
   "source": [
    "## Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     },
     "base_uri": "https://localhost:8080/",
     "height": 204.0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 255.0,
     "status": "ok",
     "timestamp": 1.529856560645E12,
     "user": {
      "displayName": "Jackson Barkstrom",
      "photoUrl": "//lh5.googleusercontent.com/-vVdnq6QxZKc/AAAAAAAAAAI/AAAAAAAAAC8/hz-Hu2YlLus/s50-c-k-no/photo.jpg",
      "userId": "100905809459765639952"
     },
     "user_tz": 240.0
    },
    "id": "EqN25BrGi6Oj",
    "outputId": "0b789ef0-61e4-4a0b-c3a3-5f5f1b160247"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "datetime    datetime64[ns]\n",
      "address             object\n",
      "cause               object\n",
      "dtype: object\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>address</th>\n",
       "      <th>cause</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2003-01-08</td>\n",
       "      <td>1002 POPLAR ST</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2002-11-07</td>\n",
       "      <td>905 FOREST ST</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002-11-14</td>\n",
       "      <td>219 5TH ST SW</td>\n",
       "      <td>NULL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2002-12-17</td>\n",
       "      <td>822 HARDY DR</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2003-01-09</td>\n",
       "      <td>1526 TRAILRIDGE RD</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    datetime             address cause\n",
       "0 2003-01-08      1002 POPLAR ST     0\n",
       "1 2002-11-07       905 FOREST ST     1\n",
       "2 2002-11-14       219 5TH ST SW  NULL\n",
       "3 2002-12-17        822 HARDY DR     1\n",
       "4 2003-01-09  1526 TRAILRIDGE RD     5"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert fire_early date column to datetime\n",
    "fire_early.datetime = pd.to_datetime(fire_early.datetime)\n",
    "\n",
    "print(fire_early.dtypes)\n",
    "\n",
    "fire_early.head()\n",
    "\n",
    "# perfect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     },
     "base_uri": "https://localhost:8080/",
     "height": 204.0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 544.0,
     "status": "ok",
     "timestamp": 1.529856569494E12,
     "user": {
      "displayName": "Jackson Barkstrom",
      "photoUrl": "//lh5.googleusercontent.com/-vVdnq6QxZKc/AAAAAAAAAAI/AAAAAAAAAC8/hz-Hu2YlLus/s50-c-k-no/photo.jpg",
      "userId": "100905809459765639952"
     },
     "user_tz": 240.0
    },
    "id": "Xw9XWhqtxAsu",
    "outputId": "ca88011d-575e-4257-dfc7-2f3b5a14fafa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date       datetime64[ns]\n",
      "cause              object\n",
      "address            object\n",
      "dtype: object\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>cause</th>\n",
       "      <th>address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-12-14</td>\n",
       "      <td>15</td>\n",
       "      <td>1155 5TH Street Southwest</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-12-15</td>\n",
       "      <td>11</td>\n",
       "      <td>517 RIDGE Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-12-20</td>\n",
       "      <td>16</td>\n",
       "      <td>10 UNIVERSITY Circle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-12-20</td>\n",
       "      <td>14</td>\n",
       "      <td>1200 5TH Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-12-21</td>\n",
       "      <td>14</td>\n",
       "      <td>1000 1ST Street South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date cause                    address\n",
       "0 2016-12-14    15  1155 5TH Street Southwest\n",
       "1 2016-12-15    11           517 RIDGE Street\n",
       "2 2016-12-20    16       10 UNIVERSITY Circle\n",
       "3 2016-12-20    14            1200 5TH Street\n",
       "4 2016-12-21    14      1000 1ST Street South"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select the columns we want from fire_late and format them properly\n",
    "\n",
    "# Select only incident type and full address columns\n",
    "fire_late_cleaned = pd.DataFrame(fire_late.iloc[:,[1,6,13]])\n",
    "\n",
    "# take only numerical values from subcategory column\n",
    "fire_late_cleaned.iloc[:,1] = fire_late_cleaned.iloc[:,1].str.replace(\"[^0-9]\", \"\")\n",
    "\n",
    "fire_late_cleaned.rename(columns={fire_late_cleaned.columns[1]: \"cause\", fire_late_cleaned.columns[0]: \"date\", fire_late_cleaned.columns[2]: \"address\"}, inplace = True)\n",
    "\n",
    "print(fire_late_cleaned.dtypes)\n",
    "\n",
    "# we won't add a column of 1's this time becuase left merge is too challenging for this dataset\n",
    "fire_late_cleaned.head()\n",
    "\n",
    "#note: st and rd versus street and road will be a big problem--going to have to correct here. \n",
    "# I noticed that some instances in the fire data don't have southwest, but some do, so we're going to have\n",
    "# to deal with imperfect matching here. I work below to fix this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     }
    },
    "colab_type": "code",
    "id": "9KCVOnNwt4X5"
   },
   "outputs": [],
   "source": [
    "# Merge the different fields of address to get one address (so that we can merge on address)\n",
    "\n",
    "residential['address'] = residential['st_number'].astype(str) + \" \" + residential['st_name'] + \" \" + residential['suffix']\n",
    "commercial['address'] = commercial['st_number'].astype(str) + \" \" + commercial['st_name'] + \" \" + commercial['suffix']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     }
    },
    "colab_type": "code",
    "id": "X_Dy08e1uZ2e"
   },
   "outputs": [],
   "source": [
    "# drop unused columns (we don't use a LOT of columns)\n",
    "residential.drop(['parcel_number','bin','objectid','st_number','st_unit','st_name','suffix', 'predir','postdir', 'unit_type','zip', 'last_edited_user', 'last_edited_date', 'master_address_id'], 1, inplace= True)\n",
    "commercial.drop(['parcel_number','bin','objectid','st_number','st_unit','st_name','suffix', 'predir','postdir', 'unit_type','zip', 'last_edited_user', 'last_edited_date', 'master_address_id'], 1, inplace= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     }
    },
    "colab_type": "code",
    "id": "akuXpZde5jAI"
   },
   "outputs": [],
   "source": [
    "# Turn style (# of stories) column into only digits (instead of \"2 stories\" it says \"2\")\n",
    "residential['style'] = residential['style'].str.extract('(^\\d*\\.*\\d*)', expand = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     }
    },
    "colab_type": "code",
    "id": "wqIJ4duEIFKY"
   },
   "outputs": [],
   "source": [
    "# Label respective properties with their appropriate title\n",
    "residential['type'] = 'Residential'\n",
    "commercial['type'] = 'Commercial'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     }
    },
    "colab_type": "code",
    "id": "abpR5qemIiCE"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>use_type</th>\n",
       "      <th>use_code</th>\n",
       "      <th>year_built</th>\n",
       "      <th>gross_area</th>\n",
       "      <th>story_height</th>\n",
       "      <th>number_of_stories</th>\n",
       "      <th>address</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>38.030637</td>\n",
       "      <td>-78.470637</td>\n",
       "      <td>R</td>\n",
       "      <td>Elementary Sch (Entire)</td>\n",
       "      <td>1930.0</td>\n",
       "      <td>815.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>402 11TH ST</td>\n",
       "      <td>Commercial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>Commercial</td>\n",
       "      <td>1963.0</td>\n",
       "      <td>1134.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>243 ZAN RD</td>\n",
       "      <td>Commercial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>R</td>\n",
       "      <td>Mini-warehouse</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>400.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>117 NORTH BAKER ST</td>\n",
       "      <td>Commercial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>R</td>\n",
       "      <td>Mini-warehouse</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>21600.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>702 WEST ST</td>\n",
       "      <td>Commercial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>38.034061</td>\n",
       "      <td>-78.489876</td>\n",
       "      <td>R</td>\n",
       "      <td>Mini-warehouse</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>22376.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>302 8TH ST</td>\n",
       "      <td>Commercial</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         lat        lon use_type                 use_code  year_built  \\\n",
       "0  38.030637 -78.470637        R  Elementary Sch (Entire)      1930.0   \n",
       "1        NaN        NaN        C               Commercial      1963.0   \n",
       "2        NaN        NaN        R           Mini-warehouse      2000.0   \n",
       "3        NaN        NaN        R           Mini-warehouse      2000.0   \n",
       "4  38.034061 -78.489876        R           Mini-warehouse      2000.0   \n",
       "\n",
       "   gross_area  story_height  number_of_stories             address        type  \n",
       "0       815.0          12.0                1.0         402 11TH ST  Commercial  \n",
       "1      1134.0          12.0                1.0          243 ZAN RD  Commercial  \n",
       "2       400.0          12.0                1.0  117 NORTH BAKER ST  Commercial  \n",
       "3     21600.0          18.0                2.0         702 WEST ST  Commercial  \n",
       "4     22376.0          18.0                2.0          302 8TH ST  Commercial  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "commercial.head()\n",
    "#commercial = commercial.drop(\"Unnamed: 0\", 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     },
     "base_uri": "https://localhost:8080/",
     "height": 426.0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 244.0,
     "status": "ok",
     "timestamp": 1.529856651475E12,
     "user": {
      "displayName": "Jackson Barkstrom",
      "photoUrl": "//lh5.googleusercontent.com/-vVdnq6QxZKc/AAAAAAAAAAI/AAAAAAAAAC8/hz-Hu2YlLus/s50-c-k-no/photo.jpg",
      "userId": "100905809459765639952"
     },
     "user_tz": 240.0
    },
    "id": "mvPH3D3pI1Jb",
    "outputId": "8b6e3ebe-458d-404e-c0e7-b685bb572f0c",
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>use_type</th>\n",
       "      <th>use_code</th>\n",
       "      <th>style</th>\n",
       "      <th>grade</th>\n",
       "      <th>ext_walls</th>\n",
       "      <th>roof</th>\n",
       "      <th>flooring</th>\n",
       "      <th>bsmt_type</th>\n",
       "      <th>...</th>\n",
       "      <th>total_rooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>half_bathrooms</th>\n",
       "      <th>full_bathrooms</th>\n",
       "      <th>basement_garage</th>\n",
       "      <th>sq_footage_finished_living</th>\n",
       "      <th>basement</th>\n",
       "      <th>finished_basement</th>\n",
       "      <th>address</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>38.064309</td>\n",
       "      <td>-78.488149</td>\n",
       "      <td>C</td>\n",
       "      <td>Duplex</td>\n",
       "      <td>2</td>\n",
       "      <td>B</td>\n",
       "      <td>Aluminum</td>\n",
       "      <td>Shingles</td>\n",
       "      <td>Other</td>\n",
       "      <td>Full Basement</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1388.0</td>\n",
       "      <td>694.0</td>\n",
       "      <td>694.0</td>\n",
       "      <td>243 ZAN RD</td>\n",
       "      <td>Residential</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>38.027193</td>\n",
       "      <td>-78.503804</td>\n",
       "      <td>R</td>\n",
       "      <td>Single Family</td>\n",
       "      <td>1</td>\n",
       "      <td>C</td>\n",
       "      <td>Vinyl</td>\n",
       "      <td>Shingles</td>\n",
       "      <td>Hardwood</td>\n",
       "      <td>Full Basement</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1020.0</td>\n",
       "      <td>1020.0</td>\n",
       "      <td>500.0</td>\n",
       "      <td>117 NORTH BAKER ST</td>\n",
       "      <td>Residential</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38.034770</td>\n",
       "      <td>-78.487978</td>\n",
       "      <td>R</td>\n",
       "      <td>Single Family</td>\n",
       "      <td>1</td>\n",
       "      <td>C</td>\n",
       "      <td>Vinyl</td>\n",
       "      <td>Shingles</td>\n",
       "      <td>Hardwood</td>\n",
       "      <td>Cellar</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>925.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>702 WEST ST</td>\n",
       "      <td>Residential</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>38.039464</td>\n",
       "      <td>-78.500183</td>\n",
       "      <td>R</td>\n",
       "      <td>Single Family</td>\n",
       "      <td>1.5</td>\n",
       "      <td>C</td>\n",
       "      <td>Stucco</td>\n",
       "      <td>Shingles</td>\n",
       "      <td>Hardwood</td>\n",
       "      <td>No Basement</td>\n",
       "      <td>...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1491.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>418 17TH ST</td>\n",
       "      <td>Residential</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-90.000000</td>\n",
       "      <td>-152.644271</td>\n",
       "      <td>R</td>\n",
       "      <td>Single Family-1 Conversion</td>\n",
       "      <td>1</td>\n",
       "      <td>C</td>\n",
       "      <td>Wood</td>\n",
       "      <td>Shingles</td>\n",
       "      <td>Hardwood</td>\n",
       "      <td>Full Basement</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>878.0</td>\n",
       "      <td>878.0</td>\n",
       "      <td>439.0</td>\n",
       "      <td>402 11TH ST</td>\n",
       "      <td>Residential</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         lat         lon use_type                    use_code style grade  \\\n",
       "0  38.064309  -78.488149        C                      Duplex     2     B   \n",
       "1  38.027193  -78.503804        R               Single Family     1     C   \n",
       "2  38.034770  -78.487978        R               Single Family     1     C   \n",
       "3  38.039464  -78.500183        R               Single Family   1.5     C   \n",
       "4 -90.000000 -152.644271        R  Single Family-1 Conversion     1     C   \n",
       "\n",
       "  ext_walls      roof  flooring      bsmt_type     ...      total_rooms  \\\n",
       "0  Aluminum  Shingles     Other  Full Basement     ...              9.0   \n",
       "1     Vinyl  Shingles  Hardwood  Full Basement     ...              5.0   \n",
       "2     Vinyl  Shingles  Hardwood         Cellar     ...              5.0   \n",
       "3    Stucco  Shingles  Hardwood    No Basement     ...              8.0   \n",
       "4      Wood  Shingles  Hardwood  Full Basement     ...              7.0   \n",
       "\n",
       "   bedrooms  half_bathrooms  full_bathrooms  basement_garage  \\\n",
       "0       5.0             1.0             3.0              0.0   \n",
       "1       2.0             0.0             2.0              0.0   \n",
       "2       3.0             0.0             1.0              0.0   \n",
       "3       5.0             0.0             1.0              0.0   \n",
       "4       3.0             0.0             2.0              0.0   \n",
       "\n",
       "   sq_footage_finished_living  basement  finished_basement  \\\n",
       "0                      1388.0     694.0              694.0   \n",
       "1                      1020.0    1020.0              500.0   \n",
       "2                       925.0     250.0                0.0   \n",
       "3                      1491.0       NaN                NaN   \n",
       "4                       878.0     878.0              439.0   \n",
       "\n",
       "              address         type  \n",
       "0          243 ZAN RD  Residential  \n",
       "1  117 NORTH BAKER ST  Residential  \n",
       "2         702 WEST ST  Residential  \n",
       "3         418 17TH ST  Residential  \n",
       "4         402 11TH ST  Residential  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "residential.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making Addresses Congruent (this is a simple, incomplete fix--some addresses have discrepencies such as \"15th st sw\" vs \"15th st\" that still need to be fixed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This just uses a dictionary to change all of the fire_late_cleaned addresses to abbreviations, like in the rest of the data. We make everything lowercase first, then do the dictionary. Then we merge."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make everything lowercase\n",
    "commercial.address = commercial.address.str.lower()\n",
    "residential.address = residential.address.str.lower()\n",
    "fire_late_cleaned.address = fire_late_cleaned.address.str.lower()\n",
    "fire_early.address = fire_early.address.str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define dictionary and apply it to all addresses for consistency\n",
    "translations = {'drive': 'dr', 'street': 'st', 'road': 'rd', 'avenue': 'ave', 'south' : 's', 'southwest': 'sw', \n",
    "                'north': 'n', 'east': 'e', 'west': 'w', 'northeast':'ne', 'northwest': 'nw', 'southeast':'se', \n",
    "                'circle':'cir', 'court': 'ct', 'lane':'ln', 'crossing': 'xing',\n",
    "               'vista':'vis', 'alley':'aly', 'anex': 'anx', 'arcade': 'arc', 'bend': 'bnd', 'bluff':'blf', \n",
    "               'bluffs': 'blfs', 'bottom': 'btm', 'boulevard': 'blvd','branch':'br', 'bridge':'brg', 'brook':'brk',\n",
    "               'bypass':'byp', 'canyon':'cyn', 'cape':'cpe', 'causeway':'cswy','center':'ctr', 'cliff':'clf', \n",
    "               'cliffs':'clfs', 'common':'cmn', 'commons':'cmns', 'corner':'cor', 'course':'crse','cove': 'cv',\n",
    "               'crescent':'cres', 'crest':'crest', 'crossroad':'xrd', 'crossroads':'xrds', 'estate':'est', \n",
    "               'expressway':'expy', 'extension':'ext','field':'fld','flats':'flts', 'forest': 'frst', \n",
    "                'forge':'frg', 'fork':'frk', 'fort':'ft', 'freeway':'fwy', 'garden':'gdn', 'gateway':'gtwy',\n",
    "               'glen':'gln', 'glens':'glns', 'green':'grn', 'grove':'grv', 'groves':'grvs', 'heights': 'hts', \n",
    "               'haven':'hvn', 'highway':'hwy', 'hill':'hl', 'hills':'hls', 'hollow':'holw', 'junction':'jct',\n",
    "               'key': 'key', 'knoll': 'knl', 'lake':'lk', 'lakes':'lks', 'meadow':'mdw', 'meadows':'mdws',\n",
    "               'mill':'ml', 'mount':'mt', 'mountain':'mtn', 'neck':'nck', 'parkway':'pkwy', 'passage':'psge',\n",
    "                'place':'pl', 'plain':'pln', 'plains':'plns', 'plaza':'plz', 'point':'pt', 'points':'pts',\n",
    "                'ridge':'rdg', 'ridges':'rdgs', 'roads':'rds', 'route':'rte', 'spring':'spg', 'springs':'spgs',\n",
    "                'square': 'sq', 'station':'sta', 'streets':'sts', 'terrace':'ter', 'throughway':'trwy',\n",
    "                'trace':'trc', 'track':'trk', 'trail':'trl', 'turnpike':'tpke', 'underpass':'upas', 'valley':'vly',\n",
    "                'view':'vw', 'village':'vlg'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply the dictionary to all addresses (looks pretty good for the most part)\n",
    "# also removes leading and trailing whitespace\n",
    "residential.address = residential.address.replace(translations, regex=True).str.strip()\n",
    "commercial.address = commercial.address.replace(translations, regex=True).str.strip()\n",
    "fire_late_cleaned.address = fire_late_cleaned.address.replace(translations, regex=True).str.strip()\n",
    "fire_early.address = fire_early.address.replace(translations, regex=True).str.strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KuhKpYjm1XWi"
   },
   "source": [
    "## Merge Fire Data to Commercial and Residential Data, one way for Training and another way for Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Vd5_gjvY-hSw"
   },
   "source": [
    "This is soley based on addresses, and we will do an exact match. In this case we are trying to see if past fire data can predict future fire data. For a true \"train\" dataset we would need to take a column with fires before 2014, and train on 2014-2016 fires, and then for a \"test\" have a column with fires before 2016, and a column for firest after 2016. Or something else like that.\n",
    "\n",
    "To start with we're just seeing if we can get any insights at all (since the project is about neural nets). One column for fires before 2016 (old data), one column for latest fire (old data), and one column for if there was a fire after 2016 (new data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-xfjC3FVa-Ae"
   },
   "source": [
    "First we will do the older data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1215 lee st           29\n",
       "500 1st st s #201     17\n",
       "1801 hydraulic rd     16\n",
       "1600 emmet st n       10\n",
       "1600 6th st se #23    10\n",
       "Name: address, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# this gives us what we need for one of the columns--number of fires before 2016\n",
    "fire_early.address.value_counts().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "address\n",
       "1 morton dr #100       2017-01-30\n",
       "1 university cir       2008-03-15\n",
       "10 university cir #1   2016-12-20\n",
       "100 alderman rd        2015-01-02\n",
       "100 e main st          2008-08-18\n",
       "Name: datetime, dtype: datetime64[ns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# this gives us what we need for the other column--latest recorded fire\n",
    "fire_early.groupby('address').datetime.max().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     }
    },
    "colab_type": "code",
    "id": "htcRyw26khet"
   },
   "outputs": [],
   "source": [
    "# first give every address a column for counts of fires before new fire data\n",
    "final_commercial = commercial.merge(pd.DataFrame(fire_early.address.value_counts()).rename(columns = {'address': 'fire_counts'}), \n",
    "                 left_on= 'address',right_index = True, how='left')\n",
    "final_residential = residential.merge(pd.DataFrame(fire_early.address.value_counts()).rename(columns = {'address': 'fire_counts'}), \n",
    "                 left_on= 'address',right_index = True, how='left')\n",
    "# fills na values with 0\n",
    "final_commercial.fire_counts = final_commercial.fire_counts.fillna(0).astype(np.int8)\n",
    "final_residential.fire_counts = final_residential.fire_counts.fillna(0).astype(np.int8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     }
    },
    "colab_type": "code",
    "id": "VPbDv8NbnOLq"
   },
   "outputs": [],
   "source": [
    "# next give every address a column for latest fire before new fire data\n",
    "# this is more because I think it might be useful, will probably with feature engineering later\n",
    "final_commercial = final_commercial.merge(pd.DataFrame(fire_early.groupby('address').datetime.max()).rename(columns = {'datetime': 'latest_fire'}), \n",
    "                 left_on= 'address',right_index = True, how='left')\n",
    "final_residential = final_residential.merge(pd.DataFrame(fire_early.groupby('address').datetime.max()).rename(columns = {'datetime': 'latest_fire'}), \n",
    "                 left_on= 'address',right_index = True, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# finally, give every address a column for number of fires in the new fire data.\n",
    "final_commercial = final_commercial.merge(pd.DataFrame(fire_late_cleaned.address.value_counts()).rename(columns = {'address': 'fire_result'}), \n",
    "                 left_on= 'address',right_index = True, how='left')\n",
    "final_residential = final_residential.merge(pd.DataFrame(fire_late_cleaned.address.value_counts()).rename(columns = {'address': 'fire_result'}), \n",
    "                 left_on= 'address',right_index = True, how='left')\n",
    "final_commercial.fire_result = final_commercial.fire_result.fillna(0).astype(np.int8)\n",
    "final_residential.fire_result = final_residential.fire_result.fillna(0).astype(np.int8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Preparation for Machine Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     }
    },
    "colab_type": "code",
    "id": "z2GbVBDx7CtP"
   },
   "outputs": [],
   "source": [
    "# Get rid of all columns where use_code displays \"Vacant Land\"--this is just not what we're looking for\n",
    "commercial_prepared   = final_commercial[final_commercial.use_code.str.contains(\"Vacant Land\") == False]\n",
    "residential_prepared = final_residential[final_residential.use_code.str.contains(\"Vacant Land\") == False]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Obviously, everything below can be modified a lot. \n",
    "\n",
    "One-hot encode and column normalize via pipeline, and don't use the less meaningful columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pipeline to Normalize or One-Hot Encode for Meangful Columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The discription below is for the columns of the residential dataset but the commercial dataset pretty much follows from this. Note that we have to figure out what to do with NaN values--they're everywhere! \n",
    "\n",
    "Stuff below just for reference. Didn't get around to using the information much, but dropped some columns bc of it.\n",
    "\n",
    "\n",
    "1) use_type (4 categories if we include nan)\n",
    "2) use_code: lots of specific values and not sure if they're useful\n",
    "2) style (7 categories if we merge '' with nan). Not sure what the numbers mean so should one hot?\n",
    "3) grade (not really sure what this means or if it's relevant at all)\n",
    "4) ext_walls: might be very helpful, should one hot, lots of columns but should be worth it\n",
    "5) flooring: same as above but less columns\n",
    "6) bsmt_type: extremely important, 8 categories including nan\n",
    "7) heating: may be helpful? a few different types\n",
    "8) fireplace: min 0 max 10, also a nan value\n",
    "9) year_built: extremely important. one value says 0 (not possible) so need to give a nan for 0 and nan. should normalize so smallest 0 largest 1? (upper bound to 2018, lower to 1700): lowest in this dataset is 1730, highest 2018.\n",
    "10) no_of_stories: min 0 max 3.75, should do a spectrum since it looks like we have all kinds of fractions here? although one hot is preferable I think we can get away with a spectrum. special place for nan?\n",
    "11) total_rooms: extreme outlier has 705, but other than that all below 23. there is a 0 and a nan, and I will treat those seperately since I don't know what they mean. I guess you can have a structure w/o a room?\n",
    "12) bedrooms: goes from 0 to 18 and nan\n",
    "13) half_bathrooms: 0 to 5 and nan. not very useful and can probably drop?\n",
    "14) full_bathrooms: 0 to 10 and nan. not that useful?\n",
    "15) basement_garage: 0 to 3 and nan, and then a few big values in the n00s. not sure what they mean\n",
    "16) sq_footage_finished_living: hugely important. max of ~8200, some nan and 0.\n",
    "17) basement: hugely important. square footage of the basement, I think, max of 5290 min 0 and nan.\n",
    "18) finished_basement is square footage of finished basement (so can make an unfinished_basement column with difference b/w basement and this). Max of 3790 and min 0 and nan.\n",
    "19) fire_counts: numbner of past fires that have occured, normalize based on max. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jwbarkstrom/anaconda3/lib/python3.6/site-packages/pandas/core/generic.py:4401: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self[name] = value\n",
      "/Users/jwbarkstrom/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# convert all the category values to numerical values for the residential data (so we can one-hot later)\n",
    "# note that NaN is given a value of -1, and we want to preserve it as a category so we add 1 to everything\n",
    "residential_prepared.use_type  = residential_prepared.use_type.astype(\"category\").cat.codes + 1\n",
    "residential_prepared.use_code  = residential_prepared.use_code.astype(\"category\").cat.codes + 1\n",
    "residential_prepared['style']     = residential_prepared['style'].astype(\"category\").cat.codes + 1\n",
    "residential_prepared.grade     = residential_prepared.grade.astype(\"category\").cat.codes + 1\n",
    "residential_prepared.ext_walls = residential_prepared.ext_walls.astype(\"category\").cat.codes + 1\n",
    "residential_prepared.flooring  = residential_prepared.flooring.astype(\"category\").cat.codes + 1\n",
    "residential_prepared.bsmt_type = residential_prepared.bsmt_type.astype(\"category\").cat.codes + 1\n",
    "residential_prepared.heating   = residential_prepared.heating.astype(\"category\").cat.codes + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert all the category values to numerical values for the commercial data (so we can one-hot later)\n",
    "# note that NaN is given a value of -1, and we want to preserve it as a category so we add 1 to everything\n",
    "commercial_prepared.use_type = commercial_prepared.use_type.astype(\"category\").cat.codes + 1\n",
    "commercial_prepared.use_code = commercial_prepared.use_code.astype(\"category\").cat.codes + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "residential_cat = residential_prepared[[\"use_type\", \"use_code\", \"style\", \"grade\", \"ext_walls\", \"flooring\", \"bsmt_type\", \"heating\"]]\n",
    "residential_num = residential_prepared[[\"year_built\", \"no_of_stories\", \"total_rooms\", \"bedrooms\", \"half_bathrooms\", \"full_bathrooms\", \"basement_garage\", \"sq_footage_finished_living\", \"basement\", \"finished_basement\", \"fire_counts\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "commercial_cat = commercial_prepared[[\"use_type\", \"use_code\"]]\n",
    "commercial_num = commercial_prepared[[\"year_built\", \"gross_area\", \"story_height\", \"number_of_stories\", \"fire_counts\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define class DataFrameSelector in order to use our pipeline\n",
    "\n",
    "class DataFrameSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, attribute_names):\n",
    "        self.attribute_names = attribute_names\n",
    "    def fit(self, X, y = None):\n",
    "        return self # nothing else to do here, just have to have a fit to fit with skleard\n",
    "    def transform(self, X):\n",
    "        return X[self.attribute_names].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run pipeline on commercial data\n",
    "num_attribs = list(commercial_num)\n",
    "cat_attribs = list(commercial_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_pipeline = Pipeline([\n",
    "    ('selector', DataFrameSelector(num_attribs)),\n",
    "    ('imputer', Imputer(strategy = \"median\")),\n",
    "    # note: will need to mess around with this strategy and see how it affects results\n",
    "    # can work on attribs_adder\n",
    "    # ('attribs_adder', CombinedAttributesAdder()),\n",
    "    # we need to use something like MinMaxScaler because it outputs values b/w 0 and 1\n",
    "    ('std_scaler', MinMaxScaler()),\n",
    "])\n",
    "\n",
    "cat_pipeline = Pipeline([\n",
    "    ('selector', DataFrameSelector(cat_attribs)),\n",
    "    # need to import this from future encoders!! how to do that I don't know!! There's a future_encoders.py\n",
    "    # file on the jupyter notebook for the textbook\n",
    "    ('cat_encoder', OneHotEncoder(sparse = False)),\n",
    "])\n",
    "\n",
    "full_pipeline = FeatureUnion(transformer_list=[\n",
    "    (\"num_pipeline\", num_pipeline),\n",
    "    (\"cat_pipeline\", cat_pipeline),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "commercial_ready_x = full_pipeline.fit_transform(commercial_prepared)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run pipeline on residential data \n",
    "num_attribs = list(residential_num)\n",
    "cat_attribs = list(residential_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cut and pasted from above so it's easier to run the code\n",
    "num_pipeline = Pipeline([\n",
    "    ('selector', DataFrameSelector(num_attribs)),\n",
    "    ('imputer', Imputer(strategy = \"median\")),\n",
    "    # note: will need to mess around with this strategy and see how it affects results\n",
    "    # can work on attribs_adder\n",
    "    # ('attribs_adder', CombinedAttributesAdder()),\n",
    "    ('std_scaler', StandardScaler()),\n",
    "])\n",
    "\n",
    "cat_pipeline = Pipeline([\n",
    "    ('selector', DataFrameSelector(cat_attribs)),\n",
    "    # need to import this from future encoders!! how to do that I don't know!! There's a future_encoders.py\n",
    "    # file on the jupyter notebook for the textbook\n",
    "    ('cat_encoder', OneHotEncoder(sparse = False)),\n",
    "])\n",
    "\n",
    "full_pipeline = FeatureUnion(transformer_list=[\n",
    "    (\"num_pipeline\", num_pipeline),\n",
    "    (\"cat_pipeline\", cat_pipeline),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "residential_ready_x = full_pipeline.fit_transform(residential_prepared)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Test Split "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Eventually, when we do this correctly, we will not have to train test split here because we will have already done so in our merge."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we take y to be whether or not there was a fire in the future (1 in 1 column if yes, otherwise 1 in other column)\n",
    "commercial_x  = commercial_ready_x\n",
    "commercial_y  = list(zip((commercial_prepared['fire_result'] > 0).astype(int), (commercial_prepared['fire_result'] == 0).astype(int)))\n",
    "residential_x = residential_ready_x\n",
    "residential_y = list(zip((residential_prepared['fire_result'] > 0).astype(int), (residential_prepared['fire_result'] == 0).astype(int)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "commercial_train_x, commercial_test_x, commercial_train_y, commercial_test_y = \\\n",
    "train_test_split(commercial_x, commercial_y, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "residential_train_x, residential_test_x, residential_train_y, residential_test_y = \\\n",
    "train_test_split(residential_x, residential_y, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1854, 82)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we have 1,854 addresses, with each address represented as 82 columns\n",
    "commercial_train_x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Machine Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural Net\n",
    "I use code where one can see the mathematics behind everything, with inspiration from the Gori textbook. I can't imagine it's ever going to be that \"sure\" of a fire, but we'll see. Starting with commercial data (smaller)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I reference http://adventuresinmachinelearning.com/tensorflow-dataset-tutorial/, https://towardsdatascience.com/how-to-use-dataset-in-tensorflow-c758ef9e4428"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 1000\n",
    "\n",
    "# placeholder to switch b/w batch sizes commented out\n",
    "BATCH_SIZE = 1000 # tf.placeholder(tf.int64)\n",
    "\n",
    "# set input/output variables\n",
    "x = tf.placeholder(tf.float32, [None, 153]) #datapoints for commercial are 82\n",
    "y = tf.placeholder(tf.float32, [None, 2]) # 2, one for no fire and one for fire\n",
    "# make them compatible with train and test\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((x, y)).batch(BATCH_SIZE).repeat()\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((x,y)).batch(BATCH_SIZE).repeat()\n",
    "\n",
    "train_data = (residential_train_x, residential_train_y)\n",
    "test_data = (residential_test_x, residential_test_y)\n",
    "\n",
    "# create the iterators from the dataset\n",
    "iter = tf.data.Iterator.from_structure(train_dataset.output_types, train_dataset.output_shapes)\n",
    "features, labels = iter.get_next()\n",
    "\n",
    "# initialization operations\n",
    "train_init_op = iter.make_initializer(train_dataset)\n",
    "test_init_op = iter.make_initializer(test_dataset)\n",
    "\n",
    "# define the weights and their initialization\n",
    "# here we go 82-->100-->1\n",
    "W1 = tf.Variable(tf.random_uniform([153, 100],minval=-0.1,\\\n",
    "                                                      maxval=0.1))\n",
    "b1 = tf.Variable(tf.random_uniform([100],minval=0,\\\n",
    "                                                      maxval=0.1))\n",
    "W2 = tf.Variable(tf.random_uniform([100, 2],minval=-0.1,\\\n",
    "                                                      maxval=0.1))\n",
    "b2 = tf.Variable(tf.zeros([1]))\n",
    "\n",
    "# define relu nonlinearity, carry out forward computation\n",
    "a1 = tf.matmul(x, W1) +b1             #activation of the input\n",
    "x2 = tf.nn.relu(a1)      #previously x2 = tf.sigmoid(a1)\n",
    "a2 = tf.matmul(x2, W2) + b2          #activation of the output\n",
    "\n",
    "# softmax processing for output layer and cross-entropy all happen at same time\n",
    "# we can essentially view the output of the softmax as percentage likelihood\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=a2))\n",
    "\n",
    "# this works because of how we formatted our y column\n",
    "correct_prediction = tf.equal(tf.argmax(a2, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "\n",
    "# learning as gradient descent on cross-entropy error function\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10325\n",
      "Accuracy: 0.951588\n"
     ]
    }
   ],
   "source": [
    "sess.run(train_init_op, feed_dict={ x: train_data[0], y: train_data[1]})\n",
    "for _ in range(EPOCHS):\n",
    "    sess.run(train_step, feed_dict={x: train_data[0], y: train_data[1]})\n",
    "results = [[1/(1+np.exp(-q)) for q in p] for p in sess.run(a2, feed_dict={x: train_data[0], y: train_data[1]})]\n",
    "print((sess.run(tf.argmax(a2, 1), feed_dict={x: train_data[0], y: train_data[1]}) == 1).sum())\n",
    "sess.run(test_init_op, feed_dict={ x: train_data[0], y: train_data[1]})\n",
    "print(\"Accuracy: %f\" % sess.run(accuracy,\\\n",
    "         feed_dict={ x: test_data[0], y: test_data[1]}))\n",
    "test_results = test_data[1]\n",
    "outputted_results = [[1/(1+np.exp(-q)) for q in p] for p in sess.run(a2, feed_dict={x: test_data[0], y: test_data[1]})]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = [item[0] for item in outputted_results]\n",
    "results = [item[0] for item in test_results]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_test = pd.DataFrame({'output':output, 'results':results})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jwbarkstrom/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.04596191726854892"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_test[output_test['output'] <0.2][output_test['output'] > 0.1]['results'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.02459016393442623"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_test[output_test['output'] <0.1]['results'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.06012269938650307"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_test[output_test['output'] >0.2]['results'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest Classifier (for comparison)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this works ok... low type 1 error (we're right about 1/3 of the time) but high type 2 error.\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# model\n",
    "forest =  RandomForestClassifier()\n",
    "\n",
    "# train (residential first)\n",
    "forest.fit(residential_train_x, residential_train_y)\n",
    "\n",
    "# predict\n",
    "forest_predictions = forest.predict(residential_test_x)\n",
    "\n",
    "y_actual = pd.Series(list(zip(*residential_test_y))[0])\n",
    "y_pred = pd.Series(list(zip(*forest_predictions))[0])\n",
    "\n",
    "from sklearn import metrics\n",
    "print(\"Accuracy:\",metrics.accuracy_score(y_actual, y_pred))\n",
    "\n",
    "confusion = pd.crosstab(y_actual, y_pred, rownames=['Actual:'], colnames = ['Predicted:'])\n",
    "\n",
    "print(confusion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YAe6MNg3-ZkZ"
   },
   "source": [
    "## Export Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     },
     "base_uri": "https://localhost:8080/",
     "height": 17.0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 535.0,
     "status": "ok",
     "timestamp": 1.529857554369E12,
     "user": {
      "displayName": "Jackson Barkstrom",
      "photoUrl": "//lh5.googleusercontent.com/-vVdnq6QxZKc/AAAAAAAAAAI/AAAAAAAAAC8/hz-Hu2YlLus/s50-c-k-no/photo.jpg",
      "userId": "100905809459765639952"
     },
     "user_tz": 240.0
    },
    "id": "sQCnMRSfxbo2",
    "outputId": "4efc454d-3c49-42ec-d523-f4de1f3a566f"
   },
   "outputs": [],
   "source": [
    "# Export data as CSVs \n",
    "# Note: the javascript code is just because we're using collaboratory,\n",
    "# if not in collaboratory we could also use a basic pandas function to output the CSV\n",
    "\n",
    "from IPython.display import Javascript\n",
    "js_download = \"\"\"\n",
    "var csv = '%s';\n",
    "\n",
    "var filename = 'results.csv';\n",
    "var blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' });\n",
    "if (navigator.msSaveBlob) { // IE 10+\n",
    "    navigator.msSaveBlob(blob, filename);\n",
    "} else {\n",
    "    var link = document.createElement(\"a\");\n",
    "    if (link.download !== undefined) { // feature detection\n",
    "        // Browsers that support HTML5 download attribute\n",
    "        var url = URL.createObjectURL(blob);\n",
    "        link.setAttribute(\"href\", url);\n",
    "        link.setAttribute(\"download\", filename);\n",
    "        link.style.visibility = 'hidden';\n",
    "        document.body.appendChild(link);\n",
    "        link.click();\n",
    "        document.body.removeChild(link);\n",
    "    }\n",
    "}\n",
    "\"\"\" % final_commercial.to_csv(index=False).replace('\\n','\\\\n').replace(\"'\",\"\\'\")\n",
    "\n",
    "Javascript(js_download)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0.0
     },
     "base_uri": "https://localhost:8080/",
     "height": 17.0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2287.0,
     "status": "ok",
     "timestamp": 1.529857556846E12,
     "user": {
      "displayName": "Jackson Barkstrom",
      "photoUrl": "//lh5.googleusercontent.com/-vVdnq6QxZKc/AAAAAAAAAAI/AAAAAAAAAC8/hz-Hu2YlLus/s50-c-k-no/photo.jpg",
      "userId": "100905809459765639952"
     },
     "user_tz": 240.0
    },
    "id": "I-7icsQyUxtG",
    "outputId": "e28a8086-e87d-4188-9e01-75a213e23903"
   },
   "outputs": [],
   "source": [
    "from IPython.display import Javascript\n",
    "js_download = \"\"\"\n",
    "var csv = '%s';\n",
    "\n",
    "var filename = 'results.csv';\n",
    "var blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' });\n",
    "if (navigator.msSaveBlob) { // IE 10+\n",
    "    navigator.msSaveBlob(blob, filename);\n",
    "} else {\n",
    "    var link = document.createElement(\"a\");\n",
    "    if (link.download !== undefined) { // feature detection\n",
    "        // Browsers that support HTML5 download attribute\n",
    "        var url = URL.createObjectURL(blob);\n",
    "        link.setAttribute(\"href\", url);\n",
    "        link.setAttribute(\"download\", filename);\n",
    "        link.style.visibility = 'hidden';\n",
    "        document.body.appendChild(link);\n",
    "        link.click();\n",
    "        document.body.removeChild(link);\n",
    "    }\n",
    "}\n",
    "\"\"\" % final_residential.to_csv(index=False).replace('\\n','\\\\n').replace(\"'\",\"\\'\")\n",
    "\n",
    "Javascript(js_download)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "Fire_merging.ipynb",
   "provenance": [
    {
     "file_id": "1jzlRGJdctEIolW920msyi7Li2eacD6-o",
     "timestamp": 1.529436192062E12
    }
   ],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}