HealthClusters.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "import os\n",
    "from sklearn import metrics\n",
    "from scipy.spatial.distance import cdist\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.mixture import GaussianMixture\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import silhouette_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(os.getcwd()+\"/Data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set max columns\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_row', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import data sets\n",
    "socio = pd.read_csv('Census_Data_-_Selected_socioeconomic_indicators_in_Chicago__2008___2012.csv')\n",
    "health = pd.read_csv('Public_Health_Statistics-_Selected_public_health_indicators_by_Chicago_community_area.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "socio = socio.loc[0:76] #filter out last nan row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#merge data sets\n",
    "sociohealth = pd.merge(left=health,right=socio,left_index=True,right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Community Area</th>\n",
       "      <th>Community Area Name</th>\n",
       "      <th>Birth Rate</th>\n",
       "      <th>General Fertility Rate</th>\n",
       "      <th>Low Birth Weight</th>\n",
       "      <th>Prenatal Care Beginning in First Trimester</th>\n",
       "      <th>Preterm Births</th>\n",
       "      <th>Teen Birth Rate</th>\n",
       "      <th>Assault (Homicide)</th>\n",
       "      <th>Breast cancer in females</th>\n",
       "      <th>Cancer (All Sites)</th>\n",
       "      <th>Colorectal Cancer</th>\n",
       "      <th>Diabetes-related</th>\n",
       "      <th>Firearm-related</th>\n",
       "      <th>Infant Mortality Rate</th>\n",
       "      <th>Lung Cancer</th>\n",
       "      <th>Prostate Cancer in Males</th>\n",
       "      <th>Stroke (Cerebrovascular Disease)</th>\n",
       "      <th>Childhood Blood Lead Level Screening</th>\n",
       "      <th>Childhood Lead Poisoning</th>\n",
       "      <th>Gonorrhea in Females</th>\n",
       "      <th>Gonorrhea in Males</th>\n",
       "      <th>Tuberculosis</th>\n",
       "      <th>Below Poverty Level</th>\n",
       "      <th>Crowded Housing</th>\n",
       "      <th>Dependency</th>\n",
       "      <th>No High School Diploma</th>\n",
       "      <th>Per Capita Income</th>\n",
       "      <th>Unemployment</th>\n",
       "      <th>Community Area Number</th>\n",
       "      <th>COMMUNITY AREA NAME</th>\n",
       "      <th>PERCENT OF HOUSING CROWDED</th>\n",
       "      <th>PERCENT HOUSEHOLDS BELOW POVERTY</th>\n",
       "      <th>PERCENT AGED 16+ UNEMPLOYED</th>\n",
       "      <th>PERCENT AGED 25+ WITHOUT HIGH SCHOOL DIPLOMA</th>\n",
       "      <th>PERCENT AGED UNDER 18 OR OVER 64</th>\n",
       "      <th>PER CAPITA INCOME</th>\n",
       "      <th>HARDSHIP INDEX</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Rogers Park</td>\n",
       "      <td>16.4</td>\n",
       "      <td>62.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>11.2</td>\n",
       "      <td>40.8</td>\n",
       "      <td>7.7</td>\n",
       "      <td>23.3</td>\n",
       "      <td>176.9</td>\n",
       "      <td>25.3</td>\n",
       "      <td>77.1</td>\n",
       "      <td>5.2</td>\n",
       "      <td>6.4</td>\n",
       "      <td>36.7</td>\n",
       "      <td>21.7</td>\n",
       "      <td>33.7</td>\n",
       "      <td>364.7</td>\n",
       "      <td>0.5</td>\n",
       "      <td>322.5</td>\n",
       "      <td>423.3</td>\n",
       "      <td>11.4</td>\n",
       "      <td>22.7</td>\n",
       "      <td>7.9</td>\n",
       "      <td>28.8</td>\n",
       "      <td>18.1</td>\n",
       "      <td>23714</td>\n",
       "      <td>7.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rogers Park</td>\n",
       "      <td>7.7</td>\n",
       "      <td>23.6</td>\n",
       "      <td>8.7</td>\n",
       "      <td>18.2</td>\n",
       "      <td>27.5</td>\n",
       "      <td>23939</td>\n",
       "      <td>39.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>West Ridge</td>\n",
       "      <td>17.3</td>\n",
       "      <td>83.3</td>\n",
       "      <td>8.1</td>\n",
       "      <td>71.1</td>\n",
       "      <td>8.3</td>\n",
       "      <td>29.9</td>\n",
       "      <td>5.8</td>\n",
       "      <td>20.2</td>\n",
       "      <td>155.9</td>\n",
       "      <td>17.3</td>\n",
       "      <td>60.5</td>\n",
       "      <td>3.7</td>\n",
       "      <td>5.1</td>\n",
       "      <td>36.0</td>\n",
       "      <td>14.2</td>\n",
       "      <td>34.7</td>\n",
       "      <td>331.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>205.7</td>\n",
       "      <td>8.9</td>\n",
       "      <td>15.1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>38.3</td>\n",
       "      <td>19.6</td>\n",
       "      <td>21375</td>\n",
       "      <td>7.9</td>\n",
       "      <td>2.0</td>\n",
       "      <td>West Ridge</td>\n",
       "      <td>7.8</td>\n",
       "      <td>17.2</td>\n",
       "      <td>8.8</td>\n",
       "      <td>20.8</td>\n",
       "      <td>38.5</td>\n",
       "      <td>23040</td>\n",
       "      <td>46.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Uptown</td>\n",
       "      <td>13.1</td>\n",
       "      <td>50.5</td>\n",
       "      <td>8.3</td>\n",
       "      <td>77.7</td>\n",
       "      <td>10.3</td>\n",
       "      <td>35.1</td>\n",
       "      <td>5.4</td>\n",
       "      <td>21.3</td>\n",
       "      <td>183.3</td>\n",
       "      <td>20.5</td>\n",
       "      <td>80.0</td>\n",
       "      <td>4.6</td>\n",
       "      <td>6.5</td>\n",
       "      <td>50.5</td>\n",
       "      <td>25.2</td>\n",
       "      <td>41.7</td>\n",
       "      <td>353.7</td>\n",
       "      <td>0.5</td>\n",
       "      <td>170.8</td>\n",
       "      <td>468.7</td>\n",
       "      <td>13.6</td>\n",
       "      <td>22.7</td>\n",
       "      <td>4.6</td>\n",
       "      <td>22.2</td>\n",
       "      <td>13.6</td>\n",
       "      <td>32355</td>\n",
       "      <td>7.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Uptown</td>\n",
       "      <td>3.8</td>\n",
       "      <td>24.0</td>\n",
       "      <td>8.9</td>\n",
       "      <td>11.8</td>\n",
       "      <td>22.2</td>\n",
       "      <td>35787</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Lincoln Square</td>\n",
       "      <td>17.1</td>\n",
       "      <td>61.0</td>\n",
       "      <td>8.1</td>\n",
       "      <td>80.5</td>\n",
       "      <td>9.7</td>\n",
       "      <td>38.4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>21.7</td>\n",
       "      <td>153.2</td>\n",
       "      <td>8.6</td>\n",
       "      <td>55.4</td>\n",
       "      <td>6.1</td>\n",
       "      <td>3.8</td>\n",
       "      <td>43.1</td>\n",
       "      <td>27.6</td>\n",
       "      <td>36.9</td>\n",
       "      <td>273.3</td>\n",
       "      <td>0.4</td>\n",
       "      <td>98.8</td>\n",
       "      <td>195.5</td>\n",
       "      <td>8.5</td>\n",
       "      <td>9.5</td>\n",
       "      <td>3.1</td>\n",
       "      <td>25.6</td>\n",
       "      <td>12.5</td>\n",
       "      <td>35503</td>\n",
       "      <td>6.8</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Lincoln Square</td>\n",
       "      <td>3.4</td>\n",
       "      <td>10.9</td>\n",
       "      <td>8.2</td>\n",
       "      <td>13.4</td>\n",
       "      <td>25.5</td>\n",
       "      <td>37524</td>\n",
       "      <td>17.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>North Center</td>\n",
       "      <td>22.4</td>\n",
       "      <td>76.2</td>\n",
       "      <td>9.1</td>\n",
       "      <td>80.4</td>\n",
       "      <td>9.8</td>\n",
       "      <td>8.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>16.6</td>\n",
       "      <td>152.1</td>\n",
       "      <td>26.1</td>\n",
       "      <td>49.8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.7</td>\n",
       "      <td>42.4</td>\n",
       "      <td>15.1</td>\n",
       "      <td>41.6</td>\n",
       "      <td>178.1</td>\n",
       "      <td>0.9</td>\n",
       "      <td>85.4</td>\n",
       "      <td>188.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>7.1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>25.5</td>\n",
       "      <td>5.4</td>\n",
       "      <td>51615</td>\n",
       "      <td>4.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>North Center</td>\n",
       "      <td>0.3</td>\n",
       "      <td>7.5</td>\n",
       "      <td>5.2</td>\n",
       "      <td>4.5</td>\n",
       "      <td>26.2</td>\n",
       "      <td>57123</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Community Area Community Area Name  Birth Rate  General Fertility Rate  \\\n",
       "0               1         Rogers Park        16.4                    62.0   \n",
       "1               2          West Ridge        17.3                    83.3   \n",
       "2               3              Uptown        13.1                    50.5   \n",
       "3               4      Lincoln Square        17.1                    61.0   \n",
       "4               5        North Center        22.4                    76.2   \n",
       "\n",
       "   Low Birth Weight  Prenatal Care Beginning in First Trimester  \\\n",
       "0              11.0                                        73.0   \n",
       "1               8.1                                        71.1   \n",
       "2               8.3                                        77.7   \n",
       "3               8.1                                        80.5   \n",
       "4               9.1                                        80.4   \n",
       "\n",
       "   Preterm Births  Teen Birth Rate  Assault (Homicide)  \\\n",
       "0            11.2             40.8                 7.7   \n",
       "1             8.3             29.9                 5.8   \n",
       "2            10.3             35.1                 5.4   \n",
       "3             9.7             38.4                 5.0   \n",
       "4             9.8              8.4                 1.0   \n",
       "\n",
       "   Breast cancer in females  Cancer (All Sites)  Colorectal Cancer  \\\n",
       "0                      23.3               176.9               25.3   \n",
       "1                      20.2               155.9               17.3   \n",
       "2                      21.3               183.3               20.5   \n",
       "3                      21.7               153.2                8.6   \n",
       "4                      16.6               152.1               26.1   \n",
       "\n",
       "   Diabetes-related  Firearm-related  Infant Mortality Rate  Lung Cancer  \\\n",
       "0              77.1              5.2                    6.4         36.7   \n",
       "1              60.5              3.7                    5.1         36.0   \n",
       "2              80.0              4.6                    6.5         50.5   \n",
       "3              55.4              6.1                    3.8         43.1   \n",
       "4              49.8              1.0                    2.7         42.4   \n",
       "\n",
       "   Prostate Cancer in Males  Stroke (Cerebrovascular Disease)  \\\n",
       "0                      21.7                              33.7   \n",
       "1                      14.2                              34.7   \n",
       "2                      25.2                              41.7   \n",
       "3                      27.6                              36.9   \n",
       "4                      15.1                              41.6   \n",
       "\n",
       "   Childhood Blood Lead Level Screening  Childhood Lead Poisoning  \\\n",
       "0                                 364.7                       0.5   \n",
       "1                                 331.4                       1.0   \n",
       "2                                 353.7                       0.5   \n",
       "3                                 273.3                       0.4   \n",
       "4                                 178.1                       0.9   \n",
       "\n",
       "   Gonorrhea in Females Gonorrhea in Males  Tuberculosis  Below Poverty Level  \\\n",
       "0                 322.5              423.3          11.4                 22.7   \n",
       "1                 141.0              205.7           8.9                 15.1   \n",
       "2                 170.8              468.7          13.6                 22.7   \n",
       "3                  98.8              195.5           8.5                  9.5   \n",
       "4                  85.4              188.6           1.9                  7.1   \n",
       "\n",
       "   Crowded Housing  Dependency  No High School Diploma  Per Capita Income  \\\n",
       "0              7.9        28.8                    18.1              23714   \n",
       "1              7.0        38.3                    19.6              21375   \n",
       "2              4.6        22.2                    13.6              32355   \n",
       "3              3.1        25.6                    12.5              35503   \n",
       "4              0.2        25.5                     5.4              51615   \n",
       "\n",
       "   Unemployment  Community Area Number COMMUNITY AREA NAME  \\\n",
       "0           7.5                    1.0         Rogers Park   \n",
       "1           7.9                    2.0          West Ridge   \n",
       "2           7.7                    3.0              Uptown   \n",
       "3           6.8                    4.0      Lincoln Square   \n",
       "4           4.5                    5.0        North Center   \n",
       "\n",
       "   PERCENT OF HOUSING CROWDED  PERCENT HOUSEHOLDS BELOW POVERTY  \\\n",
       "0                         7.7                              23.6   \n",
       "1                         7.8                              17.2   \n",
       "2                         3.8                              24.0   \n",
       "3                         3.4                              10.9   \n",
       "4                         0.3                               7.5   \n",
       "\n",
       "   PERCENT AGED 16+ UNEMPLOYED  PERCENT AGED 25+ WITHOUT HIGH SCHOOL DIPLOMA  \\\n",
       "0                          8.7                                          18.2   \n",
       "1                          8.8                                          20.8   \n",
       "2                          8.9                                          11.8   \n",
       "3                          8.2                                          13.4   \n",
       "4                          5.2                                           4.5   \n",
       "\n",
       "   PERCENT AGED UNDER 18 OR OVER 64  PER CAPITA INCOME   HARDSHIP INDEX  \n",
       "0                              27.5               23939            39.0  \n",
       "1                              38.5               23040            46.0  \n",
       "2                              22.2               35787            20.0  \n",
       "3                              25.5               37524            17.0  \n",
       "4                              26.2               57123             6.0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sociohealth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#prep data for clustering\n",
    "sociohealth.drop(['Community Area Number','COMMUNITY AREA NAME'],axis=1,inplace=True)\n",
    "sociohealth.set_index(['Community Area','Community Area Name'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#clean up gonorrhea columns\n",
    "sociohealth['Gonorrhea in Females'].fillna(0,inplace=True)\n",
    "sociohealth['Gonorrhea in Males'].replace({\".\":0},inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert object to float before data preprocessing\n",
    "sociohealth['Gonorrhea in Males'] = sociohealth['Gonorrhea in Males'].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fill any remaining nans \n",
    "sociohealth.fillna(0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/nfshome/cb4102/.conda/envs/deeplearn/lib/python3.5/site-packages/sklearn/preprocessing/data.py:617: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/nfshome/cb4102/.conda/envs/deeplearn/lib/python3.5/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    }
   ],
   "source": [
    "#standard scale data\n",
    "scaler = StandardScaler()\n",
    "sociohealth_scaled = scaler.fit_transform(sociohealth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cluster Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For n_clusters = 2, the average silhouette_score is : 0.3423426675859131\n",
      "For n_clusters = 3, the average silhouette_score is : 0.30001775456499347\n",
      "For n_clusters = 4, the average silhouette_score is : 0.25738426424833316\n",
      "For n_clusters = 5, the average silhouette_score is : 0.18869454425413498\n",
      "For n_clusters = 6, the average silhouette_score is : 0.1691462954114385\n",
      "For n_clusters = 7, the average silhouette_score is : 0.17733050162794609\n",
      "For n_clusters = 8, the average silhouette_score is : 0.15274412208363117\n",
      "For n_clusters = 9, the average silhouette_score is : 0.15125150953806152\n",
      "For n_clusters = 10, the average silhouette_score is : 0.18646196106878132\n",
      "For n_clusters = 11, the average silhouette_score is : 0.1660680413855916\n",
      "For n_clusters = 12, the average silhouette_score is : 0.16626658702230337\n",
      "For n_clusters = 13, the average silhouette_score is : 0.15070415180143923\n",
      "For n_clusters = 14, the average silhouette_score is : 0.14279914283363682\n"
     ]
    }
   ],
   "source": [
    "#test kmeans model across k range\n",
    "cluster_test = [i for i in range(2,15)] #cluster range to iterate over\n",
    "scores = []\n",
    "elbows = []\n",
    "for i in cluster_test:\n",
    "    cluster_mod = KMeans(n_clusters=i,random_state=13).fit(sociohealth_scaled)\n",
    "    silhouette_scores = silhouette_score(sociohealth_scaled,cluster_mod.labels_)\n",
    "    print(\"For n_clusters = {},\".format(i)+\" the average silhouette_score is : {}\" \\\n",
    "          .format(silhouette_scores))\n",
    "    \n",
    "    scores.append(silhouette_scores)\n",
    "    \n",
    "    elbow = sum(np.min(cdist(sociohealth_scaled, cluster_mod.cluster_centers_, \n",
    "                        'euclidean'), axis=1)) / sociohealth_scaled.shape[0]\n",
    "    elbows.append(elbow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For n_clusters = 2, the average silhouette_score is : 0.3423426675859131\n",
      "For n_clusters = 3, the average silhouette_score is : 0.1730497013159215\n",
      "For n_clusters = 4, the average silhouette_score is : 0.23252797761785282\n",
      "For n_clusters = 5, the average silhouette_score is : 0.20215898879828037\n",
      "For n_clusters = 6, the average silhouette_score is : 0.16235202493280068\n",
      "For n_clusters = 7, the average silhouette_score is : 0.1403408602384486\n",
      "For n_clusters = 8, the average silhouette_score is : 0.19245818719182162\n",
      "For n_clusters = 9, the average silhouette_score is : 0.18273268228460193\n",
      "For n_clusters = 10, the average silhouette_score is : 0.18646196106878132\n",
      "For n_clusters = 11, the average silhouette_score is : 0.14891106279035085\n",
      "For n_clusters = 12, the average silhouette_score is : 0.1482273891720097\n",
      "For n_clusters = 13, the average silhouette_score is : 0.13357318302896723\n",
      "For n_clusters = 14, the average silhouette_score is : 0.1286930487853086\n"
     ]
    }
   ],
   "source": [
    "cluster_test = [i for i in range(2,15)] #cluster range to iterate over\n",
    "\n",
    "for i in cluster_test:\n",
    "    cluster_mod = GaussianMixture(n_components=i,random_state=13).fit(sociohealth_scaled)\n",
    "    silhouette_scores = silhouette_score(sociohealth_scaled,cluster_mod.predict(sociohealth_scaled))\n",
    "    print(\"For n_clusters = {},\".format(i)+\" the average silhouette_score is : {}\".format(silhouette_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For n_clusters = 2, the average silhouette_score is : 0.2646583542194551\n",
      "For n_clusters = 3, the average silhouette_score is : 0.2584107004411183\n",
      "For n_clusters = 4, the average silhouette_score is : 0.23130301102368683\n",
      "For n_clusters = 5, the average silhouette_score is : 0.020518975356178124\n",
      "For n_clusters = 6, the average silhouette_score is : -0.043730953876489446\n",
      "For n_clusters = 7, the average silhouette_score is : -0.05555902042428653\n",
      "For n_clusters = 8, the average silhouette_score is : -0.06101447975239741\n",
      "For n_clusters = 9, the average silhouette_score is : -0.0843913899743957\n",
      "For n_clusters = 10, the average silhouette_score is : -0.09321769309941778\n",
      "For n_clusters = 11, the average silhouette_score is : -0.11890609222707499\n",
      "For n_clusters = 12, the average silhouette_score is : -0.15635285012160738\n",
      "For n_clusters = 13, the average silhouette_score is : -0.16569252974731763\n",
      "For n_clusters = 14, the average silhouette_score is : -0.1920367918524907\n"
     ]
    }
   ],
   "source": [
    "cluster_test = [i for i in range(2,15)] #cluster range to iterate over\n",
    "\n",
    "for i in cluster_test:\n",
    "    cluster_mod = AgglomerativeClustering(n_clusters=i,linkage='single').fit(sociohealth_scaled)\n",
    "    silhouette_scores = silhouette_score(sociohealth_scaled,cluster_mod.labels_)\n",
    "    print(\"For n_clusters = {},\".format(i)+\" the average silhouette_score is : {}\".format(silhouette_scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use Kmeans = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster_mod = KMeans(n_clusters=4,random_state=13).fit(sociohealth_scaled)\n",
    "cluster_labels = pd.DataFrame(cluster_mod.labels_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "sociohealth.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#add cluster labels to crime data \n",
    "sociohealth_cluster = pd.merge(left=sociohealth,right=cluster_labels,\n",
    "                         how='left',left_index=True,right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import community shape files\n",
    "community_geo = gpd.read_file('Boundaries - Community Areas (current).geojson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert merge column to appropriate data type\n",
    "community_geo['area_numbe'] = community_geo['area_numbe'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#merge df to get geometry\n",
    "sociohealth_cluster = pd.merge(left=sociohealth_cluster,right=community_geo,how='left',\n",
    "                         left_on='Community Area',right_on='area_numbe')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop unneeded columns\n",
    "sociohealth_cluster.drop(['community','area','shape_area','perimeter','area_num_1',\n",
    "'area_numbe','comarea_id','comarea','shape_len'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#output data\n",
    "sociohealth_cluster.to_csv('sociohealth_clusters.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Anomaly Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cluster_mod.fit_predict(sociohealth_scaled)\n",
    "kmeans_df = pd.DataFrame(cluster_mod.transform(sociohealth_scaled))\n",
    "kmeans_df = pd.concat((kmeans_df,pd.DataFrame(score)),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeans_df.columns =[0,1,2,3,'cluster']\n",
    "kmeans_df['dist_to_clus'] = kmeans_df.apply(lambda x: x[int(x[\"cluster\"])],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#rename columns\n",
    "kmeans_df.rename({0:'dist_0',1:'dist_1',2:'dist_2',3:\"dist_3\"},axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "health_cluster_w_anoms = pd.merge(left=sociohealth_cluster,right=kmeans_df,\n",
    "                                 left_index=True,right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "health_cluster_w_anoms.to_csv('sociohealth_clus_anomalies.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deeplearn",
   "language": "python",
   "name": "deeplearn"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}