explore_compas.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filepath = 'data/compas-scores-two-years.csv'\n",
    "\n",
    "try:\n",
    "    df = pd.read_csv(filepath, index_col='id')\n",
    "\n",
    "except IOError as err:\n",
    "    print(\"IOError: {}\".format(err))\n",
    "    print(\"To use this class, please download the following file:\")\n",
    "    print(\"\\n\\thttps://raw.githubusercontent.com/propublica/compas-analysis/master/compas-scores-two-years.csv\")\n",
    "    print(\"\\nand place it, as-is, in the folder:\")\n",
    "    print(\"\\n\\t{}\\n\".format(os.path.abspath(os.path.join(\n",
    "       os.path.abspath(__file__), '..', '..', 'data', 'raw', 'compas'))))\n",
    "    import sys\n",
    "    sys.exit(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['name', 'first', 'last', 'compas_screening_date', 'sex', 'dob', 'age',\n",
       "       'age_cat', 'race', 'juv_fel_count', 'decile_score', 'juv_misd_count',\n",
       "       'juv_other_count', 'priors_count', 'days_b_screening_arrest',\n",
       "       'c_jail_in', 'c_jail_out', 'c_case_number', 'c_offense_date',\n",
       "       'c_arrest_date', 'c_days_from_compas', 'c_charge_degree',\n",
       "       'c_charge_desc', 'is_recid', 'r_case_number', 'r_charge_degree',\n",
       "       'r_days_from_arrest', 'r_offense_date', 'r_charge_desc', 'r_jail_in',\n",
       "       'r_jail_out', 'violent_recid', 'is_violent_recid', 'vr_case_number',\n",
       "       'vr_charge_degree', 'vr_offense_date', 'vr_charge_desc',\n",
       "       'type_of_assessment', 'decile_score.1', 'score_text', 'screening_date',\n",
       "       'v_type_of_assessment', 'v_decile_score', 'v_score_text',\n",
       "       'v_screening_date', 'in_custody', 'out_custody', 'priors_count.1',\n",
       "       'start', 'end', 'event', 'two_year_recid'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "1             Aggravated Assault w/Firearm\n",
       "3           Felony Battery w/Prior Convict\n",
       "4                    Possession of Cocaine\n",
       "5                   Possession of Cannabis\n",
       "6                    arrest case no charge\n",
       "7                                  Battery\n",
       "8                Possession Burglary Tools\n",
       "9                    arrest case no charge\n",
       "10                                 Battery\n",
       "13                                   other\n",
       "14                 Poss 3,4 MDMA (Ecstasy)\n",
       "15                                 Battery\n",
       "16                                 Battery\n",
       "18          Poss3,4 Methylenedioxymethcath\n",
       "19                   arrest case no charge\n",
       "20          Felony Driving While Lic Suspd\n",
       "21                                 Battery\n",
       "22           Grand Theft in the 3rd Degree\n",
       "23           Driving While License Revoked\n",
       "24                    Possession Of Heroin\n",
       "25                   arrest case no charge\n",
       "26           Battery on Law Enforc Officer\n",
       "27           Possession Of Methamphetamine\n",
       "28                                   other\n",
       "30                                   other\n",
       "32            Susp Drivers Lic 1st Offense\n",
       "33                                 Battery\n",
       "37              Carrying Concealed Firearm\n",
       "38                   arrest case no charge\n",
       "39           Pos Cannabis W/Intent Sel/Del\n",
       "                       ...                \n",
       "10962                              Battery\n",
       "10963                arrest case no charge\n",
       "10964        Driving While License Revoked\n",
       "10965                arrest case no charge\n",
       "10966                              Battery\n",
       "10967                   Disorderly Conduct\n",
       "10969                arrest case no charge\n",
       "10971                arrest case no charge\n",
       "10972        Pos Cannabis W/Intent Sel/Del\n",
       "10975       Felony Batt(Great Bodily Harm)\n",
       "10976                              Battery\n",
       "10977                              Battery\n",
       "10979                              Battery\n",
       "10980                arrest case no charge\n",
       "10981       Felony Driving While Lic Suspd\n",
       "10982        Possession Of Methamphetamine\n",
       "10984                              Battery\n",
       "10985                Possession of Cocaine\n",
       "10987          Felony Battery (Dom Strang)\n",
       "10988         Uttering a Forged Instrument\n",
       "10989                              Battery\n",
       "10990                arrest case no charge\n",
       "10992          Grand Theft (Motor Vehicle)\n",
       "10994    Possess Cannabis/20 Grams Or Less\n",
       "10995                Possession of Cocaine\n",
       "10996                     Deliver Cannabis\n",
       "10997        Leaving the Scene of Accident\n",
       "10999        Aggravated Battery / Pregnant\n",
       "11000        Battery on Law Enforc Officer\n",
       "11001                                other\n",
       "Name: c_charge_desc, Length: 7214, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['c_charge_desc'].value_counts()\n",
    "\n",
    "# recode ones with counts below 10 to Other\n",
    "df['c_charge_desc'] = np.where(df.groupby('c_charge_desc')['c_charge_desc'].transform('size') < 10, 'other', df['c_charge_desc'])\n",
    "df['c_charge_desc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7214, 89)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one_hot = pd.get_dummies(df['c_charge_desc'], prefix='charge')\n",
    "one_hot.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.concat([df, one_hot], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "1         1\n",
       "3         3\n",
       "4         4\n",
       "5         8\n",
       "6         1\n",
       "7         1\n",
       "8         6\n",
       "9         4\n",
       "10        1\n",
       "13        3\n",
       "14        4\n",
       "15        6\n",
       "16        1\n",
       "18        4\n",
       "19        1\n",
       "20        3\n",
       "21        1\n",
       "22       10\n",
       "23        5\n",
       "24        3\n",
       "25        6\n",
       "26        9\n",
       "27        2\n",
       "28        4\n",
       "30        4\n",
       "32        1\n",
       "33        3\n",
       "37        3\n",
       "38        3\n",
       "39        7\n",
       "         ..\n",
       "10962     6\n",
       "10963     4\n",
       "10964     4\n",
       "10965     4\n",
       "10966     1\n",
       "10967     2\n",
       "10969     3\n",
       "10971     2\n",
       "10972     2\n",
       "10975     2\n",
       "10976     1\n",
       "10977     7\n",
       "10979     1\n",
       "10980     2\n",
       "10981     5\n",
       "10982     8\n",
       "10984     7\n",
       "10985    10\n",
       "10987     2\n",
       "10988     3\n",
       "10989     4\n",
       "10990    10\n",
       "10992     6\n",
       "10994     2\n",
       "10995     9\n",
       "10996     7\n",
       "10997     3\n",
       "10999     1\n",
       "11000     2\n",
       "11001     4\n",
       "Name: decile_score, Length: 7214, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['decile_score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "African-American    3696\n",
       "Caucasian           2454\n",
       "Hispanic             637\n",
       "Other                377\n",
       "Asian                 32\n",
       "Native American       18\n",
       "Name: race, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['race'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3963\n",
       "1    3251\n",
       "Name: two_year_recid, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['two_year_recid'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['name', 'first', 'last', 'compas_screening_date', 'sex', 'dob', 'age',\n",
       "       'age_cat', 'race', 'juv_fel_count',\n",
       "       ...\n",
       "       'charge_Robbery Sudd Snatch No Weapon',\n",
       "       'charge_Susp Drivers Lic 1st Offense',\n",
       "       'charge_Tamper With Witness/Victim/CI',\n",
       "       'charge_Tampering With Physical Evidence',\n",
       "       'charge_Uttering a Forged Instrument',\n",
       "       'charge_Viol Injunct Domestic Violence',\n",
       "       'charge_Viol Pretrial Release Dom Viol',\n",
       "       'charge_Viol Prot Injunc Repeat Viol', 'charge_arrest case no charge',\n",
       "       'charge_other'],\n",
       "      dtype='object', length=141)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# two things we could do:\n",
    "# (1) post-processing of the scores COMPAS gave to make them more fair\n",
    "# (2) train our own model, after-the-fact, using the actual recidivism outcomes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# what do we want to keep as features?\n",
    "features = ['sex_new', 'age', 'juv_fel_count', 'juv_misd_count', 'juv_other_count', 'priors_count', 'c_charge_degree_new']\n",
    "features = np.concatenate((features,one_hot.columns))\n",
    "features = np.concatenate((features, ['P', 'Y']))\n",
    "# our question would be: can we predict two_year_recid in a way that is fair?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['sex_new', 'age', 'juv_fel_count', 'juv_misd_count',\n",
       "       'juv_other_count', 'priors_count', 'c_charge_degree_new',\n",
       "       'charge_Agg Battery Grt/Bod/Harm',\n",
       "       'charge_Aggrav Battery w/Deadly Weapon',\n",
       "       'charge_Aggravated Assault W/Dead Weap',\n",
       "       'charge_Aggravated Assault W/dead Weap',\n",
       "       'charge_Aggravated Assault w/Firearm', 'charge_Aggravated Battery',\n",
       "       'charge_Aggravated Battery / Pregnant', 'charge_Assault',\n",
       "       'charge_Att Burgl Unoccupied Dwel', 'charge_Battery',\n",
       "       'charge_Battery on Law Enforc Officer',\n",
       "       'charge_Battery on a Person Over 65',\n",
       "       'charge_Burglary Conveyance Unoccup',\n",
       "       'charge_Burglary Dwelling Assault/Batt',\n",
       "       'charge_Burglary Dwelling Occupied',\n",
       "       'charge_Burglary Structure Unoccup',\n",
       "       'charge_Burglary Unoccupied Dwelling',\n",
       "       'charge_Carrying Concealed Firearm', 'charge_Child Abuse',\n",
       "       'charge_Corrupt Public Servant',\n",
       "       'charge_Crim Use of Personal ID Info',\n",
       "       'charge_Crimin Mischief Damage $1000+', 'charge_Criminal Mischief',\n",
       "       'charge_Criminal Mischief Damage <$200',\n",
       "       'charge_Criminal Mischief>$200<$1000',\n",
       "       'charge_Cruelty Toward Child', 'charge_DUI - Enhanced',\n",
       "       'charge_DUI Level 0.15 Or Minor In Veh',\n",
       "       'charge_DUI Property Damage/Injury',\n",
       "       'charge_Dealing in Stolen Property',\n",
       "       'charge_Deliver 3,4 Methylenediox', 'charge_Deliver Cannabis',\n",
       "       'charge_Deliver Cocaine', 'charge_Disorderly Conduct',\n",
       "       'charge_Disorderly Intoxication',\n",
       "       'charge_Driving License Suspended',\n",
       "       'charge_Driving Under The Influence',\n",
       "       'charge_Driving While License Revoked',\n",
       "       'charge_Exposes Culpable Negligence', 'charge_False Imprisonment',\n",
       "       'charge_False Ownership Info/Pawn Item',\n",
       "       'charge_Felony Batt(Great Bodily Harm)', 'charge_Felony Battery',\n",
       "       'charge_Felony Battery (Dom Strang)',\n",
       "       'charge_Felony Battery w/Prior Convict',\n",
       "       'charge_Felony Driving While Lic Suspd',\n",
       "       'charge_Felony Petit Theft',\n",
       "       'charge_Fleeing Or Attmp Eluding A Leo',\n",
       "       'charge_Grand Theft (Motor Vehicle)',\n",
       "       'charge_Grand Theft in the 3rd Degree',\n",
       "       'charge_Leave Acc/Attend Veh/More $50',\n",
       "       'charge_Leaving Acc/Unattended Veh',\n",
       "       'charge_Leaving the Scene of Accident',\n",
       "       'charge_Lewd or Lascivious Molestation',\n",
       "       'charge_Lve/Scen/Acc/Veh/Prop/Damage',\n",
       "       'charge_Neglect Child / No Bodily Harm',\n",
       "       'charge_Operating W/O Valid License', 'charge_Petit Theft',\n",
       "       'charge_Petit Theft $100- $300',\n",
       "       'charge_Pos Cannabis W/Intent Sel/Del',\n",
       "       'charge_Poss 3,4 MDMA (Ecstasy)',\n",
       "       'charge_Poss Cocaine/Intent To Del/Sel',\n",
       "       'charge_Poss Contr Subst W/o Prescript',\n",
       "       'charge_Poss Pyrrolidinovalerophenone',\n",
       "       'charge_Poss3,4 Methylenedioxymethcath',\n",
       "       'charge_Possess Cannabis/20 Grams Or Less',\n",
       "       'charge_Possession Burglary Tools',\n",
       "       'charge_Possession Of Alprazolam', 'charge_Possession Of Heroin',\n",
       "       'charge_Possession Of Methamphetamine',\n",
       "       'charge_Possession of Cannabis', 'charge_Possession of Cocaine',\n",
       "       'charge_Possession of Hydrocodone',\n",
       "       'charge_Possession of Hydromorphone',\n",
       "       'charge_Possession of Oxycodone', 'charge_Prowling/Loitering',\n",
       "       'charge_Resist Officer w/Violence',\n",
       "       'charge_Resist/Obstruct W/O Violence', 'charge_Robbery / No Weapon',\n",
       "       'charge_Robbery Sudd Snatch No Weapon',\n",
       "       'charge_Susp Drivers Lic 1st Offense',\n",
       "       'charge_Tamper With Witness/Victim/CI',\n",
       "       'charge_Tampering With Physical Evidence',\n",
       "       'charge_Uttering a Forged Instrument',\n",
       "       'charge_Viol Injunct Domestic Violence',\n",
       "       'charge_Viol Pretrial Release Dom Viol',\n",
       "       'charge_Viol Prot Injunc Repeat Viol',\n",
       "       'charge_arrest case no charge', 'charge_other', 'P', 'Y'], dtype=object)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# one-hot encode the 'c_charge_degree_new' column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['sex_new'] = df.apply(lambda row: row['sex'] == 'Male', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['c_charge_degree_new'] = df.apply(lambda row: row['c_charge_degree'] == 'F', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def protected_race(row):\n",
    "    if row['race'] == 'Caucasian':\n",
    "        return 0\n",
    "    elif row['race'] == 'African-American':\n",
    "        return 1\n",
    "    else:\n",
    "        return 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "1        3\n",
       "3        1\n",
       "4        1\n",
       "5        1\n",
       "6        3\n",
       "7        3\n",
       "8        0\n",
       "9        3\n",
       "10       0\n",
       "13       0\n",
       "14       0\n",
       "15       1\n",
       "16       0\n",
       "18       1\n",
       "19       0\n",
       "20       1\n",
       "21       3\n",
       "22       1\n",
       "23       0\n",
       "24       0\n",
       "25       1\n",
       "26       1\n",
       "27       0\n",
       "28       0\n",
       "30       3\n",
       "32       0\n",
       "33       3\n",
       "37       1\n",
       "38       0\n",
       "39       1\n",
       "        ..\n",
       "10962    0\n",
       "10963    0\n",
       "10964    1\n",
       "10965    0\n",
       "10966    0\n",
       "10967    1\n",
       "10969    1\n",
       "10971    0\n",
       "10972    0\n",
       "10975    1\n",
       "10976    0\n",
       "10977    1\n",
       "10979    1\n",
       "10980    1\n",
       "10981    1\n",
       "10982    0\n",
       "10984    1\n",
       "10985    1\n",
       "10987    3\n",
       "10988    3\n",
       "10989    1\n",
       "10990    0\n",
       "10992    0\n",
       "10994    1\n",
       "10995    1\n",
       "10996    1\n",
       "10997    1\n",
       "10999    3\n",
       "11000    1\n",
       "11001    3\n",
       "Name: P, Length: 7214, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['P'] = df.apply(lambda row: protected_race(row), axis=1)\n",
    "df['P']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6150, 144)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.loc[df['P'] != 3]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['Y'] = df['two_year_recid']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.drop(df.columns.difference(features), 1, inplace=True)\n",
    "df = df.astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>juv_fel_count</th>\n",
       "      <th>juv_misd_count</th>\n",
       "      <th>juv_other_count</th>\n",
       "      <th>priors_count</th>\n",
       "      <th>charge_Agg Battery Grt/Bod/Harm</th>\n",
       "      <th>charge_Aggrav Battery w/Deadly Weapon</th>\n",
       "      <th>charge_Aggravated Assault W/Dead Weap</th>\n",
       "      <th>charge_Aggravated Assault W/dead Weap</th>\n",
       "      <th>charge_Aggravated Assault w/Firearm</th>\n",
       "      <th>...</th>\n",
       "      <th>charge_Uttering a Forged Instrument</th>\n",
       "      <th>charge_Viol Injunct Domestic Violence</th>\n",
       "      <th>charge_Viol Pretrial Release Dom Viol</th>\n",
       "      <th>charge_Viol Prot Injunc Repeat Viol</th>\n",
       "      <th>charge_arrest case no charge</th>\n",
       "      <th>charge_other</th>\n",
       "      <th>sex_new</th>\n",
       "      <th>c_charge_degree_new</th>\n",
       "      <th>P</th>\n",
       "      <th>Y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 98 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    age  juv_fel_count  juv_misd_count  juv_other_count  priors_count  \\\n",
       "id                                                                      \n",
       "3    34              0               0                0             0   \n",
       "4    24              0               0                1             4   \n",
       "5    23              0               1                0             1   \n",
       "8    41              0               0                0            14   \n",
       "10   39              0               0                0             0   \n",
       "\n",
       "    charge_Agg Battery Grt/Bod/Harm  charge_Aggrav Battery w/Deadly Weapon  \\\n",
       "id                                                                           \n",
       "3                                 0                                      0   \n",
       "4                                 0                                      0   \n",
       "5                                 0                                      0   \n",
       "8                                 0                                      0   \n",
       "10                                0                                      0   \n",
       "\n",
       "    charge_Aggravated Assault W/Dead Weap  \\\n",
       "id                                          \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "5                                       0   \n",
       "8                                       0   \n",
       "10                                      0   \n",
       "\n",
       "    charge_Aggravated Assault W/dead Weap  \\\n",
       "id                                          \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "5                                       0   \n",
       "8                                       0   \n",
       "10                                      0   \n",
       "\n",
       "    charge_Aggravated Assault w/Firearm ...  \\\n",
       "id                                      ...   \n",
       "3                                     0 ...   \n",
       "4                                     0 ...   \n",
       "5                                     0 ...   \n",
       "8                                     0 ...   \n",
       "10                                    0 ...   \n",
       "\n",
       "    charge_Uttering a Forged Instrument  \\\n",
       "id                                        \n",
       "3                                     0   \n",
       "4                                     0   \n",
       "5                                     0   \n",
       "8                                     0   \n",
       "10                                    0   \n",
       "\n",
       "    charge_Viol Injunct Domestic Violence  \\\n",
       "id                                          \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "5                                       0   \n",
       "8                                       0   \n",
       "10                                      0   \n",
       "\n",
       "    charge_Viol Pretrial Release Dom Viol  \\\n",
       "id                                          \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "5                                       0   \n",
       "8                                       0   \n",
       "10                                      0   \n",
       "\n",
       "    charge_Viol Prot Injunc Repeat Viol  charge_arrest case no charge  \\\n",
       "id                                                                      \n",
       "3                                     0                             0   \n",
       "4                                     0                             0   \n",
       "5                                     0                             0   \n",
       "8                                     0                             0   \n",
       "10                                    0                             0   \n",
       "\n",
       "    charge_other  sex_new  c_charge_degree_new  P  Y  \n",
       "id                                                    \n",
       "3              0        1                    1  1  1  \n",
       "4              0        1                    1  1  1  \n",
       "5              0        1                    1  1  0  \n",
       "8              0        1                    1  0  1  \n",
       "10             0        0                    0  0  0  \n",
       "\n",
       "[5 rows x 98 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6009756097560975"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['P'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('compas_new.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data/german.numeric.processed') as f:\n",
    "    data_raw = np.array([list(map(float, x)) for x in map(lambda x: x.split(), f)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "col_names = []\n",
    "for i in range(23):\n",
    "    col_names.append('col_' + str(i))\n",
    "col_names.append('P')\n",
    "col_names.append('Y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "german_df = pd.DataFrame(data=data_raw, columns=col_names)\n",
    "german_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "german_df.to_csv('german_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}