8.1-checkpoint_LAB_AMF_categ.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lab Revisiting Machine Learning Case Study\n",
    "#### Aída Moure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option('display.max_columns', None)\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95412, 481)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('data/learningSet.csv')\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = data.select_dtypes(include=['object'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95412, 74)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Check for all null values in all the columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_name</th>\n",
       "      <th>nulls_percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OSOURCE</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>STATE</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ZIP</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MAILCODE</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PVASTATE</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>RFA_2A</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>MDMAUD_R</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>MDMAUD_F</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>MDMAUD_A</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>GEOCODE2</td>\n",
       "      <td>0.001383</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   column_name  nulls_percentage\n",
       "0      OSOURCE          0.000000\n",
       "1        STATE          0.000000\n",
       "2          ZIP          0.000000\n",
       "3     MAILCODE          0.000000\n",
       "4     PVASTATE          0.000000\n",
       "..         ...               ...\n",
       "69      RFA_2A          0.000000\n",
       "70    MDMAUD_R          0.000000\n",
       "71    MDMAUD_F          0.000000\n",
       "72    MDMAUD_A          0.000000\n",
       "73    GEOCODE2          0.001383\n",
       "\n",
       "[74 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nulls_percent_df = pd.DataFrame(categorical.isna().sum()/len(categorical)).reset_index()\n",
    "nulls_percent_df.columns = ['column_name', 'nulls_percentage']\n",
    "nulls_percent_df\n",
    "\n",
    "#it can't be seen all the columns..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a new empty list called drop_list. We will append this list and then drop all the columns in this list later:\n",
    "\n",
    "- OSOURCE - symbol definitions not provided, too many categories\n",
    "- ZIP CODE - we are including state already\n",
    "- Columns that over 85% missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "columns_above_threshold = nulls_percent_df[nulls_percent_df['nulls_percentage']>0.85]\n",
    "print(len(columns_above_threshold['column_name']))\n",
    "\n",
    "#we have 0 columns with more than 85% missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "columns_above_thresh_list= list(columns_above_threshold['column_name'])\n",
    "print(columns_above_thresh_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_list=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_list.append('OSOURCE')\n",
    "drop_list.append('ZIP')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(drop_list)#OK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['OSOURCE', 'ZIP']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drop_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in columns_above_thresh_list:\n",
    "    drop_list.append(column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(drop_list) #OK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reduce the number of categories in the column GENDER. The column should only have either \"M\" for males, \"F\" for females, and \"other\" for all the rest\n",
    "\n",
    "Note that there are a few null values in the column. We will first replace those null values using the code below:\n",
    "\n",
    "print(categorical['GENDER'].value_counts())\n",
    "categorical['GENDER'] = categorical['GENDER'].fillna('F')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['F', 'M', ' ', 'C', 'U', 'J', 'A'], dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical.GENDER.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GENDER\n",
      "F    51277\n",
      "M    39094\n",
      "      2957\n",
      "U     1715\n",
      "J      365\n",
      "C        2\n",
      "A        2\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(categorical['GENDER'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical['GENDER'] = categorical['GENDER'].apply(lambda x: x.replace(\" \", \"F\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['F', 'M', 'C', 'U', 'J', 'A'], dtype=object)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical.GENDER.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(data['GENDER'].value_counts())\n",
    "#data['GENDER'] = data['GENDER'].fillna('F')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GENDER\n",
      "F    54234\n",
      "M    39094\n",
      "U     1715\n",
      "J      365\n",
      "C        2\n",
      "A        2\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(categorical['GENDER'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "genero=['F', 'M']\n",
    "def clean_gender(x):\n",
    "    if x not in genero:\n",
    "        return 'other'\n",
    "    else:\n",
    "        return x\n",
    "categorical['GENDER'] = list(map(clean_gender, categorical['GENDER']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GENDER\n",
      "F        54234\n",
      "M        39094\n",
      "other     2084\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(categorical['GENDER'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ironhack",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}