Worksheet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Worksheet for Correlation Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path\n",
    "try:\n",
    "    from urllib2 import urlopen\n",
    "except ImportError:\n",
    "    from urllib.request import urlopen\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.stats import norm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download data and load into a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### LOAD THE HELPER FUNCTIONS BELOW####\n",
    "#### IMPORTANT: DONOT change these functions or your final submission will not evaluate correctly###\n",
    "\n",
    "## This downloads your datafile, Do not change this function\n",
    "def downloadFile(dataSetId):\n",
    "    fileName = '%s.csv' % (dataSetId)\n",
    "    url = 'https://s3.us-east-2.amazonaws.com/qq10-data/' + fileName\n",
    "    print(url)\n",
    "\n",
    "    response = urlopen(url)\n",
    "    status = response.getcode()\n",
    "    if status == 200:\n",
    "      print('Downloading the dataset %s' % (fileName))\n",
    "      with open(fileName, 'w') as f:\n",
    "          f.write(response.read().decode('utf8'))\n",
    "      return True\n",
    "    else:\n",
    "      logError('File not found. Please ensure you are working with correct data set Id')\n",
    "      return False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "filename = 'feature_data'\n",
    "if not os.path.isfile('%s.csv'%filename):\n",
    "    downloadFile('%s'%filename)\n",
    "df_train = pd.read_csv('%s.csv'%filename)\n",
    "df_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'target_variable_data'\n",
    "if not os.path.isfile('%s.csv'%filename):\n",
    "    downloadFile('%s'%filename)\n",
    "y_train = pd.read_csv('%s.csv'%filename)\n",
    "y_train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### View descriptive statistics of each dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.describe().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train.describe().T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explore a single target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train['A1'].plot(kind=\"hist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.distplot(y_train['A1'], fit = norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train['A1'].skew(), y_train['A1'].kurt()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Try transformations on the target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_a1 = np.log1p(y_train['A1'])\n",
    "sns.distplot(log_a1, fit = norm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Other descriptive plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train.hist(bins=50, figsize=(30,20));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explore relationships with explanatory variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "var = 'Alpha_A1_1'\n",
    "data = pd.concat([y_train['A1'], df_train[var]], axis=1)\n",
    "data.plot.scatter(x=var, y='A1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "var = 'Beta_A_1'\n",
    "data = pd.concat([y_train['A1'], df_train[var]], axis=1)\n",
    "f, ax = plt.subplots(figsize=(16, 8))\n",
    "fig = sns.boxplot(x=var, y='A1', data=data)\n",
    "plt.xticks(rotation=90);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set()\n",
    "cols = ['Alpha_A1_1', 'Alpha_A1_2', 'Alpha_A1_3', 'Alpha_A1_4', 'Alpha_A1_5', 'Alpha_A1_6', 'Alpha_A1_7', 'Alpha_A1_8', 'Alpha_A1_9', 'Alpha_A1_10']\n",
    "sns.pairplot(df_train.filter(regex='_A1'), size = 2.5)\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explore intercorrelations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrmat = df_train.filter(regex='_A1').corr(method='spearman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "var = 'Alpha_A1_1'\n",
    "f, ax = plt.subplots(figsize=(12, 10))\n",
    "k = 25 #number of variables for heatmap\n",
    "cols = corrmat.nlargest(k, var)[var].index\n",
    "cm = np.corrcoef(df_train[cols].values.T)\n",
    "sns.heatmap(cm, ax=ax, cmap=\"YlGnBu\", linewidths=0.1, yticklabels=cols.values, xticklabels=cols.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cg = sns.clustermap(cm, cmap=\"YlGnBu\", linewidths=0.1);\n",
    "plt.setp(cg.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)\n",
    "cg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explore correlations with target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tv = 'A1'\n",
    "df_tv = df_train.filter(regex='_A1').join(y_train[tv])\n",
    "corrmat = df_tv.corr(method='spearman')\n",
    "f, ax = plt.subplots(figsize=(12, 10))\n",
    "\n",
    "k = 50 #number of variables to explore\n",
    "cols = corrmat.nlargest(k, tv)[tv].index\n",
    "cm = np.corrcoef(df_tv[cols].values.T)\n",
    "sns.heatmap(cm, ax=ax, cmap=\"YlGnBu\", linewidths=0.1, yticklabels=cols.values, xticklabels=cols.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(k):\n",
    "    temp_df = pd.DataFrame(df_tv[cols[i]], index = df_tv.index, columns=[cols[i], tv])\n",
    "    temp_df[tv] = df_tv[tv]\n",
    "    print(temp_df.corr(method='pearson'))\n",
    "    plt.plot(temp_df[cols[i]], temp_df[tv], '.b')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}