debug.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hbe:  0.9490824254306831\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "from because.probability import prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(384977, 22)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('/home/arun/Desktop/dataset/Housing/housing.csv', header=0)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'url', 'region', 'region_url', 'price', 'type', 'sqfeet', 'beds',\n",
       "       'baths', 'cats_allowed', 'dogs_allowed', 'smoking_allowed',\n",
       "       'wheelchair_access', 'electric_vehicle_charge', 'comes_furnished',\n",
       "       'laundry_options', 'parking_options', 'image_url', 'description', 'lat',\n",
       "       'long', 'state'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['apartment', 'duplex', 'house', 'condo', 'flat', 'townhouse',\n",
       "       'manufactured', 'loft', 'cottage/cabin', 'in-law', 'land',\n",
       "       'assisted living'], dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  print first 5 rows of type column\n",
    "df['type'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  label encoder for type\n",
    "le = LabelEncoder()\n",
    "df['type'] = le.fit_transform(df['type'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  4,  6,  2,  5, 11, 10,  9,  3,  7,  8,  1])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  print first 5 rows of type column\n",
    "df['type'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(348244, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>sqfeet</th>\n",
       "      <th>beds</th>\n",
       "      <th>baths</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>348244.000000</td>\n",
       "      <td>348244.000000</td>\n",
       "      <td>348244.000000</td>\n",
       "      <td>348244.000000</td>\n",
       "      <td>348244.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1166.078118</td>\n",
       "      <td>998.990647</td>\n",
       "      <td>1.924711</td>\n",
       "      <td>1.491189</td>\n",
       "      <td>1.183900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>461.854198</td>\n",
       "      <td>320.029079</td>\n",
       "      <td>0.765155</td>\n",
       "      <td>0.545420</td>\n",
       "      <td>2.886026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>830.000000</td>\n",
       "      <td>775.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1050.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1395.000000</td>\n",
       "      <td>1146.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3000.000000</td>\n",
       "      <td>3000.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>11.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               price         sqfeet           beds          baths  \\\n",
       "count  348244.000000  348244.000000  348244.000000  348244.000000   \n",
       "mean     1166.078118     998.990647       1.924711       1.491189   \n",
       "std       461.854198     320.029079       0.765155       0.545420   \n",
       "min       500.000000     500.000000       1.000000       1.000000   \n",
       "25%       830.000000     775.000000       1.000000       1.000000   \n",
       "50%      1050.000000     950.000000       2.000000       1.000000   \n",
       "75%      1395.000000    1146.000000       2.000000       2.000000   \n",
       "max      3000.000000    3000.000000       4.000000       4.000000   \n",
       "\n",
       "                type  \n",
       "count  348244.000000  \n",
       "mean        1.183900  \n",
       "std         2.886026  \n",
       "min         0.000000  \n",
       "25%         0.000000  \n",
       "50%         0.000000  \n",
       "75%         0.000000  \n",
       "max        11.000000  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[np.logical_and(df.price >= 500, df.price <= 3000)]\n",
    "df = df[np.logical_and(df.sqfeet >= 500, df.sqfeet <= 3000)]\n",
    "df = df[np.logical_and(df.beds >= 1, df.beds <= 4)]\n",
    "df = df[np.logical_and(df.baths >= 1, df.baths <= 4)]\n",
    "\n",
    "# df = df[df.type == 'house']\n",
    "df = df[['price', 'sqfeet', 'beds', 'baths', 'type']]\n",
    "\n",
    "print(df.shape)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['price', 'sqfeet', 'beds', 'baths', 'type'])\n"
     ]
    }
   ],
   "source": [
    "# converting to dictionary format\n",
    "dfDict = df.to_dict('list')\n",
    "\n",
    "# print keys of the dictionary\n",
    "print(dfDict.keys())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob.DEBUG = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate prob.ProbSpace with that dataset\n",
    "myPS = prob.ProbSpace(dfDict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0000000000001072\n",
      "1.0000000000000153\n",
      "0.99999999105738\n"
     ]
    }
   ],
   "source": [
    "print(myPS.dependence('price', 'sqfeet'))\n",
    "print(myPS.dependence('price', 'sqfeet',dMethod= 'rcot'))\n",
    "print(myPS.dependence('price', 'sqfeet',dMethod= 'prob'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000000000000306"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "myPS.dependence('price', 'beds', ['type', 'sqfeet'], dMethod='rcot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000000000001839"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myPS.dependence('beds', 'price')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1166.0781176416535\n",
      "998.9906473621943\n",
      "<because.probability.pdf.PDF object at 0x7fd924c6c4e0>\n"
     ]
    }
   ],
   "source": [
    "prob.DEBUG = False\n",
    "print(myPS.E('price'))\n",
    "print(myPS.E('sqfeet'))\n",
    "\n",
    "print(myPS.distr('price'))\n",
    "prob.DEBUG = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myPS.isIndependent('price', 'beds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ProbSpace.distr: P( price | [] )\n"
     ]
    }
   ],
   "source": [
    "d = myPS.distr('price')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = list(dir(d))\n",
    "len(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E\n",
      "N\n",
      "P\n",
      "Prangex\n",
      "R\n",
      "SetHistogram\n",
      "ToHistTuple\n",
      "ToHistogram\n",
      "binCount\n",
      "binValue\n",
      "bins\n",
      "compare\n",
      "compare2\n",
      "compare3\n",
      "compare4\n",
      "compare5\n",
      "compare_ks_old\n",
      "data\n",
      "filters\n",
      "getBin\n",
      "getBinForVal\n",
      "isDiscrete\n",
      "isNull\n",
      "ksAlpha\n",
      "ksThreshold\n",
      "kurtCache\n",
      "kurtosis\n",
      "max\n",
      "maxVal\n",
      "mean\n",
      "meanCache\n",
      "median\n",
      "min\n",
      "minVal\n",
      "modality\n",
      "mode\n",
      "mvData\n",
      "percentile\n",
      "rvName\n",
      "skew\n",
      "skewCache\n",
      "stDev\n",
      "stats\n",
      "style\n",
      "var\n",
      "varCache\n"
     ]
    }
   ],
   "source": [
    "for i in p:\n",
    "    if not i.startswith('__'):\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "525.2174260975337\n"
     ]
    }
   ],
   "source": [
    "print(d.percentile(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ProbSpace.distr: P( price | [] )\n",
      "ProbSpace.distr: P( beds | [] )\n",
      "ProbSpace.distr: P( sqfeet | [] )\n"
     ]
    }
   ],
   "source": [
    "vars = ['price','beds','sqfeet']\n",
    "lim = 1\n",
    "dims = 3\n",
    "numPts = 20\n",
    "\n",
    "distrs = [myPS.distr(c) for c in vars]    # Get a list of distribution objects for each variable\n",
    "minvs = [distr.percentile(lim) for distr in distrs]\n",
    "maxvs = [distr.percentile(100-lim) for distr in distrs]\n",
    "incrs = [(maxvs[i] - minvs[i]) / (numPts-1) for i in range(dims)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[525.2174260975337, 1.0, 529.916362025036]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minvs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2702.2952196043443, 4.0, 2187.6233521657264]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxvs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[114.58304176351636, 0.15789473684210525, 87.24773632319423]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "incrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "ps = myPS\n",
    "vSpaces = []\n",
    "incrs = []\n",
    "for i in range(dims):\n",
    "    var = vars[i]\n",
    "    if ps.isDiscrete(var):\n",
    "        vSpace = ps.getMidpoints(var)\n",
    "        incrs.append(1)\n",
    "    else:\n",
    "        vSpace = list(np.linspace(minvs[i], maxvs[i], numPts))\n",
    "        incrs.append(vSpace[1] - vSpace[0])\n",
    "    vSpaces.append(vSpace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[114.58304176351635, 1, 87.2477363231942]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "incrs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[525.2174260975337,\n",
       "  639.80046786105,\n",
       "  754.3835096245664,\n",
       "  868.9665513880827,\n",
       "  983.5495931515991,\n",
       "  1098.1326349151154,\n",
       "  1212.715676678632,\n",
       "  1327.2987184421481,\n",
       "  1441.8817602056647,\n",
       "  1556.464801969181,\n",
       "  1671.0478437326974,\n",
       "  1785.6308854962138,\n",
       "  1900.21392725973,\n",
       "  2014.7969690232464,\n",
       "  2129.380010786763,\n",
       "  2243.963052550279,\n",
       "  2358.5460943137955,\n",
       "  2473.129136077312,\n",
       "  2587.712177840828,\n",
       "  2702.2952196043443],\n",
       " [1.0, 2.0, 3.0, 4.0],\n",
       " [529.916362025036,\n",
       "  617.1640983482303,\n",
       "  704.4118346714245,\n",
       "  791.6595709946188,\n",
       "  878.907307317813,\n",
       "  966.1550436410072,\n",
       "  1053.4027799642015,\n",
       "  1140.6505162873957,\n",
       "  1227.89825261059,\n",
       "  1315.145988933784,\n",
       "  1402.3937252569783,\n",
       "  1489.6414615801727,\n",
       "  1576.889197903367,\n",
       "  1664.1369342265612,\n",
       "  1751.3846705497554,\n",
       "  1838.6324068729496,\n",
       "  1925.8801431961438,\n",
       "  2013.127879519338,\n",
       "  2100.375615842532,\n",
       "  2187.6233521657264]]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vSpaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def findPoint(minv, maxv, indx):\n",
    "    val = ((maxv - minv) / (numPts) ) * indx + minv\n",
    "    return val\n",
    "\n",
    "gridSet = []\n",
    "\n",
    "for i in range(20):\n",
    "    y = (i-1) % 20\n",
    "    gridSet.append(findPoint(minvs[0], maxvs[0], y))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}