01-intro/homework.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['VendorID', 'tpep_pickup_datetime', 'tpep_dropoff_datetime',\n",
       "       'passenger_count', 'trip_distance', 'RatecodeID', 'store_and_fwd_flag',\n",
       "       'PULocationID', 'DOLocationID', 'payment_type', 'fare_amount', 'extra',\n",
       "       'mta_tax', 'tip_amount', 'tolls_amount', 'improvement_surcharge',\n",
       "       'total_amount', 'congestion_surcharge', 'airport_fee', 'duration'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q1: nb columns = 19\n",
      "Q2: duration STd = 42.594351241920904\n",
      "Q3: fraction of the records 98.1220282212598 %\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_parquet('./data/yellow_tripdata_2023-01.parquet')\n",
    "#Q1\n",
    "print(f\"Q1: nb columns = {len(df.columns)}\")\n",
    "\n",
    "\n",
    "df['duration'] = df.tpep_dropoff_datetime - df.tpep_pickup_datetime\n",
    "df.duration = df.duration.apply(lambda td: td.total_seconds() / 60)\n",
    "print(f\"Q2: duration STd = {df['duration'].std()}\")\n",
    "\n",
    "before_len = len(df)\n",
    "df = df[(df.duration >= 1) & (df.duration <= 60)]\n",
    "after_len = len(df)\n",
    "\n",
    "print(f\"Q3: fraction of the records {after_len/before_len *100} %\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['PULocationID', 'DOLocationID']\n",
    "\n",
    "df[categorical] = df[categorical].astype(str)\n",
    "\n",
    "train_dicts = df[categorical].to_dict(orient='records')\n",
    "\n",
    "dv = DictVectorizer()\n",
    "X_train = dv.fit_transform(train_dicts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q4: the dimensionality(number of columns): 515\n"
     ]
    }
   ],
   "source": [
    "print(f\"Q4: the dimensionality(number of columns): {X_train.shape[1]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q5: RMSE on train: 7.649261927686161\n"
     ]
    }
   ],
   "source": [
    "target = 'duration'\n",
    "y_train = df[target].values\n",
    "\n",
    "lr = LinearRegression()\n",
    "lr.fit(X_train, y_train)\n",
    "\n",
    "y_pred = lr.predict(X_train)\n",
    "\n",
    "print(f\"Q5: RMSE on train: {mean_squared_error(y_train, y_pred, squared=False)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q6: RMSE on val: 13.322327465985968\n"
     ]
    }
   ],
   "source": [
    "df2 = pd.read_parquet('./data/yellow_tripdata_2023-02.parquet')\n",
    "df2['duration'] = (df2.tpep_dropoff_datetime - df2.tpep_pickup_datetime).apply(lambda td: td.total_seconds() / 60)\n",
    "df2 = df2[(df2.duration >= 1) & (df2.duration <= 60)]\n",
    "\n",
    "X_val = dv.transform(df2[categorical].to_dict(orient='records'))\n",
    "\n",
    "\n",
    "y_val = df2[target].values\n",
    "y_p_val = lr.predict(X_val)\n",
    "\n",
    "print(f\"Q6: RMSE on val: {mean_squared_error(y_val, y_p_val, squared=False)}\")"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}