6. Speed modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original NE: POINT (31.5 66.8)\n",
      "Original SW: POINT (8.5 53)\n",
      "Transformed NE: (697681.2863658136, 7416224.9393330095)\n",
      "Transformed SW: (-735260.4063631196, 6033964.466017238)\n",
      "col_count 286\n",
      "row_count 276\n",
      "grid_size 78936\n",
      "CPU times: user 2.76 ms, sys: 2.98 ms, total: 5.74 ms\n",
      "Wall time: 6.35 ms\n"
     ]
    }
   ],
   "source": [
    "from pygradu import gridify\n",
    "import shapely.geometry\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import importlib\n",
    "\n",
    "%time grid = gridify.area_to_grid(side_length=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:2785: DtypeWarning: Columns (15) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>mmsi</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>speed</th>\n",
       "      <th>course</th>\n",
       "      <th>heading</th>\n",
       "      <th>turnrate</th>\n",
       "      <th>breadth</th>\n",
       "      <th>vessel_type</th>\n",
       "      <th>vessel_max_speed</th>\n",
       "      <th>draft</th>\n",
       "      <th>power</th>\n",
       "      <th>dwt</th>\n",
       "      <th>iceclass</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2017-11-02 11:19:07</td>\n",
       "      <td>205366000</td>\n",
       "      <td>54.347242</td>\n",
       "      <td>9.991140</td>\n",
       "      <td>4.01</td>\n",
       "      <td>56.6</td>\n",
       "      <td>56.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.33</td>\n",
       "      <td>T</td>\n",
       "      <td>16.0</td>\n",
       "      <td>8.18</td>\n",
       "      <td>5820.0</td>\n",
       "      <td>13289.0</td>\n",
       "      <td>IA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2017-11-02 11:30:58</td>\n",
       "      <td>205366000</td>\n",
       "      <td>54.360348</td>\n",
       "      <td>10.025337</td>\n",
       "      <td>4.17</td>\n",
       "      <td>77.5</td>\n",
       "      <td>81.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.33</td>\n",
       "      <td>T</td>\n",
       "      <td>16.0</td>\n",
       "      <td>8.18</td>\n",
       "      <td>5820.0</td>\n",
       "      <td>13289.0</td>\n",
       "      <td>IA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2017-11-02 11:37:58</td>\n",
       "      <td>205366000</td>\n",
       "      <td>54.359573</td>\n",
       "      <td>10.049203</td>\n",
       "      <td>4.48</td>\n",
       "      <td>92.4</td>\n",
       "      <td>92.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.33</td>\n",
       "      <td>T</td>\n",
       "      <td>16.0</td>\n",
       "      <td>8.18</td>\n",
       "      <td>5820.0</td>\n",
       "      <td>13289.0</td>\n",
       "      <td>IA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2017-11-02 11:44:07</td>\n",
       "      <td>205366000</td>\n",
       "      <td>54.365497</td>\n",
       "      <td>10.071320</td>\n",
       "      <td>3.81</td>\n",
       "      <td>44.9</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.33</td>\n",
       "      <td>T</td>\n",
       "      <td>16.0</td>\n",
       "      <td>8.18</td>\n",
       "      <td>5820.0</td>\n",
       "      <td>13289.0</td>\n",
       "      <td>IA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2017-11-02 11:56:08</td>\n",
       "      <td>205366000</td>\n",
       "      <td>54.370437</td>\n",
       "      <td>10.109498</td>\n",
       "      <td>2.73</td>\n",
       "      <td>103.6</td>\n",
       "      <td>103.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.33</td>\n",
       "      <td>T</td>\n",
       "      <td>16.0</td>\n",
       "      <td>8.18</td>\n",
       "      <td>5820.0</td>\n",
       "      <td>13289.0</td>\n",
       "      <td>IA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0           timestamp       mmsi        lat        lon  speed  \\\n",
       "0           0 2017-11-02 11:19:07  205366000  54.347242   9.991140   4.01   \n",
       "1           1 2017-11-02 11:30:58  205366000  54.360348  10.025337   4.17   \n",
       "2           2 2017-11-02 11:37:58  205366000  54.359573  10.049203   4.48   \n",
       "3           3 2017-11-02 11:44:07  205366000  54.365497  10.071320   3.81   \n",
       "4           4 2017-11-02 11:56:08  205366000  54.370437  10.109498   2.73   \n",
       "\n",
       "   course  heading  turnrate  breadth vessel_type  vessel_max_speed  draft  \\\n",
       "0    56.6     56.0       0.0    21.33           T              16.0   8.18   \n",
       "1    77.5     81.0       0.0    21.33           T              16.0   8.18   \n",
       "2    92.4     92.0       0.0    21.33           T              16.0   8.18   \n",
       "3    44.9     46.0       0.0    21.33           T              16.0   8.18   \n",
       "4   103.6    103.0       0.0    21.33           T              16.0   8.18   \n",
       "\n",
       "    power      dwt iceclass  \n",
       "0  5820.0  13289.0       IA  \n",
       "1  5820.0  13289.0       IA  \n",
       "2  5820.0  13289.0       IA  \n",
       "3  5820.0  13289.0       IA  \n",
       "4  5820.0  13289.0       IA  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import dataset\n",
    "\n",
    "ais = pd.read_csv('data/datasets/training_set.csv', parse_dates = ['timestamp'])\n",
    "ais = ais.sort_values(by=['mmsi', 'timestamp'])\n",
    "ais.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ais = ais[ais.speed > 0.1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 185 ms, sys: 72.5 ms, total: 258 ms\n",
      "Wall time: 258 ms\n"
     ]
    }
   ],
   "source": [
    "# Remove unnecessary columns\n",
    "%time ais_subset = ais[['timestamp', 'mmsi', 'lat', 'lon', 'speed', 'vessel_type']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5min 32s, sys: 5.81 s, total: 5min 38s\n",
      "Wall time: 5min 42s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "%time ais_subset['grid_point'] = ais_subset.apply(lambda row: grid.get_grid_point(row.lat, row.lon), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6min 56s, sys: 26.5 s, total: 7min 22s\n",
      "Wall time: 7min 52s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "%time ais_subset['node'] = ais_subset.apply(lambda row: grid.get_grid_position(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 51.6 s, sys: 1.98 s, total: 53.6 s\n",
      "Wall time: 1min 1s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>-200</th>\n",
       "      <th>-197</th>\n",
       "      <th>-196</th>\n",
       "      <th>-195</th>\n",
       "      <th>-194</th>\n",
       "      <th>-193</th>\n",
       "      <th>76</th>\n",
       "      <th>77</th>\n",
       "      <th>78</th>\n",
       "      <th>79</th>\n",
       "      <th>...</th>\n",
       "      <th>68947</th>\n",
       "      <th>68948</th>\n",
       "      <th>69212</th>\n",
       "      <th>69213</th>\n",
       "      <th>69219</th>\n",
       "      <th>69487</th>\n",
       "      <th>69488</th>\n",
       "      <th>69495</th>\n",
       "      <th>69763</th>\n",
       "      <th>71160</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CONT</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.281563</td>\n",
       "      <td>6.134489</td>\n",
       "      <td>5.101567</td>\n",
       "      <td>5.633553</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.680829</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GC</th>\n",
       "      <td>1.394444</td>\n",
       "      <td>4.195000</td>\n",
       "      <td>5.298432</td>\n",
       "      <td>5.314470</td>\n",
       "      <td>4.799957</td>\n",
       "      <td>5.213959</td>\n",
       "      <td>3.704</td>\n",
       "      <td>4.106</td>\n",
       "      <td>4.1125</td>\n",
       "      <td>4.263636</td>\n",
       "      <td>...</td>\n",
       "      <td>1.199451</td>\n",
       "      <td>3.545102</td>\n",
       "      <td>5.569362</td>\n",
       "      <td>5.540676</td>\n",
       "      <td>2.613311</td>\n",
       "      <td>3.509783</td>\n",
       "      <td>4.548291</td>\n",
       "      <td>0.437899</td>\n",
       "      <td>1.026824</td>\n",
       "      <td>22.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAS</th>\n",
       "      <td>NaN</td>\n",
       "      <td>4.833333</td>\n",
       "      <td>4.728333</td>\n",
       "      <td>4.965000</td>\n",
       "      <td>4.224000</td>\n",
       "      <td>4.320000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.6300</td>\n",
       "      <td>4.722222</td>\n",
       "      <td>...</td>\n",
       "      <td>1.780000</td>\n",
       "      <td>3.885000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RORO</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.837040</td>\n",
       "      <td>5.819038</td>\n",
       "      <td>4.903045</td>\n",
       "      <td>5.430000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.717500</td>\n",
       "      <td>5.884000</td>\n",
       "      <td>2.265556</td>\n",
       "      <td>2.830000</td>\n",
       "      <td>4.531250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.230000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 15679 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        -200      -197      -196      -195      -194      -193     76     \\\n",
       "CONT       NaN       NaN  6.281563  6.134489  5.101567  5.633553     NaN   \n",
       "GC    1.394444  4.195000  5.298432  5.314470  4.799957  5.213959   3.704   \n",
       "PAS        NaN  4.833333  4.728333  4.965000  4.224000  4.320000     NaN   \n",
       "RORO       NaN       NaN       NaN       NaN       NaN       NaN     NaN   \n",
       "T          NaN       NaN  5.837040  5.819038  4.903045  5.430000     NaN   \n",
       "\n",
       "       77      78        79      ...       68947     68948     69212  \\\n",
       "CONT     NaN     NaN       NaN   ...         NaN       NaN       NaN   \n",
       "GC     4.106  4.1125  4.263636   ...    1.199451  3.545102  5.569362   \n",
       "PAS      NaN  4.6300  4.722222   ...    1.780000  3.885000       NaN   \n",
       "RORO     NaN     NaN       NaN   ...         NaN       NaN       NaN   \n",
       "T        NaN     NaN       NaN   ...         NaN       NaN  5.717500   \n",
       "\n",
       "         69213     69219     69487     69488     69495     69763   71160  \n",
       "CONT       NaN  2.680829       NaN       NaN       NaN       NaN     NaN  \n",
       "GC    5.540676  2.613311  3.509783  4.548291  0.437899  1.026824   22.33  \n",
       "PAS        NaN       NaN       NaN       NaN       NaN       NaN     NaN  \n",
       "RORO       NaN       NaN       NaN       NaN       NaN       NaN     NaN  \n",
       "T     5.884000  2.265556  2.830000  4.531250       NaN  1.230000     NaN  \n",
       "\n",
       "[5 rows x 15679 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "importlib.reload(gridify)\n",
    "\n",
    "%time avg_speeds = gridify.get_avg_speeds(ais_subset)\n",
    "avg_speeds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CONT': nan,\n",
       " 'GC': 4.114166666666667,\n",
       " 'PAS': 4.644285714285715,\n",
       " 'RORO': nan,\n",
       " 'T': nan}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_speeds.to_dict()[354]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#avg_speeds.to_csv('speed_model_5km.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}