CNN_Second_iteration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load in required libraries, below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data \n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "from sklearn.datasets import make_moons\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## SageMaker Resources\n",
    "\n",
    "The below cell stores the SageMaker session and role (for creating estimators and models), and creates a default S3 bucket. After creating this bucket, locally stored data can be uploaded to S3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sagemaker\n",
    "import boto3\n",
    "import sagemaker\n",
    "from sagemaker import get_execution_role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# SageMaker session and role\n",
    "sagemaker_session = sagemaker.Session()\n",
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "# default S3 bucket\n",
    "bucket = sagemaker_session.default_bucket()\n",
    "prefix='cnn-wendy-data-2'\n",
    "prefix_output='cnn-wendy-model-2'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we retrieve the dataset of images and we upload it to S3\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!wget -nc https://da-youtube-ml.s3.eu-central-1.amazonaws.com/wendy-cnn/frames/wendy_cnn_frames_data_2.zip\n",
    "#!unzip -qq -n wendy_cnn_frames_data_2.zip -d wendy_cnn_frames_data_2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uploading data to cnn-wendy-data-2\n",
      "Data uploaded to s3://sagemaker-eu-central-1-283211002347/cnn-wendy-data-2\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# upload to S3. Skip if already uploaded. This can take a while.\n",
    "print('Uploading data to {}'.format(prefix))\n",
    "input_data = sagemaker_session.upload_data(path='wendy_cnn_frames_data_2', bucket=bucket, key_prefix=prefix)\n",
    "print('Data uploaded to {}'.format(input_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# location to input data can be written down here, if known\n",
    "input_data='s3://sagemaker-eu-central-1-283211002347/cnn-wendy-data-2'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After uploading images to S3, we can define and train the estimator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output path for models is s3://sagemaker-eu-central-1-283211002347/cnn-wendy-model-2\n"
     ]
    }
   ],
   "source": [
    "# import a PyTorch wrapper\n",
    "from sagemaker.pytorch import PyTorch\n",
    "\n",
    "# specify an output path\n",
    "\n",
    "output_path = 's3://{}/{}'.format(bucket, prefix_output)\n",
    "print('Output path for models is {}'.format(output_path))\n",
    "\n",
    "# instantiate a pytorch estimator\n",
    "estimator = PyTorch(entry_point='train.py',\n",
    "                    source_dir='letsplay_classifier',\n",
    "                    role=role,\n",
    "                    framework_version='1.6',\n",
    "                    train_instance_count=1,\n",
    "                    train_instance_type='ml.p2.8xlarge',\n",
    "                    train_volume_size = 10,\n",
    "                    output_path=output_path,\n",
    "                    sagemaker_session=sagemaker_session,\n",
    "                    hyperparameters={\n",
    "                        'img-width': 160,\n",
    "                        'img-height': 90,\n",
    "                        'batch-size': 32,\n",
    "                        'layer-cfg': 'B',\n",
    "                        'epochs': 6\n",
    "                    })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the Estimator\n",
    "\n",
    "After instantiating the estimator, we train it with a call to `.fit()`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'create_image_uri' will be deprecated in favor of 'ImageURIProvider' class in SageMaker Python SDK v2.\n",
      "'s3_input' class will be renamed to 'TrainingInput' in SageMaker Python SDK v2.\n",
      "'create_image_uri' will be deprecated in favor of 'ImageURIProvider' class in SageMaker Python SDK v2.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-10-23 03:39:01 Starting - Starting the training job...\n",
      "2020-10-23 03:39:04 Starting - Launching requested ML instances.........\n",
      "2020-10-23 03:40:39 Starting - Preparing the instances for training......\n",
      "2020-10-23 03:41:48 Downloading - Downloading input data...........................\n",
      "2020-10-23 03:46:21 Training - Downloading the training image.\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2020-10-23 03:46:35,767 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2020-10-23 03:46:35,844 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2020-10-23 03:46:35,851 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2020-10-23 03:46:36,226 sagemaker-training-toolkit INFO     Installing dependencies from requirements.txt:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python -m pip install -r requirements.txt\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.6/site-packages (from -r requirements.txt (line 1)) (0.23.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch==1.6.0 in /opt/conda/lib/python3.6/site-packages (from -r requirements.txt (line 2)) (1.6.0)\u001b[0m\n",
      "\u001b[34mCollecting torchdata==0.2.0\n",
      "  Downloading torchdata-0.2.0-py3-none-any.whl (27 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torchvision==0.7.0 in /opt/conda/lib/python3.6/site-packages (from -r requirements.txt (line 4)) (0.7.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.6/site-packages (from scikit-learn->-r requirements.txt (line 1)) (0.17.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy>=1.13.3 in /opt/conda/lib/python3.6/site-packages (from scikit-learn->-r requirements.txt (line 1)) (1.19.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from scikit-learn->-r requirements.txt (line 1)) (2.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.6/site-packages (from scikit-learn->-r requirements.txt (line 1)) (1.5.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: future in /opt/conda/lib/python3.6/site-packages (from torch==1.6.0->-r requirements.txt (line 2)) (0.18.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow>=4.1.1 in /opt/conda/lib/python3.6/site-packages (from torchvision==0.7.0->-r requirements.txt (line 4)) (7.2.0)\u001b[0m\n",
      "\u001b[34mInstalling collected packages: torchdata\u001b[0m\n",
      "\u001b[34mSuccessfully installed torchdata-0.2.0\u001b[0m\n",
      "\u001b[34m2020-10-23 03:46:38,204 sagemaker-training-toolkit INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[34mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"train\": \"/opt/ml/input/data/train\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"batch-size\": 32,\n",
      "        \"layer-cfg\": \"B\",\n",
      "        \"img-width\": 160,\n",
      "        \"epochs\": 6,\n",
      "        \"img-height\": 90\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"train\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": true,\n",
      "    \"job_name\": \"pytorch-training-2020-10-23-03-39-01-406\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-eu-central-1-283211002347/pytorch-training-2020-10-23-03-39-01-406/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"train\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 32,\n",
      "    \"num_gpus\": 8,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"train.py\"\u001b[0m\n",
      "\u001b[34m}\n",
      "\u001b[0m\n",
      "\u001b[34mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={\"batch-size\":32,\"epochs\":6,\"img-height\":90,\"img-width\":160,\"layer-cfg\":\"B\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=train.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"train\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=train\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=32\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=8\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-eu-central-1-283211002347/pytorch-training-2020-10-23-03-39-01-406/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"train\":\"/opt/ml/input/data/train\"},\"current_host\":\"algo-1\",\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"batch-size\":32,\"epochs\":6,\"img-height\":90,\"img-width\":160,\"layer-cfg\":\"B\"},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"pytorch-training-2020-10-23-03-39-01-406\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-eu-central-1-283211002347/pytorch-training-2020-10-23-03-39-01-406/source/sourcedir.tar.gz\",\"module_name\":\"train\",\"network_interface_name\":\"eth0\",\"num_cpus\":32,\"num_gpus\":8,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"train.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[\"--batch-size\",\"32\",\"--epochs\",\"6\",\"--img-height\",\"90\",\"--img-width\",\"160\",\"--layer-cfg\",\"B\"]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_TRAIN=/opt/ml/input/data/train\u001b[0m\n",
      "\u001b[34mSM_HP_BATCH-SIZE=32\u001b[0m\n",
      "\u001b[34mSM_HP_LAYER-CFG=B\u001b[0m\n",
      "\u001b[34mSM_HP_IMG-WIDTH=160\u001b[0m\n",
      "\u001b[34mSM_HP_EPOCHS=6\u001b[0m\n",
      "\u001b[34mSM_HP_IMG-HEIGHT=90\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python36.zip:/opt/conda/lib/python3.6:/opt/conda/lib/python3.6/lib-dynload:/opt/conda/lib/python3.6/site-packages\n",
      "\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python train.py --batch-size 32 --epochs 6 --img-height 90 --img-width 160 --layer-cfg B\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 0/6\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mTraining batch 0/1072\u001b[0m\n",
      "\u001b[34m[2020-10-23 03:46:48.136 algo-1:31 INFO json_config.py:90] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\u001b[0m\n",
      "\u001b[34m[2020-10-23 03:46:48.137 algo-1:31 INFO hook.py:193] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\u001b[0m\n",
      "\u001b[34m[2020-10-23 03:46:48.137 algo-1:31 INFO hook.py:238] Saving to /opt/ml/output/tensors\u001b[0m\n",
      "\u001b[34m[2020-10-23 03:46:48.137 algo-1:31 INFO state_store.py:67] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\u001b[0m\n",
      "\u001b[34m[2020-10-23 03:46:48.160 algo-1:31 INFO hook.py:398] Monitoring the collections: losses\u001b[0m\n",
      "\u001b[34m[2020-10-23 03:46:48.160 algo-1:31 INFO hook.py:459] Hook is writing from the hook with pid: 31\n",
      "\u001b[0m\n",
      "\n",
      "2020-10-23 03:46:35 Training - Training image download completed. Training in progress.\u001b[34mTraining batch 100/1072\u001b[0m\n",
      "\u001b[34mTraining batch 200/1072\u001b[0m\n",
      "\u001b[34mTraining batch 300/1072\u001b[0m\n",
      "\u001b[34mTraining batch 400/1072\u001b[0m\n",
      "\u001b[34mTraining batch 500/1072\u001b[0m\n",
      "\u001b[34mTraining batch 600/1072\u001b[0m\n",
      "\u001b[34mTraining batch 700/1072\u001b[0m\n",
      "\u001b[34mTraining batch 800/1072\u001b[0m\n",
      "\u001b[34mTraining batch 900/1072\u001b[0m\n",
      "\u001b[34mTraining batch 1000/1072\u001b[0m\n",
      "\u001b[34mValidation batch 0/358\u001b[0m\n",
      "\u001b[34mValidation batch 100/358\u001b[0m\n",
      "\u001b[34mValidation batch 200/358\u001b[0m\n",
      "\u001b[34mValidation batch 300/358\u001b[0m\n",
      "\u001b[34mEpoch 0 result: \u001b[0m\n",
      "\u001b[34mAvg loss (train): 0.0088\u001b[0m\n",
      "\u001b[34mAvg acc (train): 0.9070\u001b[0m\n",
      "\u001b[34mAvg loss (val): 0.0053\u001b[0m\n",
      "\u001b[34mAvg acc (val): 0.9464\u001b[0m\n",
      "\u001b[34m----------\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 1/6\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mTraining batch 0/1072\u001b[0m\n",
      "\u001b[34mTraining batch 100/1072\u001b[0m\n",
      "\u001b[34mTraining batch 200/1072\u001b[0m\n",
      "\u001b[34mTraining batch 300/1072\u001b[0m\n",
      "\u001b[34mTraining batch 400/1072\u001b[0m\n",
      "\u001b[34mTraining batch 500/1072\u001b[0m\n",
      "\u001b[34mTraining batch 600/1072\u001b[0m\n",
      "\u001b[34mTraining batch 700/1072\u001b[0m\n",
      "\u001b[34mTraining batch 800/1072\u001b[0m\n",
      "\u001b[34mTraining batch 900/1072\u001b[0m\n",
      "\u001b[34mTraining batch 1000/1072\u001b[0m\n",
      "\u001b[34mValidation batch 0/358\u001b[0m\n",
      "\u001b[34mValidation batch 100/358\u001b[0m\n",
      "\u001b[34mValidation batch 200/358\u001b[0m\n",
      "\u001b[34mValidation batch 300/358\u001b[0m\n",
      "\u001b[34mEpoch 1 result: \u001b[0m\n",
      "\u001b[34mAvg loss (train): 0.0048\u001b[0m\n",
      "\u001b[34mAvg acc (train): 0.9500\u001b[0m\n",
      "\u001b[34mAvg loss (val): 0.0042\u001b[0m\n",
      "\u001b[34mAvg acc (val): 0.9633\u001b[0m\n",
      "\u001b[34m----------\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 2/6\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mTraining batch 0/1072\u001b[0m\n",
      "\u001b[34mTraining batch 100/1072\u001b[0m\n",
      "\u001b[34mTraining batch 200/1072\u001b[0m\n",
      "\u001b[34mTraining batch 300/1072\u001b[0m\n",
      "\u001b[34mTraining batch 400/1072\u001b[0m\n",
      "\u001b[34mTraining batch 500/1072\u001b[0m\n",
      "\u001b[34mTraining batch 600/1072\u001b[0m\n",
      "\u001b[34mTraining batch 700/1072\u001b[0m\n",
      "\u001b[34mTraining batch 800/1072\u001b[0m\n",
      "\u001b[34mTraining batch 900/1072\u001b[0m\n",
      "\u001b[34mTraining batch 1000/1072\u001b[0m\n",
      "\u001b[34mValidation batch 0/358\u001b[0m\n",
      "\u001b[34mValidation batch 100/358\u001b[0m\n",
      "\u001b[34mValidation batch 200/358\u001b[0m\n",
      "\u001b[34mValidation batch 300/358\u001b[0m\n",
      "\u001b[34mEpoch 2 result: \u001b[0m\n",
      "\u001b[34mAvg loss (train): 0.0036\u001b[0m\n",
      "\u001b[34mAvg acc (train): 0.9645\u001b[0m\n",
      "\u001b[34mAvg loss (val): 0.0043\u001b[0m\n",
      "\u001b[34mAvg acc (val): 0.9572\u001b[0m\n",
      "\u001b[34m----------\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 3/6\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mTraining batch 0/1072\u001b[0m\n",
      "\u001b[34mTraining batch 100/1072\u001b[0m\n",
      "\u001b[34mTraining batch 200/1072\u001b[0m\n",
      "\u001b[34mTraining batch 300/1072\u001b[0m\n",
      "\u001b[34mTraining batch 400/1072\u001b[0m\n",
      "\u001b[34mTraining batch 500/1072\u001b[0m\n",
      "\u001b[34mTraining batch 600/1072\u001b[0m\n",
      "\u001b[34mTraining batch 700/1072\u001b[0m\n",
      "\u001b[34mTraining batch 800/1072\u001b[0m\n",
      "\u001b[34mTraining batch 900/1072\u001b[0m\n",
      "\u001b[34mTraining batch 1000/1072\u001b[0m\n",
      "\u001b[34mValidation batch 0/358\u001b[0m\n",
      "\u001b[34mValidation batch 100/358\u001b[0m\n",
      "\u001b[34mValidation batch 200/358\u001b[0m\n",
      "\u001b[34mValidation batch 300/358\u001b[0m\n",
      "\u001b[34mEpoch 3 result: \u001b[0m\n",
      "\u001b[34mAvg loss (train): 0.0029\u001b[0m\n",
      "\u001b[34mAvg acc (train): 0.9712\u001b[0m\n",
      "\u001b[34mAvg loss (val): 0.0025\u001b[0m\n",
      "\u001b[34mAvg acc (val): 0.9771\u001b[0m\n",
      "\u001b[34m----------\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 4/6\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mTraining batch 0/1072\u001b[0m\n",
      "\u001b[34mTraining batch 100/1072\u001b[0m\n",
      "\u001b[34mTraining batch 200/1072\u001b[0m\n",
      "\u001b[34mTraining batch 300/1072\u001b[0m\n",
      "\u001b[34mTraining batch 400/1072\u001b[0m\n",
      "\u001b[34mTraining batch 500/1072\u001b[0m\n",
      "\u001b[34mTraining batch 600/1072\u001b[0m\n",
      "\u001b[34mTraining batch 700/1072\u001b[0m\n",
      "\u001b[34mTraining batch 800/1072\u001b[0m\n",
      "\u001b[34mTraining batch 900/1072\u001b[0m\n",
      "\u001b[34mTraining batch 1000/1072\u001b[0m\n",
      "\u001b[34mValidation batch 0/358\u001b[0m\n",
      "\u001b[34mValidation batch 100/358\u001b[0m\n",
      "\u001b[34mValidation batch 200/358\u001b[0m\n",
      "\u001b[34mValidation batch 300/358\u001b[0m\n",
      "\u001b[34mEpoch 4 result: \u001b[0m\n",
      "\u001b[34mAvg loss (train): 0.0025\u001b[0m\n",
      "\u001b[34mAvg acc (train): 0.9760\u001b[0m\n",
      "\u001b[34mAvg loss (val): 0.0027\u001b[0m\n",
      "\u001b[34mAvg acc (val): 0.9762\u001b[0m\n",
      "\u001b[34m----------\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 5/6\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mTraining batch 0/1072\u001b[0m\n",
      "\u001b[34mTraining batch 100/1072\u001b[0m\n",
      "\u001b[34mTraining batch 200/1072\u001b[0m\n",
      "\u001b[34mTraining batch 300/1072\u001b[0m\n",
      "\u001b[34mTraining batch 400/1072\u001b[0m\n",
      "\u001b[34mTraining batch 500/1072\u001b[0m\n",
      "\u001b[34mTraining batch 600/1072\u001b[0m\n",
      "\u001b[34mTraining batch 700/1072\u001b[0m\n",
      "\u001b[34mTraining batch 800/1072\u001b[0m\n",
      "\u001b[34mTraining batch 900/1072\u001b[0m\n",
      "\u001b[34mTraining batch 1000/1072\u001b[0m\n",
      "\u001b[34mValidation batch 0/358\u001b[0m\n",
      "\u001b[34mValidation batch 100/358\u001b[0m\n",
      "\u001b[34mValidation batch 200/358\u001b[0m\n",
      "\u001b[34mValidation batch 300/358\u001b[0m\n",
      "\u001b[34mEpoch 5 result: \u001b[0m\n",
      "\u001b[34mAvg loss (train): 0.0021\u001b[0m\n",
      "\u001b[34mAvg acc (train): 0.9779\u001b[0m\n",
      "\u001b[34mAvg loss (val): 0.0028\u001b[0m\n",
      "\u001b[34mAvg acc (val): 0.9774\u001b[0m\n",
      "\u001b[34m----------\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[34mTraining completed in 59m 3s\u001b[0m\n",
      "\u001b[34mBest acc: 0.9774\u001b[0m\n",
      "\u001b[34mSaving the model.\u001b[0m\n",
      "\u001b[34m2020-10-23 04:45:53,154 sagemaker-training-toolkit INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2020-10-23 04:46:57 Uploading - Uploading generated training model\n",
      "2020-10-23 04:48:05 Completed - Training job completed\n",
      "Training seconds: 3977\n",
      "Billable seconds: 3977\n",
      "CPU times: user 8.9 s, sys: 284 ms, total: 9.18 s\n",
      "Wall time: 1h 9min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# train the estimator on S3 training data\n",
    "estimator.fit({'train': input_data})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3://sagemaker-eu-central-1-283211002347/cnn-wendy-model-2/pytorch-training-2020-10-23-03-39-01-406/output/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "print(estimator.model_data)\n",
    "model_data = estimator.model_data\n",
    "# model_data = 's3://sagemaker-eu-central-1-283211002347.s3.eu-central-1.amazonaws.com/cnn-wendy-model-2/pytorch-training-2020-10-23-01-41-38-537/output/model.tar.gz'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We set up a model that can predict the class of an image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deploy the trained model\n",
    "\n",
    "We deploy our model to create a predictor. We'll use this to make predictions on our data and evaluate the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Parameter image will be renamed to image_uri in SageMaker Python SDK v2.\n"
     ]
    }
   ],
   "source": [
    "# importing PyTorchModel\n",
    "from sagemaker.pytorch import PyTorchModel\n",
    "\n",
    "# Create a model from the trained estimator data\n",
    "# And point to the prediction script\n",
    "model = PyTorchModel(model_data=model_data,\n",
    "                     role = role,\n",
    "                     framework_version='1.6',\n",
    "                     entry_point='predict.py',\n",
    "                     source_dir='letsplay_classifier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'create_image_uri' will be deprecated in favor of 'ImageURIProvider' class in SageMaker Python SDK v2.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------!CPU times: user 37.7 s, sys: 4.51 s, total: 42.2 s\n",
      "Wall time: 10min 43s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# deploy and create a predictor\n",
    "              \n",
    "predictor = model.deploy(initial_instance_count=1, instance_type='ml.p2.xlarge')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the endpoint where the predictor is located\n",
    "endpoint_name = predictor.endpoint\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that the model is deployed, we check how the predictor performs on our full dataset,\n",
    "ensuring that the predictions make sense. We produce a classification report.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pytorch-inference-2020-10-23-05-01-33-314\n"
     ]
    }
   ],
   "source": [
    "print(endpoint_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50 processed up to 1784\n",
      "100 processed up to 2897\n",
      "150 processed up to 4410\n",
      "200 processed up to 5601\n",
      "250 processed up to 6777\n",
      "300 processed up to 8105\n",
      "350 processed up to 9541\n",
      "400 processed up to 11087\n",
      "450 processed up to 12379\n",
      "500 processed up to 13735\n",
      "550 processed up to 15134\n",
      "600 processed up to 16094\n",
      "650 processed up to 17007\n",
      "700 processed up to 17797\n",
      "750 processed up to 18984\n",
      "800 processed up to 20236\n",
      "850 processed up to 21478\n",
      "900 processed up to 22318\n",
      "950 processed up to 23435\n",
      "1000 processed up to 24561\n",
      "1050 processed up to 26057\n",
      "1100 processed up to 27221\n",
      "1150 processed up to 28414\n",
      "1200 processed up to 29488\n",
      "1250 processed up to 30775\n",
      "1300 processed up to 31661\n",
      "1350 processed up to 32807\n",
      "1400 processed up to 33724\n",
      "1450 processed up to 34755\n",
      "1500 processed up to 36029\n",
      "1550 processed up to 37662\n",
      "1600 processed up to 38817\n",
      "1650 processed up to 40092\n",
      "1700 processed up to 41033\n",
      "1750 processed up to 41976\n",
      "1800 processed up to 43382\n",
      "1850 processed up to 44646\n",
      "1900 processed up to 45597\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.99       108\n",
      "           1       0.96      1.00      0.98        23\n",
      "           2       0.99      0.99      0.99       667\n",
      "           3       1.00      1.00      1.00         4\n",
      "           4       0.99      0.96      0.98       152\n",
      "\n",
      "    accuracy                           0.99       954\n",
      "   macro avg       0.98      0.99      0.99       954\n",
      "weighted avg       0.99      0.99      0.99       954\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#endpoint_name='pytorch-inference-2020-10-20-02-20-28-656'\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from letsplay_classifier.endpoint import evaluate\n",
    "y_true, y_pred = evaluate(endpoint_name, 'wendy_cnn_frames_data_2', 0.02)\n",
    "report = classification_report(y_true=y_true, y_pred=y_pred)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report = classification_report(y_true=y_true, y_pred=y_pred)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_actu, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delete the Endpoint\n",
    "\n",
    "Finally, I've added a convenience function to delete prediction endpoints after we're done with them. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Accepts a predictor endpoint as input\n",
    "# And deletes the endpoint by name\n",
    "def delete_endpoint(predictor):\n",
    "        try:\n",
    "            boto3.client('sagemaker').delete_endpoint(EndpointName=endpoint_name)\n",
    "            print('Deleted {}'.format(predictor.endpoint))\n",
    "        except:\n",
    "            print('Already deleted: {}'.format(predictor.endpoint))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleted pytorch-inference-2020-10-23-05-01-33-314\n"
     ]
    }
   ],
   "source": [
    "# delete the predictor endpoint \n",
    "delete_endpoint(predictor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}