tf-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T20:33:09.270252Z",
     "start_time": "2020-11-18T20:33:09.183765Z"
    },
    "code_folding": [
     0
    ],
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorflow.python.ops.standard_ops'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-3e8b8d9877ca>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0monnx\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0monnx_tf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mprepare\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0monnx_model\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0monnx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"reusr.onnx\"\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# load onnx model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\onnx_tf\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mbackend\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mversion\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mversion\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0m__version__\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\onnx_tf\\backend.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0monnx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrunner\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mBackendIsNotSupposedToImplementIt\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0monnx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhelper\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmake_opsetid\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     22\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0monnx_tf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackend_rep\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mTensorflowRep\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     39\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0msys\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0m_sys\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 41\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtools\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmodule_util\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0m_module_util\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     42\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlazy_loader\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mLazyLoader\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0m_LazyLoader\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     72\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     73\u001b[0m \u001b[1;31m# Ops\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstandard_ops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     75\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m \u001b[1;31m# Namespaces\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow.python.ops.standard_ops'"
     ]
    }
   ],
   "source": [
    "#REMOVE\n",
    "# import onnx\n",
    "\n",
    "# from onnx_tf.backend import prepare\n",
    "\n",
    "# onnx_model = onnx.load(\"reusr.onnx\")  # load onnx model\n",
    "# tf_rep = prepare(onnx_model)  # prepare tf representation\n",
    "# tf_rep.export_graph(\"reusr-tf.pb\")  # export the model\n",
    "\n",
    "# import tensorflow.compat.v1 as tf\n",
    "# tf.disable_v2_behavior()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:00:09.025518Z",
     "start_time": "2020-11-21T00:00:06.254949Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import re\n",
    "import tensorflow as tf\n",
    "\n",
    "# from dotenv import load_dotenv\n",
    "# load_dotenv()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configure, Load Dataset "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset of **469** images scraped from google belonging to 6 categories: **cardboard, glass bottles, plastic containers, paper, glass jars and plastic bottles**\n",
    "\n",
    "Train dataset: 378 images\n",
    "Test dataset: 91 images\n",
    "\n",
    "![](https://snap-n-reuse.netlify.app/assets/img/landing/f3.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:00:09.041473Z",
     "start_time": "2020-11-21T00:00:09.030504Z"
    }
   },
   "outputs": [],
   "source": [
    "data_dir = 'data'\n",
    "TRAIN_DATASET_PATH = data_dir + '/train'\n",
    "VALIDATION_DATASET_PATH = data_dir + '/test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:00:09.056477Z",
     "start_time": "2020-11-21T00:00:09.042471Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data/train'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TRAIN_DATASET_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:00:09.072391Z",
     "start_time": "2020-11-21T00:00:09.059460Z"
    }
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 8\n",
    "N_CLASSES = 6\n",
    "TOTAL_EPOCHS = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:00:09.683784Z",
     "start_time": "2020-11-21T00:00:09.442174Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 378 images belonging to 6 classes.\n",
      "Found 91 images belonging to 6 classes.\n"
     ]
    }
   ],
   "source": [
    "# Generators\n",
    "train_generator = tf.keras.preprocessing.image.ImageDataGenerator(\n",
    "    data_format='channels_last',\n",
    "    rescale=1. / 255,\n",
    "    rotation_range=40,\n",
    "    width_shift_range=0.2,\n",
    "    height_shift_range=0.2,\n",
    "    shear_range=0.2,\n",
    "    zoom_range=0.2,\n",
    "    horizontal_flip=True\n",
    ")\n",
    "\n",
    "train_batches = train_generator.flow_from_directory(\n",
    "    batch_size=BATCH_SIZE,\n",
    "    directory=TRAIN_DATASET_PATH,\n",
    "    target_size=[224, 224],\n",
    "    class_mode='categorical'\n",
    ")\n",
    "\n",
    "val_generator = tf.keras.preprocessing.image.ImageDataGenerator(\n",
    "    data_format='channels_last',\n",
    "    rescale=1. / 255\n",
    ")\n",
    "\n",
    "val_batches = train_generator.flow_from_directory(\n",
    "    batch_size=BATCH_SIZE,\n",
    "    directory=VALIDATION_DATASET_PATH,\n",
    "    target_size=[224, 224],\n",
    "    class_mode='categorical'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:00:15.196684Z",
     "start_time": "2020-11-21T00:00:11.946832Z"
    }
   },
   "outputs": [],
   "source": [
    "# Model\n",
    "kernel_initializer = tf.keras.initializers.glorot_uniform(seed=1337)\n",
    "trained_model = tf.keras.applications.mobilenet_v2.MobileNetV2(\n",
    "                      include_top=False,\n",
    "                      weights='imagenet',\n",
    "                      classes=N_CLASSES,\n",
    "                      alpha=0.5,\n",
    "                      input_shape=[224, 224, 3],\n",
    "                      pooling='max')\n",
    "output = tf.keras.layers.Dense(N_CLASSES, activation='softmax', kernel_initializer=kernel_initializer)(trained_model.output)\n",
    "model = tf.keras.Model(inputs=trained_model.input, outputs=output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:00:15.211511Z",
     "start_time": "2020-11-21T00:00:15.197526Z"
    }
   },
   "outputs": [],
   "source": [
    "# Callback to save weights, based on val_acc\n",
    "model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
    "  './checkpoints/{epoch:02d}_{val_accuracy:.4f}.h5',\n",
    "  save_weights_only=False,\n",
    "  verbose=1,\n",
    "  monitor='val_accuracy',\n",
    "  save_best_only=True,\n",
    "#     save_freq='epoch',\n",
    "  mode='max'\n",
    ")\n",
    "\n",
    "# Callbackto plot data on TensorBoard\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(\n",
    "  log_dir='./logs/reusr',\n",
    "  histogram_freq=0,\n",
    "#   batch_size=BATCH_SIZE\n",
    ")\n",
    "\n",
    "# Callback to reduce learning rate after plateaus\n",
    "reduce_lr_callback = tf.keras.callbacks.ReduceLROnPlateau(\n",
    "  monitor='val_accuracy',\n",
    "  factor=0.5,\n",
    "  patience=4,\n",
    "  min_lr=1e-6\n",
    ")\n",
    "\n",
    "early_stopping_callback = tf.keras.callbacks.EarlyStopping(\n",
    "  monitor='val_accuracy',\n",
    "  patience=20,\n",
    "  mode='max',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:00:17.223360Z",
     "start_time": "2020-11-21T00:00:17.205377Z"
    }
   },
   "outputs": [],
   "source": [
    "TRAIN_DATASET_SIZE = len(train_batches)\n",
    "VAL_DATASET_SIZE   = len(val_batches)\n",
    "\n",
    "# Weighted losses for class equilibrium\n",
    "unique, counts = np.unique(train_batches.classes, return_counts=True)\n",
    "class_weigths = dict(zip(unique, np.true_divide(counts.sum(), N_CLASSES*counts)))\n",
    "\n",
    "\n",
    "if Path('./checkpoints/').exists():\n",
    "  epoch_number_array = []\n",
    "  val_accuracy_array = []\n",
    "  file_name_array = []\n",
    "  for file in os.listdir('./checkpoints/'):\n",
    "    epoch, val_acc = re.search(r'(\\d\\d)_(\\d\\.\\d{4})\\.h5', file).group(1,2)\n",
    "    epoch_number_array.append(int(epoch))\n",
    "    val_accuracy_array.append(float(val_acc))\n",
    "    file_name_array.append(file)\n",
    "\n",
    "  if len(val_accuracy_array) == 0:\n",
    "    INITIAL_EPOCH = 0\n",
    "  else:\n",
    "    highest_acc = val_accuracy_array.index(max(val_accuracy_array))\n",
    "    INITIAL_EPOCH = epoch_number_array[highest_acc]\n",
    "    model_checkpoint_callback.best = val_accuracy_array[highest_acc]\n",
    "    model.load_weights('./checkpoints/'+file_name_array[highest_acc])\n",
    "else:\n",
    "  os.makedirs('./checkpoints/')\n",
    "  INITIAL_EPOCH = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:00:19.023113Z",
     "start_time": "2020-11-21T00:00:18.986552Z"
    }
   },
   "outputs": [],
   "source": [
    "# Prepare model to run\n",
    "model.compile(optimizer = tf.keras.optimizers.Adam(),\n",
    "              loss = 'categorical_crossentropy',\n",
    "              metrics = ['accuracy']\n",
    "              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T00:52:03.901413Z",
     "start_time": "2020-11-21T00:00:23.520707Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 4.0101 - accuracy: 0.4471\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.25275, saving model to ./checkpoints/01_0.2527.h5\n",
      "48/48 [==============================] - 49s 1s/step - loss: 4.0101 - accuracy: 0.4471 - val_loss: 5.4826 - val_accuracy: 0.2527 - lr: 0.0010\n",
      "Epoch 2/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 3.7789 - accuracy: 0.5212\n",
      "Epoch 00002: val_accuracy did not improve from 0.25275\n",
      "48/48 [==============================] - 40s 843ms/step - loss: 3.7789 - accuracy: 0.5212 - val_loss: 6.0274 - val_accuracy: 0.1538 - lr: 0.0010\n",
      "Epoch 3/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 3.0154 - accuracy: 0.5476\n",
      "Epoch 00003: val_accuracy did not improve from 0.25275\n",
      "48/48 [==============================] - 35s 739ms/step - loss: 3.0154 - accuracy: 0.5476 - val_loss: 7.9196 - val_accuracy: 0.2088 - lr: 0.0010\n",
      "Epoch 4/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 2.8251 - accuracy: 0.5820\n",
      "Epoch 00004: val_accuracy did not improve from 0.25275\n",
      "48/48 [==============================] - 34s 701ms/step - loss: 2.8251 - accuracy: 0.5820 - val_loss: 7.8702 - val_accuracy: 0.1758 - lr: 0.0010\n",
      "Epoch 5/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 2.2478 - accuracy: 0.6455\n",
      "Epoch 00005: val_accuracy did not improve from 0.25275\n",
      "48/48 [==============================] - 32s 672ms/step - loss: 2.2478 - accuracy: 0.6455 - val_loss: 10.0859 - val_accuracy: 0.1538 - lr: 0.0010\n",
      "Epoch 6/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 1.8795 - accuracy: 0.6587\n",
      "Epoch 00006: val_accuracy did not improve from 0.25275\n",
      "48/48 [==============================] - 34s 718ms/step - loss: 1.8795 - accuracy: 0.6587 - val_loss: 8.9941 - val_accuracy: 0.2418 - lr: 5.0000e-04\n",
      "Epoch 7/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.9737 - accuracy: 0.7646\n",
      "Epoch 00007: val_accuracy improved from 0.25275 to 0.28571, saving model to ./checkpoints/07_0.2857.h5\n",
      "48/48 [==============================] - 31s 643ms/step - loss: 0.9737 - accuracy: 0.7646 - val_loss: 5.9876 - val_accuracy: 0.2857 - lr: 5.0000e-04\n",
      "Epoch 8/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 1.0510 - accuracy: 0.7434\n",
      "Epoch 00008: val_accuracy did not improve from 0.28571\n",
      "48/48 [==============================] - 32s 670ms/step - loss: 1.0510 - accuracy: 0.7434 - val_loss: 6.2260 - val_accuracy: 0.2527 - lr: 5.0000e-04\n",
      "Epoch 9/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.9747 - accuracy: 0.7513\n",
      "Epoch 00009: val_accuracy improved from 0.28571 to 0.29670, saving model to ./checkpoints/09_0.2967.h5\n",
      "48/48 [==============================] - 33s 689ms/step - loss: 0.9747 - accuracy: 0.7513 - val_loss: 4.0965 - val_accuracy: 0.2967 - lr: 5.0000e-04\n",
      "Epoch 10/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 1.0278 - accuracy: 0.7593\n",
      "Epoch 00010: val_accuracy did not improve from 0.29670\n",
      "48/48 [==============================] - 34s 706ms/step - loss: 1.0278 - accuracy: 0.7593 - val_loss: 3.4642 - val_accuracy: 0.2857 - lr: 5.0000e-04\n",
      "Epoch 11/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.7478 - accuracy: 0.8175\n",
      "Epoch 00011: val_accuracy did not improve from 0.29670\n",
      "48/48 [==============================] - 42s 872ms/step - loss: 0.7478 - accuracy: 0.8175 - val_loss: 6.2252 - val_accuracy: 0.1648 - lr: 5.0000e-04\n",
      "Epoch 12/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.6630 - accuracy: 0.8280\n",
      "Epoch 00012: val_accuracy did not improve from 0.29670\n",
      "48/48 [==============================] - 32s 673ms/step - loss: 0.6630 - accuracy: 0.8280 - val_loss: 7.6079 - val_accuracy: 0.1868 - lr: 5.0000e-04\n",
      "Epoch 13/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.6295 - accuracy: 0.8519\n",
      "Epoch 00013: val_accuracy did not improve from 0.29670\n",
      "48/48 [==============================] - 31s 639ms/step - loss: 0.6295 - accuracy: 0.8519 - val_loss: 5.6715 - val_accuracy: 0.1868 - lr: 5.0000e-04\n",
      "Epoch 14/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.5057 - accuracy: 0.8492\n",
      "Epoch 00014: val_accuracy improved from 0.29670 to 0.31868, saving model to ./checkpoints/14_0.3187.h5\n",
      "48/48 [==============================] - 33s 689ms/step - loss: 0.5057 - accuracy: 0.8492 - val_loss: 3.3279 - val_accuracy: 0.3187 - lr: 2.5000e-04\n",
      "Epoch 15/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.4816 - accuracy: 0.8651\n",
      "Epoch 00015: val_accuracy did not improve from 0.31868\n",
      "48/48 [==============================] - 34s 712ms/step - loss: 0.4816 - accuracy: 0.8651 - val_loss: 4.2622 - val_accuracy: 0.3187 - lr: 2.5000e-04\n",
      "Epoch 16/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.2663 - accuracy: 0.9074\n",
      "Epoch 00016: val_accuracy improved from 0.31868 to 0.38462, saving model to ./checkpoints/16_0.3846.h5\n",
      "48/48 [==============================] - 34s 717ms/step - loss: 0.2663 - accuracy: 0.9074 - val_loss: 2.9057 - val_accuracy: 0.3846 - lr: 2.5000e-04\n",
      "Epoch 17/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.2896 - accuracy: 0.9233\n",
      "Epoch 00017: val_accuracy improved from 0.38462 to 0.47253, saving model to ./checkpoints/17_0.4725.h5\n",
      "48/48 [==============================] - 32s 665ms/step - loss: 0.2896 - accuracy: 0.9233 - val_loss: 2.6030 - val_accuracy: 0.4725 - lr: 2.5000e-04\n",
      "Epoch 18/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.3855 - accuracy: 0.8889\n",
      "Epoch 00018: val_accuracy improved from 0.47253 to 0.61538, saving model to ./checkpoints/18_0.6154.h5\n",
      "48/48 [==============================] - 34s 699ms/step - loss: 0.3855 - accuracy: 0.8889 - val_loss: 1.3233 - val_accuracy: 0.6154 - lr: 2.5000e-04\n",
      "Epoch 19/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.2788 - accuracy: 0.9101\n",
      "Epoch 00019: val_accuracy did not improve from 0.61538\n",
      "48/48 [==============================] - 31s 648ms/step - loss: 0.2788 - accuracy: 0.9101 - val_loss: 2.3657 - val_accuracy: 0.5165 - lr: 2.5000e-04\n",
      "Epoch 20/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.3017 - accuracy: 0.9127\n",
      "Epoch 00020: val_accuracy did not improve from 0.61538\n",
      "48/48 [==============================] - 32s 675ms/step - loss: 0.3017 - accuracy: 0.9127 - val_loss: 1.5887 - val_accuracy: 0.5604 - lr: 2.5000e-04\n",
      "Epoch 21/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.2436 - accuracy: 0.9259\n",
      "Epoch 00021: val_accuracy improved from 0.61538 to 0.63736, saving model to ./checkpoints/21_0.6374.h5\n",
      "48/48 [==============================] - 37s 765ms/step - loss: 0.2436 - accuracy: 0.9259 - val_loss: 1.1768 - val_accuracy: 0.6374 - lr: 2.5000e-04\n",
      "Epoch 22/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.3036 - accuracy: 0.8968\n",
      "Epoch 00022: val_accuracy did not improve from 0.63736\n",
      "48/48 [==============================] - 37s 770ms/step - loss: 0.3036 - accuracy: 0.8968 - val_loss: 1.3604 - val_accuracy: 0.5824 - lr: 2.5000e-04\n",
      "Epoch 23/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.2425 - accuracy: 0.9180\n",
      "Epoch 00023: val_accuracy improved from 0.63736 to 0.73626, saving model to ./checkpoints/23_0.7363.h5\n",
      "48/48 [==============================] - 37s 771ms/step - loss: 0.2425 - accuracy: 0.9180 - val_loss: 1.1376 - val_accuracy: 0.7363 - lr: 2.5000e-04\n",
      "Epoch 24/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.2698 - accuracy: 0.9101\n",
      "Epoch 00024: val_accuracy did not improve from 0.73626\n",
      "48/48 [==============================] - 35s 727ms/step - loss: 0.2698 - accuracy: 0.9101 - val_loss: 1.8672 - val_accuracy: 0.6044 - lr: 2.5000e-04\n",
      "Epoch 25/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.2719 - accuracy: 0.9206\n",
      "Epoch 00025: val_accuracy did not improve from 0.73626\n",
      "48/48 [==============================] - 37s 765ms/step - loss: 0.2719 - accuracy: 0.9206 - val_loss: 1.2482 - val_accuracy: 0.6923 - lr: 2.5000e-04\n",
      "Epoch 26/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.3015 - accuracy: 0.9021\n",
      "Epoch 00026: val_accuracy did not improve from 0.73626\n",
      "48/48 [==============================] - 37s 772ms/step - loss: 0.3015 - accuracy: 0.9021 - val_loss: 1.3695 - val_accuracy: 0.6593 - lr: 2.5000e-04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1434 - accuracy: 0.9497\n",
      "Epoch 00027: val_accuracy did not improve from 0.73626\n",
      "48/48 [==============================] - 34s 713ms/step - loss: 0.1434 - accuracy: 0.9497 - val_loss: 1.2916 - val_accuracy: 0.6923 - lr: 2.5000e-04\n",
      "Epoch 28/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.2020 - accuracy: 0.9339\n",
      "Epoch 00028: val_accuracy did not improve from 0.73626\n",
      "48/48 [==============================] - 35s 720ms/step - loss: 0.2020 - accuracy: 0.9339 - val_loss: 1.4804 - val_accuracy: 0.6593 - lr: 1.2500e-04\n",
      "Epoch 29/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1996 - accuracy: 0.9444\n",
      "Epoch 00029: val_accuracy did not improve from 0.73626\n",
      "48/48 [==============================] - 34s 718ms/step - loss: 0.1996 - accuracy: 0.9444 - val_loss: 1.5267 - val_accuracy: 0.6813 - lr: 1.2500e-04\n",
      "Epoch 30/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1376 - accuracy: 0.9365\n",
      "Epoch 00030: val_accuracy did not improve from 0.73626\n",
      "48/48 [==============================] - 31s 646ms/step - loss: 0.1376 - accuracy: 0.9365 - val_loss: 1.7027 - val_accuracy: 0.6264 - lr: 1.2500e-04\n",
      "Epoch 31/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1425 - accuracy: 0.9418- ETA: 8s - l\n",
      "Epoch 00031: val_accuracy did not improve from 0.73626\n",
      "48/48 [==============================] - 32s 666ms/step - loss: 0.1425 - accuracy: 0.9418 - val_loss: 1.3019 - val_accuracy: 0.6923 - lr: 1.2500e-04\n",
      "Epoch 32/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1441 - accuracy: 0.9471\n",
      "Epoch 00032: val_accuracy did not improve from 0.73626\n",
      "48/48 [==============================] - 32s 663ms/step - loss: 0.1441 - accuracy: 0.9471 - val_loss: 1.3170 - val_accuracy: 0.6923 - lr: 6.2500e-05\n",
      "Epoch 33/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0986 - accuracy: 0.9735\n",
      "Epoch 00033: val_accuracy improved from 0.73626 to 0.75824, saving model to ./checkpoints/33_0.7582.h5\n",
      "48/48 [==============================] - 31s 650ms/step - loss: 0.0986 - accuracy: 0.9735 - val_loss: 1.2272 - val_accuracy: 0.7582 - lr: 6.2500e-05\n",
      "Epoch 34/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1295 - accuracy: 0.9471\n",
      "Epoch 00034: val_accuracy did not improve from 0.75824\n",
      "48/48 [==============================] - 30s 630ms/step - loss: 0.1295 - accuracy: 0.9471 - val_loss: 1.2049 - val_accuracy: 0.6813 - lr: 6.2500e-05\n",
      "Epoch 35/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1071 - accuracy: 0.9735\n",
      "Epoch 00035: val_accuracy improved from 0.75824 to 0.79121, saving model to ./checkpoints/35_0.7912.h5\n",
      "48/48 [==============================] - 32s 668ms/step - loss: 0.1071 - accuracy: 0.9735 - val_loss: 0.9468 - val_accuracy: 0.7912 - lr: 6.2500e-05\n",
      "Epoch 36/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1157 - accuracy: 0.9577\n",
      "Epoch 00036: val_accuracy did not improve from 0.79121\n",
      "48/48 [==============================] - 31s 654ms/step - loss: 0.1157 - accuracy: 0.9577 - val_loss: 0.8616 - val_accuracy: 0.7802 - lr: 6.2500e-05\n",
      "Epoch 37/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1655 - accuracy: 0.9471\n",
      "Epoch 00037: val_accuracy improved from 0.79121 to 0.81319, saving model to ./checkpoints/37_0.8132.h5\n",
      "48/48 [==============================] - 33s 693ms/step - loss: 0.1655 - accuracy: 0.9471 - val_loss: 0.9793 - val_accuracy: 0.8132 - lr: 6.2500e-05\n",
      "Epoch 38/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1492 - accuracy: 0.9418\n",
      "Epoch 00038: val_accuracy did not improve from 0.81319\n",
      "48/48 [==============================] - 32s 658ms/step - loss: 0.1492 - accuracy: 0.9418 - val_loss: 0.9079 - val_accuracy: 0.8022 - lr: 6.2500e-05\n",
      "Epoch 39/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1903 - accuracy: 0.9365\n",
      "Epoch 00039: val_accuracy did not improve from 0.81319\n",
      "48/48 [==============================] - 31s 647ms/step - loss: 0.1903 - accuracy: 0.9365 - val_loss: 1.0513 - val_accuracy: 0.7802 - lr: 6.2500e-05\n",
      "Epoch 40/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1630 - accuracy: 0.9471\n",
      "Epoch 00040: val_accuracy did not improve from 0.81319\n",
      "48/48 [==============================] - 37s 764ms/step - loss: 0.1630 - accuracy: 0.9471 - val_loss: 0.7978 - val_accuracy: 0.7912 - lr: 6.2500e-05\n",
      "Epoch 41/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0811 - accuracy: 0.9630\n",
      "Epoch 00041: val_accuracy did not improve from 0.81319\n",
      "48/48 [==============================] - 34s 714ms/step - loss: 0.0811 - accuracy: 0.9630 - val_loss: 1.1254 - val_accuracy: 0.7802 - lr: 6.2500e-05\n",
      "Epoch 42/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0722 - accuracy: 0.9709\n",
      "Epoch 00042: val_accuracy did not improve from 0.81319\n",
      "48/48 [==============================] - 32s 662ms/step - loss: 0.0722 - accuracy: 0.9709 - val_loss: 1.0478 - val_accuracy: 0.8022 - lr: 3.1250e-05\n",
      "Epoch 43/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0839 - accuracy: 0.9709\n",
      "Epoch 00043: val_accuracy improved from 0.81319 to 0.82418, saving model to ./checkpoints/43_0.8242.h5\n",
      "48/48 [==============================] - 36s 754ms/step - loss: 0.0839 - accuracy: 0.9709 - val_loss: 0.8149 - val_accuracy: 0.8242 - lr: 3.1250e-05\n",
      "Epoch 44/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1069 - accuracy: 0.9683\n",
      "Epoch 00044: val_accuracy improved from 0.82418 to 0.85714, saving model to ./checkpoints/44_0.8571.h5\n",
      "48/48 [==============================] - 35s 728ms/step - loss: 0.1069 - accuracy: 0.9683 - val_loss: 0.6244 - val_accuracy: 0.8571 - lr: 3.1250e-05\n",
      "Epoch 45/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1582 - accuracy: 0.9524\n",
      "Epoch 00045: val_accuracy improved from 0.85714 to 0.86813, saving model to ./checkpoints/45_0.8681.h5\n",
      "48/48 [==============================] - 35s 724ms/step - loss: 0.1582 - accuracy: 0.9524 - val_loss: 0.5815 - val_accuracy: 0.8681 - lr: 3.1250e-05\n",
      "Epoch 46/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1038 - accuracy: 0.9683\n",
      "Epoch 00046: val_accuracy did not improve from 0.86813\n",
      "48/48 [==============================] - 32s 667ms/step - loss: 0.1038 - accuracy: 0.9683 - val_loss: 0.6452 - val_accuracy: 0.8132 - lr: 3.1250e-05\n",
      "Epoch 47/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1066 - accuracy: 0.9709\n",
      "Epoch 00047: val_accuracy did not improve from 0.86813\n",
      "48/48 [==============================] - 37s 770ms/step - loss: 0.1066 - accuracy: 0.9709 - val_loss: 0.6163 - val_accuracy: 0.8242 - lr: 3.1250e-05\n",
      "Epoch 48/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1107 - accuracy: 0.9577\n",
      "Epoch 00048: val_accuracy did not improve from 0.86813\n",
      "48/48 [==============================] - 34s 712ms/step - loss: 0.1107 - accuracy: 0.9577 - val_loss: 0.6787 - val_accuracy: 0.8462 - lr: 3.1250e-05\n",
      "Epoch 49/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1330 - accuracy: 0.9471\n",
      "Epoch 00049: val_accuracy did not improve from 0.86813\n",
      "48/48 [==============================] - 41s 855ms/step - loss: 0.1330 - accuracy: 0.9471 - val_loss: 0.5626 - val_accuracy: 0.8352 - lr: 3.1250e-05\n",
      "Epoch 50/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0679 - accuracy: 0.9735\n",
      "Epoch 00050: val_accuracy improved from 0.86813 to 0.87912, saving model to ./checkpoints/50_0.8791.h5\n",
      "48/48 [==============================] - 32s 673ms/step - loss: 0.0679 - accuracy: 0.9735 - val_loss: 0.6191 - val_accuracy: 0.8791 - lr: 1.5625e-05\n",
      "Epoch 51/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0592 - accuracy: 0.9762\n",
      "Epoch 00051: val_accuracy did not improve from 0.87912\n",
      "48/48 [==============================] - 31s 652ms/step - loss: 0.0592 - accuracy: 0.9762 - val_loss: 0.4493 - val_accuracy: 0.8681 - lr: 1.5625e-05\n",
      "Epoch 52/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1221 - accuracy: 0.9524\n",
      "Epoch 00052: val_accuracy did not improve from 0.87912\n",
      "48/48 [==============================] - 34s 707ms/step - loss: 0.1221 - accuracy: 0.9524 - val_loss: 0.4844 - val_accuracy: 0.8571 - lr: 1.5625e-05\n",
      "Epoch 53/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0624 - accuracy: 0.9762\n",
      "Epoch 00053: val_accuracy did not improve from 0.87912\n",
      "48/48 [==============================] - 31s 651ms/step - loss: 0.0624 - accuracy: 0.9762 - val_loss: 0.5711 - val_accuracy: 0.8571 - lr: 1.5625e-05\n",
      "Epoch 54/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0812 - accuracy: 0.9709\n",
      "Epoch 00054: val_accuracy did not improve from 0.87912\n",
      "48/48 [==============================] - 46s 960ms/step - loss: 0.0812 - accuracy: 0.9709 - val_loss: 0.5983 - val_accuracy: 0.8571 - lr: 1.5625e-05\n",
      "Epoch 55/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1227 - accuracy: 0.9577\n",
      "Epoch 00055: val_accuracy did not improve from 0.87912\n",
      "48/48 [==============================] - 35s 727ms/step - loss: 0.1227 - accuracy: 0.9577 - val_loss: 0.7631 - val_accuracy: 0.7912 - lr: 7.8125e-06\n",
      "Epoch 56/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0951 - accuracy: 0.9550\n",
      "Epoch 00056: val_accuracy did not improve from 0.87912\n",
      "48/48 [==============================] - 32s 676ms/step - loss: 0.0951 - accuracy: 0.9550 - val_loss: 0.8206 - val_accuracy: 0.8462 - lr: 7.8125e-06\n",
      "Epoch 57/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0436 - accuracy: 0.9841\n",
      "Epoch 00057: val_accuracy did not improve from 0.87912\n",
      "48/48 [==============================] - 34s 714ms/step - loss: 0.0436 - accuracy: 0.9841 - val_loss: 0.4688 - val_accuracy: 0.8791 - lr: 7.8125e-06\n",
      "Epoch 58/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0946 - accuracy: 0.9630\n",
      "Epoch 00058: val_accuracy did not improve from 0.87912\n",
      "48/48 [==============================] - 40s 843ms/step - loss: 0.0946 - accuracy: 0.9630 - val_loss: 0.7315 - val_accuracy: 0.8352 - lr: 7.8125e-06\n",
      "Epoch 59/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0995 - accuracy: 0.9762\n",
      "Epoch 00059: val_accuracy improved from 0.87912 to 0.89011, saving model to ./checkpoints/59_0.8901.h5\n",
      "48/48 [==============================] - 52s 1s/step - loss: 0.0995 - accuracy: 0.9762 - val_loss: 0.4521 - val_accuracy: 0.8901 - lr: 3.9063e-06\n",
      "Epoch 60/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1505 - accuracy: 0.9577\n",
      "Epoch 00060: val_accuracy did not improve from 0.89011\n",
      "48/48 [==============================] - 39s 818ms/step - loss: 0.1505 - accuracy: 0.9577 - val_loss: 0.6969 - val_accuracy: 0.8132 - lr: 3.9063e-06\n",
      "Epoch 61/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0812 - accuracy: 0.9735\n",
      "Epoch 00061: val_accuracy did not improve from 0.89011\n",
      "48/48 [==============================] - 38s 788ms/step - loss: 0.0812 - accuracy: 0.9735 - val_loss: 0.4815 - val_accuracy: 0.8791 - lr: 3.9063e-06\n",
      "Epoch 62/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1054 - accuracy: 0.9630\n",
      "Epoch 00062: val_accuracy did not improve from 0.89011\n",
      "48/48 [==============================] - 35s 720ms/step - loss: 0.1054 - accuracy: 0.9630 - val_loss: 0.5205 - val_accuracy: 0.8681 - lr: 3.9063e-06\n",
      "Epoch 63/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0617 - accuracy: 0.9762\n",
      "Epoch 00063: val_accuracy did not improve from 0.89011\n",
      "48/48 [==============================] - 36s 742ms/step - loss: 0.0617 - accuracy: 0.9762 - val_loss: 0.6380 - val_accuracy: 0.8132 - lr: 3.9063e-06\n",
      "Epoch 64/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1320 - accuracy: 0.9550\n",
      "Epoch 00064: val_accuracy did not improve from 0.89011\n",
      "48/48 [==============================] - 32s 658ms/step - loss: 0.1320 - accuracy: 0.9550 - val_loss: 0.6962 - val_accuracy: 0.8462 - lr: 1.9531e-06\n",
      "Epoch 65/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0561 - accuracy: 0.9815\n",
      "Epoch 00065: val_accuracy improved from 0.89011 to 0.90110, saving model to ./checkpoints/65_0.9011.h5\n",
      "48/48 [==============================] - 33s 679ms/step - loss: 0.0561 - accuracy: 0.9815 - val_loss: 0.3455 - val_accuracy: 0.9011 - lr: 1.9531e-06\n",
      "Epoch 66/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0759 - accuracy: 0.9841\n",
      "Epoch 00066: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 36s 747ms/step - loss: 0.0759 - accuracy: 0.9841 - val_loss: 0.5810 - val_accuracy: 0.8571 - lr: 1.9531e-06\n",
      "Epoch 67/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0730 - accuracy: 0.9656\n",
      "Epoch 00067: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 35s 719ms/step - loss: 0.0730 - accuracy: 0.9656 - val_loss: 0.5200 - val_accuracy: 0.8901 - lr: 1.9531e-06\n",
      "Epoch 68/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1184 - accuracy: 0.9656\n",
      "Epoch 00068: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 35s 729ms/step - loss: 0.1184 - accuracy: 0.9656 - val_loss: 0.7851 - val_accuracy: 0.8022 - lr: 1.9531e-06\n",
      "Epoch 69/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0763 - accuracy: 0.9735\n",
      "Epoch 00069: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 36s 758ms/step - loss: 0.0763 - accuracy: 0.9735 - val_loss: 0.6356 - val_accuracy: 0.8571 - lr: 1.9531e-06\n",
      "Epoch 70/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0578 - accuracy: 0.9815\n",
      "Epoch 00070: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 38s 801ms/step - loss: 0.0578 - accuracy: 0.9815 - val_loss: 0.4965 - val_accuracy: 0.8352 - lr: 1.0000e-06\n",
      "Epoch 71/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1183 - accuracy: 0.9683\n",
      "Epoch 00071: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 34s 698ms/step - loss: 0.1183 - accuracy: 0.9683 - val_loss: 0.4094 - val_accuracy: 0.8791 - lr: 1.0000e-06\n",
      "Epoch 72/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0708 - accuracy: 0.9683\n",
      "Epoch 00072: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 32s 666ms/step - loss: 0.0708 - accuracy: 0.9683 - val_loss: 0.4652 - val_accuracy: 0.8901 - lr: 1.0000e-06\n",
      "Epoch 73/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0821 - accuracy: 0.9630\n",
      "Epoch 00073: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 53s 1s/step - loss: 0.0821 - accuracy: 0.9630 - val_loss: 0.3317 - val_accuracy: 0.8901 - lr: 1.0000e-06\n",
      "Epoch 74/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0842 - accuracy: 0.9630\n",
      "Epoch 00074: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 57s 1s/step - loss: 0.0842 - accuracy: 0.9630 - val_loss: 0.5246 - val_accuracy: 0.8791 - lr: 1.0000e-06\n",
      "Epoch 75/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0588 - accuracy: 0.9788\n",
      "Epoch 00075: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 36s 742ms/step - loss: 0.0588 - accuracy: 0.9788 - val_loss: 0.5401 - val_accuracy: 0.8462 - lr: 1.0000e-06\n",
      "Epoch 76/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0708 - accuracy: 0.9815\n",
      "Epoch 00076: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 36s 744ms/step - loss: 0.0708 - accuracy: 0.9815 - val_loss: 0.4951 - val_accuracy: 0.8791 - lr: 1.0000e-06\n",
      "Epoch 77/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0696 - accuracy: 0.9709\n",
      "Epoch 00077: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 34s 699ms/step - loss: 0.0696 - accuracy: 0.9709 - val_loss: 0.4164 - val_accuracy: 0.8681 - lr: 1.0000e-06\n",
      "Epoch 78/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0757 - accuracy: 0.9683\n",
      "Epoch 00078: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 33s 697ms/step - loss: 0.0757 - accuracy: 0.9683 - val_loss: 0.7957 - val_accuracy: 0.8132 - lr: 1.0000e-06\n",
      "Epoch 79/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0820 - accuracy: 0.9709\n",
      "Epoch 00079: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 34s 715ms/step - loss: 0.0820 - accuracy: 0.9709 - val_loss: 0.5600 - val_accuracy: 0.8462 - lr: 1.0000e-06\n",
      "Epoch 80/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48/48 [==============================] - ETA: 0s - loss: 0.0613 - accuracy: 0.9762\n",
      "Epoch 00080: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 34s 704ms/step - loss: 0.0613 - accuracy: 0.9762 - val_loss: 0.5815 - val_accuracy: 0.8681 - lr: 1.0000e-06\n",
      "Epoch 81/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0687 - accuracy: 0.9709\n",
      "Epoch 00081: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 32s 659ms/step - loss: 0.0687 - accuracy: 0.9709 - val_loss: 0.7098 - val_accuracy: 0.8352 - lr: 1.0000e-06\n",
      "Epoch 82/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0542 - accuracy: 0.9815\n",
      "Epoch 00082: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 32s 675ms/step - loss: 0.0542 - accuracy: 0.9815 - val_loss: 0.4342 - val_accuracy: 0.8901 - lr: 1.0000e-06\n",
      "Epoch 83/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.0561 - accuracy: 0.9788\n",
      "Epoch 00083: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 32s 668ms/step - loss: 0.0561 - accuracy: 0.9788 - val_loss: 0.5601 - val_accuracy: 0.8462 - lr: 1.0000e-06\n",
      "Epoch 84/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1153 - accuracy: 0.9709\n",
      "Epoch 00084: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 54s 1s/step - loss: 0.1153 - accuracy: 0.9709 - val_loss: 0.6606 - val_accuracy: 0.8352 - lr: 1.0000e-06\n",
      "Epoch 85/100\n",
      "48/48 [==============================] - ETA: 0s - loss: 0.1204 - accuracy: 0.9630\n",
      "Epoch 00085: val_accuracy did not improve from 0.90110\n",
      "48/48 [==============================] - 37s 770ms/step - loss: 0.1204 - accuracy: 0.9630 - val_loss: 0.6157 - val_accuracy: 0.8571 - lr: 1.0000e-06\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1ad7f9adec8>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Starts training the model\n",
    "model.fit(train_batches,\n",
    "        epochs=TOTAL_EPOCHS,\n",
    "        verbose=1,\n",
    "        steps_per_epoch=TRAIN_DATASET_SIZE,\n",
    "        validation_data=val_batches,\n",
    "        validation_steps=VAL_DATASET_SIZE,\n",
    "#         validation_split=0.1,\n",
    "        initial_epoch=INITIAL_EPOCH,\n",
    "        class_weight=class_weigths,\n",
    "#         use_multiprocessing=True,\n",
    "        workers=4,\n",
    "        callbacks=[model_checkpoint_callback, tensorboard_callback, reduce_lr_callback, early_stopping_callback]\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "~90% validation accuracy achieved. Trained tensorflow model saved in h5 format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T01:03:01.903586Z",
     "start_time": "2020-11-21T01:03:01.588302Z"
    }
   },
   "outputs": [],
   "source": [
    "MODEL_PATH = 'models/reusr-keras.h5'\n",
    "tf.keras.models.save_model(\n",
    "model,\n",
    "MODEL_PATH,\n",
    "overwrite=True,\n",
    "include_optimizer=True,\n",
    "save_format=None\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorflowJS model generator code\n",
    "\n",
    "converting tensorflow model to tfjs model for easy use in web app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-21T01:03:07.462788Z",
     "start_time": "2020-11-21T01:03:02.221439Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-11-21 06:33:02.665413: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudart64_101.dll\n"
     ]
    }
   ],
   "source": [
    "!tensorflowjs_converter \\\n",
    "--input_format=keras \\\n",
    "models/reusr-keras.h5 \\\n",
    "models/reusr-tfjs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2 files: model.json and shard files are generated from the above code"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": false,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}