Both addition and subtraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making Your Model Learn Addition and Subtraction!\n",
    "## Part 1: Introduction\n",
    "\n",
    "Given the string \"54+7\", the model should return a prediction: \"61\". The addition example was too easy for the model so I added subtraction in the mix as well. So, please ensure you read the code carefully as it's a little different than the hands on project."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tested with tensorflow version 2.0.1\n",
      "Using tensorflow version: 1.12.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import TimeDistributed, Dense, Dropout, SimpleRNN, RepeatVector\n",
    "from tensorflow.keras.callbacks import EarlyStopping, LambdaCallback\n",
    "\n",
    "from termcolor import colored\n",
    "\n",
    "print('Tested with tensorflow version 2.0.1')\n",
    "print('Using tensorflow version:', tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 2: Generate Data\n",
    "___"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "hP-OyQrivBNl",
    "outputId": "70a8ce77-3241-4ea9-dbc7-d9bebb3789f3"
   },
   "outputs": [],
   "source": [
    "all_chars = '0123456789+-'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features: 12\n"
     ]
    }
   ],
   "source": [
    "num_features = len(all_chars)\n",
    "\n",
    "char_to_index = dict((c, i) for i, c in enumerate(all_chars))\n",
    "index_to_char = dict((i, c) for i, c in enumerate(all_chars))\n",
    "\n",
    "print('Number of features:', num_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "L2b1Xg9hvBNr"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('68+68', '136')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def generate_data():\n",
    "    first_num = np.random.randint(low=0,high=100)\n",
    "    second_num = np.random.randint(low=0,high=100)\n",
    "    add = np.squeeze(np.random.randint(low=0, high=100)) > 50.\n",
    "    if add:\n",
    "        example = str(first_num) + '+' + str(second_num)\n",
    "        label = str(first_num+second_num)\n",
    "    else:\n",
    "        example = str(first_num) + '-' + str(second_num)\n",
    "        label = str(first_num-second_num)\n",
    "    return example, label\n",
    "\n",
    "generate_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 3: Create the Model\n",
    "___\n",
    "Consider these two reviews:\n",
    "\n",
    "Review 1: This movie is not terrible at all.\n",
    "\n",
    "Review 2: This movie is pretty decent."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 377
    },
    "colab_type": "code",
    "id": "iZKl0LtdvBNy",
    "outputId": "34a3c344-5985-4e3b-e1ac-392c18739308"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "simple_rnn (SimpleRNN)       (None, 128)               18048     \n",
      "_________________________________________________________________\n",
      "repeat_vector (RepeatVector) (None, 5, 128)            0         \n",
      "_________________________________________________________________\n",
      "simple_rnn_1 (SimpleRNN)     (None, 5, 128)            32896     \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 5, 12)             1548      \n",
      "=================================================================\n",
      "Total params: 52,492\n",
      "Trainable params: 52,492\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "hidden_units = 128\n",
    "max_time_steps = 5\n",
    "\n",
    "model = Sequential([\n",
    "    SimpleRNN(hidden_units, input_shape=(None, num_features)),\n",
    "    RepeatVector(max_time_steps),\n",
    "    SimpleRNN(hidden_units, return_sequences=True),\n",
    "    TimeDistributed(Dense(num_features, activation='softmax'))\n",
    "])\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 4: Vectorize and De-Vectorize Data\n",
    "___"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ci29YaA6vBN1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text Example and Label: 86-74 12\n",
      "Vectorized Example and Label Shapes: (5, 12) (5, 12)\n"
     ]
    }
   ],
   "source": [
    "def vectorize_example(example, label):\n",
    "    \n",
    "    x = np.zeros((max_time_steps, num_features))\n",
    "    y = np.zeros((max_time_steps, num_features))\n",
    "    \n",
    "    diff_x = max_time_steps - len(example)\n",
    "    diff_y = max_time_steps - len(label)\n",
    "    \n",
    "    for i, c in enumerate(example):\n",
    "        x[diff_x+i, char_to_index[c]] = 1\n",
    "    for i in range(diff_x):\n",
    "        x[i, char_to_index['0']] = 1\n",
    "    for i, c in enumerate(label):\n",
    "        y[diff_y+i, char_to_index[c]] = 1\n",
    "    for i in range(diff_y):\n",
    "        y[i, char_to_index['0']] = 1\n",
    "        \n",
    "    return x, y\n",
    "\n",
    "e, l = generate_data()\n",
    "print('Text Example and Label:', e, l)\n",
    "x, y = vectorize_example(e, l)\n",
    "print('Vectorized Example and Label Shapes:', x.shape, y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_nARKq-bvBN9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'86-74'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def devectorize_example(example):\n",
    "    result = [index_to_char[np.argmax(vec)] for i, vec in enumerate(example)]\n",
    "    return ''.join(result)\n",
    "\n",
    "def strip_padding(example):\n",
    "    encountered_non_zero = False\n",
    "    output = ''\n",
    "    for c in example:\n",
    "        if not encountered_non_zero and c == '0':\n",
    "            continue\n",
    "        if c == '+' or c == '-':\n",
    "            encountered_non_zero = False\n",
    "        else:\n",
    "            encountered_non_zero = True\n",
    "        output += c\n",
    "    return output\n",
    "\n",
    "devectorize_example(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "KCGUxNwNvBOA",
    "outputId": "a721451a-7f7c-42be-bdf0-0c96381f2eb2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00012 : 12\n"
     ]
    }
   ],
   "source": [
    "print(devectorize_example(y), ':', strip_padding(devectorize_example(y)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 5: Create Dataset\n",
    "___"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "T5rdrhiVvBOI",
    "outputId": "426b3722-78cf-494f-8bbf-1cdf7eab3113"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000, 5, 12) (20000, 5, 12)\n"
     ]
    }
   ],
   "source": [
    "def create_dataset(num_examples=2000):\n",
    "\n",
    "    x_train = np.zeros((num_examples, max_time_steps, num_features))\n",
    "    y_train = np.zeros((num_examples, max_time_steps, num_features))\n",
    "\n",
    "    for i in range(num_examples):\n",
    "        e, l = generate_data()\n",
    "        x, y = vectorize_example(e, l)\n",
    "        x_train[i] = x\n",
    "        y_train[i] = y\n",
    "    \n",
    "    return x_train, y_train\n",
    "\n",
    "x_train, y_train = create_dataset(20000)\n",
    "print(x_train.shape, y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "RRwiuwcQvBOL",
    "outputId": "07591064-5c57-4adb-ccf6-c226457d2263"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'31-73'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "devectorize_example(x_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "cRUvtd0IvBOO",
    "outputId": "d07ce279-06ba-44c4-de91-0ec4fb551877"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'00-42'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "devectorize_example(y_train[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 6: Training the Model\n",
    "___\n",
    "Please note that the hyper-parameters are a little different . This model should easily get to 100% accuracy even with 2 operations (+ and -) instead of just 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 2686
    },
    "colab_type": "code",
    "id": "S8HWRdiOvBOS",
    "outputId": "2365b11f-30e8-48cc-e940-4d3267771fbe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.55 _ 0.62 _ 0.63 _ 0.63 _ 0.64 _ 0.64 _ 0.65 _ 0.67 _ 0.67 _ 0.68 _ 0.70 _ 0.70 _ 0.71 _ 0.72 _ 0.73 _ 0.74 _ 0.75 _ 0.75 _ 0.76 _ 0.77 _ 0.77 _ 0.77 _ 0.79 _ 0.79 _ 0.80 _ 0.80 _ 0.81 _ 0.82 _ 0.83 _ 0.83 _ 0.85 _ 0.86 _ 0.87 _ 0.88 _ 0.88 _ 0.90 _ 0.90 _ 0.92 _ 0.92 _ 0.93 _ 0.94 _ 0.94 _ 0.95 _ 0.95 _ 0.95 _ 0.95 _ 0.96 _ 0.96 _ 0.96 _ 0.97 _ 0.97 _ 0.97 _ 0.97 _ 0.97 _ 0.97 _ 0.98 _ 0.98 _ 0.98 _ 0.98 _ 0.98 _ 0.98 _ 0.98 _ 0.98 _ 0.98 _ 0.98 _ 0.98 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 1.00 _ 1.00 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ 0.99 _ "
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1bce6428ef0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_logger = LambdaCallback(\n",
    "    on_epoch_end=lambda e, l: print('{:.2f}'.format(l['val_acc']), end=' _ ')\n",
    ")\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=10)\n",
    "\n",
    "model.fit(x_train, y_train, epochs=100, validation_split=0.2, verbose=False,\n",
    "         batch_size=1024, callbacks=[simple_logger, early_stopping])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a test set and look at some predictions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 187
    },
    "colab_type": "code",
    "id": "tlNtu_9ZvBOY",
    "outputId": "82198555-f0e8-4a7a-9125-cc61cf4f4a26"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mInput: 42-92, Out: -50, Pred: -50\u001b[0m\n",
      "\u001b[32mInput: 3+10, Out: 13, Pred: 13\u001b[0m\n",
      "\u001b[32mInput: 72-70, Out: 2, Pred: 2\u001b[0m\n",
      "\u001b[32mInput: 45+96, Out: 141, Pred: 141\u001b[0m\n",
      "\u001b[32mInput: 86+69, Out: 155, Pred: 155\u001b[0m\n",
      "\u001b[32mInput: 59-11, Out: 48, Pred: 48\u001b[0m\n",
      "\u001b[32mInput: 62+62, Out: 124, Pred: 124\u001b[0m\n",
      "\u001b[32mInput: 64+54, Out: 118, Pred: 118\u001b[0m\n",
      "\u001b[32mInput: 81+71, Out: 152, Pred: 152\u001b[0m\n",
      "\u001b[32mInput: 18-3, Out: 15, Pred: 15\u001b[0m\n",
      "\u001b[32mInput: 4-52, Out: -48, Pred: -48\u001b[0m\n",
      "\u001b[32mInput: 65+45, Out: 110, Pred: 110\u001b[0m\n",
      "\u001b[32mInput: 78+32, Out: 110, Pred: 110\u001b[0m\n",
      "\u001b[32mInput: 14-75, Out: -61, Pred: -61\u001b[0m\n",
      "\u001b[32mInput: 4+25, Out: 29, Pred: 29\u001b[0m\n",
      "\u001b[32mInput: 7-21, Out: -14, Pred: -14\u001b[0m\n",
      "\u001b[32mInput: 38-60, Out: -22, Pred: -22\u001b[0m\n",
      "\u001b[32mInput: 61-84, Out: -23, Pred: -23\u001b[0m\n",
      "\u001b[32mInput: 69+42, Out: 111, Pred: 111\u001b[0m\n",
      "\u001b[32mInput: 28-67, Out: -39, Pred: -39\u001b[0m\n",
      "\n",
      "Full sequence accuracy: 100.000 %\n"
     ]
    }
   ],
   "source": [
    "x_test, y_test = create_dataset(num_examples=20)\n",
    "preds = model.predict(x_test)\n",
    "full_seq_acc = 0\n",
    "\n",
    "for i, pred in enumerate(preds):\n",
    "    pred_str = strip_padding(devectorize_example(pred))\n",
    "    y_test_str = strip_padding(devectorize_example(y_test[i]))\n",
    "    x_test_str = strip_padding(devectorize_example(x_test[i]))\n",
    "    col = 'green' if pred_str == y_test_str else 'red'\n",
    "    full_seq_acc += 1/len(preds) * int(pred_str == y_test_str)\n",
    "    outstring = 'Input: {}, Out: {}, Pred: {}'.format(x_test_str, y_test_str, pred_str)\n",
    "    print(colored(outstring, col))\n",
    "print('\\nFull sequence accuracy: {:.3f} %'.format(100 * full_seq_acc))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "addition_simpleRNN.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}