Multilayer neural network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\eric\\Anaconda2\\envs\\exts-aml2\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "% matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr: (546, 9)\n",
      "X_te: (137, 9)\n",
      "y_tr: (546,)\n",
      "y_te: (137,)\n"
     ]
    }
   ],
   "source": [
    "# import the data\n",
    "bcw_data = pd.read_csv(os.path.join(\"data\", \"uci\", \"breast-cancer-wisconsin.data\"), na_values=\"?\", header=None)\n",
    "\n",
    "# set the column names\n",
    "bcw_data.columns = [\"ID\",\"THICKNESS\",\"CELL_SIZE_UNIFORMITY\",\"CELL_SHAPE_UNIFORMITY\",\"MARGINAL_ADHESION\",\"EPI_CELL_SIZE\",\"BARE_NUCLEI\",\"BLAND_CHROMATIN\",\"NORMAL_NUCLEOLI\",\"MITOSES\",\"CLASS\"]\n",
    "\n",
    "# remove NAs\n",
    "bcw_data = bcw_data.dropna(axis=0, how=\"any\")\n",
    "bcw_data = bcw_data.drop(\"ID\", axis=1)\n",
    "\n",
    "y1 = bcw_data.pop(\"CLASS\").values\n",
    "y1 = (y1 / 2) - 1\n",
    "X1 = bcw_data.values\n",
    "\n",
    "# split the data\n",
    "X_tr, X_te, y_tr, y_te = train_test_split(X1, y1, test_size=0.2, random_state=0)\n",
    "print(\"X_tr:\", X_tr.shape)\n",
    "print(\"X_te:\", X_te.shape)\n",
    "print(\"y_tr:\", y_tr.shape)\n",
    "print(\"y_te:\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1.])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_classes = len(np.unique(y1))\n",
    "batch_size = 64\n",
    "steps_per_epoch = X_tr.shape[0] / batch_size\n",
    "\n",
    "# Batch generator\n",
    "def get_batches(X, y, batch_size):\n",
    "    # Shuffle X,y\n",
    "    shuffled_idx = np.arange(len(y))\n",
    "    np.random.shuffle(shuffled_idx)\n",
    "\n",
    "    # Enumerate indexes by steps of batch_size\n",
    "    for i in range(0, len(y), batch_size):\n",
    "        # Batch indexes\n",
    "        batch_idx = shuffled_idx[i:i+batch_size]\n",
    "        \n",
    "        yield X[batch_idx], y[batch_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## MODEL 7.20.2.9j\n",
    "# Create new graph\n",
    "graph = tf.Graph()\n",
    "model_name = \"model_0.0.1\"\n",
    "\n",
    "with graph.as_default():\n",
    "    # Placeholders\n",
    "    X = tf.placeholder(dtype=tf.float32, shape=[None, 9])\n",
    "    y = tf.placeholder(dtype=tf.int32, shape=[None])\n",
    "    training = tf.placeholder(dtype=tf.bool)\n",
    "    \n",
    "    # create global step for decaying learning rate\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    \n",
    "    # learning rate o\n",
    "    epochs_per_decay = 20\n",
    "    starting_rate = 0.1\n",
    "    decay_factor = 0.80\n",
    "    staircase = True\n",
    "    \n",
    "    learning_rate = tf.train.exponential_decay(starting_rate,                 # start at 0.003\n",
    "                                               global_step, \n",
    "                                               steps_per_epoch * epochs_per_decay,       # 100 epochs\n",
    "                                               decay_factor,                   # 0.5 decrease\n",
    "                                               staircase=staircase) \n",
    "    \n",
    "    # Hidden layer with 64 units\n",
    "    with tf.name_scope('local1') as scope:\n",
    "        hidden = tf.layers.dense(\n",
    "            X,                              # input\n",
    "            64,                             # 64 units\n",
    "            activation=tf.nn.relu,          # activation\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=0),  # kernel initializer\n",
    "            bias_initializer=tf.zeros_initializer(), # bias\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "            name='hidden'                   # name\n",
    "        )\n",
    "\n",
    "        # Apply dropout\n",
    "        #hidden = tf.layers.dropout(hidden, rate=0.5, seed=0, training=training)\n",
    "    \n",
    "    with tf.name_scope('local2') as scope:\n",
    "        hidden2 = tf.layers.dense(\n",
    "            hidden,                         # input\n",
    "            48,                             # 48 units\n",
    "            activation=tf.nn.relu,          # activation\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=0),  # kernel initializer\n",
    "            bias_initializer=tf.zeros_initializer(), # bias\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "            name='hidden2'                  # name\n",
    "        )\n",
    "    \n",
    "    # Output layer\n",
    "    logits = tf.layers.dense(\n",
    "        hidden2,                       # input\n",
    "        num_classes,                             # 4 units\n",
    "        activation=None,               # No activation function\n",
    "        kernel_initializer=tf.variance_scaling_initializer(scale=1, seed=0),\n",
    "        bias_initializer=tf.zeros_initializer(),\n",
    "        kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "        name='output'\n",
    "    )\n",
    "    \n",
    "    # Loss fuction: mean cross-entropy\n",
    "    mean_ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "    \n",
    "    # easier way to handle regularization loss\n",
    "    loss = mean_ce + tf.losses.get_regularization_loss()\n",
    "    \n",
    "    # Gradient descent\n",
    "    gd = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "    # Minimize loss\n",
    "    train_op = gd.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # Compute predictions and accuracy\n",
    "    predictions = tf.argmax(logits, axis=1, output_type=tf.int32)\n",
    "    is_correct = tf.equal(y, predictions)\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct, dtype=tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 - valid: 0.825 train: 0.685 (mean) learning rate 0.100\n",
      "Epoch 6 - valid: 0.905 train: 0.941 (mean) learning rate 0.100\n",
      "Epoch 11 - valid: 0.956 train: 0.955 (mean) learning rate 0.100\n",
      "Epoch 16 - valid: 0.942 train: 0.967 (mean) learning rate 0.100\n",
      "Epoch 21 - valid: 0.949 train: 0.972 (mean) learning rate 0.080\n",
      "Epoch 26 - valid: 0.971 train: 0.979 (mean) learning rate 0.080\n",
      "Epoch 31 - valid: 0.942 train: 0.981 (mean) learning rate 0.080\n",
      "Epoch 36 - valid: 0.971 train: 0.981 (mean) learning rate 0.080\n",
      "Epoch 41 - valid: 0.942 train: 0.981 (mean) learning rate 0.064\n",
      "Epoch 46 - valid: 0.964 train: 0.975 (mean) learning rate 0.064\n",
      "Epoch 51 - valid: 0.971 train: 0.978 (mean) learning rate 0.064\n",
      "Epoch 56 - valid: 0.964 train: 0.970 (mean) learning rate 0.064\n",
      "Epoch 61 - valid: 0.956 train: 0.979 (mean) learning rate 0.051\n",
      "Epoch 66 - valid: 0.964 train: 0.984 (mean) learning rate 0.051\n",
      "Epoch 71 - valid: 0.956 train: 0.981 (mean) learning rate 0.051\n",
      "Epoch 76 - valid: 0.956 train: 0.981 (mean) learning rate 0.041\n",
      "Epoch 81 - valid: 0.964 train: 0.979 (mean) learning rate 0.041\n",
      "Epoch 86 - valid: 0.949 train: 0.984 (mean) learning rate 0.041\n",
      "Epoch 91 - valid: 0.949 train: 0.981 (mean) learning rate 0.041\n",
      "Epoch 96 - valid: 0.964 train: 0.979 (mean) learning rate 0.033\n",
      "Test accuracy: 0.96350366\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0,\n",
       "       1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1,\n",
       "       1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0,\n",
       "       1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0,\n",
       "       1, 0, 0, 1, 0])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Validation accuracy\n",
    "valid_acc_values = []\n",
    "tr_acc_values = []\n",
    "lr_values = []\n",
    "cv_cost_values = []\n",
    "tr_cost_values = []\n",
    "config = tf.ConfigProto(device_count = {'GPU': 0})\n",
    "\n",
    "with tf.Session(graph=graph, config=config) as sess:\n",
    "    # Initialize variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    saver = tf.train.Saver()\n",
    "    \n",
    "    # Set seed\n",
    "    np.random.seed(0)\n",
    "\n",
    "    # Train several epochs\n",
    "    for epoch in range(100):\n",
    "        # Accuracy values (train) after each batch\n",
    "        batch_acc = []\n",
    "        batch_cost = []\n",
    "\n",
    "        # Get batches of data\n",
    "        for X_batch, y_batch in get_batches(X_tr, y_tr, batch_size):\n",
    "            # Run training and evaluate accuracy\n",
    "            _, acc_value, lr, tr_cost = sess.run([train_op, accuracy, learning_rate, loss], feed_dict={\n",
    "                X: X_batch,\n",
    "                y: y_batch,\n",
    "                training: True # Apply dropout\n",
    "            })\n",
    "\n",
    "            # Save accuracy (current batch)\n",
    "            batch_acc.append(acc_value)\n",
    "            batch_cost.append(tr_cost)\n",
    "\n",
    "            # Evaluate validation accuracy\n",
    "            valid_acc, cv_cost = sess.run([accuracy, loss], feed_dict={\n",
    "                X: X_te,\n",
    "                y: y_te,\n",
    "                training: False # Do not apply dropout\n",
    "            })\n",
    "            valid_acc_values.append(valid_acc)\n",
    "            tr_acc_values.append(np.mean(batch_acc))\n",
    "            cv_cost_values.append(cv_cost)\n",
    "            tr_cost_values.append(np.mean(batch_cost))\n",
    "            lr_values.append(lr)\n",
    "\n",
    "        # Print progress\n",
    "        if epoch % 5 == 0:\n",
    "            print('Epoch {} - valid: {:.3f} train: {:.3f} (mean) learning rate {:.3f}'.format(\n",
    "                epoch+1, valid_acc, np.mean(batch_acc), lr\n",
    "            ))\n",
    "    \n",
    "    save_path = saver.save(sess, \"./model/nn_model.ckpt\")\n",
    "    \n",
    "    test_acc, yhat = sess.run([accuracy, predictions], feed_dict = {\n",
    "        X: X_te,\n",
    "        y: y_te,\n",
    "        training: False\n",
    "    })\n",
    "    \n",
    "print(\"Test accuracy:\", test_acc)\n",
    "yhat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wisconsin Diagnostic Breast Cancer (WDBC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr: (455, 30)\n",
      "X_te: (114, 30)\n",
      "y_tr: (455,)\n",
      "y_te: (114,)\n"
     ]
    }
   ],
   "source": [
    "# import the data\n",
    "wdbc_data = pd.read_csv(os.path.join(\"data\", \"uci\", \"wdbc.data\"), na_values=\"?\", header=None)\n",
    "\n",
    "# set the column names\n",
    "wdbc_data=wdbc_data.rename(columns = {0:'ID', 1:\"CLASS\", 2: \"MEAN_RADIUS\", 3: \"MEAN_TEXTURE\", 4: \"MEAN_PERIMETER\", 5: \"MEAN_AREA\", 6: \"MEAN_SMOOTHNESS\", 7: \"MEAN_COMPACTNESS\", 8: \"MEAN_CONCAVITY\", 9:\"MEAN_CONCAVE_POINTS\", 10: \"MEAN_SYMMETRY\", 11: \"MEAN_FRACTAL_DIMENSIONS\", 12: \"RADIUS_SE\", 13: \"TEXTURE_SE\", 14: \"PERIMETER_SE\", 15: \"AREA_SE\", 16: \"SMOOTHNESS_SE\", 17: \"COMPACTNESS_SE\", 18: \"CONCAVITY_SE\", 19: \"CONCAVE_POINTS_SE\", 20: \"SYMMETRY_SE\",21: \"FRACTAL_DIMENSIONS_SE\", 22: \"WORST_RADIUS\", 23: \"WORST_TEXTURE\", 24: \"WORST_PERIMETER\", 25: \"WORST_AREA\", 26: \"WORST_SMOOTHNESS\", 27: \"WORST_COMPACTNESS\", 28: \"WORST_CONCAVITY\", 29: \"WORST_CONCAVE_POINTS\", 30: \"WORST_SYMMETRY\", 31: \"WORST_FRACTAL_DIMENSIONS\"})\n",
    "wdbc_data = wdbc_data.dropna(axis=0, how=\"any\")\n",
    "\n",
    "y2 = wdbc_data.pop(\"CLASS\").values\n",
    "labels = np.zeros(len(y2))\n",
    "labels[y2 == 'M'] = 1\n",
    "X2 = wdbc_data.drop([\"ID\"], axis=1).values\n",
    "\n",
    "# split the data\n",
    "X_tr, X_te, y_tr, y_te = train_test_split(X2, labels, test_size=0.2, random_state=0)\n",
    "print(\"X_tr:\", X_tr.shape)\n",
    "print(\"X_te:\", X_te.shape)\n",
    "print(\"y_tr:\", y_tr.shape)\n",
    "print(\"y_te:\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    0.627417\n",
       "1.0    0.372583\n",
       "dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(labels, normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create new graph\n",
    "graph = tf.Graph()\n",
    "model_name = \"model_0.0.2\"\n",
    "\n",
    "with graph.as_default():\n",
    "    # Placeholders\n",
    "    X = tf.placeholder(dtype=tf.float32, shape=[None, 30])\n",
    "    y = tf.placeholder(dtype=tf.int32, shape=[None])\n",
    "    training = tf.placeholder(dtype=tf.bool)\n",
    "    \n",
    "    # create global step for decaying learning rate\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    \n",
    "    # learning rate o\n",
    "    epochs_per_decay = 30\n",
    "    starting_rate = 0.1\n",
    "    decay_factor = 0.80\n",
    "    staircase = True\n",
    "    \n",
    "    learning_rate = tf.train.exponential_decay(starting_rate,                 # start at 0.003\n",
    "                                               global_step, \n",
    "                                               steps_per_epoch * epochs_per_decay,       # 100 epochs\n",
    "                                               decay_factor,                   # 0.5 decrease\n",
    "                                               staircase=staircase) \n",
    "    \n",
    "    # Hidden layer with 64 units\n",
    "    with tf.name_scope('local1') as scope:\n",
    "        hidden = tf.layers.dense(\n",
    "            X,                              # input\n",
    "            64,                             # 64 units\n",
    "            activation=tf.nn.relu,          # activation\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=0),  # kernel initializer\n",
    "            bias_initializer=tf.zeros_initializer(), # bias\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "            name='hidden'                   # name\n",
    "        )\n",
    "\n",
    "        # Apply dropout\n",
    "        #hidden = tf.layers.dropout(hidden, rate=0.5, seed=0, training=training)\n",
    "    \n",
    "    with tf.name_scope('local2') as scope:\n",
    "        hidden2 = tf.layers.dense(\n",
    "            hidden,                         # input\n",
    "            32,                             # 48 units\n",
    "            activation=tf.nn.relu,          # activation\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=0),  # kernel initializer\n",
    "            bias_initializer=tf.zeros_initializer(), # bias\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "            name='hidden2'                  # name\n",
    "        )\n",
    "    \n",
    "    with tf.name_scope('local3') as scope:\n",
    "        hidden3 = tf.layers.dense(\n",
    "            hidden2,                         # input\n",
    "            32,                             # 48 units\n",
    "            activation=tf.nn.relu,          # activation\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=0),  # kernel initializer\n",
    "            bias_initializer=tf.zeros_initializer(), # bias\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "            name='hidden3'                  # name\n",
    "        )\n",
    "        \n",
    "    # Output layer\n",
    "    logits = tf.layers.dense(\n",
    "        hidden3,                       # input\n",
    "        num_classes,                             # 4 units\n",
    "        activation=None,               # No activation function\n",
    "        kernel_initializer=tf.variance_scaling_initializer(scale=1, seed=0),\n",
    "        bias_initializer=tf.zeros_initializer(),\n",
    "        kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "        name='output'\n",
    "    )\n",
    "    \n",
    "    # Loss fuction: mean cross-entropy\n",
    "    mean_ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "    \n",
    "    # easier way to handle regularization loss\n",
    "    loss = mean_ce + tf.losses.get_regularization_loss()\n",
    "    \n",
    "    # Gradient descent\n",
    "    gd = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "    # Minimize loss\n",
    "    train_op = gd.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # Compute predictions and accuracy\n",
    "    predictions = tf.argmax(logits, axis=1, output_type=tf.int32)\n",
    "    is_correct = tf.equal(y, predictions)\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct, dtype=tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 - valid: 0.588 train: 0.498 (mean) learning rate 0.100\n",
      "Epoch 6 - valid: 0.588 train: 0.614 (mean) learning rate 0.100\n",
      "Epoch 11 - valid: 0.588 train: 0.630 (mean) learning rate 0.100\n",
      "Epoch 16 - valid: 0.588 train: 0.646 (mean) learning rate 0.100\n",
      "Epoch 21 - valid: 0.588 train: 0.630 (mean) learning rate 0.100\n",
      "Epoch 26 - valid: 0.588 train: 0.662 (mean) learning rate 0.100\n",
      "Epoch 31 - valid: 0.588 train: 0.630 (mean) learning rate 0.100\n",
      "Epoch 36 - valid: 0.588 train: 0.646 (mean) learning rate 0.080\n",
      "Epoch 41 - valid: 0.588 train: 0.646 (mean) learning rate 0.080\n",
      "Epoch 46 - valid: 0.588 train: 0.646 (mean) learning rate 0.080\n",
      "Epoch 51 - valid: 0.588 train: 0.678 (mean) learning rate 0.080\n",
      "Epoch 56 - valid: 0.588 train: 0.630 (mean) learning rate 0.080\n",
      "Epoch 61 - valid: 0.588 train: 0.662 (mean) learning rate 0.080\n",
      "Epoch 66 - valid: 0.588 train: 0.678 (mean) learning rate 0.064\n",
      "Epoch 71 - valid: 0.588 train: 0.630 (mean) learning rate 0.064\n",
      "Epoch 76 - valid: 0.588 train: 0.614 (mean) learning rate 0.064\n",
      "Epoch 81 - valid: 0.588 train: 0.646 (mean) learning rate 0.064\n",
      "Epoch 86 - valid: 0.588 train: 0.662 (mean) learning rate 0.064\n",
      "Epoch 91 - valid: 0.588 train: 0.646 (mean) learning rate 0.064\n",
      "Epoch 96 - valid: 0.588 train: 0.662 (mean) learning rate 0.064\n",
      "Test accuracy: 0.5877193\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Validation accuracy\n",
    "valid_acc_values = []\n",
    "tr_acc_values = []\n",
    "lr_values = []\n",
    "cv_cost_values = []\n",
    "tr_cost_values = []\n",
    "config = tf.ConfigProto(device_count = {'GPU': 0})\n",
    "\n",
    "with tf.Session(graph=graph, config=config) as sess:\n",
    "    # Initialize variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    saver = tf.train.Saver()\n",
    "    \n",
    "    # Set seed\n",
    "    np.random.seed(0)\n",
    "\n",
    "    # Train several epochs\n",
    "    for epoch in range(100):\n",
    "        # Accuracy values (train) after each batch\n",
    "        batch_acc = []\n",
    "        batch_cost = []\n",
    "\n",
    "        # Get batches of data\n",
    "        for X_batch, y_batch in get_batches(X_tr, y_tr, batch_size):\n",
    "            # Run training and evaluate accuracy\n",
    "            _, acc_value, lr, tr_cost = sess.run([train_op, accuracy, learning_rate, loss], feed_dict={\n",
    "                X: X_batch,\n",
    "                y: y_batch,\n",
    "                training: True # Apply dropout\n",
    "            })\n",
    "\n",
    "            # Save accuracy (current batch)\n",
    "            batch_acc.append(acc_value)\n",
    "            batch_cost.append(tr_cost)\n",
    "\n",
    "            # Evaluate validation accuracy\n",
    "            valid_acc, cv_cost = sess.run([accuracy, loss], feed_dict={\n",
    "                X: X_te,\n",
    "                y: y_te,\n",
    "                training: False # Do not apply dropout\n",
    "            })\n",
    "            valid_acc_values.append(valid_acc)\n",
    "            tr_acc_values.append(np.mean(batch_acc))\n",
    "            cv_cost_values.append(cv_cost)\n",
    "            tr_cost_values.append(np.mean(batch_cost))\n",
    "            lr_values.append(lr)\n",
    "\n",
    "        # Print progress\n",
    "        if epoch % 5 == 0:\n",
    "            print('Epoch {} - valid: {:.3f} train: {:.3f} (mean) learning rate {:.3f}'.format(\n",
    "                epoch+1, valid_acc, np.mean(batch_acc), lr\n",
    "            ))\n",
    "    \n",
    "    save_path = saver.save(sess, \"./model/nn_model.ckpt\")\n",
    "    \n",
    "    test_acc, yhat = sess.run([accuracy, predictions], feed_dict = {\n",
    "        X: X_te,\n",
    "        y: y_te,\n",
    "        training: False\n",
    "    })\n",
    "    \n",
    "print(\"Test accuracy:\", test_acc)\n",
    "yhat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Wisconsin Prognostic Breast Cancer (WPBC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr: (155, 32)\n",
      "X_te: (39, 32)\n",
      "y_tr: (155,)\n",
      "y_te: (39,)\n",
      "y_class_tr: (155,)\n",
      "y_class_te: (39,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\eric\\Anaconda2\\envs\\exts-aml2\\lib\\site-packages\\ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    }
   ],
   "source": [
    "# import the data\n",
    "wpbc_data = pd.read_csv(os.path.join(\"data\", \"uci\", \"wpbc.data\"), na_values=\"?\", header=None)\n",
    "\n",
    "# set the column names\n",
    "wpbc_data=wpbc_data.rename(columns = {0:'ID', 1:\"CLASS\", 2: \"TIME\", 3: \"MEAN_RADIUS\", 4: \"MEAN_TEXTURE\", 5: \"MEAN_PERIMITER\", 6: \"MEAN_AREA\", 7: \"MEAN_SMOOTHNESS\", 8: \"MEAN_COMPACTNESS\", 9: \"MEAN_CONCAVITY\", 10:\"MEAN_CONCAVE_POINTS\", 11: \"MEAN_SYMMETRY\", 12: \"MEAN_FRACTAL_DIMENSIONS\", 13: \"RADIUS_SE\", 14: \"TEXTURE_SE\", 15: \"PERIMETER_SE\", 16: \"AREA_SE\", 17: \"SMOOTHNESS_SE\", 18: \"COMPACTNESS_SE\", 19: \"CONCAVITY_SE\", 20: \"CONCAVE_POINTS_SE\", 21: \"SYMMETRY_SE\",22: \"FRACTAL_DIMENSIONS_SE\", 23: \"WORST_RADIUS\", 24: \"WORST_TEXTURE\", 25: \"WORST_PERIMETER\", 26: \"WORST_AREA\", 27: \"WORST_SMOOTHNESS\", 28: \"WORST_COMPACTNESS\", 29: \"WORST_CONCAVITY\", 30: \"WORST_CONCAVE_POINTS\", 31: \"WORST_SYMMETRY\", 32: \"WORST_FRACTAL_DIMENSIONS\", 33: \"TUMOR_SIZE\", 34: \"LYMPH_STATUS\"})\n",
    "\n",
    "wpbc_data = wpbc_data.dropna(axis=0, how=\"any\")\n",
    "\n",
    "wpbc_data['OUTCOME'] = 0\n",
    "wpbc_data['OUTCOME'][(wpbc_data.CLASS == \"R\") & (wpbc_data.TIME <= 24)] = 1\n",
    "\n",
    "y_class = wpbc_data.pop(\"CLASS\").values\n",
    "y3 = wpbc_data.pop(\"OUTCOME\").values\n",
    "X3 = wpbc_data.drop([\"ID\",\"TIME\"], axis=1)\n",
    "\n",
    "# split the data\n",
    "X_tr, X_te, y_tr, y_te, y_class_tr, y_class_te = train_test_split(X3.values, y3, y_class, test_size=0.2, random_state=1)\n",
    "print(\"X_tr:\", X_tr.shape)\n",
    "print(\"X_te:\", X_te.shape)\n",
    "print(\"y_tr:\", y_tr.shape)\n",
    "print(\"y_te:\", y_te.shape)\n",
    "print(\"y_class_tr:\", y_class_tr.shape)\n",
    "print(\"y_class_te:\", y_class_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.85567\n",
       "1    0.14433\n",
       "dtype: float64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(y3, normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.820513\n",
       "1    0.179487\n",
       "dtype: float64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(y_te, normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## MODEL 7.20.2.9j\n",
    "# Create new graph\n",
    "graph = tf.Graph()\n",
    "model_name = \"model_0.0.3\"\n",
    "\n",
    "with graph.as_default():\n",
    "    # Placeholders\n",
    "    X = tf.placeholder(dtype=tf.float32, shape=[None, 32])\n",
    "    y = tf.placeholder(dtype=tf.int32, shape=[None])\n",
    "    training = tf.placeholder(dtype=tf.bool)\n",
    "    \n",
    "    # create global step for decaying learning rate\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    \n",
    "    # learning rate o\n",
    "    epochs_per_decay = 20\n",
    "    starting_rate = 0.1\n",
    "    decay_factor = 0.80\n",
    "    staircase = True\n",
    "    \n",
    "    learning_rate = tf.train.exponential_decay(starting_rate,                 # start at 0.003\n",
    "                                               global_step, \n",
    "                                               steps_per_epoch * epochs_per_decay,       # 100 epochs\n",
    "                                               decay_factor,                   # 0.5 decrease\n",
    "                                               staircase=staircase) \n",
    "    \n",
    "    # Hidden layer with 64 units\n",
    "    with tf.name_scope('local1') as scope:\n",
    "        hidden = tf.layers.dense(\n",
    "            X,                              # input\n",
    "            64,                             # 64 units\n",
    "            activation=tf.nn.relu,          # activation\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=0),  # kernel initializer\n",
    "            bias_initializer=tf.zeros_initializer(), # bias\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "            name='hidden'                   # name\n",
    "        )\n",
    "\n",
    "        # Apply dropout\n",
    "        hidden = tf.layers.dropout(hidden, rate=0.5, seed=0, training=training)\n",
    "    \n",
    "    with tf.name_scope('local2') as scope:\n",
    "        hidden2 = tf.layers.dense(\n",
    "            hidden,                         # input\n",
    "            48,                             # 48 units\n",
    "            activation=tf.nn.relu,          # activation\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=0),  # kernel initializer\n",
    "            bias_initializer=tf.zeros_initializer(), # bias\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "            name='hidden2'                  # name\n",
    "        )\n",
    "        \n",
    "        hidden2 = tf.layers.dropout(hidden2, rate=0.5, seed=0, training=training)\n",
    "        \n",
    "    # Output layer\n",
    "    logits = tf.layers.dense(\n",
    "        hidden2,                       # input\n",
    "        num_classes,                             # 4 units\n",
    "        activation=None,               # No activation function\n",
    "        kernel_initializer=tf.variance_scaling_initializer(scale=1, seed=0),\n",
    "        bias_initializer=tf.zeros_initializer(),\n",
    "        kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.01),\n",
    "        name='output'\n",
    "    )\n",
    "    \n",
    "    # Loss fuction: mean cross-entropy\n",
    "    mean_ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "    \n",
    "    # easier way to handle regularization loss\n",
    "    loss = mean_ce + tf.losses.get_regularization_loss()\n",
    "    \n",
    "    # Gradient descent\n",
    "    gd = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "    # Minimize loss\n",
    "    train_op = gd.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # Compute predictions and accuracy\n",
    "    predictions = tf.argmax(logits, axis=1, output_type=tf.int32)\n",
    "    is_correct = tf.equal(y, predictions)\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct, dtype=tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 - valid: 0.744 train: 0.581 (mean) learning rate 0.100\n",
      "Epoch 6 - valid: 0.821 train: 0.876 (mean) learning rate 0.100\n",
      "Epoch 11 - valid: 0.821 train: 0.869 (mean) learning rate 0.100\n",
      "Epoch 16 - valid: 0.821 train: 0.869 (mean) learning rate 0.100\n",
      "Epoch 21 - valid: 0.821 train: 0.862 (mean) learning rate 0.100\n",
      "Epoch 26 - valid: 0.821 train: 0.862 (mean) learning rate 0.100\n",
      "Epoch 31 - valid: 0.821 train: 0.862 (mean) learning rate 0.100\n",
      "Epoch 36 - valid: 0.821 train: 0.848 (mean) learning rate 0.100\n",
      "Epoch 41 - valid: 0.821 train: 0.862 (mean) learning rate 0.100\n",
      "Epoch 46 - valid: 0.821 train: 0.869 (mean) learning rate 0.100\n",
      "Epoch 51 - valid: 0.821 train: 0.855 (mean) learning rate 0.100\n",
      "Epoch 56 - valid: 0.821 train: 0.855 (mean) learning rate 0.100\n",
      "Epoch 61 - valid: 0.821 train: 0.869 (mean) learning rate 0.080\n",
      "Epoch 66 - valid: 0.821 train: 0.869 (mean) learning rate 0.080\n",
      "Epoch 71 - valid: 0.821 train: 0.862 (mean) learning rate 0.080\n",
      "Epoch 76 - valid: 0.821 train: 0.869 (mean) learning rate 0.080\n",
      "Epoch 81 - valid: 0.821 train: 0.862 (mean) learning rate 0.080\n",
      "Epoch 86 - valid: 0.821 train: 0.855 (mean) learning rate 0.080\n",
      "Epoch 91 - valid: 0.821 train: 0.855 (mean) learning rate 0.080\n",
      "Epoch 96 - valid: 0.821 train: 0.869 (mean) learning rate 0.080\n",
      "Test accuracy: 0.82051283\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Validation accuracy\n",
    "valid_acc_values = []\n",
    "tr_acc_values = []\n",
    "lr_values = []\n",
    "cv_cost_values = []\n",
    "tr_cost_values = []\n",
    "config = tf.ConfigProto(device_count = {'GPU': 0})\n",
    "\n",
    "with tf.Session(graph=graph, config=config) as sess:\n",
    "    # Initialize variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    saver = tf.train.Saver()\n",
    "    \n",
    "    # Set seed\n",
    "    np.random.seed(0)\n",
    "\n",
    "    # Train several epochs\n",
    "    for epoch in range(100):\n",
    "        # Accuracy values (train) after each batch\n",
    "        batch_acc = []\n",
    "        batch_cost = []\n",
    "\n",
    "        # Get batches of data\n",
    "        for X_batch, y_batch in get_batches(X_tr, y_tr, batch_size):\n",
    "            # Run training and evaluate accuracy\n",
    "            _, acc_value, lr, tr_cost = sess.run([train_op, accuracy, learning_rate, loss], feed_dict={\n",
    "                X: X_batch,\n",
    "                y: y_batch,\n",
    "                training: True # Apply dropout\n",
    "            })\n",
    "\n",
    "            # Save accuracy (current batch)\n",
    "            batch_acc.append(acc_value)\n",
    "            batch_cost.append(tr_cost)\n",
    "\n",
    "            # Evaluate validation accuracy\n",
    "            valid_acc, cv_cost = sess.run([accuracy, loss], feed_dict={\n",
    "                X: X_te,\n",
    "                y: y_te,\n",
    "                training: False # Do not apply dropout\n",
    "            })\n",
    "            valid_acc_values.append(valid_acc)\n",
    "            tr_acc_values.append(np.mean(batch_acc))\n",
    "            cv_cost_values.append(cv_cost)\n",
    "            tr_cost_values.append(np.mean(batch_cost))\n",
    "            lr_values.append(lr)\n",
    "\n",
    "        # Print progress\n",
    "        if epoch % 5 == 0:\n",
    "            print('Epoch {} - valid: {:.3f} train: {:.3f} (mean) learning rate {:.3f}'.format(\n",
    "                epoch+1, valid_acc, np.mean(batch_acc), lr\n",
    "            ))\n",
    "    \n",
    "    save_path = saver.save(sess, \"./model/nn_model.ckpt\")\n",
    "    \n",
    "    test_acc, yhat = sess.run([accuracy, predictions], feed_dict = {\n",
    "        X: X_te,\n",
    "        y: y_te,\n",
    "        training: False\n",
    "    })\n",
    "    \n",
    "print(\"Test accuracy:\", test_acc)\n",
    "yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}