Main_scripts_01.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center>MobileNet - Pytorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Step 1: Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# MobileNet-Pytorch\n",
    "import argparse \n",
    "import torch\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "from torchvision import datasets, transforms\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "from sklearn.metrics import accuracy_score\n",
    "from mobilenets import mobilenet\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "use_cudause_cud  = torch.cuda.is_available()\n",
    "dtype = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# Train, Validate, Test. Heavily inspired by Kevinzakka https://github.com/kevinzakka/DenseNet/blob/master/data_loader.py\n",
    "\n",
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                 std=[0.229, 0.224, 0.225])\n",
    "\n",
    "valid_size=0.1\n",
    "\n",
    "# define transforms\n",
    "valid_transform = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        normalize\n",
    "])\n",
    "\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomCrop(32, padding=4),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    normalize\n",
    "])\n",
    "\n",
    "\n",
    "# load the dataset\n",
    "train_dataset = datasets.CIFAR10(root=\"data\", train=True, \n",
    "            download=True, transform=train_transform)\n",
    "\n",
    "valid_dataset = datasets.CIFAR10(root=\"data\", train=True, \n",
    "            download=True, transform=valid_transform)\n",
    "\n",
    "num_train = len(train_dataset)\n",
    "indices = list(range(num_train))\n",
    "split = int(np.floor(valid_size * num_train)) #5w张图片的10%用来当做验证集\n",
    "\n",
    "\n",
    "np.random.seed(22)\n",
    "np.random.shuffle(indices) # 随机乱序[0,1,...,49999]\n",
    "\n",
    "train_idx, valid_idx = indices[split:], indices[:split]\n",
    "\n",
    "\n",
    "train_sampler = SubsetRandomSampler(train_idx) # 这个很有意思\n",
    "valid_sampler = SubsetRandomSampler(valid_idx)\n",
    "\n",
    "\n",
    "# ------------------------- 使用不同的批次大小 ------------------------------------\n",
    "show_step=10  # 批次大，show_step就小点\n",
    "max_epoch=50  # 训练最大epoch数目\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, \n",
    "                batch_size=256, sampler=train_sampler)\n",
    "\n",
    "valid_loader = torch.utils.data.DataLoader(valid_dataset, \n",
    "                batch_size=256, sampler=valid_sampler)\n",
    "\n",
    "\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(), normalize\n",
    "])\n",
    "\n",
    "test_dataset = datasets.CIFAR10(root=\"data\", \n",
    "                                train=False, \n",
    "                                download=True,transform=test_transform)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, \n",
    "                                          batch_size=64, \n",
    "                                          shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2: Model Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# \n",
    "model = mobilenet(num_classes=10, large_img=False)\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.01)\n",
    "scheduler = StepLR(optimizer, step_size=10, gamma=0.5)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Implement validation\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    writer = SummaryWriter()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        if use_cuda:\n",
    "            data, target = data.cuda(), target.cuda()\n",
    "        data, target = Variable(data), Variable(target)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        correct = 0\n",
    "        pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
    "        correct += pred.eq(target.data.view_as(pred)).sum()\n",
    "        \n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        accuracy = 100. * (correct.cpu().numpy()/ len(output))\n",
    "        optimizer.step()\n",
    "        if batch_idx % show_step == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}, Accuracy: {:.2f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item(), accuracy))\n",
    "            writer.add_scalar('Loss/Loss', loss.item(), epoch)\n",
    "            writer.add_scalar('Accuracy/Accuracy', accuracy, epoch)\n",
    "    scheduler.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def validate(epoch):\n",
    "    model.eval()\n",
    "    writer = SummaryWriter()\n",
    "    valid_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in valid_loader:\n",
    "        if use_cuda:\n",
    "            data, target = data.cuda(), target.cuda()\n",
    "        data, target = Variable(data), Variable(target)\n",
    "        output = model(data)\n",
    "        valid_loss += F.cross_entropy(output, target, size_average=False).item() # sum up batch loss\n",
    "        pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
    "        correct += pred.eq(target.data.view_as(pred)).sum()\n",
    "\n",
    "    valid_loss /= len(valid_idx)\n",
    "    accuracy = 100. * correct.cpu().numpy() / len(valid_idx)\n",
    "    print('\\nValidation set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\\n'.format(\n",
    "        valid_loss, correct, len(valid_idx),\n",
    "        100. * correct / len(valid_idx)))\n",
    "    writer.add_scalar('Loss/Validation_Loss', valid_loss, epoch)\n",
    "    writer.add_scalar('Accuracy/Validation_Accuracy', accuracy, epoch)\n",
    "    return valid_loss, accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Fix best model\n",
    "\n",
    "def test(epoch):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        if use_cuda:\n",
    "            data, target = data.cuda(), target.cuda()\n",
    "        data, target = Variable(data), Variable(target)\n",
    "        output = model(data)\n",
    "        test_loss += F.cross_entropy(output, target, size_average=False).item() # sum up batch loss\n",
    "        pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
    "        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct.cpu().numpy() / len(test_loader.dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_best(loss, accuracy, best_loss, best_acc):\n",
    "    if best_loss == None:\n",
    "        best_loss = loss\n",
    "        best_acc = accuracy\n",
    "        file = 'saved_models/best_save_model.p'\n",
    "        torch.save(model.state_dict(), file)\n",
    "        \n",
    "    elif loss < best_loss and accuracy > best_acc:\n",
    "        best_loss = loss\n",
    "        best_acc = accuracy\n",
    "        file = 'saved_models/best_save_model.p'\n",
    "        torch.save(model.state_dict(), file)\n",
    "    return best_loss, best_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 0 [0/50000 (0%)]\tLoss: 2.304640, Accuracy: 7.81\n",
      "Train Epoch: 0 [2560/50000 (6%)]\tLoss: 2.375479, Accuracy: 13.67\n",
      "Train Epoch: 0 [5120/50000 (11%)]\tLoss: 1.988270, Accuracy: 25.78\n",
      "Train Epoch: 0 [7680/50000 (17%)]\tLoss: 2.038945, Accuracy: 27.34\n",
      "Train Epoch: 0 [10240/50000 (23%)]\tLoss: 1.903131, Accuracy: 26.56\n",
      "Train Epoch: 0 [12800/50000 (28%)]\tLoss: 1.943959, Accuracy: 28.52\n",
      "Train Epoch: 0 [15360/50000 (34%)]\tLoss: 2.084677, Accuracy: 24.22\n",
      "Train Epoch: 0 [17920/50000 (40%)]\tLoss: 1.904895, Accuracy: 26.56\n",
      "Train Epoch: 0 [20480/50000 (45%)]\tLoss: 1.815676, Accuracy: 29.69\n",
      "Train Epoch: 0 [23040/50000 (51%)]\tLoss: 1.820209, Accuracy: 31.64\n",
      "Train Epoch: 0 [25600/50000 (57%)]\tLoss: 1.765703, Accuracy: 32.03\n",
      "Train Epoch: 0 [28160/50000 (62%)]\tLoss: 1.795561, Accuracy: 32.81\n",
      "Train Epoch: 0 [30720/50000 (68%)]\tLoss: 1.691612, Accuracy: 38.67\n",
      "Train Epoch: 0 [33280/50000 (74%)]\tLoss: 1.778997, Accuracy: 35.55\n",
      "Train Epoch: 0 [35840/50000 (80%)]\tLoss: 1.712279, Accuracy: 38.67\n",
      "Train Epoch: 0 [38400/50000 (85%)]\tLoss: 1.680794, Accuracy: 33.20\n",
      "Train Epoch: 0 [40960/50000 (91%)]\tLoss: 1.667910, Accuracy: 37.89\n",
      "Train Epoch: 0 [43520/50000 (97%)]\tLoss: 1.713743, Accuracy: 39.45\n",
      "\n",
      "Validation set: Average loss: 1.9943, Accuracy: 1465/5000 (29.00%)\n",
      "\n",
      "the time of this epoch:[46.00657057762146 s]\n",
      "Train Epoch: 1 [0/50000 (0%)]\tLoss: 1.632339, Accuracy: 41.02\n",
      "Train Epoch: 1 [2560/50000 (6%)]\tLoss: 1.647108, Accuracy: 38.28\n",
      "Train Epoch: 1 [5120/50000 (11%)]\tLoss: 1.681082, Accuracy: 35.16\n",
      "Train Epoch: 1 [7680/50000 (17%)]\tLoss: 1.606355, Accuracy: 44.14\n",
      "Train Epoch: 1 [10240/50000 (23%)]\tLoss: 1.580529, Accuracy: 41.41\n",
      "Train Epoch: 1 [12800/50000 (28%)]\tLoss: 1.610758, Accuracy: 36.72\n",
      "Train Epoch: 1 [15360/50000 (34%)]\tLoss: 1.568583, Accuracy: 44.14\n",
      "Train Epoch: 1 [17920/50000 (40%)]\tLoss: 1.585046, Accuracy: 42.58\n",
      "Train Epoch: 1 [20480/50000 (45%)]\tLoss: 1.517433, Accuracy: 46.88\n",
      "Train Epoch: 1 [23040/50000 (51%)]\tLoss: 1.429016, Accuracy: 42.19\n",
      "Train Epoch: 1 [25600/50000 (57%)]\tLoss: 1.589997, Accuracy: 44.53\n",
      "Train Epoch: 1 [28160/50000 (62%)]\tLoss: 1.453393, Accuracy: 45.70\n",
      "Train Epoch: 1 [30720/50000 (68%)]\tLoss: 1.347473, Accuracy: 47.66\n",
      "Train Epoch: 1 [33280/50000 (74%)]\tLoss: 1.450169, Accuracy: 45.70\n",
      "Train Epoch: 1 [35840/50000 (80%)]\tLoss: 1.513361, Accuracy: 45.31\n",
      "Train Epoch: 1 [38400/50000 (85%)]\tLoss: 1.360406, Accuracy: 53.52\n",
      "Train Epoch: 1 [40960/50000 (91%)]\tLoss: 1.355815, Accuracy: 52.34\n",
      "Train Epoch: 1 [43520/50000 (97%)]\tLoss: 1.351413, Accuracy: 49.61\n",
      "\n",
      "Validation set: Average loss: 2.1610, Accuracy: 1799/5000 (35.00%)\n",
      "\n",
      "the time of this epoch:[45.624205112457275 s]\n",
      "Train Epoch: 2 [0/50000 (0%)]\tLoss: 1.345893, Accuracy: 50.00\n",
      "Train Epoch: 2 [2560/50000 (6%)]\tLoss: 1.357609, Accuracy: 47.27\n",
      "Train Epoch: 2 [5120/50000 (11%)]\tLoss: 1.319342, Accuracy: 52.73\n",
      "Train Epoch: 2 [7680/50000 (17%)]\tLoss: 1.299884, Accuracy: 57.42\n",
      "Train Epoch: 2 [10240/50000 (23%)]\tLoss: 1.477730, Accuracy: 49.22\n",
      "Train Epoch: 2 [12800/50000 (28%)]\tLoss: 1.236399, Accuracy: 50.78\n",
      "Train Epoch: 2 [15360/50000 (34%)]\tLoss: 1.266518, Accuracy: 56.25\n",
      "Train Epoch: 2 [17920/50000 (40%)]\tLoss: 1.252718, Accuracy: 53.52\n",
      "Train Epoch: 2 [20480/50000 (45%)]\tLoss: 1.229239, Accuracy: 56.25\n",
      "Train Epoch: 2 [23040/50000 (51%)]\tLoss: 1.255580, Accuracy: 56.25\n",
      "Train Epoch: 2 [25600/50000 (57%)]\tLoss: 1.362134, Accuracy: 52.73\n",
      "Train Epoch: 2 [28160/50000 (62%)]\tLoss: 1.288565, Accuracy: 53.52\n",
      "Train Epoch: 2 [30720/50000 (68%)]\tLoss: 1.279981, Accuracy: 51.56\n",
      "Train Epoch: 2 [33280/50000 (74%)]\tLoss: 1.231764, Accuracy: 57.42\n",
      "Train Epoch: 2 [35840/50000 (80%)]\tLoss: 1.133125, Accuracy: 57.42\n",
      "Train Epoch: 2 [38400/50000 (85%)]\tLoss: 1.364199, Accuracy: 50.78\n",
      "Train Epoch: 2 [40960/50000 (91%)]\tLoss: 1.161319, Accuracy: 55.47\n",
      "Train Epoch: 2 [43520/50000 (97%)]\tLoss: 1.325220, Accuracy: 52.73\n",
      "\n",
      "Validation set: Average loss: 1.8665, Accuracy: 2298/5000 (45.00%)\n",
      "\n",
      "the time of this epoch:[45.44239068031311 s]\n",
      "Train Epoch: 3 [0/50000 (0%)]\tLoss: 1.232820, Accuracy: 53.52\n",
      "Train Epoch: 3 [2560/50000 (6%)]\tLoss: 1.146858, Accuracy: 57.42\n",
      "Train Epoch: 3 [5120/50000 (11%)]\tLoss: 1.142073, Accuracy: 58.98\n",
      "Train Epoch: 3 [7680/50000 (17%)]\tLoss: 1.156909, Accuracy: 59.38\n",
      "Train Epoch: 3 [10240/50000 (23%)]\tLoss: 1.103985, Accuracy: 61.72\n",
      "Train Epoch: 3 [12800/50000 (28%)]\tLoss: 1.042347, Accuracy: 63.28\n",
      "Train Epoch: 3 [15360/50000 (34%)]\tLoss: 1.101361, Accuracy: 62.89\n",
      "Train Epoch: 3 [17920/50000 (40%)]\tLoss: 1.120274, Accuracy: 61.33\n",
      "Train Epoch: 3 [20480/50000 (45%)]\tLoss: 1.098087, Accuracy: 61.72\n",
      "Train Epoch: 3 [23040/50000 (51%)]\tLoss: 1.034163, Accuracy: 61.72\n",
      "Train Epoch: 3 [25600/50000 (57%)]\tLoss: 1.092200, Accuracy: 58.59\n",
      "Train Epoch: 3 [28160/50000 (62%)]\tLoss: 1.134420, Accuracy: 59.38\n",
      "Train Epoch: 3 [30720/50000 (68%)]\tLoss: 1.190365, Accuracy: 53.52\n",
      "Train Epoch: 3 [33280/50000 (74%)]\tLoss: 1.091210, Accuracy: 61.72\n",
      "Train Epoch: 3 [35840/50000 (80%)]\tLoss: 1.137656, Accuracy: 57.42\n",
      "Train Epoch: 3 [38400/50000 (85%)]\tLoss: 1.096919, Accuracy: 60.55\n",
      "Train Epoch: 3 [40960/50000 (91%)]\tLoss: 1.015445, Accuracy: 60.16\n",
      "Train Epoch: 3 [43520/50000 (97%)]\tLoss: 1.007591, Accuracy: 63.28\n",
      "\n",
      "Validation set: Average loss: 1.6995, Accuracy: 2558/5000 (51.00%)\n",
      "\n",
      "the time of this epoch:[45.68873310089111 s]\n",
      "Train Epoch: 4 [0/50000 (0%)]\tLoss: 1.071004, Accuracy: 61.33\n",
      "Train Epoch: 4 [2560/50000 (6%)]\tLoss: 0.961048, Accuracy: 65.23\n",
      "Train Epoch: 4 [5120/50000 (11%)]\tLoss: 1.152082, Accuracy: 54.69\n",
      "Train Epoch: 4 [7680/50000 (17%)]\tLoss: 1.047876, Accuracy: 62.11\n",
      "Train Epoch: 4 [10240/50000 (23%)]\tLoss: 1.115256, Accuracy: 61.72\n",
      "Train Epoch: 4 [12800/50000 (28%)]\tLoss: 1.072117, Accuracy: 62.50\n",
      "Train Epoch: 4 [15360/50000 (34%)]\tLoss: 1.138838, Accuracy: 61.33\n",
      "Train Epoch: 4 [17920/50000 (40%)]\tLoss: 1.097900, Accuracy: 57.81\n",
      "Train Epoch: 4 [20480/50000 (45%)]\tLoss: 1.085955, Accuracy: 61.72\n",
      "Train Epoch: 4 [23040/50000 (51%)]\tLoss: 1.159088, Accuracy: 62.11\n",
      "Train Epoch: 4 [25600/50000 (57%)]\tLoss: 0.992548, Accuracy: 62.11\n",
      "Train Epoch: 4 [28160/50000 (62%)]\tLoss: 1.019503, Accuracy: 63.28\n",
      "Train Epoch: 4 [30720/50000 (68%)]\tLoss: 0.990608, Accuracy: 63.28\n",
      "Train Epoch: 4 [33280/50000 (74%)]\tLoss: 1.023596, Accuracy: 64.84\n",
      "Train Epoch: 4 [35840/50000 (80%)]\tLoss: 1.035349, Accuracy: 65.62\n",
      "Train Epoch: 4 [38400/50000 (85%)]\tLoss: 0.961257, Accuracy: 66.41\n",
      "Train Epoch: 4 [40960/50000 (91%)]\tLoss: 0.921044, Accuracy: 64.45\n",
      "Train Epoch: 4 [43520/50000 (97%)]\tLoss: 0.911353, Accuracy: 64.84\n",
      "\n",
      "Validation set: Average loss: 1.2715, Accuracy: 2890/5000 (57.00%)\n",
      "\n",
      "the time of this epoch:[45.690197467803955 s]\n",
      "Train Epoch: 5 [0/50000 (0%)]\tLoss: 1.021438, Accuracy: 60.94\n",
      "Train Epoch: 5 [2560/50000 (6%)]\tLoss: 1.030170, Accuracy: 62.50\n",
      "Train Epoch: 5 [5120/50000 (11%)]\tLoss: 1.070339, Accuracy: 61.33\n",
      "Train Epoch: 5 [7680/50000 (17%)]\tLoss: 1.068951, Accuracy: 61.33\n",
      "Train Epoch: 5 [10240/50000 (23%)]\tLoss: 0.960264, Accuracy: 67.58\n",
      "Train Epoch: 5 [12800/50000 (28%)]\tLoss: 0.969358, Accuracy: 64.45\n",
      "Train Epoch: 5 [15360/50000 (34%)]\tLoss: 0.964646, Accuracy: 63.67\n",
      "Train Epoch: 5 [17920/50000 (40%)]\tLoss: 1.002027, Accuracy: 61.72\n",
      "Train Epoch: 5 [20480/50000 (45%)]\tLoss: 1.003622, Accuracy: 63.28\n",
      "Train Epoch: 5 [23040/50000 (51%)]\tLoss: 0.850504, Accuracy: 68.75\n",
      "Train Epoch: 5 [25600/50000 (57%)]\tLoss: 0.953952, Accuracy: 68.75\n",
      "Train Epoch: 5 [28160/50000 (62%)]\tLoss: 1.058722, Accuracy: 61.72\n",
      "Train Epoch: 5 [30720/50000 (68%)]\tLoss: 0.941002, Accuracy: 67.58\n",
      "Train Epoch: 5 [33280/50000 (74%)]\tLoss: 0.857589, Accuracy: 67.97\n",
      "Train Epoch: 5 [35840/50000 (80%)]\tLoss: 0.857287, Accuracy: 70.70\n",
      "Train Epoch: 5 [38400/50000 (85%)]\tLoss: 0.932361, Accuracy: 66.80\n",
      "Train Epoch: 5 [40960/50000 (91%)]\tLoss: 0.968800, Accuracy: 67.58\n",
      "Train Epoch: 5 [43520/50000 (97%)]\tLoss: 0.878204, Accuracy: 69.14\n",
      "\n",
      "Validation set: Average loss: 0.9977, Accuracy: 3244/5000 (64.00%)\n",
      "\n",
      "the time of this epoch:[45.64286732673645 s]\n",
      "Train Epoch: 6 [0/50000 (0%)]\tLoss: 0.873158, Accuracy: 71.09\n",
      "Train Epoch: 6 [2560/50000 (6%)]\tLoss: 0.931225, Accuracy: 69.53\n",
      "Train Epoch: 6 [5120/50000 (11%)]\tLoss: 0.894971, Accuracy: 68.75\n",
      "Train Epoch: 6 [7680/50000 (17%)]\tLoss: 0.898539, Accuracy: 67.97\n",
      "Train Epoch: 6 [10240/50000 (23%)]\tLoss: 0.968493, Accuracy: 66.80\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 6 [12800/50000 (28%)]\tLoss: 0.879068, Accuracy: 67.58\n",
      "Train Epoch: 6 [15360/50000 (34%)]\tLoss: 0.871549, Accuracy: 68.36\n",
      "Train Epoch: 6 [17920/50000 (40%)]\tLoss: 0.957298, Accuracy: 67.19\n",
      "Train Epoch: 6 [20480/50000 (45%)]\tLoss: 0.890900, Accuracy: 69.92\n",
      "Train Epoch: 6 [23040/50000 (51%)]\tLoss: 0.914453, Accuracy: 68.75\n",
      "Train Epoch: 6 [25600/50000 (57%)]\tLoss: 0.768623, Accuracy: 73.05\n",
      "Train Epoch: 6 [28160/50000 (62%)]\tLoss: 0.793022, Accuracy: 73.05\n",
      "Train Epoch: 6 [30720/50000 (68%)]\tLoss: 1.031054, Accuracy: 60.94\n",
      "Train Epoch: 6 [33280/50000 (74%)]\tLoss: 1.017033, Accuracy: 62.50\n",
      "Train Epoch: 6 [35840/50000 (80%)]\tLoss: 0.963585, Accuracy: 70.31\n",
      "Train Epoch: 6 [38400/50000 (85%)]\tLoss: 0.841889, Accuracy: 71.48\n",
      "Train Epoch: 6 [40960/50000 (91%)]\tLoss: 0.723412, Accuracy: 73.83\n",
      "Train Epoch: 6 [43520/50000 (97%)]\tLoss: 0.898974, Accuracy: 66.80\n",
      "\n",
      "Validation set: Average loss: 1.1921, Accuracy: 3115/5000 (62.00%)\n",
      "\n",
      "the time of this epoch:[45.59594202041626 s]\n",
      "Train Epoch: 7 [0/50000 (0%)]\tLoss: 0.797428, Accuracy: 72.27\n",
      "Train Epoch: 7 [2560/50000 (6%)]\tLoss: 0.875929, Accuracy: 65.23\n",
      "Train Epoch: 7 [5120/50000 (11%)]\tLoss: 0.755426, Accuracy: 72.66\n",
      "Train Epoch: 7 [7680/50000 (17%)]\tLoss: 0.758174, Accuracy: 72.27\n",
      "Train Epoch: 7 [10240/50000 (23%)]\tLoss: 0.837481, Accuracy: 71.88\n",
      "Train Epoch: 7 [12800/50000 (28%)]\tLoss: 0.743150, Accuracy: 73.05\n",
      "Train Epoch: 7 [15360/50000 (34%)]\tLoss: 0.927124, Accuracy: 65.62\n",
      "Train Epoch: 7 [17920/50000 (40%)]\tLoss: 0.974580, Accuracy: 64.06\n",
      "Train Epoch: 7 [20480/50000 (45%)]\tLoss: 0.915633, Accuracy: 70.31\n",
      "Train Epoch: 7 [23040/50000 (51%)]\tLoss: 0.779973, Accuracy: 71.48\n",
      "Train Epoch: 7 [25600/50000 (57%)]\tLoss: 0.787671, Accuracy: 75.39\n",
      "Train Epoch: 7 [28160/50000 (62%)]\tLoss: 0.735420, Accuracy: 74.61\n",
      "Train Epoch: 7 [30720/50000 (68%)]\tLoss: 0.980843, Accuracy: 65.62\n",
      "Train Epoch: 7 [33280/50000 (74%)]\tLoss: 0.735105, Accuracy: 76.17\n",
      "Train Epoch: 7 [35840/50000 (80%)]\tLoss: 0.852449, Accuracy: 68.75\n",
      "Train Epoch: 7 [38400/50000 (85%)]\tLoss: 0.686216, Accuracy: 76.56\n",
      "Train Epoch: 7 [40960/50000 (91%)]\tLoss: 0.818543, Accuracy: 68.36\n",
      "Train Epoch: 7 [43520/50000 (97%)]\tLoss: 0.798227, Accuracy: 73.83\n",
      "\n",
      "Validation set: Average loss: 1.0784, Accuracy: 3094/5000 (61.00%)\n",
      "\n",
      "the time of this epoch:[45.59801530838013 s]\n",
      "Train Epoch: 8 [0/50000 (0%)]\tLoss: 0.844532, Accuracy: 64.84\n",
      "Train Epoch: 8 [2560/50000 (6%)]\tLoss: 0.840039, Accuracy: 69.92\n",
      "Train Epoch: 8 [5120/50000 (11%)]\tLoss: 0.979897, Accuracy: 64.84\n",
      "Train Epoch: 8 [7680/50000 (17%)]\tLoss: 0.692914, Accuracy: 75.00\n",
      "Train Epoch: 8 [10240/50000 (23%)]\tLoss: 0.724727, Accuracy: 74.22\n",
      "Train Epoch: 8 [12800/50000 (28%)]\tLoss: 0.882555, Accuracy: 69.53\n",
      "Train Epoch: 8 [15360/50000 (34%)]\tLoss: 0.901278, Accuracy: 66.41\n",
      "Train Epoch: 8 [17920/50000 (40%)]\tLoss: 0.777695, Accuracy: 72.66\n",
      "Train Epoch: 8 [20480/50000 (45%)]\tLoss: 0.622746, Accuracy: 77.73\n",
      "Train Epoch: 8 [23040/50000 (51%)]\tLoss: 0.811691, Accuracy: 69.92\n",
      "Train Epoch: 8 [25600/50000 (57%)]\tLoss: 0.853677, Accuracy: 67.97\n",
      "Train Epoch: 8 [28160/50000 (62%)]\tLoss: 0.790524, Accuracy: 71.88\n",
      "Train Epoch: 8 [30720/50000 (68%)]\tLoss: 0.719153, Accuracy: 73.83\n",
      "Train Epoch: 8 [33280/50000 (74%)]\tLoss: 0.895000, Accuracy: 70.31\n",
      "Train Epoch: 8 [35840/50000 (80%)]\tLoss: 0.747145, Accuracy: 67.97\n",
      "Train Epoch: 8 [38400/50000 (85%)]\tLoss: 0.680042, Accuracy: 74.61\n",
      "Train Epoch: 8 [40960/50000 (91%)]\tLoss: 0.649794, Accuracy: 78.91\n",
      "Train Epoch: 8 [43520/50000 (97%)]\tLoss: 0.798714, Accuracy: 72.66\n",
      "\n",
      "Validation set: Average loss: 0.8808, Accuracy: 3470/5000 (69.00%)\n",
      "\n",
      "the time of this epoch:[45.5941903591156 s]\n",
      "Train Epoch: 9 [0/50000 (0%)]\tLoss: 0.751900, Accuracy: 72.66\n",
      "Train Epoch: 9 [2560/50000 (6%)]\tLoss: 0.751358, Accuracy: 73.05\n",
      "Train Epoch: 9 [5120/50000 (11%)]\tLoss: 0.637127, Accuracy: 77.34\n",
      "Train Epoch: 9 [7680/50000 (17%)]\tLoss: 0.894856, Accuracy: 71.88\n",
      "Train Epoch: 9 [10240/50000 (23%)]\tLoss: 0.796597, Accuracy: 73.05\n",
      "Train Epoch: 9 [12800/50000 (28%)]\tLoss: 0.652884, Accuracy: 76.17\n",
      "Train Epoch: 9 [15360/50000 (34%)]\tLoss: 0.819650, Accuracy: 67.97\n",
      "Train Epoch: 9 [17920/50000 (40%)]\tLoss: 0.854984, Accuracy: 68.75\n",
      "Train Epoch: 9 [20480/50000 (45%)]\tLoss: 0.695605, Accuracy: 76.56\n",
      "Train Epoch: 9 [23040/50000 (51%)]\tLoss: 0.845717, Accuracy: 70.70\n",
      "Train Epoch: 9 [25600/50000 (57%)]\tLoss: 0.712806, Accuracy: 75.39\n",
      "Train Epoch: 9 [28160/50000 (62%)]\tLoss: 0.843894, Accuracy: 73.83\n",
      "Train Epoch: 9 [30720/50000 (68%)]\tLoss: 0.799597, Accuracy: 70.70\n",
      "Train Epoch: 9 [33280/50000 (74%)]\tLoss: 0.807656, Accuracy: 71.09\n",
      "Train Epoch: 9 [35840/50000 (80%)]\tLoss: 0.784108, Accuracy: 72.27\n",
      "Train Epoch: 9 [38400/50000 (85%)]\tLoss: 0.741232, Accuracy: 72.66\n",
      "Train Epoch: 9 [40960/50000 (91%)]\tLoss: 0.787422, Accuracy: 71.48\n",
      "Train Epoch: 9 [43520/50000 (97%)]\tLoss: 0.607615, Accuracy: 80.47\n",
      "\n",
      "Validation set: Average loss: 0.8649, Accuracy: 3541/5000 (70.00%)\n",
      "\n",
      "the time of this epoch:[45.304527759552 s]\n",
      "Train Epoch: 10 [0/50000 (0%)]\tLoss: 0.693572, Accuracy: 75.00\n",
      "Train Epoch: 10 [2560/50000 (6%)]\tLoss: 0.773178, Accuracy: 71.48\n",
      "Train Epoch: 10 [5120/50000 (11%)]\tLoss: 0.809231, Accuracy: 71.48\n",
      "Train Epoch: 10 [7680/50000 (17%)]\tLoss: 0.824328, Accuracy: 70.70\n",
      "Train Epoch: 10 [10240/50000 (23%)]\tLoss: 0.606630, Accuracy: 80.86\n",
      "Train Epoch: 10 [12800/50000 (28%)]\tLoss: 0.728803, Accuracy: 76.95\n",
      "Train Epoch: 10 [15360/50000 (34%)]\tLoss: 0.671966, Accuracy: 74.61\n",
      "Train Epoch: 10 [17920/50000 (40%)]\tLoss: 0.685895, Accuracy: 75.39\n",
      "Train Epoch: 10 [20480/50000 (45%)]\tLoss: 0.717080, Accuracy: 70.70\n",
      "Train Epoch: 10 [23040/50000 (51%)]\tLoss: 0.563309, Accuracy: 79.69\n",
      "Train Epoch: 10 [25600/50000 (57%)]\tLoss: 0.748091, Accuracy: 73.83\n",
      "Train Epoch: 10 [28160/50000 (62%)]\tLoss: 0.720833, Accuracy: 75.00\n",
      "Train Epoch: 10 [30720/50000 (68%)]\tLoss: 0.823620, Accuracy: 69.92\n",
      "Train Epoch: 10 [33280/50000 (74%)]\tLoss: 0.727509, Accuracy: 75.78\n",
      "Train Epoch: 10 [35840/50000 (80%)]\tLoss: 0.816477, Accuracy: 71.09\n",
      "Train Epoch: 10 [38400/50000 (85%)]\tLoss: 0.668511, Accuracy: 78.12\n",
      "Train Epoch: 10 [40960/50000 (91%)]\tLoss: 0.790450, Accuracy: 72.66\n",
      "Train Epoch: 10 [43520/50000 (97%)]\tLoss: 0.667439, Accuracy: 79.30\n",
      "\n",
      "Validation set: Average loss: 0.8093, Accuracy: 3620/5000 (72.00%)\n",
      "\n",
      "the time of this epoch:[45.15373134613037 s]\n",
      "Train Epoch: 11 [0/50000 (0%)]\tLoss: 0.704481, Accuracy: 72.66\n",
      "Train Epoch: 11 [2560/50000 (6%)]\tLoss: 0.559917, Accuracy: 80.08\n",
      "Train Epoch: 11 [5120/50000 (11%)]\tLoss: 0.626401, Accuracy: 77.34\n",
      "Train Epoch: 11 [7680/50000 (17%)]\tLoss: 0.577667, Accuracy: 76.95\n",
      "Train Epoch: 11 [10240/50000 (23%)]\tLoss: 0.590236, Accuracy: 76.56\n",
      "Train Epoch: 11 [12800/50000 (28%)]\tLoss: 0.692517, Accuracy: 73.83\n",
      "Train Epoch: 11 [15360/50000 (34%)]\tLoss: 0.580948, Accuracy: 80.08\n",
      "Train Epoch: 11 [17920/50000 (40%)]\tLoss: 0.603340, Accuracy: 78.52\n",
      "Train Epoch: 11 [20480/50000 (45%)]\tLoss: 0.675969, Accuracy: 76.56\n",
      "Train Epoch: 11 [23040/50000 (51%)]\tLoss: 0.539162, Accuracy: 82.42\n",
      "Train Epoch: 11 [25600/50000 (57%)]\tLoss: 0.633920, Accuracy: 79.69\n",
      "Train Epoch: 11 [28160/50000 (62%)]\tLoss: 0.501537, Accuracy: 82.81\n",
      "Train Epoch: 11 [30720/50000 (68%)]\tLoss: 0.608916, Accuracy: 77.73\n",
      "Train Epoch: 11 [33280/50000 (74%)]\tLoss: 0.610011, Accuracy: 76.56\n",
      "Train Epoch: 11 [35840/50000 (80%)]\tLoss: 0.687126, Accuracy: 76.95\n",
      "Train Epoch: 11 [38400/50000 (85%)]\tLoss: 0.540769, Accuracy: 82.81\n",
      "Train Epoch: 11 [40960/50000 (91%)]\tLoss: 0.568021, Accuracy: 79.69\n",
      "Train Epoch: 11 [43520/50000 (97%)]\tLoss: 0.593505, Accuracy: 78.52\n",
      "\n",
      "Validation set: Average loss: 0.6152, Accuracy: 3926/5000 (78.00%)\n",
      "\n",
      "the time of this epoch:[45.10688924789429 s]\n",
      "Train Epoch: 12 [0/50000 (0%)]\tLoss: 0.475969, Accuracy: 84.38\n",
      "Train Epoch: 12 [2560/50000 (6%)]\tLoss: 0.529017, Accuracy: 82.03\n",
      "Train Epoch: 12 [5120/50000 (11%)]\tLoss: 0.531822, Accuracy: 81.64\n",
      "Train Epoch: 12 [7680/50000 (17%)]\tLoss: 0.580279, Accuracy: 76.95\n",
      "Train Epoch: 12 [10240/50000 (23%)]\tLoss: 0.669334, Accuracy: 75.39\n",
      "Train Epoch: 12 [12800/50000 (28%)]\tLoss: 0.573532, Accuracy: 83.98\n",
      "Train Epoch: 12 [15360/50000 (34%)]\tLoss: 0.505478, Accuracy: 83.59\n",
      "Train Epoch: 12 [17920/50000 (40%)]\tLoss: 0.563034, Accuracy: 78.52\n",
      "Train Epoch: 12 [20480/50000 (45%)]\tLoss: 0.469307, Accuracy: 83.20\n",
      "Train Epoch: 12 [23040/50000 (51%)]\tLoss: 0.588324, Accuracy: 79.30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 12 [25600/50000 (57%)]\tLoss: 0.545958, Accuracy: 80.86\n",
      "Train Epoch: 12 [28160/50000 (62%)]\tLoss: 0.557921, Accuracy: 78.91\n",
      "Train Epoch: 12 [30720/50000 (68%)]\tLoss: 0.562885, Accuracy: 80.08\n",
      "Train Epoch: 12 [33280/50000 (74%)]\tLoss: 0.573832, Accuracy: 80.47\n",
      "Train Epoch: 12 [35840/50000 (80%)]\tLoss: 0.565834, Accuracy: 78.91\n",
      "Train Epoch: 12 [38400/50000 (85%)]\tLoss: 0.584349, Accuracy: 80.86\n",
      "Train Epoch: 12 [40960/50000 (91%)]\tLoss: 0.662645, Accuracy: 78.52\n",
      "Train Epoch: 12 [43520/50000 (97%)]\tLoss: 0.601087, Accuracy: 77.34\n",
      "\n",
      "Validation set: Average loss: 0.7937, Accuracy: 3693/5000 (73.00%)\n",
      "\n",
      "the time of this epoch:[45.13844347000122 s]\n",
      "Train Epoch: 13 [0/50000 (0%)]\tLoss: 0.642801, Accuracy: 75.78\n",
      "Train Epoch: 13 [2560/50000 (6%)]\tLoss: 0.491489, Accuracy: 81.25\n",
      "Train Epoch: 13 [5120/50000 (11%)]\tLoss: 0.574425, Accuracy: 80.86\n",
      "Train Epoch: 13 [7680/50000 (17%)]\tLoss: 0.621274, Accuracy: 77.34\n",
      "Train Epoch: 13 [10240/50000 (23%)]\tLoss: 0.533319, Accuracy: 82.42\n",
      "Train Epoch: 13 [12800/50000 (28%)]\tLoss: 0.580936, Accuracy: 78.12\n",
      "Train Epoch: 13 [15360/50000 (34%)]\tLoss: 0.602989, Accuracy: 79.30\n",
      "Train Epoch: 13 [17920/50000 (40%)]\tLoss: 0.538015, Accuracy: 80.47\n",
      "Train Epoch: 13 [20480/50000 (45%)]\tLoss: 0.565666, Accuracy: 83.20\n",
      "Train Epoch: 13 [23040/50000 (51%)]\tLoss: 0.538433, Accuracy: 80.08\n",
      "Train Epoch: 13 [25600/50000 (57%)]\tLoss: 0.584321, Accuracy: 78.52\n",
      "Train Epoch: 13 [28160/50000 (62%)]\tLoss: 0.601698, Accuracy: 81.64\n",
      "Train Epoch: 13 [30720/50000 (68%)]\tLoss: 0.612790, Accuracy: 82.81\n",
      "Train Epoch: 13 [33280/50000 (74%)]\tLoss: 0.615119, Accuracy: 78.91\n",
      "Train Epoch: 13 [35840/50000 (80%)]\tLoss: 0.526592, Accuracy: 81.64\n",
      "Train Epoch: 13 [38400/50000 (85%)]\tLoss: 0.515560, Accuracy: 83.59\n",
      "Train Epoch: 13 [40960/50000 (91%)]\tLoss: 0.707819, Accuracy: 76.56\n",
      "Train Epoch: 13 [43520/50000 (97%)]\tLoss: 0.609903, Accuracy: 79.30\n",
      "\n",
      "Validation set: Average loss: 0.6081, Accuracy: 3970/5000 (79.00%)\n",
      "\n",
      "the time of this epoch:[45.469340562820435 s]\n",
      "Train Epoch: 14 [0/50000 (0%)]\tLoss: 0.531992, Accuracy: 80.47\n",
      "Train Epoch: 14 [2560/50000 (6%)]\tLoss: 0.584443, Accuracy: 81.64\n",
      "Train Epoch: 14 [5120/50000 (11%)]\tLoss: 0.541948, Accuracy: 81.64\n",
      "Train Epoch: 14 [7680/50000 (17%)]\tLoss: 0.507530, Accuracy: 81.25\n",
      "Train Epoch: 14 [10240/50000 (23%)]\tLoss: 0.572173, Accuracy: 78.52\n",
      "Train Epoch: 14 [12800/50000 (28%)]\tLoss: 0.487041, Accuracy: 83.98\n",
      "Train Epoch: 14 [15360/50000 (34%)]\tLoss: 0.466065, Accuracy: 85.16\n",
      "Train Epoch: 14 [17920/50000 (40%)]\tLoss: 0.585286, Accuracy: 79.69\n",
      "Train Epoch: 14 [20480/50000 (45%)]\tLoss: 0.562429, Accuracy: 80.86\n",
      "Train Epoch: 14 [23040/50000 (51%)]\tLoss: 0.524772, Accuracy: 82.03\n",
      "Train Epoch: 14 [25600/50000 (57%)]\tLoss: 0.461829, Accuracy: 82.81\n",
      "Train Epoch: 14 [28160/50000 (62%)]\tLoss: 0.551098, Accuracy: 80.47\n",
      "Train Epoch: 14 [30720/50000 (68%)]\tLoss: 0.577891, Accuracy: 81.64\n",
      "Train Epoch: 14 [33280/50000 (74%)]\tLoss: 0.500008, Accuracy: 80.86\n",
      "Train Epoch: 14 [35840/50000 (80%)]\tLoss: 0.571969, Accuracy: 82.42\n",
      "Train Epoch: 14 [38400/50000 (85%)]\tLoss: 0.556755, Accuracy: 81.25\n",
      "Train Epoch: 14 [40960/50000 (91%)]\tLoss: 0.542305, Accuracy: 83.98\n",
      "Train Epoch: 14 [43520/50000 (97%)]\tLoss: 0.538964, Accuracy: 80.08\n",
      "\n",
      "Validation set: Average loss: 0.6126, Accuracy: 3980/5000 (79.00%)\n",
      "\n",
      "the time of this epoch:[45.6427857875824 s]\n",
      "Train Epoch: 15 [0/50000 (0%)]\tLoss: 0.452394, Accuracy: 83.98\n",
      "Train Epoch: 15 [2560/50000 (6%)]\tLoss: 0.569112, Accuracy: 79.30\n",
      "Train Epoch: 15 [5120/50000 (11%)]\tLoss: 0.636094, Accuracy: 80.86\n",
      "Train Epoch: 15 [7680/50000 (17%)]\tLoss: 0.578781, Accuracy: 81.64\n",
      "Train Epoch: 15 [10240/50000 (23%)]\tLoss: 0.623331, Accuracy: 76.17\n",
      "Train Epoch: 15 [12800/50000 (28%)]\tLoss: 0.468346, Accuracy: 83.59\n",
      "Train Epoch: 15 [15360/50000 (34%)]\tLoss: 0.503794, Accuracy: 80.86\n",
      "Train Epoch: 15 [17920/50000 (40%)]\tLoss: 0.508419, Accuracy: 82.03\n",
      "Train Epoch: 15 [20480/50000 (45%)]\tLoss: 0.494817, Accuracy: 84.38\n",
      "Train Epoch: 15 [23040/50000 (51%)]\tLoss: 0.595943, Accuracy: 79.30\n",
      "Train Epoch: 15 [25600/50000 (57%)]\tLoss: 0.582131, Accuracy: 78.12\n",
      "Train Epoch: 15 [28160/50000 (62%)]\tLoss: 0.487143, Accuracy: 82.42\n",
      "Train Epoch: 15 [30720/50000 (68%)]\tLoss: 0.656751, Accuracy: 78.12\n",
      "Train Epoch: 15 [33280/50000 (74%)]\tLoss: 0.568990, Accuracy: 82.03\n",
      "Train Epoch: 15 [35840/50000 (80%)]\tLoss: 0.564707, Accuracy: 81.64\n",
      "Train Epoch: 15 [38400/50000 (85%)]\tLoss: 0.514646, Accuracy: 82.42\n",
      "Train Epoch: 15 [40960/50000 (91%)]\tLoss: 0.567725, Accuracy: 79.30\n",
      "Train Epoch: 15 [43520/50000 (97%)]\tLoss: 0.477492, Accuracy: 83.59\n",
      "\n",
      "Validation set: Average loss: 0.5651, Accuracy: 4044/5000 (80.00%)\n",
      "\n",
      "the time of this epoch:[45.64967727661133 s]\n",
      "Train Epoch: 16 [0/50000 (0%)]\tLoss: 0.519301, Accuracy: 85.16\n",
      "Train Epoch: 16 [2560/50000 (6%)]\tLoss: 0.475860, Accuracy: 82.42\n",
      "Train Epoch: 16 [5120/50000 (11%)]\tLoss: 0.483279, Accuracy: 82.03\n",
      "Train Epoch: 16 [7680/50000 (17%)]\tLoss: 0.467799, Accuracy: 82.03\n",
      "Train Epoch: 16 [10240/50000 (23%)]\tLoss: 0.518702, Accuracy: 81.25\n",
      "Train Epoch: 16 [12800/50000 (28%)]\tLoss: 0.543581, Accuracy: 81.25\n",
      "Train Epoch: 16 [15360/50000 (34%)]\tLoss: 0.579655, Accuracy: 81.25\n",
      "Train Epoch: 16 [17920/50000 (40%)]\tLoss: 0.553627, Accuracy: 82.03\n",
      "Train Epoch: 16 [20480/50000 (45%)]\tLoss: 0.443329, Accuracy: 83.20\n",
      "Train Epoch: 16 [23040/50000 (51%)]\tLoss: 0.564346, Accuracy: 80.08\n",
      "Train Epoch: 16 [25600/50000 (57%)]\tLoss: 0.584027, Accuracy: 80.86\n",
      "Train Epoch: 16 [28160/50000 (62%)]\tLoss: 0.485441, Accuracy: 85.55\n",
      "Train Epoch: 16 [30720/50000 (68%)]\tLoss: 0.487872, Accuracy: 82.42\n",
      "Train Epoch: 16 [33280/50000 (74%)]\tLoss: 0.519631, Accuracy: 82.81\n",
      "Train Epoch: 16 [35840/50000 (80%)]\tLoss: 0.460591, Accuracy: 84.38\n",
      "Train Epoch: 16 [38400/50000 (85%)]\tLoss: 0.471610, Accuracy: 83.59\n",
      "Train Epoch: 16 [40960/50000 (91%)]\tLoss: 0.456408, Accuracy: 84.38\n",
      "Train Epoch: 16 [43520/50000 (97%)]\tLoss: 0.553805, Accuracy: 80.86\n",
      "\n",
      "Validation set: Average loss: 0.5891, Accuracy: 4021/5000 (80.00%)\n",
      "\n",
      "the time of this epoch:[45.64809703826904 s]\n",
      "Train Epoch: 17 [0/50000 (0%)]\tLoss: 0.542435, Accuracy: 85.16\n",
      "Train Epoch: 17 [2560/50000 (6%)]\tLoss: 0.509896, Accuracy: 82.42\n",
      "Train Epoch: 17 [5120/50000 (11%)]\tLoss: 0.439023, Accuracy: 85.94\n",
      "Train Epoch: 17 [7680/50000 (17%)]\tLoss: 0.490951, Accuracy: 79.30\n",
      "Train Epoch: 17 [10240/50000 (23%)]\tLoss: 0.547840, Accuracy: 81.64\n",
      "Train Epoch: 17 [12800/50000 (28%)]\tLoss: 0.486635, Accuracy: 82.42\n",
      "Train Epoch: 17 [15360/50000 (34%)]\tLoss: 0.402739, Accuracy: 83.98\n",
      "Train Epoch: 17 [17920/50000 (40%)]\tLoss: 0.504857, Accuracy: 81.64\n",
      "Train Epoch: 17 [20480/50000 (45%)]\tLoss: 0.472825, Accuracy: 80.86\n",
      "Train Epoch: 17 [23040/50000 (51%)]\tLoss: 0.462817, Accuracy: 84.38\n",
      "Train Epoch: 17 [25600/50000 (57%)]\tLoss: 0.535185, Accuracy: 81.64\n",
      "Train Epoch: 17 [28160/50000 (62%)]\tLoss: 0.517251, Accuracy: 80.47\n",
      "Train Epoch: 17 [30720/50000 (68%)]\tLoss: 0.566323, Accuracy: 78.12\n",
      "Train Epoch: 17 [33280/50000 (74%)]\tLoss: 0.591291, Accuracy: 76.95\n",
      "Train Epoch: 17 [35840/50000 (80%)]\tLoss: 0.572353, Accuracy: 81.64\n",
      "Train Epoch: 17 [38400/50000 (85%)]\tLoss: 0.495859, Accuracy: 81.64\n",
      "Train Epoch: 17 [40960/50000 (91%)]\tLoss: 0.630718, Accuracy: 76.56\n",
      "Train Epoch: 17 [43520/50000 (97%)]\tLoss: 0.480965, Accuracy: 81.25\n",
      "\n",
      "Validation set: Average loss: 0.6115, Accuracy: 3971/5000 (79.00%)\n",
      "\n",
      "the time of this epoch:[45.61922478675842 s]\n",
      "Train Epoch: 18 [0/50000 (0%)]\tLoss: 0.522519, Accuracy: 82.03\n",
      "Train Epoch: 18 [2560/50000 (6%)]\tLoss: 0.503672, Accuracy: 81.64\n",
      "Train Epoch: 18 [5120/50000 (11%)]\tLoss: 0.383841, Accuracy: 86.33\n",
      "Train Epoch: 18 [7680/50000 (17%)]\tLoss: 0.542960, Accuracy: 83.98\n",
      "Train Epoch: 18 [10240/50000 (23%)]\tLoss: 0.591047, Accuracy: 80.08\n",
      "Train Epoch: 18 [12800/50000 (28%)]\tLoss: 0.486118, Accuracy: 83.59\n",
      "Train Epoch: 18 [15360/50000 (34%)]\tLoss: 0.436351, Accuracy: 83.98\n",
      "Train Epoch: 18 [17920/50000 (40%)]\tLoss: 0.445548, Accuracy: 82.81\n",
      "Train Epoch: 18 [20480/50000 (45%)]\tLoss: 0.466834, Accuracy: 83.98\n",
      "Train Epoch: 18 [23040/50000 (51%)]\tLoss: 0.522172, Accuracy: 81.64\n",
      "Train Epoch: 18 [25600/50000 (57%)]\tLoss: 0.423315, Accuracy: 86.33\n",
      "Train Epoch: 18 [28160/50000 (62%)]\tLoss: 0.409587, Accuracy: 84.38\n",
      "Train Epoch: 18 [30720/50000 (68%)]\tLoss: 0.403201, Accuracy: 86.33\n",
      "Train Epoch: 18 [33280/50000 (74%)]\tLoss: 0.467869, Accuracy: 83.98\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 18 [35840/50000 (80%)]\tLoss: 0.435257, Accuracy: 83.98\n",
      "Train Epoch: 18 [38400/50000 (85%)]\tLoss: 0.562506, Accuracy: 81.64\n",
      "Train Epoch: 18 [40960/50000 (91%)]\tLoss: 0.451398, Accuracy: 86.33\n",
      "Train Epoch: 18 [43520/50000 (97%)]\tLoss: 0.531987, Accuracy: 80.08\n",
      "\n",
      "Validation set: Average loss: 0.6196, Accuracy: 3907/5000 (78.00%)\n",
      "\n",
      "the time of this epoch:[45.64635920524597 s]\n",
      "Train Epoch: 19 [0/50000 (0%)]\tLoss: 0.369795, Accuracy: 85.94\n",
      "Train Epoch: 19 [2560/50000 (6%)]\tLoss: 0.422641, Accuracy: 84.38\n",
      "Train Epoch: 19 [5120/50000 (11%)]\tLoss: 0.453015, Accuracy: 83.98\n",
      "Train Epoch: 19 [7680/50000 (17%)]\tLoss: 0.378072, Accuracy: 85.94\n",
      "Train Epoch: 19 [10240/50000 (23%)]\tLoss: 0.479218, Accuracy: 84.38\n",
      "Train Epoch: 19 [12800/50000 (28%)]\tLoss: 0.446828, Accuracy: 82.42\n",
      "Train Epoch: 19 [15360/50000 (34%)]\tLoss: 0.461119, Accuracy: 85.16\n",
      "Train Epoch: 19 [17920/50000 (40%)]\tLoss: 0.400610, Accuracy: 87.11\n",
      "Train Epoch: 19 [20480/50000 (45%)]\tLoss: 0.383897, Accuracy: 87.50\n",
      "Train Epoch: 19 [23040/50000 (51%)]\tLoss: 0.487120, Accuracy: 82.81\n",
      "Train Epoch: 19 [25600/50000 (57%)]\tLoss: 0.437091, Accuracy: 83.98\n",
      "Train Epoch: 19 [28160/50000 (62%)]\tLoss: 0.520573, Accuracy: 81.64\n",
      "Train Epoch: 19 [30720/50000 (68%)]\tLoss: 0.472925, Accuracy: 82.42\n",
      "Train Epoch: 19 [33280/50000 (74%)]\tLoss: 0.494787, Accuracy: 82.03\n",
      "Train Epoch: 19 [35840/50000 (80%)]\tLoss: 0.558490, Accuracy: 80.47\n",
      "Train Epoch: 19 [38400/50000 (85%)]\tLoss: 0.458902, Accuracy: 81.64\n",
      "Train Epoch: 19 [40960/50000 (91%)]\tLoss: 0.473104, Accuracy: 83.98\n",
      "Train Epoch: 19 [43520/50000 (97%)]\tLoss: 0.531574, Accuracy: 81.64\n",
      "\n",
      "Validation set: Average loss: 0.7974, Accuracy: 3777/5000 (75.00%)\n",
      "\n",
      "the time of this epoch:[45.61947989463806 s]\n",
      "Train Epoch: 20 [0/50000 (0%)]\tLoss: 0.374331, Accuracy: 85.16\n",
      "Train Epoch: 20 [2560/50000 (6%)]\tLoss: 0.360563, Accuracy: 87.89\n",
      "Train Epoch: 20 [5120/50000 (11%)]\tLoss: 0.320878, Accuracy: 89.45\n",
      "Train Epoch: 20 [7680/50000 (17%)]\tLoss: 0.399911, Accuracy: 86.33\n",
      "Train Epoch: 20 [10240/50000 (23%)]\tLoss: 0.456367, Accuracy: 84.38\n",
      "Train Epoch: 20 [12800/50000 (28%)]\tLoss: 0.524351, Accuracy: 83.59\n",
      "Train Epoch: 20 [15360/50000 (34%)]\tLoss: 0.561968, Accuracy: 79.30\n",
      "Train Epoch: 20 [17920/50000 (40%)]\tLoss: 0.434807, Accuracy: 85.94\n",
      "Train Epoch: 20 [20480/50000 (45%)]\tLoss: 0.505653, Accuracy: 82.42\n",
      "Train Epoch: 20 [23040/50000 (51%)]\tLoss: 0.461581, Accuracy: 80.47\n",
      "Train Epoch: 20 [25600/50000 (57%)]\tLoss: 0.476720, Accuracy: 83.59\n",
      "Train Epoch: 20 [28160/50000 (62%)]\tLoss: 0.489480, Accuracy: 82.03\n",
      "Train Epoch: 20 [30720/50000 (68%)]\tLoss: 0.392589, Accuracy: 85.55\n",
      "Train Epoch: 20 [33280/50000 (74%)]\tLoss: 0.468390, Accuracy: 85.16\n",
      "Train Epoch: 20 [35840/50000 (80%)]\tLoss: 0.478645, Accuracy: 82.42\n",
      "Train Epoch: 20 [38400/50000 (85%)]\tLoss: 0.551363, Accuracy: 80.08\n",
      "Train Epoch: 20 [40960/50000 (91%)]\tLoss: 0.475384, Accuracy: 84.38\n",
      "Train Epoch: 20 [43520/50000 (97%)]\tLoss: 0.486273, Accuracy: 80.47\n",
      "\n",
      "Validation set: Average loss: 0.5114, Accuracy: 4147/5000 (82.00%)\n",
      "\n",
      "the time of this epoch:[45.70154166221619 s]\n",
      "Train Epoch: 21 [0/50000 (0%)]\tLoss: 0.337077, Accuracy: 85.55\n",
      "Train Epoch: 21 [2560/50000 (6%)]\tLoss: 0.456871, Accuracy: 84.38\n",
      "Train Epoch: 21 [5120/50000 (11%)]\tLoss: 0.384466, Accuracy: 85.55\n",
      "Train Epoch: 21 [7680/50000 (17%)]\tLoss: 0.426526, Accuracy: 85.94\n",
      "Train Epoch: 21 [10240/50000 (23%)]\tLoss: 0.361839, Accuracy: 88.28\n",
      "Train Epoch: 21 [12800/50000 (28%)]\tLoss: 0.285973, Accuracy: 91.41\n",
      "Train Epoch: 21 [15360/50000 (34%)]\tLoss: 0.369028, Accuracy: 87.50\n",
      "Train Epoch: 21 [17920/50000 (40%)]\tLoss: 0.370362, Accuracy: 88.28\n",
      "Train Epoch: 21 [20480/50000 (45%)]\tLoss: 0.381936, Accuracy: 88.28\n",
      "Train Epoch: 21 [23040/50000 (51%)]\tLoss: 0.399898, Accuracy: 85.55\n",
      "Train Epoch: 21 [25600/50000 (57%)]\tLoss: 0.436110, Accuracy: 86.33\n",
      "Train Epoch: 21 [28160/50000 (62%)]\tLoss: 0.237531, Accuracy: 93.36\n",
      "Train Epoch: 21 [30720/50000 (68%)]\tLoss: 0.441644, Accuracy: 85.55\n",
      "Train Epoch: 21 [33280/50000 (74%)]\tLoss: 0.362645, Accuracy: 84.38\n",
      "Train Epoch: 21 [35840/50000 (80%)]\tLoss: 0.368197, Accuracy: 87.50\n",
      "Train Epoch: 21 [38400/50000 (85%)]\tLoss: 0.440625, Accuracy: 84.77\n",
      "Train Epoch: 21 [40960/50000 (91%)]\tLoss: 0.369940, Accuracy: 86.72\n",
      "Train Epoch: 21 [43520/50000 (97%)]\tLoss: 0.449684, Accuracy: 86.33\n",
      "\n",
      "Validation set: Average loss: 0.4740, Accuracy: 4245/5000 (84.00%)\n",
      "\n",
      "the time of this epoch:[45.6160352230072 s]\n",
      "Train Epoch: 22 [0/50000 (0%)]\tLoss: 0.426529, Accuracy: 85.55\n",
      "Train Epoch: 22 [2560/50000 (6%)]\tLoss: 0.321122, Accuracy: 88.67\n",
      "Train Epoch: 22 [5120/50000 (11%)]\tLoss: 0.374373, Accuracy: 86.33\n",
      "Train Epoch: 22 [7680/50000 (17%)]\tLoss: 0.295425, Accuracy: 89.45\n",
      "Train Epoch: 22 [10240/50000 (23%)]\tLoss: 0.401714, Accuracy: 87.11\n",
      "Train Epoch: 22 [12800/50000 (28%)]\tLoss: 0.352343, Accuracy: 87.11\n",
      "Train Epoch: 22 [15360/50000 (34%)]\tLoss: 0.502391, Accuracy: 79.30\n",
      "Train Epoch: 22 [17920/50000 (40%)]\tLoss: 0.488217, Accuracy: 83.98\n",
      "Train Epoch: 22 [20480/50000 (45%)]\tLoss: 0.313931, Accuracy: 89.45\n",
      "Train Epoch: 22 [23040/50000 (51%)]\tLoss: 0.448672, Accuracy: 85.55\n",
      "Train Epoch: 22 [25600/50000 (57%)]\tLoss: 0.477486, Accuracy: 84.38\n",
      "Train Epoch: 22 [28160/50000 (62%)]\tLoss: 0.299882, Accuracy: 90.23\n",
      "Train Epoch: 22 [30720/50000 (68%)]\tLoss: 0.379868, Accuracy: 84.77\n",
      "Train Epoch: 22 [33280/50000 (74%)]\tLoss: 0.371414, Accuracy: 87.50\n",
      "Train Epoch: 22 [35840/50000 (80%)]\tLoss: 0.548627, Accuracy: 82.03\n",
      "Train Epoch: 22 [38400/50000 (85%)]\tLoss: 0.342880, Accuracy: 87.50\n",
      "Train Epoch: 22 [40960/50000 (91%)]\tLoss: 0.438913, Accuracy: 84.77\n",
      "Train Epoch: 22 [43520/50000 (97%)]\tLoss: 0.365705, Accuracy: 87.11\n",
      "\n",
      "Validation set: Average loss: 0.5098, Accuracy: 4158/5000 (83.00%)\n",
      "\n",
      "the time of this epoch:[45.66106176376343 s]\n",
      "Train Epoch: 23 [0/50000 (0%)]\tLoss: 0.348902, Accuracy: 86.72\n",
      "Train Epoch: 23 [2560/50000 (6%)]\tLoss: 0.344047, Accuracy: 87.11\n",
      "Train Epoch: 23 [5120/50000 (11%)]\tLoss: 0.301243, Accuracy: 88.67\n",
      "Train Epoch: 23 [7680/50000 (17%)]\tLoss: 0.344460, Accuracy: 86.72\n",
      "Train Epoch: 23 [10240/50000 (23%)]\tLoss: 0.409550, Accuracy: 85.55\n",
      "Train Epoch: 23 [12800/50000 (28%)]\tLoss: 0.292376, Accuracy: 87.89\n",
      "Train Epoch: 23 [15360/50000 (34%)]\tLoss: 0.381073, Accuracy: 85.55\n",
      "Train Epoch: 23 [17920/50000 (40%)]\tLoss: 0.361941, Accuracy: 88.28\n",
      "Train Epoch: 23 [20480/50000 (45%)]\tLoss: 0.457104, Accuracy: 84.77\n",
      "Train Epoch: 23 [23040/50000 (51%)]\tLoss: 0.288655, Accuracy: 89.45\n",
      "Train Epoch: 23 [25600/50000 (57%)]\tLoss: 0.301375, Accuracy: 90.62\n",
      "Train Epoch: 23 [28160/50000 (62%)]\tLoss: 0.451707, Accuracy: 83.20\n",
      "Train Epoch: 23 [30720/50000 (68%)]\tLoss: 0.326264, Accuracy: 88.28\n",
      "Train Epoch: 23 [33280/50000 (74%)]\tLoss: 0.372078, Accuracy: 88.28\n",
      "Train Epoch: 23 [35840/50000 (80%)]\tLoss: 0.328576, Accuracy: 88.28\n",
      "Train Epoch: 23 [38400/50000 (85%)]\tLoss: 0.270795, Accuracy: 92.19\n",
      "Train Epoch: 23 [40960/50000 (91%)]\tLoss: 0.310832, Accuracy: 88.28\n",
      "Train Epoch: 23 [43520/50000 (97%)]\tLoss: 0.367098, Accuracy: 88.28\n",
      "\n",
      "Validation set: Average loss: 0.5056, Accuracy: 4167/5000 (83.00%)\n",
      "\n",
      "the time of this epoch:[45.63466477394104 s]\n",
      "Train Epoch: 24 [0/50000 (0%)]\tLoss: 0.347934, Accuracy: 89.45\n",
      "Train Epoch: 24 [2560/50000 (6%)]\tLoss: 0.359330, Accuracy: 87.89\n",
      "Train Epoch: 24 [5120/50000 (11%)]\tLoss: 0.331798, Accuracy: 89.84\n",
      "Train Epoch: 24 [7680/50000 (17%)]\tLoss: 0.284257, Accuracy: 89.84\n",
      "Train Epoch: 24 [10240/50000 (23%)]\tLoss: 0.442609, Accuracy: 87.50\n",
      "Train Epoch: 24 [12800/50000 (28%)]\tLoss: 0.302670, Accuracy: 89.45\n",
      "Train Epoch: 24 [15360/50000 (34%)]\tLoss: 0.398307, Accuracy: 87.89\n",
      "Train Epoch: 24 [17920/50000 (40%)]\tLoss: 0.381640, Accuracy: 86.72\n",
      "Train Epoch: 24 [20480/50000 (45%)]\tLoss: 0.481579, Accuracy: 83.20\n",
      "Train Epoch: 24 [23040/50000 (51%)]\tLoss: 0.448514, Accuracy: 84.38\n",
      "Train Epoch: 24 [25600/50000 (57%)]\tLoss: 0.345171, Accuracy: 87.50\n",
      "Train Epoch: 24 [28160/50000 (62%)]\tLoss: 0.307425, Accuracy: 87.11\n",
      "Train Epoch: 24 [30720/50000 (68%)]\tLoss: 0.414094, Accuracy: 82.81\n",
      "Train Epoch: 24 [33280/50000 (74%)]\tLoss: 0.303037, Accuracy: 89.45\n",
      "Train Epoch: 24 [35840/50000 (80%)]\tLoss: 0.358834, Accuracy: 86.33\n",
      "Train Epoch: 24 [38400/50000 (85%)]\tLoss: 0.317464, Accuracy: 90.23\n",
      "Train Epoch: 24 [40960/50000 (91%)]\tLoss: 0.457532, Accuracy: 85.94\n",
      "Train Epoch: 24 [43520/50000 (97%)]\tLoss: 0.330755, Accuracy: 90.23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Validation set: Average loss: 0.4787, Accuracy: 4249/5000 (84.00%)\n",
      "\n",
      "the time of this epoch:[45.619932889938354 s]\n",
      "Train Epoch: 25 [0/50000 (0%)]\tLoss: 0.231624, Accuracy: 91.41\n",
      "Train Epoch: 25 [2560/50000 (6%)]\tLoss: 0.362761, Accuracy: 89.84\n",
      "Train Epoch: 25 [5120/50000 (11%)]\tLoss: 0.258635, Accuracy: 91.41\n",
      "Train Epoch: 25 [7680/50000 (17%)]\tLoss: 0.373098, Accuracy: 85.94\n",
      "Train Epoch: 25 [10240/50000 (23%)]\tLoss: 0.315651, Accuracy: 89.84\n",
      "Train Epoch: 25 [12800/50000 (28%)]\tLoss: 0.236082, Accuracy: 91.80\n",
      "Train Epoch: 25 [15360/50000 (34%)]\tLoss: 0.355143, Accuracy: 89.06\n",
      "Train Epoch: 25 [17920/50000 (40%)]\tLoss: 0.208655, Accuracy: 91.80\n",
      "Train Epoch: 25 [20480/50000 (45%)]\tLoss: 0.280318, Accuracy: 90.23\n",
      "Train Epoch: 25 [23040/50000 (51%)]\tLoss: 0.352141, Accuracy: 89.06\n",
      "Train Epoch: 25 [25600/50000 (57%)]\tLoss: 0.398950, Accuracy: 86.33\n",
      "Train Epoch: 25 [28160/50000 (62%)]\tLoss: 0.304791, Accuracy: 90.23\n",
      "Train Epoch: 25 [30720/50000 (68%)]\tLoss: 0.369378, Accuracy: 85.94\n",
      "Train Epoch: 25 [33280/50000 (74%)]\tLoss: 0.352082, Accuracy: 85.94\n",
      "Train Epoch: 25 [35840/50000 (80%)]\tLoss: 0.372787, Accuracy: 85.16\n",
      "Train Epoch: 25 [38400/50000 (85%)]\tLoss: 0.372095, Accuracy: 89.06\n",
      "Train Epoch: 25 [40960/50000 (91%)]\tLoss: 0.269732, Accuracy: 90.23\n",
      "Train Epoch: 25 [43520/50000 (97%)]\tLoss: 0.394019, Accuracy: 87.50\n",
      "\n",
      "Validation set: Average loss: 0.4499, Accuracy: 4265/5000 (85.00%)\n",
      "\n",
      "the time of this epoch:[45.56515145301819 s]\n",
      "Train Epoch: 26 [0/50000 (0%)]\tLoss: 0.286515, Accuracy: 89.45\n",
      "Train Epoch: 26 [2560/50000 (6%)]\tLoss: 0.258311, Accuracy: 90.62\n",
      "Train Epoch: 26 [5120/50000 (11%)]\tLoss: 0.310435, Accuracy: 88.67\n",
      "Train Epoch: 26 [7680/50000 (17%)]\tLoss: 0.408220, Accuracy: 85.94\n",
      "Train Epoch: 26 [10240/50000 (23%)]\tLoss: 0.208633, Accuracy: 93.36\n",
      "Train Epoch: 26 [12800/50000 (28%)]\tLoss: 0.325035, Accuracy: 89.06\n",
      "Train Epoch: 26 [15360/50000 (34%)]\tLoss: 0.347650, Accuracy: 87.89\n",
      "Train Epoch: 26 [17920/50000 (40%)]\tLoss: 0.364210, Accuracy: 86.72\n",
      "Train Epoch: 26 [20480/50000 (45%)]\tLoss: 0.292624, Accuracy: 90.23\n",
      "Train Epoch: 26 [23040/50000 (51%)]\tLoss: 0.271857, Accuracy: 89.06\n",
      "Train Epoch: 26 [25600/50000 (57%)]\tLoss: 0.328786, Accuracy: 89.45\n",
      "Train Epoch: 26 [28160/50000 (62%)]\tLoss: 0.289591, Accuracy: 90.62\n",
      "Train Epoch: 26 [30720/50000 (68%)]\tLoss: 0.317636, Accuracy: 89.06\n",
      "Train Epoch: 26 [33280/50000 (74%)]\tLoss: 0.429133, Accuracy: 86.33\n",
      "Train Epoch: 26 [35840/50000 (80%)]\tLoss: 0.385510, Accuracy: 87.50\n",
      "Train Epoch: 26 [38400/50000 (85%)]\tLoss: 0.384539, Accuracy: 86.72\n",
      "Train Epoch: 26 [40960/50000 (91%)]\tLoss: 0.231507, Accuracy: 90.62\n",
      "Train Epoch: 26 [43520/50000 (97%)]\tLoss: 0.377464, Accuracy: 86.33\n",
      "\n",
      "Validation set: Average loss: 0.5020, Accuracy: 4210/5000 (84.00%)\n",
      "\n",
      "the time of this epoch:[45.44961452484131 s]\n",
      "Train Epoch: 27 [0/50000 (0%)]\tLoss: 0.215298, Accuracy: 92.97\n",
      "Train Epoch: 27 [2560/50000 (6%)]\tLoss: 0.304370, Accuracy: 89.45\n",
      "Train Epoch: 27 [5120/50000 (11%)]\tLoss: 0.293848, Accuracy: 89.06\n",
      "Train Epoch: 27 [7680/50000 (17%)]\tLoss: 0.282063, Accuracy: 89.84\n",
      "Train Epoch: 27 [10240/50000 (23%)]\tLoss: 0.248694, Accuracy: 89.84\n",
      "Train Epoch: 27 [12800/50000 (28%)]\tLoss: 0.315912, Accuracy: 90.62\n",
      "Train Epoch: 27 [15360/50000 (34%)]\tLoss: 0.277374, Accuracy: 87.89\n",
      "Train Epoch: 27 [17920/50000 (40%)]\tLoss: 0.295977, Accuracy: 89.84\n",
      "Train Epoch: 27 [20480/50000 (45%)]\tLoss: 0.371237, Accuracy: 87.89\n",
      "Train Epoch: 27 [23040/50000 (51%)]\tLoss: 0.378639, Accuracy: 85.94\n",
      "Train Epoch: 27 [25600/50000 (57%)]\tLoss: 0.384599, Accuracy: 84.77\n",
      "Train Epoch: 27 [28160/50000 (62%)]\tLoss: 0.301942, Accuracy: 91.80\n",
      "Train Epoch: 27 [30720/50000 (68%)]\tLoss: 0.259683, Accuracy: 92.19\n",
      "Train Epoch: 27 [33280/50000 (74%)]\tLoss: 0.332540, Accuracy: 87.11\n",
      "Train Epoch: 27 [35840/50000 (80%)]\tLoss: 0.316695, Accuracy: 86.72\n",
      "Train Epoch: 27 [38400/50000 (85%)]\tLoss: 0.281558, Accuracy: 91.41\n",
      "Train Epoch: 27 [40960/50000 (91%)]\tLoss: 0.300397, Accuracy: 88.67\n",
      "Train Epoch: 27 [43520/50000 (97%)]\tLoss: 0.358155, Accuracy: 87.89\n",
      "\n",
      "Validation set: Average loss: 0.4775, Accuracy: 4235/5000 (84.00%)\n",
      "\n",
      "the time of this epoch:[45.08314037322998 s]\n",
      "Train Epoch: 28 [0/50000 (0%)]\tLoss: 0.307235, Accuracy: 88.28\n",
      "Train Epoch: 28 [2560/50000 (6%)]\tLoss: 0.277648, Accuracy: 92.19\n",
      "Train Epoch: 28 [5120/50000 (11%)]\tLoss: 0.326727, Accuracy: 90.23\n",
      "Train Epoch: 28 [7680/50000 (17%)]\tLoss: 0.280559, Accuracy: 90.62\n",
      "Train Epoch: 28 [10240/50000 (23%)]\tLoss: 0.303760, Accuracy: 91.02\n",
      "Train Epoch: 28 [12800/50000 (28%)]\tLoss: 0.262466, Accuracy: 90.23\n",
      "Train Epoch: 28 [15360/50000 (34%)]\tLoss: 0.312166, Accuracy: 89.45\n",
      "Train Epoch: 28 [17920/50000 (40%)]\tLoss: 0.224823, Accuracy: 90.62\n",
      "Train Epoch: 28 [20480/50000 (45%)]\tLoss: 0.328333, Accuracy: 87.11\n",
      "Train Epoch: 28 [23040/50000 (51%)]\tLoss: 0.250631, Accuracy: 91.80\n",
      "Train Epoch: 28 [25600/50000 (57%)]\tLoss: 0.309835, Accuracy: 89.84\n",
      "Train Epoch: 28 [28160/50000 (62%)]\tLoss: 0.337038, Accuracy: 87.89\n",
      "Train Epoch: 28 [30720/50000 (68%)]\tLoss: 0.302061, Accuracy: 89.45\n",
      "Train Epoch: 28 [33280/50000 (74%)]\tLoss: 0.295286, Accuracy: 89.45\n",
      "Train Epoch: 28 [35840/50000 (80%)]\tLoss: 0.314364, Accuracy: 89.06\n",
      "Train Epoch: 28 [38400/50000 (85%)]\tLoss: 0.324559, Accuracy: 91.02\n",
      "Train Epoch: 28 [40960/50000 (91%)]\tLoss: 0.290112, Accuracy: 89.84\n",
      "Train Epoch: 28 [43520/50000 (97%)]\tLoss: 0.418147, Accuracy: 85.55\n",
      "\n",
      "Validation set: Average loss: 0.4676, Accuracy: 4230/5000 (84.00%)\n",
      "\n",
      "the time of this epoch:[45.07322716712952 s]\n",
      "Train Epoch: 29 [0/50000 (0%)]\tLoss: 0.249243, Accuracy: 91.02\n",
      "Train Epoch: 29 [2560/50000 (6%)]\tLoss: 0.304194, Accuracy: 89.84\n",
      "Train Epoch: 29 [5120/50000 (11%)]\tLoss: 0.329098, Accuracy: 87.11\n",
      "Train Epoch: 29 [7680/50000 (17%)]\tLoss: 0.312368, Accuracy: 89.06\n",
      "Train Epoch: 29 [10240/50000 (23%)]\tLoss: 0.280326, Accuracy: 91.02\n",
      "Train Epoch: 29 [12800/50000 (28%)]\tLoss: 0.202140, Accuracy: 91.80\n",
      "Train Epoch: 29 [15360/50000 (34%)]\tLoss: 0.268177, Accuracy: 90.23\n",
      "Train Epoch: 29 [17920/50000 (40%)]\tLoss: 0.280375, Accuracy: 91.02\n",
      "Train Epoch: 29 [20480/50000 (45%)]\tLoss: 0.303207, Accuracy: 89.84\n",
      "Train Epoch: 29 [23040/50000 (51%)]\tLoss: 0.308194, Accuracy: 88.67\n",
      "Train Epoch: 29 [25600/50000 (57%)]\tLoss: 0.275763, Accuracy: 90.23\n",
      "Train Epoch: 29 [28160/50000 (62%)]\tLoss: 0.296548, Accuracy: 88.28\n",
      "Train Epoch: 29 [30720/50000 (68%)]\tLoss: 0.364663, Accuracy: 88.67\n",
      "Train Epoch: 29 [33280/50000 (74%)]\tLoss: 0.333396, Accuracy: 88.67\n",
      "Train Epoch: 29 [35840/50000 (80%)]\tLoss: 0.273858, Accuracy: 89.06\n",
      "Train Epoch: 29 [38400/50000 (85%)]\tLoss: 0.302447, Accuracy: 90.23\n",
      "Train Epoch: 29 [40960/50000 (91%)]\tLoss: 0.353836, Accuracy: 87.50\n",
      "Train Epoch: 29 [43520/50000 (97%)]\tLoss: 0.362942, Accuracy: 87.50\n",
      "\n",
      "Validation set: Average loss: 0.5212, Accuracy: 4200/5000 (84.00%)\n",
      "\n",
      "the time of this epoch:[45.04032588005066 s]\n",
      "Train Epoch: 30 [0/50000 (0%)]\tLoss: 0.281258, Accuracy: 90.62\n",
      "Train Epoch: 30 [2560/50000 (6%)]\tLoss: 0.261040, Accuracy: 89.84\n",
      "Train Epoch: 30 [5120/50000 (11%)]\tLoss: 0.213682, Accuracy: 92.19\n",
      "Train Epoch: 30 [7680/50000 (17%)]\tLoss: 0.323423, Accuracy: 89.45\n",
      "Train Epoch: 30 [10240/50000 (23%)]\tLoss: 0.268140, Accuracy: 90.23\n",
      "Train Epoch: 30 [12800/50000 (28%)]\tLoss: 0.295669, Accuracy: 91.41\n",
      "Train Epoch: 30 [15360/50000 (34%)]\tLoss: 0.281239, Accuracy: 90.23\n",
      "Train Epoch: 30 [17920/50000 (40%)]\tLoss: 0.252351, Accuracy: 91.02\n",
      "Train Epoch: 30 [20480/50000 (45%)]\tLoss: 0.332053, Accuracy: 89.06\n",
      "Train Epoch: 30 [23040/50000 (51%)]\tLoss: 0.214457, Accuracy: 92.97\n",
      "Train Epoch: 30 [25600/50000 (57%)]\tLoss: 0.320653, Accuracy: 90.23\n",
      "Train Epoch: 30 [28160/50000 (62%)]\tLoss: 0.335825, Accuracy: 86.72\n",
      "Train Epoch: 30 [30720/50000 (68%)]\tLoss: 0.222470, Accuracy: 92.97\n",
      "Train Epoch: 30 [33280/50000 (74%)]\tLoss: 0.319261, Accuracy: 87.89\n",
      "Train Epoch: 30 [35840/50000 (80%)]\tLoss: 0.331649, Accuracy: 89.06\n",
      "Train Epoch: 30 [38400/50000 (85%)]\tLoss: 0.295179, Accuracy: 89.06\n",
      "Train Epoch: 30 [40960/50000 (91%)]\tLoss: 0.292835, Accuracy: 90.62\n",
      "Train Epoch: 30 [43520/50000 (97%)]\tLoss: 0.325246, Accuracy: 90.23\n",
      "\n",
      "Validation set: Average loss: 0.4732, Accuracy: 4243/5000 (84.00%)\n",
      "\n",
      "the time of this epoch:[45.15553617477417 s]\n",
      "Train Epoch: 31 [0/50000 (0%)]\tLoss: 0.265771, Accuracy: 92.19\n",
      "Train Epoch: 31 [2560/50000 (6%)]\tLoss: 0.262800, Accuracy: 91.41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 31 [5120/50000 (11%)]\tLoss: 0.200296, Accuracy: 92.97\n",
      "Train Epoch: 31 [7680/50000 (17%)]\tLoss: 0.259731, Accuracy: 89.84\n",
      "Train Epoch: 31 [10240/50000 (23%)]\tLoss: 0.325326, Accuracy: 89.45\n",
      "Train Epoch: 31 [12800/50000 (28%)]\tLoss: 0.187430, Accuracy: 94.53\n",
      "Train Epoch: 31 [15360/50000 (34%)]\tLoss: 0.244801, Accuracy: 90.23\n",
      "Train Epoch: 31 [17920/50000 (40%)]\tLoss: 0.214703, Accuracy: 91.02\n",
      "Train Epoch: 31 [20480/50000 (45%)]\tLoss: 0.194716, Accuracy: 94.14\n",
      "Train Epoch: 31 [23040/50000 (51%)]\tLoss: 0.180351, Accuracy: 93.36\n",
      "Train Epoch: 31 [25600/50000 (57%)]\tLoss: 0.261332, Accuracy: 89.84\n",
      "Train Epoch: 31 [28160/50000 (62%)]\tLoss: 0.185743, Accuracy: 94.53\n",
      "Train Epoch: 31 [30720/50000 (68%)]\tLoss: 0.221609, Accuracy: 91.41\n",
      "Train Epoch: 31 [33280/50000 (74%)]\tLoss: 0.213749, Accuracy: 91.80\n",
      "Train Epoch: 31 [35840/50000 (80%)]\tLoss: 0.240427, Accuracy: 91.41\n",
      "Train Epoch: 31 [38400/50000 (85%)]\tLoss: 0.245609, Accuracy: 91.80\n",
      "Train Epoch: 31 [40960/50000 (91%)]\tLoss: 0.205142, Accuracy: 94.53\n",
      "Train Epoch: 31 [43520/50000 (97%)]\tLoss: 0.281710, Accuracy: 89.84\n",
      "\n",
      "Validation set: Average loss: 0.4144, Accuracy: 4360/5000 (87.00%)\n",
      "\n",
      "the time of this epoch:[45.58424949645996 s]\n",
      "Train Epoch: 32 [0/50000 (0%)]\tLoss: 0.270801, Accuracy: 90.23\n",
      "Train Epoch: 32 [2560/50000 (6%)]\tLoss: 0.217135, Accuracy: 92.58\n",
      "Train Epoch: 32 [5120/50000 (11%)]\tLoss: 0.197183, Accuracy: 92.58\n",
      "Train Epoch: 32 [7680/50000 (17%)]\tLoss: 0.275761, Accuracy: 91.02\n",
      "Train Epoch: 32 [10240/50000 (23%)]\tLoss: 0.228092, Accuracy: 93.36\n",
      "Train Epoch: 32 [12800/50000 (28%)]\tLoss: 0.168692, Accuracy: 93.36\n",
      "Train Epoch: 32 [15360/50000 (34%)]\tLoss: 0.150817, Accuracy: 96.09\n",
      "Train Epoch: 32 [17920/50000 (40%)]\tLoss: 0.208992, Accuracy: 91.80\n",
      "Train Epoch: 32 [20480/50000 (45%)]\tLoss: 0.267958, Accuracy: 91.41\n",
      "Train Epoch: 32 [23040/50000 (51%)]\tLoss: 0.177213, Accuracy: 94.14\n",
      "Train Epoch: 32 [25600/50000 (57%)]\tLoss: 0.290437, Accuracy: 90.23\n",
      "Train Epoch: 32 [28160/50000 (62%)]\tLoss: 0.218367, Accuracy: 91.41\n",
      "Train Epoch: 32 [30720/50000 (68%)]\tLoss: 0.272131, Accuracy: 90.62\n",
      "Train Epoch: 32 [33280/50000 (74%)]\tLoss: 0.213988, Accuracy: 92.58\n",
      "Train Epoch: 32 [35840/50000 (80%)]\tLoss: 0.356406, Accuracy: 87.11\n",
      "Train Epoch: 32 [38400/50000 (85%)]\tLoss: 0.288617, Accuracy: 91.80\n",
      "Train Epoch: 32 [40960/50000 (91%)]\tLoss: 0.245544, Accuracy: 92.97\n",
      "Train Epoch: 32 [43520/50000 (97%)]\tLoss: 0.279884, Accuracy: 90.23\n",
      "\n",
      "Validation set: Average loss: 0.4212, Accuracy: 4359/5000 (87.00%)\n",
      "\n",
      "the time of this epoch:[45.550692558288574 s]\n",
      "Train Epoch: 33 [0/50000 (0%)]\tLoss: 0.201486, Accuracy: 92.19\n",
      "Train Epoch: 33 [2560/50000 (6%)]\tLoss: 0.266828, Accuracy: 91.02\n",
      "Train Epoch: 33 [5120/50000 (11%)]\tLoss: 0.239060, Accuracy: 92.58\n",
      "Train Epoch: 33 [7680/50000 (17%)]\tLoss: 0.170466, Accuracy: 94.53\n",
      "Train Epoch: 33 [10240/50000 (23%)]\tLoss: 0.220699, Accuracy: 92.58\n",
      "Train Epoch: 33 [12800/50000 (28%)]\tLoss: 0.202649, Accuracy: 94.14\n",
      "Train Epoch: 33 [15360/50000 (34%)]\tLoss: 0.251099, Accuracy: 92.97\n",
      "Train Epoch: 33 [17920/50000 (40%)]\tLoss: 0.153564, Accuracy: 94.53\n",
      "Train Epoch: 33 [20480/50000 (45%)]\tLoss: 0.208257, Accuracy: 91.80\n",
      "Train Epoch: 33 [23040/50000 (51%)]\tLoss: 0.146825, Accuracy: 94.53\n",
      "Train Epoch: 33 [25600/50000 (57%)]\tLoss: 0.185741, Accuracy: 93.36\n",
      "Train Epoch: 33 [28160/50000 (62%)]\tLoss: 0.169990, Accuracy: 94.14\n",
      "Train Epoch: 33 [30720/50000 (68%)]\tLoss: 0.233750, Accuracy: 91.41\n",
      "Train Epoch: 33 [33280/50000 (74%)]\tLoss: 0.195221, Accuracy: 93.36\n",
      "Train Epoch: 33 [35840/50000 (80%)]\tLoss: 0.206053, Accuracy: 91.80\n",
      "Train Epoch: 33 [38400/50000 (85%)]\tLoss: 0.227404, Accuracy: 92.58\n",
      "Train Epoch: 33 [40960/50000 (91%)]\tLoss: 0.285405, Accuracy: 88.67\n",
      "Train Epoch: 33 [43520/50000 (97%)]\tLoss: 0.128687, Accuracy: 96.09\n",
      "\n",
      "Validation set: Average loss: 0.4345, Accuracy: 4323/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.58608913421631 s]\n",
      "Train Epoch: 34 [0/50000 (0%)]\tLoss: 0.195399, Accuracy: 94.14\n",
      "Train Epoch: 34 [2560/50000 (6%)]\tLoss: 0.178640, Accuracy: 94.14\n",
      "Train Epoch: 34 [5120/50000 (11%)]\tLoss: 0.193448, Accuracy: 93.75\n",
      "Train Epoch: 34 [7680/50000 (17%)]\tLoss: 0.260489, Accuracy: 91.02\n",
      "Train Epoch: 34 [10240/50000 (23%)]\tLoss: 0.271650, Accuracy: 90.23\n",
      "Train Epoch: 34 [12800/50000 (28%)]\tLoss: 0.171175, Accuracy: 93.36\n",
      "Train Epoch: 34 [15360/50000 (34%)]\tLoss: 0.232796, Accuracy: 92.19\n",
      "Train Epoch: 34 [17920/50000 (40%)]\tLoss: 0.214303, Accuracy: 92.58\n",
      "Train Epoch: 34 [20480/50000 (45%)]\tLoss: 0.146070, Accuracy: 94.92\n",
      "Train Epoch: 34 [23040/50000 (51%)]\tLoss: 0.188179, Accuracy: 94.92\n",
      "Train Epoch: 34 [25600/50000 (57%)]\tLoss: 0.336992, Accuracy: 88.67\n",
      "Train Epoch: 34 [28160/50000 (62%)]\tLoss: 0.264238, Accuracy: 91.80\n",
      "Train Epoch: 34 [30720/50000 (68%)]\tLoss: 0.131973, Accuracy: 94.92\n",
      "Train Epoch: 34 [33280/50000 (74%)]\tLoss: 0.121116, Accuracy: 96.48\n",
      "Train Epoch: 34 [35840/50000 (80%)]\tLoss: 0.193556, Accuracy: 92.97\n",
      "Train Epoch: 34 [38400/50000 (85%)]\tLoss: 0.119482, Accuracy: 96.48\n",
      "Train Epoch: 34 [40960/50000 (91%)]\tLoss: 0.187438, Accuracy: 95.31\n",
      "Train Epoch: 34 [43520/50000 (97%)]\tLoss: 0.183595, Accuracy: 93.36\n",
      "\n",
      "Validation set: Average loss: 0.4401, Accuracy: 4347/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.65585112571716 s]\n",
      "Train Epoch: 35 [0/50000 (0%)]\tLoss: 0.173481, Accuracy: 94.14\n",
      "Train Epoch: 35 [2560/50000 (6%)]\tLoss: 0.162310, Accuracy: 94.53\n",
      "Train Epoch: 35 [5120/50000 (11%)]\tLoss: 0.191946, Accuracy: 92.58\n",
      "Train Epoch: 35 [7680/50000 (17%)]\tLoss: 0.179992, Accuracy: 94.14\n",
      "Train Epoch: 35 [10240/50000 (23%)]\tLoss: 0.203921, Accuracy: 92.58\n",
      "Train Epoch: 35 [12800/50000 (28%)]\tLoss: 0.215836, Accuracy: 92.97\n",
      "Train Epoch: 35 [15360/50000 (34%)]\tLoss: 0.188703, Accuracy: 93.36\n",
      "Train Epoch: 35 [17920/50000 (40%)]\tLoss: 0.232766, Accuracy: 92.97\n",
      "Train Epoch: 35 [20480/50000 (45%)]\tLoss: 0.198910, Accuracy: 93.36\n",
      "Train Epoch: 35 [23040/50000 (51%)]\tLoss: 0.117595, Accuracy: 96.48\n",
      "Train Epoch: 35 [25600/50000 (57%)]\tLoss: 0.260015, Accuracy: 90.62\n",
      "Train Epoch: 35 [28160/50000 (62%)]\tLoss: 0.172382, Accuracy: 94.92\n",
      "Train Epoch: 35 [30720/50000 (68%)]\tLoss: 0.169752, Accuracy: 94.14\n",
      "Train Epoch: 35 [33280/50000 (74%)]\tLoss: 0.180364, Accuracy: 94.92\n",
      "Train Epoch: 35 [35840/50000 (80%)]\tLoss: 0.176287, Accuracy: 94.92\n",
      "Train Epoch: 35 [38400/50000 (85%)]\tLoss: 0.257739, Accuracy: 91.02\n",
      "Train Epoch: 35 [40960/50000 (91%)]\tLoss: 0.291930, Accuracy: 89.45\n",
      "Train Epoch: 35 [43520/50000 (97%)]\tLoss: 0.242690, Accuracy: 91.80\n",
      "\n",
      "Validation set: Average loss: 0.4374, Accuracy: 4356/5000 (87.00%)\n",
      "\n",
      "the time of this epoch:[45.542335510253906 s]\n",
      "Train Epoch: 36 [0/50000 (0%)]\tLoss: 0.199717, Accuracy: 91.41\n",
      "Train Epoch: 36 [2560/50000 (6%)]\tLoss: 0.127777, Accuracy: 95.70\n",
      "Train Epoch: 36 [5120/50000 (11%)]\tLoss: 0.197513, Accuracy: 92.58\n",
      "Train Epoch: 36 [7680/50000 (17%)]\tLoss: 0.241914, Accuracy: 92.19\n",
      "Train Epoch: 36 [10240/50000 (23%)]\tLoss: 0.176955, Accuracy: 92.97\n",
      "Train Epoch: 36 [12800/50000 (28%)]\tLoss: 0.141308, Accuracy: 94.92\n",
      "Train Epoch: 36 [15360/50000 (34%)]\tLoss: 0.221718, Accuracy: 92.97\n",
      "Train Epoch: 36 [17920/50000 (40%)]\tLoss: 0.148588, Accuracy: 95.31\n",
      "Train Epoch: 36 [20480/50000 (45%)]\tLoss: 0.168432, Accuracy: 93.36\n",
      "Train Epoch: 36 [23040/50000 (51%)]\tLoss: 0.192471, Accuracy: 92.97\n",
      "Train Epoch: 36 [25600/50000 (57%)]\tLoss: 0.180665, Accuracy: 92.97\n",
      "Train Epoch: 36 [28160/50000 (62%)]\tLoss: 0.241041, Accuracy: 92.19\n",
      "Train Epoch: 36 [30720/50000 (68%)]\tLoss: 0.200317, Accuracy: 91.80\n",
      "Train Epoch: 36 [33280/50000 (74%)]\tLoss: 0.203264, Accuracy: 92.19\n",
      "Train Epoch: 36 [35840/50000 (80%)]\tLoss: 0.178367, Accuracy: 92.97\n",
      "Train Epoch: 36 [38400/50000 (85%)]\tLoss: 0.230679, Accuracy: 91.80\n",
      "Train Epoch: 36 [40960/50000 (91%)]\tLoss: 0.180948, Accuracy: 94.14\n",
      "Train Epoch: 36 [43520/50000 (97%)]\tLoss: 0.204056, Accuracy: 94.92\n",
      "\n",
      "Validation set: Average loss: 0.4669, Accuracy: 4319/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.67924952507019 s]\n",
      "Train Epoch: 37 [0/50000 (0%)]\tLoss: 0.152142, Accuracy: 96.09\n",
      "Train Epoch: 37 [2560/50000 (6%)]\tLoss: 0.201661, Accuracy: 91.41\n",
      "Train Epoch: 37 [5120/50000 (11%)]\tLoss: 0.186195, Accuracy: 93.36\n",
      "Train Epoch: 37 [7680/50000 (17%)]\tLoss: 0.233872, Accuracy: 92.58\n",
      "Train Epoch: 37 [10240/50000 (23%)]\tLoss: 0.212995, Accuracy: 92.58\n",
      "Train Epoch: 37 [12800/50000 (28%)]\tLoss: 0.224819, Accuracy: 92.97\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 37 [15360/50000 (34%)]\tLoss: 0.213533, Accuracy: 91.80\n",
      "Train Epoch: 37 [17920/50000 (40%)]\tLoss: 0.210548, Accuracy: 92.19\n",
      "Train Epoch: 37 [20480/50000 (45%)]\tLoss: 0.253925, Accuracy: 91.80\n",
      "Train Epoch: 37 [23040/50000 (51%)]\tLoss: 0.215963, Accuracy: 92.19\n",
      "Train Epoch: 37 [25600/50000 (57%)]\tLoss: 0.171909, Accuracy: 93.36\n",
      "Train Epoch: 37 [28160/50000 (62%)]\tLoss: 0.224016, Accuracy: 94.92\n",
      "Train Epoch: 37 [30720/50000 (68%)]\tLoss: 0.151392, Accuracy: 94.92\n",
      "Train Epoch: 37 [33280/50000 (74%)]\tLoss: 0.125350, Accuracy: 94.92\n",
      "Train Epoch: 37 [35840/50000 (80%)]\tLoss: 0.191303, Accuracy: 93.36\n",
      "Train Epoch: 37 [38400/50000 (85%)]\tLoss: 0.183572, Accuracy: 94.14\n",
      "Train Epoch: 37 [40960/50000 (91%)]\tLoss: 0.179196, Accuracy: 93.75\n",
      "Train Epoch: 37 [43520/50000 (97%)]\tLoss: 0.193141, Accuracy: 93.75\n",
      "\n",
      "Validation set: Average loss: 0.4682, Accuracy: 4332/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.55517935752869 s]\n",
      "Train Epoch: 38 [0/50000 (0%)]\tLoss: 0.169538, Accuracy: 92.19\n",
      "Train Epoch: 38 [2560/50000 (6%)]\tLoss: 0.186190, Accuracy: 93.75\n",
      "Train Epoch: 38 [5120/50000 (11%)]\tLoss: 0.235603, Accuracy: 90.23\n",
      "Train Epoch: 38 [7680/50000 (17%)]\tLoss: 0.112146, Accuracy: 96.09\n",
      "Train Epoch: 38 [10240/50000 (23%)]\tLoss: 0.135916, Accuracy: 95.31\n",
      "Train Epoch: 38 [12800/50000 (28%)]\tLoss: 0.194997, Accuracy: 92.97\n",
      "Train Epoch: 38 [15360/50000 (34%)]\tLoss: 0.168524, Accuracy: 93.75\n",
      "Train Epoch: 38 [17920/50000 (40%)]\tLoss: 0.157594, Accuracy: 95.70\n",
      "Train Epoch: 38 [20480/50000 (45%)]\tLoss: 0.222537, Accuracy: 90.62\n",
      "Train Epoch: 38 [23040/50000 (51%)]\tLoss: 0.182072, Accuracy: 92.58\n",
      "Train Epoch: 38 [25600/50000 (57%)]\tLoss: 0.182055, Accuracy: 94.14\n",
      "Train Epoch: 38 [28160/50000 (62%)]\tLoss: 0.166654, Accuracy: 94.53\n",
      "Train Epoch: 38 [30720/50000 (68%)]\tLoss: 0.093684, Accuracy: 97.27\n",
      "Train Epoch: 38 [33280/50000 (74%)]\tLoss: 0.140571, Accuracy: 96.09\n",
      "Train Epoch: 38 [35840/50000 (80%)]\tLoss: 0.208057, Accuracy: 92.97\n",
      "Train Epoch: 38 [38400/50000 (85%)]\tLoss: 0.214443, Accuracy: 91.80\n",
      "Train Epoch: 38 [40960/50000 (91%)]\tLoss: 0.209735, Accuracy: 93.75\n",
      "Train Epoch: 38 [43520/50000 (97%)]\tLoss: 0.220803, Accuracy: 91.02\n",
      "\n",
      "Validation set: Average loss: 0.4529, Accuracy: 4328/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.511467933654785 s]\n",
      "Train Epoch: 39 [0/50000 (0%)]\tLoss: 0.219202, Accuracy: 91.41\n",
      "Train Epoch: 39 [2560/50000 (6%)]\tLoss: 0.192145, Accuracy: 93.75\n",
      "Train Epoch: 39 [5120/50000 (11%)]\tLoss: 0.199473, Accuracy: 93.75\n",
      "Train Epoch: 39 [7680/50000 (17%)]\tLoss: 0.178895, Accuracy: 94.53\n",
      "Train Epoch: 39 [10240/50000 (23%)]\tLoss: 0.193942, Accuracy: 94.53\n",
      "Train Epoch: 39 [12800/50000 (28%)]\tLoss: 0.119124, Accuracy: 95.31\n",
      "Train Epoch: 39 [15360/50000 (34%)]\tLoss: 0.159233, Accuracy: 94.92\n",
      "Train Epoch: 39 [17920/50000 (40%)]\tLoss: 0.201624, Accuracy: 92.97\n",
      "Train Epoch: 39 [20480/50000 (45%)]\tLoss: 0.156176, Accuracy: 94.53\n",
      "Train Epoch: 39 [23040/50000 (51%)]\tLoss: 0.179873, Accuracy: 94.14\n",
      "Train Epoch: 39 [25600/50000 (57%)]\tLoss: 0.184660, Accuracy: 94.14\n",
      "Train Epoch: 39 [28160/50000 (62%)]\tLoss: 0.127192, Accuracy: 94.53\n",
      "Train Epoch: 39 [30720/50000 (68%)]\tLoss: 0.209395, Accuracy: 91.02\n",
      "Train Epoch: 39 [33280/50000 (74%)]\tLoss: 0.171000, Accuracy: 94.14\n",
      "Train Epoch: 39 [35840/50000 (80%)]\tLoss: 0.201088, Accuracy: 92.58\n",
      "Train Epoch: 39 [38400/50000 (85%)]\tLoss: 0.133072, Accuracy: 94.92\n",
      "Train Epoch: 39 [40960/50000 (91%)]\tLoss: 0.178143, Accuracy: 93.36\n",
      "Train Epoch: 39 [43520/50000 (97%)]\tLoss: 0.213150, Accuracy: 93.36\n",
      "\n",
      "Validation set: Average loss: 0.4936, Accuracy: 4308/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.567169427871704 s]\n",
      "Train Epoch: 40 [0/50000 (0%)]\tLoss: 0.187140, Accuracy: 91.41\n",
      "Train Epoch: 40 [2560/50000 (6%)]\tLoss: 0.246733, Accuracy: 91.80\n",
      "Train Epoch: 40 [5120/50000 (11%)]\tLoss: 0.167691, Accuracy: 93.36\n",
      "Train Epoch: 40 [7680/50000 (17%)]\tLoss: 0.092219, Accuracy: 96.48\n",
      "Train Epoch: 40 [10240/50000 (23%)]\tLoss: 0.224092, Accuracy: 91.80\n",
      "Train Epoch: 40 [12800/50000 (28%)]\tLoss: 0.141728, Accuracy: 94.92\n",
      "Train Epoch: 40 [15360/50000 (34%)]\tLoss: 0.177842, Accuracy: 92.97\n",
      "Train Epoch: 40 [17920/50000 (40%)]\tLoss: 0.150751, Accuracy: 94.92\n",
      "Train Epoch: 40 [20480/50000 (45%)]\tLoss: 0.121539, Accuracy: 96.09\n",
      "Train Epoch: 40 [23040/50000 (51%)]\tLoss: 0.227488, Accuracy: 91.80\n",
      "Train Epoch: 40 [25600/50000 (57%)]\tLoss: 0.170062, Accuracy: 93.75\n",
      "Train Epoch: 40 [28160/50000 (62%)]\tLoss: 0.117123, Accuracy: 95.31\n",
      "Train Epoch: 40 [30720/50000 (68%)]\tLoss: 0.164737, Accuracy: 95.70\n",
      "Train Epoch: 40 [33280/50000 (74%)]\tLoss: 0.149473, Accuracy: 94.14\n",
      "Train Epoch: 40 [35840/50000 (80%)]\tLoss: 0.211677, Accuracy: 93.75\n",
      "Train Epoch: 40 [38400/50000 (85%)]\tLoss: 0.194328, Accuracy: 92.97\n",
      "Train Epoch: 40 [40960/50000 (91%)]\tLoss: 0.229130, Accuracy: 92.58\n",
      "Train Epoch: 40 [43520/50000 (97%)]\tLoss: 0.128513, Accuracy: 95.70\n",
      "\n",
      "Validation set: Average loss: 0.4839, Accuracy: 4327/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.513254165649414 s]\n",
      "Train Epoch: 41 [0/50000 (0%)]\tLoss: 0.121540, Accuracy: 95.70\n",
      "Train Epoch: 41 [2560/50000 (6%)]\tLoss: 0.189811, Accuracy: 93.75\n",
      "Train Epoch: 41 [5120/50000 (11%)]\tLoss: 0.188141, Accuracy: 94.14\n",
      "Train Epoch: 41 [7680/50000 (17%)]\tLoss: 0.140362, Accuracy: 94.53\n",
      "Train Epoch: 41 [10240/50000 (23%)]\tLoss: 0.140409, Accuracy: 94.14\n",
      "Train Epoch: 41 [12800/50000 (28%)]\tLoss: 0.152054, Accuracy: 96.48\n",
      "Train Epoch: 41 [15360/50000 (34%)]\tLoss: 0.185862, Accuracy: 92.97\n",
      "Train Epoch: 41 [17920/50000 (40%)]\tLoss: 0.143757, Accuracy: 94.92\n",
      "Train Epoch: 41 [20480/50000 (45%)]\tLoss: 0.191554, Accuracy: 93.36\n",
      "Train Epoch: 41 [23040/50000 (51%)]\tLoss: 0.144627, Accuracy: 95.31\n",
      "Train Epoch: 41 [25600/50000 (57%)]\tLoss: 0.181043, Accuracy: 92.97\n",
      "Train Epoch: 41 [28160/50000 (62%)]\tLoss: 0.154821, Accuracy: 95.31\n",
      "Train Epoch: 41 [30720/50000 (68%)]\tLoss: 0.156581, Accuracy: 95.70\n",
      "Train Epoch: 41 [33280/50000 (74%)]\tLoss: 0.110503, Accuracy: 96.09\n",
      "Train Epoch: 41 [35840/50000 (80%)]\tLoss: 0.122720, Accuracy: 94.53\n",
      "Train Epoch: 41 [38400/50000 (85%)]\tLoss: 0.126287, Accuracy: 96.09\n",
      "Train Epoch: 41 [40960/50000 (91%)]\tLoss: 0.172418, Accuracy: 94.53\n",
      "Train Epoch: 41 [43520/50000 (97%)]\tLoss: 0.137474, Accuracy: 96.88\n",
      "\n",
      "Validation set: Average loss: 0.4644, Accuracy: 4363/5000 (87.00%)\n",
      "\n",
      "the time of this epoch:[45.59950661659241 s]\n",
      "Train Epoch: 42 [0/50000 (0%)]\tLoss: 0.145455, Accuracy: 94.92\n",
      "Train Epoch: 42 [2560/50000 (6%)]\tLoss: 0.184278, Accuracy: 93.36\n",
      "Train Epoch: 42 [5120/50000 (11%)]\tLoss: 0.137742, Accuracy: 94.92\n",
      "Train Epoch: 42 [7680/50000 (17%)]\tLoss: 0.113860, Accuracy: 96.48\n",
      "Train Epoch: 42 [10240/50000 (23%)]\tLoss: 0.193811, Accuracy: 93.75\n",
      "Train Epoch: 42 [12800/50000 (28%)]\tLoss: 0.184680, Accuracy: 94.53\n",
      "Train Epoch: 42 [15360/50000 (34%)]\tLoss: 0.158169, Accuracy: 94.92\n",
      "Train Epoch: 42 [17920/50000 (40%)]\tLoss: 0.162425, Accuracy: 94.53\n",
      "Train Epoch: 42 [20480/50000 (45%)]\tLoss: 0.099235, Accuracy: 97.27\n",
      "Train Epoch: 42 [23040/50000 (51%)]\tLoss: 0.165685, Accuracy: 94.14\n",
      "Train Epoch: 42 [25600/50000 (57%)]\tLoss: 0.090099, Accuracy: 95.70\n",
      "Train Epoch: 42 [28160/50000 (62%)]\tLoss: 0.134280, Accuracy: 94.92\n",
      "Train Epoch: 42 [30720/50000 (68%)]\tLoss: 0.195921, Accuracy: 93.36\n",
      "Train Epoch: 42 [33280/50000 (74%)]\tLoss: 0.184564, Accuracy: 93.36\n",
      "Train Epoch: 42 [35840/50000 (80%)]\tLoss: 0.156864, Accuracy: 95.31\n",
      "Train Epoch: 42 [38400/50000 (85%)]\tLoss: 0.144501, Accuracy: 94.53\n",
      "Train Epoch: 42 [40960/50000 (91%)]\tLoss: 0.117453, Accuracy: 96.48\n",
      "Train Epoch: 42 [43520/50000 (97%)]\tLoss: 0.122548, Accuracy: 96.09\n",
      "\n",
      "Validation set: Average loss: 0.4828, Accuracy: 4336/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.562594175338745 s]\n",
      "Train Epoch: 43 [0/50000 (0%)]\tLoss: 0.134125, Accuracy: 96.48\n",
      "Train Epoch: 43 [2560/50000 (6%)]\tLoss: 0.152054, Accuracy: 95.31\n",
      "Train Epoch: 43 [5120/50000 (11%)]\tLoss: 0.149180, Accuracy: 94.92\n",
      "Train Epoch: 43 [7680/50000 (17%)]\tLoss: 0.142696, Accuracy: 94.53\n",
      "Train Epoch: 43 [10240/50000 (23%)]\tLoss: 0.081875, Accuracy: 96.88\n",
      "Train Epoch: 43 [12800/50000 (28%)]\tLoss: 0.168013, Accuracy: 94.14\n",
      "Train Epoch: 43 [15360/50000 (34%)]\tLoss: 0.140116, Accuracy: 95.31\n",
      "Train Epoch: 43 [17920/50000 (40%)]\tLoss: 0.172080, Accuracy: 94.53\n",
      "Train Epoch: 43 [20480/50000 (45%)]\tLoss: 0.143026, Accuracy: 94.53\n",
      "Train Epoch: 43 [23040/50000 (51%)]\tLoss: 0.128789, Accuracy: 97.27\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 43 [25600/50000 (57%)]\tLoss: 0.087618, Accuracy: 97.27\n",
      "Train Epoch: 43 [28160/50000 (62%)]\tLoss: 0.187053, Accuracy: 93.36\n",
      "Train Epoch: 43 [30720/50000 (68%)]\tLoss: 0.105647, Accuracy: 96.09\n",
      "Train Epoch: 43 [33280/50000 (74%)]\tLoss: 0.169502, Accuracy: 95.31\n",
      "Train Epoch: 43 [35840/50000 (80%)]\tLoss: 0.123194, Accuracy: 95.31\n",
      "Train Epoch: 43 [38400/50000 (85%)]\tLoss: 0.110913, Accuracy: 96.48\n",
      "Train Epoch: 43 [40960/50000 (91%)]\tLoss: 0.150977, Accuracy: 95.70\n",
      "Train Epoch: 43 [43520/50000 (97%)]\tLoss: 0.175949, Accuracy: 93.75\n",
      "\n",
      "Validation set: Average loss: 0.4792, Accuracy: 4348/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.52462816238403 s]\n",
      "Train Epoch: 44 [0/50000 (0%)]\tLoss: 0.133412, Accuracy: 93.75\n",
      "Train Epoch: 44 [2560/50000 (6%)]\tLoss: 0.096028, Accuracy: 96.88\n",
      "Train Epoch: 44 [5120/50000 (11%)]\tLoss: 0.098115, Accuracy: 96.09\n",
      "Train Epoch: 44 [7680/50000 (17%)]\tLoss: 0.098042, Accuracy: 96.88\n",
      "Train Epoch: 44 [10240/50000 (23%)]\tLoss: 0.074757, Accuracy: 98.05\n",
      "Train Epoch: 44 [12800/50000 (28%)]\tLoss: 0.113175, Accuracy: 95.70\n",
      "Train Epoch: 44 [15360/50000 (34%)]\tLoss: 0.088559, Accuracy: 98.05\n",
      "Train Epoch: 44 [17920/50000 (40%)]\tLoss: 0.114741, Accuracy: 96.48\n",
      "Train Epoch: 44 [20480/50000 (45%)]\tLoss: 0.126955, Accuracy: 94.53\n",
      "Train Epoch: 44 [23040/50000 (51%)]\tLoss: 0.059957, Accuracy: 98.44\n",
      "Train Epoch: 44 [25600/50000 (57%)]\tLoss: 0.122719, Accuracy: 94.92\n",
      "Train Epoch: 44 [28160/50000 (62%)]\tLoss: 0.079351, Accuracy: 96.88\n",
      "Train Epoch: 44 [30720/50000 (68%)]\tLoss: 0.076182, Accuracy: 96.88\n",
      "Train Epoch: 44 [33280/50000 (74%)]\tLoss: 0.177493, Accuracy: 94.14\n",
      "Train Epoch: 44 [35840/50000 (80%)]\tLoss: 0.143659, Accuracy: 94.14\n",
      "Train Epoch: 44 [38400/50000 (85%)]\tLoss: 0.147556, Accuracy: 95.31\n",
      "Train Epoch: 44 [40960/50000 (91%)]\tLoss: 0.152792, Accuracy: 94.92\n",
      "Train Epoch: 44 [43520/50000 (97%)]\tLoss: 0.166099, Accuracy: 94.53\n",
      "\n",
      "Validation set: Average loss: 0.4919, Accuracy: 4343/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.52489900588989 s]\n",
      "Train Epoch: 45 [0/50000 (0%)]\tLoss: 0.141479, Accuracy: 95.31\n",
      "Train Epoch: 45 [2560/50000 (6%)]\tLoss: 0.132724, Accuracy: 96.48\n",
      "Train Epoch: 45 [5120/50000 (11%)]\tLoss: 0.083910, Accuracy: 97.27\n",
      "Train Epoch: 45 [7680/50000 (17%)]\tLoss: 0.156828, Accuracy: 93.75\n",
      "Train Epoch: 45 [10240/50000 (23%)]\tLoss: 0.097889, Accuracy: 96.09\n",
      "Train Epoch: 45 [12800/50000 (28%)]\tLoss: 0.130425, Accuracy: 96.48\n",
      "Train Epoch: 45 [15360/50000 (34%)]\tLoss: 0.146009, Accuracy: 94.53\n",
      "Train Epoch: 45 [17920/50000 (40%)]\tLoss: 0.122212, Accuracy: 97.27\n",
      "Train Epoch: 45 [20480/50000 (45%)]\tLoss: 0.094537, Accuracy: 97.66\n",
      "Train Epoch: 45 [23040/50000 (51%)]\tLoss: 0.128877, Accuracy: 94.53\n",
      "Train Epoch: 45 [25600/50000 (57%)]\tLoss: 0.155790, Accuracy: 94.92\n",
      "Train Epoch: 45 [28160/50000 (62%)]\tLoss: 0.103953, Accuracy: 96.48\n",
      "Train Epoch: 45 [30720/50000 (68%)]\tLoss: 0.126388, Accuracy: 96.09\n",
      "Train Epoch: 45 [33280/50000 (74%)]\tLoss: 0.104876, Accuracy: 97.27\n",
      "Train Epoch: 45 [35840/50000 (80%)]\tLoss: 0.117038, Accuracy: 95.70\n",
      "Train Epoch: 45 [38400/50000 (85%)]\tLoss: 0.207228, Accuracy: 93.75\n",
      "Train Epoch: 45 [40960/50000 (91%)]\tLoss: 0.114103, Accuracy: 96.48\n",
      "Train Epoch: 45 [43520/50000 (97%)]\tLoss: 0.171332, Accuracy: 94.92\n",
      "\n",
      "Validation set: Average loss: 0.5112, Accuracy: 4307/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.51470351219177 s]\n",
      "Train Epoch: 46 [0/50000 (0%)]\tLoss: 0.160613, Accuracy: 94.92\n",
      "Train Epoch: 46 [2560/50000 (6%)]\tLoss: 0.142201, Accuracy: 94.53\n",
      "Train Epoch: 46 [5120/50000 (11%)]\tLoss: 0.110722, Accuracy: 96.88\n",
      "Train Epoch: 46 [7680/50000 (17%)]\tLoss: 0.134642, Accuracy: 95.70\n",
      "Train Epoch: 46 [10240/50000 (23%)]\tLoss: 0.178009, Accuracy: 94.14\n",
      "Train Epoch: 46 [12800/50000 (28%)]\tLoss: 0.100416, Accuracy: 96.48\n",
      "Train Epoch: 46 [15360/50000 (34%)]\tLoss: 0.152494, Accuracy: 95.70\n",
      "Train Epoch: 46 [17920/50000 (40%)]\tLoss: 0.161598, Accuracy: 94.92\n",
      "Train Epoch: 46 [20480/50000 (45%)]\tLoss: 0.115757, Accuracy: 95.70\n",
      "Train Epoch: 46 [23040/50000 (51%)]\tLoss: 0.092570, Accuracy: 96.48\n",
      "Train Epoch: 46 [25600/50000 (57%)]\tLoss: 0.105404, Accuracy: 95.70\n",
      "Train Epoch: 46 [28160/50000 (62%)]\tLoss: 0.176416, Accuracy: 93.36\n",
      "Train Epoch: 46 [30720/50000 (68%)]\tLoss: 0.164780, Accuracy: 94.53\n",
      "Train Epoch: 46 [33280/50000 (74%)]\tLoss: 0.143669, Accuracy: 93.75\n",
      "Train Epoch: 46 [35840/50000 (80%)]\tLoss: 0.122929, Accuracy: 95.31\n",
      "Train Epoch: 46 [38400/50000 (85%)]\tLoss: 0.164436, Accuracy: 93.36\n",
      "Train Epoch: 46 [40960/50000 (91%)]\tLoss: 0.149865, Accuracy: 93.75\n",
      "Train Epoch: 46 [43520/50000 (97%)]\tLoss: 0.127003, Accuracy: 94.92\n",
      "\n",
      "Validation set: Average loss: 0.4897, Accuracy: 4340/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.49495553970337 s]\n",
      "Train Epoch: 47 [0/50000 (0%)]\tLoss: 0.122726, Accuracy: 95.70\n",
      "Train Epoch: 47 [2560/50000 (6%)]\tLoss: 0.157962, Accuracy: 93.75\n",
      "Train Epoch: 47 [5120/50000 (11%)]\tLoss: 0.096001, Accuracy: 96.09\n",
      "Train Epoch: 47 [7680/50000 (17%)]\tLoss: 0.148490, Accuracy: 95.70\n",
      "Train Epoch: 47 [10240/50000 (23%)]\tLoss: 0.097717, Accuracy: 96.88\n",
      "Train Epoch: 47 [12800/50000 (28%)]\tLoss: 0.162485, Accuracy: 94.14\n",
      "Train Epoch: 47 [15360/50000 (34%)]\tLoss: 0.092552, Accuracy: 96.88\n",
      "Train Epoch: 47 [17920/50000 (40%)]\tLoss: 0.147760, Accuracy: 95.31\n",
      "Train Epoch: 47 [20480/50000 (45%)]\tLoss: 0.139640, Accuracy: 93.75\n",
      "Train Epoch: 47 [23040/50000 (51%)]\tLoss: 0.181079, Accuracy: 95.31\n",
      "Train Epoch: 47 [25600/50000 (57%)]\tLoss: 0.175674, Accuracy: 93.75\n",
      "Train Epoch: 47 [28160/50000 (62%)]\tLoss: 0.114516, Accuracy: 95.70\n",
      "Train Epoch: 47 [30720/50000 (68%)]\tLoss: 0.098856, Accuracy: 96.09\n",
      "Train Epoch: 47 [33280/50000 (74%)]\tLoss: 0.096192, Accuracy: 96.48\n",
      "Train Epoch: 47 [35840/50000 (80%)]\tLoss: 0.104711, Accuracy: 97.66\n",
      "Train Epoch: 47 [38400/50000 (85%)]\tLoss: 0.172701, Accuracy: 94.14\n",
      "Train Epoch: 47 [40960/50000 (91%)]\tLoss: 0.138267, Accuracy: 94.14\n",
      "Train Epoch: 47 [43520/50000 (97%)]\tLoss: 0.181492, Accuracy: 93.36\n",
      "\n",
      "Validation set: Average loss: 0.5786, Accuracy: 4256/5000 (85.00%)\n",
      "\n",
      "the time of this epoch:[45.538442611694336 s]\n",
      "Train Epoch: 48 [0/50000 (0%)]\tLoss: 0.056466, Accuracy: 96.88\n",
      "Train Epoch: 48 [2560/50000 (6%)]\tLoss: 0.102953, Accuracy: 96.09\n",
      "Train Epoch: 48 [5120/50000 (11%)]\tLoss: 0.092365, Accuracy: 98.83\n",
      "Train Epoch: 48 [7680/50000 (17%)]\tLoss: 0.131485, Accuracy: 95.70\n",
      "Train Epoch: 48 [10240/50000 (23%)]\tLoss: 0.107197, Accuracy: 95.31\n",
      "Train Epoch: 48 [12800/50000 (28%)]\tLoss: 0.153116, Accuracy: 94.53\n",
      "Train Epoch: 48 [15360/50000 (34%)]\tLoss: 0.140875, Accuracy: 96.09\n",
      "Train Epoch: 48 [17920/50000 (40%)]\tLoss: 0.130033, Accuracy: 95.31\n",
      "Train Epoch: 48 [20480/50000 (45%)]\tLoss: 0.138602, Accuracy: 95.31\n",
      "Train Epoch: 48 [23040/50000 (51%)]\tLoss: 0.066072, Accuracy: 97.66\n",
      "Train Epoch: 48 [25600/50000 (57%)]\tLoss: 0.157790, Accuracy: 94.53\n",
      "Train Epoch: 48 [28160/50000 (62%)]\tLoss: 0.096780, Accuracy: 97.66\n",
      "Train Epoch: 48 [30720/50000 (68%)]\tLoss: 0.094081, Accuracy: 95.70\n",
      "Train Epoch: 48 [33280/50000 (74%)]\tLoss: 0.173208, Accuracy: 94.53\n",
      "Train Epoch: 48 [35840/50000 (80%)]\tLoss: 0.128546, Accuracy: 96.88\n",
      "Train Epoch: 48 [38400/50000 (85%)]\tLoss: 0.101090, Accuracy: 96.88\n",
      "Train Epoch: 48 [40960/50000 (91%)]\tLoss: 0.116240, Accuracy: 94.92\n",
      "Train Epoch: 48 [43520/50000 (97%)]\tLoss: 0.115083, Accuracy: 94.92\n",
      "\n",
      "Validation set: Average loss: 0.5406, Accuracy: 4326/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.52974820137024 s]\n",
      "Train Epoch: 49 [0/50000 (0%)]\tLoss: 0.186016, Accuracy: 93.36\n",
      "Train Epoch: 49 [2560/50000 (6%)]\tLoss: 0.125049, Accuracy: 95.31\n",
      "Train Epoch: 49 [5120/50000 (11%)]\tLoss: 0.129749, Accuracy: 95.70\n",
      "Train Epoch: 49 [7680/50000 (17%)]\tLoss: 0.108347, Accuracy: 97.27\n",
      "Train Epoch: 49 [10240/50000 (23%)]\tLoss: 0.075657, Accuracy: 97.66\n",
      "Train Epoch: 49 [12800/50000 (28%)]\tLoss: 0.112178, Accuracy: 95.70\n",
      "Train Epoch: 49 [15360/50000 (34%)]\tLoss: 0.067775, Accuracy: 96.48\n",
      "Train Epoch: 49 [17920/50000 (40%)]\tLoss: 0.132861, Accuracy: 95.70\n",
      "Train Epoch: 49 [20480/50000 (45%)]\tLoss: 0.130666, Accuracy: 96.09\n",
      "Train Epoch: 49 [23040/50000 (51%)]\tLoss: 0.156162, Accuracy: 94.53\n",
      "Train Epoch: 49 [25600/50000 (57%)]\tLoss: 0.104501, Accuracy: 96.09\n",
      "Train Epoch: 49 [28160/50000 (62%)]\tLoss: 0.083544, Accuracy: 97.66\n",
      "Train Epoch: 49 [30720/50000 (68%)]\tLoss: 0.126312, Accuracy: 94.14\n",
      "Train Epoch: 49 [33280/50000 (74%)]\tLoss: 0.118379, Accuracy: 96.48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 49 [35840/50000 (80%)]\tLoss: 0.137849, Accuracy: 94.92\n",
      "Train Epoch: 49 [38400/50000 (85%)]\tLoss: 0.109643, Accuracy: 96.88\n",
      "Train Epoch: 49 [40960/50000 (91%)]\tLoss: 0.099273, Accuracy: 96.09\n",
      "Train Epoch: 49 [43520/50000 (97%)]\tLoss: 0.183204, Accuracy: 95.31\n",
      "\n",
      "Validation set: Average loss: 0.5065, Accuracy: 4334/5000 (86.00%)\n",
      "\n",
      "the time of this epoch:[45.60560369491577 s]\n"
     ]
    }
   ],
   "source": [
    "# Fantastic logger for tensorboard and pytorch, \n",
    "# run tensorboard by opening a new terminal and run \"tensorboard --logdir runs\"\n",
    "# open tensorboard at http://localhost:6006/\n",
    "from tensorboardX import SummaryWriter\n",
    "best_loss = None\n",
    "best_acc = None\n",
    "\n",
    "import time \n",
    "SINCE=time.time()\n",
    "\n",
    "for epoch in range(max_epoch):\n",
    "    train(epoch)\n",
    "    loss, accuracy = validate(epoch)\n",
    "    best_loss, best_acc = save_best(loss, accuracy, best_loss, best_acc)\n",
    "    \n",
    "    NOW=time.time() \n",
    "    DURINGS=NOW-SINCE\n",
    "    SINCE=NOW\n",
    "    print(\"the time of this epoch:[{} s]\".format(DURINGS))\n",
    "    \n",
    "writer = SummaryWriter()\n",
    "writer.export_scalars_to_json(\"./all_scalars.json\")\n",
    "\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3: Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Average loss: 0.5929, Accuracy: 8561/10000 (85.61%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test(epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}