15_迁移学习CIFAR10.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "15_迁移学习CIFAR10.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mengcius/pytorch-learn/blob/master/15_%E8%BF%81%E7%A7%BB%E5%AD%A6%E4%B9%A0CIFAR10.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yPQkxVHgB5n9",
        "colab_type": "text"
      },
      "source": [
        "## 15_迁移学习CIFAR10\n",
        "ResNet18迁移学习cifar10\n",
        "\n",
        "1、测试1 batchsz128，可能不适合cifar\n",
        "\n",
        "0 loss: 1.0104621648788452\n",
        "0 test acc: 0.7064\n",
        "\n",
        "1 loss: 0.6776887774467468\n",
        "1 test acc: 0.7688\n",
        "\n",
        "2 loss: 0.8195689916610718\n",
        "2 test acc: 0.7795\n",
        "\n",
        "10 loss: 0.5256583094596863\n",
        "10 test acc: 0.8258\n",
        "\n",
        "22 loss: 0.24958983063697815\n",
        "22 test acc: 0.8587\n",
        "\n",
        "70 loss: 0.07972583919763565\n",
        "70 test acc: 0.8629\n",
        "\n",
        "99 loss: 0.09527455270290375\n",
        "99 test acc: 0.8724\n",
        "\n",
        "2、测试2 batchsz64，差不多一样\n",
        "\n",
        "0 loss: 2.024899482727051\n",
        "0 test acc: 0.7201\n",
        "\n",
        "1 loss: 0.6208065748214722\n",
        "1 test acc: 0.7446\n",
        "\n",
        "2 loss: 1.001968264579773\n",
        "2 test acc: 0.7767\n",
        "\n",
        "10 loss: 3.493842601776123\n",
        "10 test acc: 0.8219\n",
        "\n",
        "19 loss: 0.3018224537372589\n",
        "19 test acc: 0.8537\n",
        "\n",
        "3、模拟验证集测试集 \n",
        "\n",
        "epoch: 0 loss: 0.8336920738220215\n",
        "epoch: 0 val acc: 0.7181\n",
        "\n",
        "epoch: 1 loss: 1.0085989236831665\n",
        "epoch: 1 val acc: 0.7728\n",
        "\n",
        "epoch: 2 loss: 0.6742492318153381\n",
        "epoch: 2 val acc: 0.7753\n",
        "\n",
        "epoch: 3 loss: 0.5457809567451477\n",
        "epoch: 3 val acc: 0.8057\n",
        "\n",
        "epoch: 4 loss: 0.6048915982246399\n",
        "epoch: 4 val acc: 0.8054\n",
        "\n",
        "val best epoch: 3 val best acc: 0.8057\n",
        "\n",
        "loaded from ckpt!\n",
        "\n",
        "test acc: 0.8057"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XeDBUFQaF33E",
        "colab_type": "text"
      },
      "source": [
        "### 安装visdom"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v8FxqZFOFnbF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "outputId": "ff41e195-6d0e-4162-f196-cc1070041482"
      },
      "source": [
        "# !wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "# !unzip ngrok-stable-linux-amd64.zip\n",
        "get_ipython().system_raw('python3 -m pip install visdom')\n",
        "get_ipython().system_raw('python3 -m visdom.server -port 6006 >> visdomlog.txt 2>&1 &')\n",
        "get_ipython().system_raw('ssh -o ServerAliveInterval=60 -o StrictHostKeyChecking=no -R YOURDOMAINHERE:80:localhost:6006 serveo.net &')\n",
        "print('Visdom view: {}'.format('https://YOURDOMAINHERE.serveo.net/'))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Visdom view: https://YOURDOMAINHERE.serveo.net/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LMqO5tYACGuj",
        "colab_type": "text"
      },
      "source": [
        "### utils.py"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2N3WWkTpA0WJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from    matplotlib import pyplot as plt\n",
        "import  torch\n",
        "from    torch import nn\n",
        "\n",
        "class Flatten(nn.Module):\n",
        "\n",
        "    def __init__(self):\n",
        "        super(Flatten, self).__init__()\n",
        "\n",
        "    def forward(self, x):\n",
        "        shape = torch.prod(torch.tensor(x.shape[1:])).item() # 取c h w\n",
        "        return x.view(-1, shape) # 占了第一个维度\n",
        "\n",
        "\n",
        "def plot_image(img, label, name):\n",
        "\n",
        "    fig = plt.figure()\n",
        "    for i in range(6):\n",
        "        plt.subplot(2, 3, i + 1)\n",
        "        plt.tight_layout()\n",
        "        plt.imshow(img[i][0]*0.3081+0.1307, cmap='gray', interpolation='none')\n",
        "        plt.title(\"{}: {}\".format(name, label[i].item()))\n",
        "        plt.xticks([])\n",
        "        plt.yticks([])\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2uKhyAOMCMyz",
        "colab_type": "text"
      },
      "source": [
        "### main.py"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OaN1m5o7APkH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "18a49450-d811-4190-bc53-0755d4133ee3"
      },
      "source": [
        "import  torch\n",
        "from    torch.utils.data import DataLoader\n",
        "from    torchvision import datasets\n",
        "from    torchvision import transforms\n",
        "from    torch import nn, optim\n",
        "import  visdom\n",
        "import  torchvision\n",
        "\n",
        "from    torchvision.models import resnet18 # 现有模型\n",
        "# from    utils import Flatten\n",
        "\n",
        "\n",
        "batchsz = 128\n",
        "lr = 1e-3\n",
        "epochs = 5\n",
        "\n",
        "device = torch.device('cuda')\n",
        "torch.manual_seed(1234) # 随机种子，来保证实验可以复现出来\n",
        "viz = visdom.Visdom(port='6006')\n",
        "\n",
        "\n",
        "# 加载数据集\n",
        "cifar_train = datasets.CIFAR10('cifar', True, transform=transforms.Compose([\n",
        "    transforms.Resize((32, 32)),\n",
        "    transforms.RandomCrop(32, padding=4),\n",
        "    transforms.RandomHorizontalFlip(),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=[0.485, 0.456, 0.406], # 数据归一化，R G B上的标准差\n",
        "                            std=[0.229, 0.224, 0.225]) # R G B上的方标准差\n",
        "]), download=True)\n",
        "cifar_train = DataLoader(cifar_train, batch_size=batchsz, shuffle=True)\n",
        "\n",
        "cifar_test = datasets.CIFAR10('cifar', False, transform=transforms.Compose([\n",
        "    transforms.Resize((32, 32)),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
        "                            std=[0.229, 0.224, 0.225])\n",
        "]), download=True)\n",
        "cifar_test = DataLoader(cifar_test, batch_size=batchsz, shuffle=True)\n",
        "\n",
        "x, label = iter(cifar_train).next() # 一个batch\n",
        "print('x:', x.shape, 'label:', label.shape)\n",
        "# x:torch.Size([32,3,32,32]) label:torch.Size([32])\n",
        "\n",
        "\n",
        "# 评估ACC函数，验证或测试一样\n",
        "def evalute(model, loader):\n",
        "    model.eval() # test也是\n",
        "    \n",
        "    correct = 0 # 预测正确的数量\n",
        "    total = len(loader.dataset)\n",
        "\n",
        "    for x,y in loader:\n",
        "        x,y = x.to(device), y.to(device)\n",
        "        with torch.no_grad(): # 不需要反向传播\n",
        "            logits = model(x)\n",
        "            pred = logits.argmax(dim=1)\n",
        "        correct += torch.eq(pred, y).sum().float().item() # item转为numpy\n",
        "\n",
        "    return correct / total\n",
        "\n",
        "\n",
        "def main():\n",
        "\n",
        "    # 构建自定义网络\n",
        "    # model = Lenet5().to(device)\n",
        "    # model = ResNet18().to(device)\n",
        "    # print(model) # 打印结构（初始化类里的结构，forward里打印不出来）\n",
        "\n",
        "    # 构建迁移学习模型\n",
        "    trained_model = resnet18(pretrained=True) # 使用已有模型训练好的各层参数（最后一层除外）\n",
        "    model = nn.Sequential(*list(trained_model.children())[:-1], # 迁移，[b, 512, 1, 1]，去掉了最后一层\n",
        "                          Flatten(), # 拉平c h w，[b, 512, 1, 1] => [b, 512]\n",
        "                          nn.Linear(512, 10) # 新要学习的全连接层，5分类\n",
        "                          ).to(device)\n",
        "    # x = torch.randn(2, 3, 224, 224)\n",
        "    # print(model(x).shape) # 迁移后[2,10]\n",
        "    print(model)\n",
        "\n",
        "\n",
        "    optimizer = optim.Adam(model.parameters(), lr=lr)\n",
        "    criteon = nn.CrossEntropyLoss().to(device)\n",
        "\n",
        "    best_acc, best_epoch = 0, 0\n",
        "    global_step = 0 # x坐标\n",
        "    viz.line([0], [-1], win='loss', opts=dict(title='loss'))\n",
        "    viz.line([0], [-1], win='val_acc', opts=dict(title='val_acc'))\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        # 训练\n",
        "        for step, (x,y) in enumerate(cifar_train): \n",
        "            \n",
        "            x, y = x.to(device), y.to(device) # x: [b, 3, 224, 224], y: [b]\n",
        "            \n",
        "            model.train()\n",
        "            logits = model(x)\n",
        "            loss = criteon(logits, y) # CrossEntropy内部做one hot，直接传入y\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            viz.line([loss.item()], [global_step], win='loss', update='append')\n",
        "            global_step += 1\n",
        "\n",
        "        # 这打印的是此epoch中最后一个batch的loss，loss数据仅供参考，主要看TEST ACC\n",
        "        print('epoch:', epoch, ' loss:', loss.item()) # loss是tensor标量，要通过item转为numpy打印出来\n",
        "\n",
        "        # 验证\n",
        "        if epoch % 1 == 0: \n",
        "\n",
        "            val_acc = evalute(model, cifar_test) # 评估验证集的ACC\n",
        "            if val_acc > best_acc:\n",
        "                best_epoch = epoch\n",
        "                best_acc = val_acc\n",
        "\n",
        "                torch.save(model.state_dict(), 'best.mdl') # 保存模型状态checkpoint，文件后缀无所谓\n",
        "\n",
        "                viz.line([val_acc], [global_step], win='val_acc', update='append')\n",
        "            \n",
        "            print('epoch:', epoch, ' val acc:', val_acc) \n",
        "\n",
        "    print('val best epoch:', best_epoch, ' val best acc:', best_acc)\n",
        "\n",
        "    # 加载checkpoint\n",
        "    model.load_state_dict(torch.load('best.mdl')) # 加载模型checkpoint\n",
        "    print('loaded from ckpt!')\n",
        "    # best_acc = checkpoint['acc']\n",
        "    # best_epoch = checkpoint['epoch']\n",
        "    # print('checkpoint best acc:', best_acc, 'checkpoint best epoch:', best_epoch)\n",
        "\n",
        "    # 测试\n",
        "    test_acc = evalute(model, cifar_test) # 评估测试集的ACC（cifar没有分验证集和测试集，这里就用一样的）\n",
        "    print('test acc:', test_acc)\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    main()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting up a new session...\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "x: torch.Size([128, 3, 32, 32]) label: torch.Size([128])\n",
            "Sequential(\n",
            "  (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
            "  (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (2): ReLU(inplace)\n",
            "  (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
            "  (4): Sequential(\n",
            "    (0): BasicBlock(\n",
            "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    )\n",
            "    (1): BasicBlock(\n",
            "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    )\n",
            "  )\n",
            "  (5): Sequential(\n",
            "    (0): BasicBlock(\n",
            "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (downsample): Sequential(\n",
            "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
            "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (1): BasicBlock(\n",
            "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    )\n",
            "  )\n",
            "  (6): Sequential(\n",
            "    (0): BasicBlock(\n",
            "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (downsample): Sequential(\n",
            "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
            "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (1): BasicBlock(\n",
            "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    )\n",
            "  )\n",
            "  (7): Sequential(\n",
            "    (0): BasicBlock(\n",
            "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (downsample): Sequential(\n",
            "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
            "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (1): BasicBlock(\n",
            "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace)\n",
            "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    )\n",
            "  )\n",
            "  (8): AdaptiveAvgPool2d(output_size=(1, 1))\n",
            "  (9): Flatten()\n",
            "  (10): Linear(in_features=512, out_features=10, bias=True)\n",
            ")\n",
            "epoch: 0 loss: 0.8336920738220215\n",
            "epoch: 0 val acc: 0.7181\n",
            "epoch: 1 loss: 1.0085989236831665\n",
            "epoch: 1 val acc: 0.7728\n",
            "epoch: 2 loss: 0.6742492318153381\n",
            "epoch: 2 val acc: 0.7753\n",
            "epoch: 3 loss: 0.5457809567451477\n",
            "epoch: 3 val acc: 0.8057\n",
            "epoch: 4 loss: 0.6048915982246399\n",
            "epoch: 4 val acc: 0.8054\n",
            "val best epoch: 3 val best acc: 0.8057\n",
            "loaded from ckpt!\n",
            "test acc: 0.8057\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}