pytorch_gpu_intro.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pytorch_gpu_intro.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KJ3OIjdrEeoT"
      },
      "source": [
        "2021 Takahiro Shinozaki @ Tokyo Tech\n",
        "\n",
        "Quick introduction of pytorch GPU/CPU utilization\n",
        "\n",
        "References: \n",
        "*   https://pytorch.org/tutorials/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C2bgINAcEvMu"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import time"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Alv6Ht1MFQrJ"
      },
      "source": [
        "To use GPU, your computer have to have it.\n",
        "\n",
        "If you are using Google colab, you can request to use a computer with a GPU from the top menu : Runtime -> Change Tuntime Type -> GPU"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "giye2BlT1lhR"
      },
      "source": [
        "# Check if a GPU is available"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "plGehiWVEOp3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dd68f115-2f64-4aaf-f9e9-13611b2b7be7"
      },
      "source": [
        "if torch.cuda.is_available():\n",
        "    print('CUDA（GPU）is available')\n",
        "else:\n",
        "    print('CUDA（GPU）is not available')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CUDA（GPU）is available\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S8BLVTqNGw4-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "39aad02b-571a-4dcc-e78c-30d1eea429e6"
      },
      "source": [
        "# Check the GPU status\n",
        "! nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Wed Nov 17 04:11:57 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 495.44       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   32C    P8    29W / 149W |      3MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6DSh9n-CGUiN"
      },
      "source": [
        "# Switch a device of a tensor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jT-e-CalHsDF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e0736bd-4285-4dd5-a531-5595419ed7f5"
      },
      "source": [
        "x = torch.tensor([[1.0, 2.0], [3.0, 4.0]])\n",
        "print(x)\n",
        "print(x.device)\n",
        "\n",
        "x = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device='cpu')\n",
        "print(x)\n",
        "print(x.device)\n",
        "\n",
        "x = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device='cuda')\n",
        "print(x)\n",
        "print(x.device)\n",
        "\n",
        "y = x.to('cpu')\n",
        "print(y)\n",
        "print(y.device)\n",
        "\n",
        "z = y.to('cuda')\n",
        "print(z)\n",
        "print(z.device)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[1., 2.],\n",
            "        [3., 4.]])\n",
            "cpu\n",
            "tensor([[1., 2.],\n",
            "        [3., 4.]])\n",
            "cpu\n",
            "tensor([[1., 2.],\n",
            "        [3., 4.]], device='cuda:0')\n",
            "cuda:0\n",
            "tensor([[1., 2.],\n",
            "        [3., 4.]])\n",
            "cpu\n",
            "tensor([[1., 2.],\n",
            "        [3., 4.]], device='cuda:0')\n",
            "cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_az4dQRHY9SK"
      },
      "source": [
        "Speed up by using GPU depends on the type and size of the computation.\n",
        "\n",
        "Since using GPU involves some overhead, GPU is usually more advantageous when handling larger problems."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gOWTAyTHHSxb"
      },
      "source": [
        "# Compare a computation speed of CPU/GPU\n",
        "\n",
        "## Matrix product"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ps2J9tr5CUKb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cb403dd9-02fc-41d2-c609-c88dd775a21a"
      },
      "source": [
        "dim=50\n",
        "print('dim= ', dim)\n",
        "\n",
        "xcpu = torch.rand(dim, dim, device='cpu')\n",
        "start_time = time.time()\n",
        "y = torch.mm(xcpu, xcpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (CPU) =', elapsed_time, 'sec')\n",
        "\n",
        "xgpu = torch.rand(dim, dim, device='cuda')\n",
        "start_time = time.time()\n",
        "y = torch.mm(xgpu, xgpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (GPU) =', elapsed_time, 'sec')\n",
        "\n",
        "dim=5000\n",
        "print('dim= ', dim)\n",
        "\n",
        "xcpu = torch.rand(dim, dim, device='cpu')\n",
        "start_time = time.time()\n",
        "y = torch.mm(xcpu, xcpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (CPU) =', elapsed_time, 'sec')\n",
        "\n",
        "xgpu = torch.rand(dim, dim, device='cuda')\n",
        "start_time = time.time()\n",
        "y = torch.mm(xgpu, xgpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (GPU) =', elapsed_time, 'sec')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dim=  50\n",
            "elapsed_time (CPU) = 0.0003695487976074219 sec\n",
            "elapsed_time (GPU) = 0.0005223751068115234 sec\n",
            "dim=  5000\n",
            "elapsed_time (CPU) = 3.503896713256836 sec\n",
            "elapsed_time (GPU) = 0.0044498443603515625 sec\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VImxUJmNIEn4"
      },
      "source": [
        "## Matrix determinant"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cN7kKY97Wruw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "69eee69d-6342-461f-f8c5-344cb5763602"
      },
      "source": [
        "dim=50\n",
        "print('dim= ', dim)\n",
        "\n",
        "xcpu = torch.rand(dim, dim, device='cpu')\n",
        "start_time = time.time()\n",
        "y = torch.linalg.det(xcpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (CPU) =', elapsed_time, 'sec')\n",
        "\n",
        "xgpu = torch.rand(dim, dim, device='cuda')\n",
        "start_time = time.time()\n",
        "y = torch.linalg.det(xgpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (GPU) =', elapsed_time, 'sec')\n",
        "\n",
        "dim=5000\n",
        "print('dim= ', dim)\n",
        "\n",
        "xcpu = torch.rand(dim, dim, device='cpu')\n",
        "start_time = time.time()\n",
        "y = torch.linalg.det(xcpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (CPU) =', elapsed_time, 'sec')\n",
        "\n",
        "xgpu = torch.rand(dim, dim, device='cuda')\n",
        "start_time = time.time()\n",
        "y = torch.linalg.det(xgpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (GPU) =', elapsed_time, 'sec')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dim=  50\n",
            "elapsed_time (CPU) = 0.05368828773498535 sec\n",
            "elapsed_time (GPU) = 0.07685184478759766 sec\n",
            "dim=  5000\n",
            "elapsed_time (CPU) = 1.45981764793396 sec\n",
            "elapsed_time (GPU) = 0.3555586338043213 sec\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SGp6SCvAILT1"
      },
      "source": [
        "## Eigenvalues"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cYzG5Pm1Jl4s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "80b0a1a1-09d9-4555-d8eb-99dfd0b5dea2"
      },
      "source": [
        "x = torch.tensor([[1.0,0.0],[0.0,3.0]])\n",
        "e,v = torch.linalg.eig(x)\n",
        "print(e)\n",
        "print(v)\n",
        "\n",
        "dim=50\n",
        "print('dim= ', dim)\n",
        "\n",
        "xcpu = torch.rand(dim, dim, device='cpu')\n",
        "start_time = time.time()\n",
        "e, v = torch.linalg.eig(xcpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (CPU) =', elapsed_time, 'sec')\n",
        "\n",
        "xgpu = torch.rand(dim, dim, device='cuda')\n",
        "start_time = time.time()\n",
        "e, v = torch.linalg.eig(xgpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (GPU) =', elapsed_time, 'sec')\n",
        "\n",
        "dim=5000\n",
        "print('dim= ', dim)\n",
        "\n",
        "xcpu = torch.rand(dim, dim, device='cpu')\n",
        "start_time = time.time()\n",
        "e, v = torch.linalg.eig(xcpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (CPU) =', elapsed_time, 'sec')\n",
        "\n",
        "xgpu = torch.rand(dim, dim, device='cuda')\n",
        "start_time = time.time()\n",
        "e, v = torch.linalg.eig(xgpu)\n",
        "elapsed_time = time.time() - start_time\n",
        "print('elapsed_time (GPU) =', elapsed_time, 'sec')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([1.+0.j, 3.+0.j])\n",
            "tensor([[1.+0.j, 0.+0.j],\n",
            "        [0.+0.j, 1.+0.j]])\n",
            "dim=  50\n",
            "elapsed_time (CPU) = 0.028529882431030273 sec\n",
            "elapsed_time (GPU) = 0.002012014389038086 sec\n",
            "dim=  5000\n",
            "elapsed_time (CPU) = 43.454078912734985 sec\n",
            "elapsed_time (GPU) = 23.43065309524536 sec\n"
          ]
        }
      ]
    }
  ]
}