Object_Detection_Workshop_Using_OpenCV_solution.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Object_Detection_Workshop_Using_OpenCV_solution.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "AFzE6b6WkS6y",
        "gRQgOxhikNsd",
        "pugSRpm0kQPm",
        "Omlj6WbKQZUq",
        "ZHU24FP-QqX_",
        "K3qmSEpkQ5HZ",
        "fOdCPEjtePTb",
        "bEnVxratU5x_"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AFzE6b6WkS6y",
        "colab_type": "text"
      },
      "source": [
        "# Install Darknet"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j-IZCz2mkLz3",
        "colab_type": "text"
      },
      "source": [
        "Run the cell below to setup the environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ArGr3sqduIdZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt-get update\n",
        "!apt-get upgrade\n",
        "!apt-get install build-essential\n",
        "!apt-get install cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev\n",
        "!apt-get install libavcodec-dev libavformat-dev libswscale-d\n",
        "!apt-get -y install cmake\n",
        "!apt-get install libopencv-dev\n",
        "!git clone https://github.com/AlexeyAB/darknet/\n",
        "!apt-get install vim\n",
        "%cd darknet\n",
        "!ls\n",
        "!wget https://pjreddie.com/media/files/yolov3.weights\n",
        "!ls\n",
        "!sed -i 's/OPENCV=0/OPENCV=1/g' Makefile\n",
        "!sed -i 's/CUDNN=0/CUDNN=1/g' Makefile\n",
        "!sed -i 's/GPU=0/GPU=1/g' Makefile\n",
        "#!vim Makefile\n",
        "!ls\n",
        "%cd ../\n",
        "!ls\n",
        "!apt install g++-5\n",
        "!apt install gcc-5\n",
        "\n",
        "!update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 10\n",
        "!update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 20\n",
        "!update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 10\n",
        "!update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 20\n",
        "!update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30\n",
        "!update-alternatives --set cc /usr/bin/gcc\n",
        "!update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30\n",
        "!update-alternatives --set c++ /usr/bin/g++\n",
        "!apt update -qq;\n",
        "!wget https://developer.nvidia.com/compute/cuda/8.0/Prod2/local_installers/cuda-repo-ubuntu1604-8-0-local-ga2_8.0.61-1_amd64-deb\n",
        "!dpkg -i cuda-repo-ubuntu1604-8-0-local-ga2_8.0.61-1_amd64-deb\n",
        "!apt-get update -qq\n",
        "\n",
        "!apt-get install cuda -y -qq #gcc-5 g++-5 \n",
        "!apt update\n",
        "!apt upgrade\n",
        "!apt install cuda-8.0 -y\n",
        "%cd darknet\n",
        "!make\n",
        "!ls\n",
        "!./darknet detect cfg/yolov3.cfg yolov3.weights data/person.jpg"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gRQgOxhikNsd",
        "colab_type": "text"
      },
      "source": [
        "# Import Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qcdCeU_eFOxI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import cv2 as cv\n",
        "import numpy as np\n",
        "import os.path\n",
        "import matplotlib.pyplot as plt\n",
        "from google.colab import files\n",
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pugSRpm0kQPm",
        "colab_type": "text"
      },
      "source": [
        "# Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3SOhbz5FoHUw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def img_show(path):\n",
        "    image = cv.imread(path)\n",
        "    height, width = image.shape[:2]\n",
        "    resized_image = cv.resize(image,(3*width, 3*height), interpolation = cv.INTER_CUBIC)\n",
        "\n",
        "    fig = plt.gcf()\n",
        "    fig.set_size_inches(18, 10)\n",
        "    plt.axis(\"off\")\n",
        "    #plt.rcParams['figure.figsize'] = [10, 5]\n",
        "    plt.imshow(cv.cvtColor(resized_image, cv.COLOR_BGR2RGB))\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3VVnAPP4Ator",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def upload():\n",
        "    uploaded = files.upload() \n",
        "    for name, data in uploaded.items():\n",
        "        with open(name, 'wb') as f:\n",
        "            f.write(data)\n",
        "        print ('saved file', name)\n",
        "\n",
        "def download(path):\n",
        "    files.download(path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Omlj6WbKQZUq",
        "colab_type": "text"
      },
      "source": [
        "# Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_bM4z47eoQco",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img_show('predictions.jpg')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eX-4qwysQdXa",
        "colab_type": "text"
      },
      "source": [
        "# YOLO - OpenCV Implementation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZHU24FP-QqX_",
        "colab_type": "text"
      },
      "source": [
        "## Hyper Params and Network Settings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4yaCOEbVFi0a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# YOLO Params\n",
        "conf_threshold = 0.5\n",
        "nms_threshold = 0.4\n",
        "input_width = 416\n",
        "input_height = 416\n",
        "\n",
        "\n",
        "# Load label names\n",
        "classFile = 'data/coco.names'\n",
        "classes = None\n",
        "\n",
        "with open(classFile, 'rt') as f:\n",
        "    classes = f.read().rstrip('\\n').split('\\n')\n",
        "\n",
        "# get config and weight files\n",
        "model_config = 'cfg/yolov3.cfg'\n",
        "model_weights = 'yolov3.weights'\n",
        "\n",
        "# create the network using the loaded files\n",
        "net = cv.dnn.readNetFromDarknet(model_config, model_weights)\n",
        "net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV)\n",
        "net.setPreferableTarget(cv.dnn.DNN_TARGET_OPENCL)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K3qmSEpkQ5HZ",
        "colab_type": "text"
      },
      "source": [
        "## Get the label names"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SjXPSyeKF1X4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_output_names(net):\n",
        "    # Grab the names of all the layers in the network\n",
        "    layer_names = net.getLayerNames()\n",
        "    # Get the names from the output layer\n",
        "    return [layer_names[i[0] - 1] for i in net.getUnconnectedOutLayers()]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fOdCPEjtePTb",
        "colab_type": "text"
      },
      "source": [
        "## Drawing the Bounding Boxes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k4sR3wM7F477",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def draw_pred(frame, class_id, conf, left, top, right, bot):\n",
        "    # Draw the bounding box\n",
        "    frame = cv.rectangle(frame, (left, top), (right, bot), (255,178,50), 3)\n",
        "    label = '%.2f' % conf\n",
        "\n",
        "    # Get the label and confidence score\n",
        "\n",
        "    if classes:\n",
        "        assert(class_id < len(classes))\n",
        "        label = f'{classes[class_id]} : {label}'\n",
        "\n",
        "\n",
        "    # Display the label at the top of the bounding box\n",
        "    label_size, base_line = cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, 0.5, 1)\n",
        "    top = max(top, label_size[1])\n",
        "    frame = cv.rectangle(frame, (left, top - round(1.5 * label_size[1])),\n",
        "                        (left + round(1.5 * label_size[0]), top + base_line),\n",
        "                        (255,255,255), cv.FILLED)\n",
        "    frame = cv.putText(frame, label, (left, top), cv.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 0), 1)\n",
        "    return frame"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "504nlo5qYla7",
        "colab_type": "text"
      },
      "source": [
        "## Removing Bounding Boxes with Low Confidence"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YFuNk8EzYmNu",
        "colab_type": "text"
      },
      "source": [
        "**PARAMS**\n",
        "frame(image matrix) = The pixel data of the current frame being analyzed\n",
        "outputs(ND-Array) = All the predicted bounding boxes\n",
        "\n",
        "**OUTPUT**\n",
        "\n",
        "The current frame with the detected bounding boxes drawn on top.\n",
        "\n",
        "**TODO**\n",
        "\n",
        "1.) For each Bounding Box get the highest scoring class label.\n",
        "\n",
        "2.) Get the confidence score for the winning label.\n",
        "\n",
        "3.) Check that the confidence score is higher than the set threshold.\n",
        "\n",
        "4.) Get the location of the predicted box.\n",
        "\n",
        "5.) Use Non-Maximum Suppression to get rid of redundant overlapping boxes.\n",
        "\n",
        "6.) Call draw_pred to draw the bounding boxes.\n",
        "\n",
        "7.) return the frame with all the bounding boxes added."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FQN8AFlZGFU1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def postprocess(frame, outputs):\n",
        "    frame_height = frame.shape[0]\n",
        "    frame_width = frame.shape[1]\n",
        "\n",
        "    '''\n",
        "    Scan through all the bounding boxes output from the network and \n",
        "    keep the ones with a high confidence score. Assign the box's class label as\n",
        "    the class with the highest score\n",
        "    '''\n",
        "\n",
        "    class_ids = []\n",
        "    confidences = []\n",
        "    boxes = []\n",
        "\n",
        "    for out in outputs:\n",
        "        for detection in out:\n",
        "            scores = detection[5:]\n",
        "            class_id = np.argmax(scores)\n",
        "            confidence = scores[class_id]\n",
        "            # Checking whether the score meets our confidence threshold\n",
        "            if confidence > conf_threshold:\n",
        "                center_x = int(detection[0] * frame_width)\n",
        "                center_y = int(detection[1] * frame_height)\n",
        "                width = int(detection[2] * frame_width)\n",
        "                height = int(detection[3] * frame_height)\n",
        "                left = int(center_x - width / 2)\n",
        "                top = int(center_y - height / 2)\n",
        "                class_ids.append(class_id)\n",
        "                confidences.append(float(confidence))\n",
        "                boxes.append([left, top, width, height])\n",
        "\n",
        "    # Perform Non-Maximum Suppression to eliminate redundant overlapping boxes\n",
        "    indices = cv.dnn.NMSBoxes(boxes, confidences, conf_threshold, nms_threshold)\n",
        "    for i in indices:\n",
        "        i = i[0]\n",
        "        box = boxes[i]\n",
        "        left = box[0]\n",
        "        top = box[1]\n",
        "        width = box[2]\n",
        "        height = box[3]\n",
        "        draw_pred(class_ids[i], confidences[i], left, top, left + width, top + height)\n",
        "        \n",
        "    return frame"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JBnUWtTwUORd",
        "colab_type": "text"
      },
      "source": [
        "## Preprocess Input"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rwe_XweZUzmd",
        "colab_type": "text"
      },
      "source": [
        "**PARAMS**\n",
        "\n",
        "input_file(str) = name of the file to be processed. Location should be part of the string if file is not in the root directory.\n",
        "\n",
        "        ex: 'dog.jpg' or 'data/cat.jpg'\n",
        "        \n",
        "file_type(str) = The type of the input file. It can be either 'image' or 'video'\n",
        "\n",
        "**OUTPUT**\n",
        "\n",
        "Return True if the successful, False otherwise\n",
        "\n",
        "**TODO:**\n",
        "\n",
        "1.) Check whether file_type is 'image' or 'video' and set the output to the suitable format.\n",
        "\n",
        "2.) Get the current frame pixel data.\n",
        "\n",
        "3.) Create a blob with the correct dimensions to feed into the network\n",
        "\n",
        "4.) Set the input to the network using the blob\n",
        "\n",
        "5.) Run the a forward pass through the network to get the predicted boxes\n",
        "\n",
        "6.) Run all the predicted boxes through postprocess to filter out the weak predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CZdSO5FSD3G5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def yolo(input_file, file_type='video'):\n",
        "    output_file = ''\n",
        "    \n",
        "    if not os.path.isfile(input_file):\n",
        "        print('Input file ', input_file, ' wasn\\'t found')\n",
        "        return False\n",
        "\n",
        "    # capture the raw input\n",
        "    cap = cv.VideoCapture(input_file)\n",
        "    \n",
        "    # set the correct file type for the output\n",
        "    if file_type == 'image':\n",
        "        output_file = input_file[:-4] + '_yolo_output.jpg'\n",
        "\n",
        "    elif file_type == 'video':\n",
        "        output_file = input_file[:-4] + '_yolo_output.avi'\n",
        "        vid_writer = cv.VideoWriter(output_file,\n",
        "                                    cv.VideoWriter_fourcc('M', 'J', 'P', 'G'),\n",
        "                                    24, (round(cap.get(cv.CAP_PROP_FRAME_WIDTH)),\n",
        "                                         round(cap.get(cv.CAP_PROP_FRAME_HEIGHT))))\n",
        "\n",
        "    else:\n",
        "        print('Invalid file type')\n",
        "        return False\n",
        "        \n",
        "\n",
        "    while True:\n",
        "        has_frame, frame = cap.read()\n",
        "\n",
        "        if not has_frame:\n",
        "            print('Done processing:')\n",
        "            print('Output file is stored as: ', output_file)\n",
        "            cap.release()\n",
        "            break\n",
        "\n",
        "        # preprocess the input so that it can be fed to the network\n",
        "        blob = cv.dnn.blobFromImage(frame, 1/255, (input_width, input_height),\n",
        "                                    [0, 0, 0], 1, crop=False)\n",
        "        net.setInput(blob)\n",
        "        # Do a forward pass through the network to get all the predictions\n",
        "        outs = net.forward(get_output_names(net))\n",
        "        # filter out weak and redundant predictions and draw the winning bounding boxes\n",
        "        frame = postprocess(frame, outs)\n",
        "\n",
        "\n",
        "        # get efficiency information\n",
        "        t, _ = net.getPerfProfile()\n",
        "        label = 'Inference time: %.2f ms' % (t * 1000.0 / cv.getTickFrequency())\n",
        "        cv.putText(frame, label, (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255))\n",
        "\n",
        "        # Write the frame with the detection boxes\n",
        "        if file_type == 'image':\n",
        "            cv.imwrite(output_file, frame.astype(np.uint8))\n",
        "        else:\n",
        "            vid_writer.write(frame.astype(np.uint8))\n",
        "\n",
        "    return True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bEnVxratU5x_",
        "colab_type": "text"
      },
      "source": [
        "# Image Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8M4nkS0hoSh7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd data\n",
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Px6MMNRjohMq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd ..\n",
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DoC7vcW_wZQ8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "yolo('data/eagle.jpg', 'image')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BUzhGEmLxvha",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "imShow('data/eagle_yolo_output.jpg')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OSczGeKnTRkh",
        "colab_type": "text"
      },
      "source": [
        "Use the cell below to process videos with YOLO on darknet. Use the upload() helper function to upload the video you'd like to run through the network and then use download() to get the output back from colab."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSmZ98zcqo8g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!./darknet detector demo cfg/coco.data cfg/yolov3.cfg yolov3.weights -dont_show video.mp4 -i 0 -out_filename yolo_out.avi"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}