wrangle_act.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Project: Wrangling and Analyze Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Gathering\n",
    "In the cell below, gather **all** three pieces of data for this project and load them in the notebook. **Note:** the methods required to gather each data are different.\n",
    "1. Directly download the WeRateDogs Twitter archive data (twitter_archive_enhanced.csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-08-01 16:23:56 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Phineas. He's a mystical boy. Only eve...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Phineas</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-08-01 00:17:27 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Tilly. She's just checking pup on you....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Tilly</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-31 00:18:03 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Archie. He is a rare Norwegian Pouncin...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891815181...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Archie</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-30 15:58:51 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Darla. She commenced a snooze mid meal...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891689557...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Darla</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-29 16:00:24 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Franklin. He would like you to stop ca...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891327558...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Franklin</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id  in_reply_to_status_id  in_reply_to_user_id  \\\n",
       "0  892420643555336193                    NaN                  NaN   \n",
       "1  892177421306343426                    NaN                  NaN   \n",
       "2  891815181378084864                    NaN                  NaN   \n",
       "3  891689557279858688                    NaN                  NaN   \n",
       "4  891327558926688256                    NaN                  NaN   \n",
       "\n",
       "                   timestamp  \\\n",
       "0  2017-08-01 16:23:56 +0000   \n",
       "1  2017-08-01 00:17:27 +0000   \n",
       "2  2017-07-31 00:18:03 +0000   \n",
       "3  2017-07-30 15:58:51 +0000   \n",
       "4  2017-07-29 16:00:24 +0000   \n",
       "\n",
       "                                              source  \\\n",
       "0  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "3  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "4  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                text  retweeted_status_id  \\\n",
       "0  This is Phineas. He's a mystical boy. Only eve...                  NaN   \n",
       "1  This is Tilly. She's just checking pup on you....                  NaN   \n",
       "2  This is Archie. He is a rare Norwegian Pouncin...                  NaN   \n",
       "3  This is Darla. She commenced a snooze mid meal...                  NaN   \n",
       "4  This is Franklin. He would like you to stop ca...                  NaN   \n",
       "\n",
       "   retweeted_status_user_id retweeted_status_timestamp  \\\n",
       "0                       NaN                        NaN   \n",
       "1                       NaN                        NaN   \n",
       "2                       NaN                        NaN   \n",
       "3                       NaN                        NaN   \n",
       "4                       NaN                        NaN   \n",
       "\n",
       "                                       expanded_urls  rating_numerator  \\\n",
       "0  https://twitter.com/dog_rates/status/892420643...                13   \n",
       "1  https://twitter.com/dog_rates/status/892177421...                13   \n",
       "2  https://twitter.com/dog_rates/status/891815181...                12   \n",
       "3  https://twitter.com/dog_rates/status/891689557...                13   \n",
       "4  https://twitter.com/dog_rates/status/891327558...                12   \n",
       "\n",
       "   rating_denominator      name doggo floofer pupper puppo  \n",
       "0                  10   Phineas  None    None   None  None  \n",
       "1                  10     Tilly  None    None   None  None  \n",
       "2                  10    Archie  None    None   None  None  \n",
       "3                  10     Darla  None    None   None  None  \n",
       "4                  10  Franklin  None    None   None  None  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import tweepy\n",
    "import glob\n",
    "import numpy as np\n",
    "import os\n",
    "import json\n",
    "from bs4 import BeautifulSoup\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "df = pd.read_csv('twitter-archive-enhanced.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Use the Requests library to download the tweet image prediction (image_predictions.tsv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_url=[' https://d17h27t6h515a5.cloudfront.net/topher/2017/August/599fd2ad_image-predictions/image-predictions.tsv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_name = 'tweet_image'\n",
    "if not os.path.exists(folder_name):\n",
    "    os.makedirs(folder_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "for url in tweet_url :\n",
    "    response = requests.get(url)\n",
    "    with open(os.path.join(folder_name,url.split('/')[-1]),mode=\"wb\") as file :\n",
    "        file.write(response.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>jpg_url</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>673612854080196609</td>\n",
       "      <td>https://pbs.twimg.com/media/CVknUTlVEAARjU5.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Newfoundland</td>\n",
       "      <td>0.223101</td>\n",
       "      <td>True</td>\n",
       "      <td>Leonberg</td>\n",
       "      <td>0.111106</td>\n",
       "      <td>True</td>\n",
       "      <td>shovel</td>\n",
       "      <td>0.085626</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1233</th>\n",
       "      <td>746131877086527488</td>\n",
       "      <td>https://pbs.twimg.com/media/ClrK-rGWAAENcAa.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>chow</td>\n",
       "      <td>0.575637</td>\n",
       "      <td>True</td>\n",
       "      <td>Pomeranian</td>\n",
       "      <td>0.195950</td>\n",
       "      <td>True</td>\n",
       "      <td>Norwich_terrier</td>\n",
       "      <td>0.141224</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>899</th>\n",
       "      <td>699788877217865730</td>\n",
       "      <td>https://pbs.twimg.com/media/CbYmRHyWEAASNzm.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Border_terrier</td>\n",
       "      <td>0.355060</td>\n",
       "      <td>True</td>\n",
       "      <td>toy_poodle</td>\n",
       "      <td>0.169736</td>\n",
       "      <td>True</td>\n",
       "      <td>Norwegian_elkhound</td>\n",
       "      <td>0.099884</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1736</th>\n",
       "      <td>821813639212650496</td>\n",
       "      <td>https://pbs.twimg.com/media/CtVAvX-WIAAcGTf.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Saint_Bernard</td>\n",
       "      <td>0.995143</td>\n",
       "      <td>True</td>\n",
       "      <td>Cardigan</td>\n",
       "      <td>0.003044</td>\n",
       "      <td>True</td>\n",
       "      <td>English_springer</td>\n",
       "      <td>0.001050</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>666073100786774016</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5d9DZXAAALcwe.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Walker_hound</td>\n",
       "      <td>0.260857</td>\n",
       "      <td>True</td>\n",
       "      <td>English_foxhound</td>\n",
       "      <td>0.175382</td>\n",
       "      <td>True</td>\n",
       "      <td>Ibizan_hound</td>\n",
       "      <td>0.097471</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id                                          jpg_url  \\\n",
       "394   673612854080196609  https://pbs.twimg.com/media/CVknUTlVEAARjU5.jpg   \n",
       "1233  746131877086527488  https://pbs.twimg.com/media/ClrK-rGWAAENcAa.jpg   \n",
       "899   699788877217865730  https://pbs.twimg.com/media/CbYmRHyWEAASNzm.jpg   \n",
       "1736  821813639212650496  https://pbs.twimg.com/media/CtVAvX-WIAAcGTf.jpg   \n",
       "12    666073100786774016  https://pbs.twimg.com/media/CT5d9DZXAAALcwe.jpg   \n",
       "\n",
       "      img_num              p1   p1_conf  p1_dog                p2   p2_conf  \\\n",
       "394         1    Newfoundland  0.223101    True          Leonberg  0.111106   \n",
       "1233        1            chow  0.575637    True        Pomeranian  0.195950   \n",
       "899         1  Border_terrier  0.355060    True        toy_poodle  0.169736   \n",
       "1736        1   Saint_Bernard  0.995143    True          Cardigan  0.003044   \n",
       "12          1    Walker_hound  0.260857    True  English_foxhound  0.175382   \n",
       "\n",
       "      p2_dog                  p3   p3_conf  p3_dog  \n",
       "394     True              shovel  0.085626   False  \n",
       "1233    True     Norwich_terrier  0.141224    True  \n",
       "899     True  Norwegian_elkhound  0.099884    True  \n",
       "1736    True    English_springer  0.001050    True  \n",
       "12      True        Ibizan_hound  0.097471    True  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images_df = pd.read_csv('image-predictions.tsv',sep='\\t')\n",
    "images_df.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. Use the Tweepy library to query additional data via the Twitter API (tweet_json.txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets=[]\n",
    "with open ('tweet-json.txt', 'r') as outfile:\n",
    "    for line in outfile:\n",
    "        data =json.loads(line)\n",
    "        tweets.append({'Tweet_id':data['id'],'Retweets': data['retweet_count'],'Favorites':data['favorite_count']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_df = pd.DataFrame(tweets, columns = ['Tweet_id','Retweets','Favorites'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet_id</th>\n",
       "      <th>Retweets</th>\n",
       "      <th>Favorites</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>8853</td>\n",
       "      <td>39467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>6514</td>\n",
       "      <td>33819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>4328</td>\n",
       "      <td>25461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>8964</td>\n",
       "      <td>42908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>9774</td>\n",
       "      <td>41048</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Tweet_id  Retweets  Favorites\n",
       "0  892420643555336193      8853      39467\n",
       "1  892177421306343426      6514      33819\n",
       "2  891815181378084864      4328      25461\n",
       "3  891689557279858688      8964      42908\n",
       "4  891327558926688256      9774      41048"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 4,
        "hidden": false,
        "row": 28,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "## Assessing Data\n",
    "In this section, detect and document at least **eight (8) quality issues and two (2) tidiness issue**. You must use **both** visual assessment\n",
    "programmatic assessement to assess the data.\n",
    "\n",
    "**Note:** pay attention to the following key points when you access the data.\n",
    "\n",
    "* You only want original ratings (no retweets) that have images. Though there are 5000+ tweets in the dataset, not all are dog ratings and some are retweets.\n",
    "* Assessing and cleaning the entire dataset completely would require a lot of time, and is not necessary to practice and demonstrate your skills in data wrangling. Therefore, the requirements of this project are only to assess and clean at least 8 quality issues and at least 2 tidiness issues in this dataset.\n",
    "* The fact that the rating numerators are greater than the denominators does not need to be cleaned. This [unique rating system](http://knowyourmeme.com/memes/theyre-good-dogs-brent) is a big part of the popularity of WeRateDogs.\n",
    "* You do not need to gather the tweets beyond August 1st, 2017. You can, but note that you won't be able to gather the image predictions for these tweets since you don't have access to the algorithm used.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2356 entries, 0 to 2355\n",
      "Data columns (total 17 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   tweet_id                    2356 non-null   int64  \n",
      " 1   in_reply_to_status_id       78 non-null     float64\n",
      " 2   in_reply_to_user_id         78 non-null     float64\n",
      " 3   timestamp                   2356 non-null   object \n",
      " 4   source                      2356 non-null   object \n",
      " 5   text                        2356 non-null   object \n",
      " 6   retweeted_status_id         181 non-null    float64\n",
      " 7   retweeted_status_user_id    181 non-null    float64\n",
      " 8   retweeted_status_timestamp  181 non-null    object \n",
      " 9   expanded_urls               2297 non-null   object \n",
      " 10  rating_numerator            2356 non-null   int64  \n",
      " 11  rating_denominator          2356 non-null   int64  \n",
      " 12  name                        2356 non-null   object \n",
      " 13  doggo                       2356 non-null   object \n",
      " 14  floofer                     2356 non-null   object \n",
      " 15  pupper                      2356 non-null   object \n",
      " 16  puppo                       2356 non-null   object \n",
      "dtypes: float64(4), int64(3), object(10)\n",
      "memory usage: 220.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2278"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df.in_reply_to_status_id.isnull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "None     2326\n",
       "puppo      30\n",
       "Name: puppo, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.puppo.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "None      2099\n",
       "pupper     257\n",
       "Name: pupper, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.pupper.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10     2333\n",
       "11        3\n",
       "50        3\n",
       "80        2\n",
       "20        2\n",
       "2         1\n",
       "16        1\n",
       "40        1\n",
       "70        1\n",
       "15        1\n",
       "90        1\n",
       "110       1\n",
       "120       1\n",
       "130       1\n",
       "150       1\n",
       "170       1\n",
       "7         1\n",
       "0         1\n",
       "Name: rating_denominator, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.rating_denominator.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "None     2259\n",
       "doggo      97\n",
       "Name: doggo, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.doggo.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "None       745\n",
       "a           55\n",
       "Charlie     12\n",
       "Lucy        11\n",
       "Cooper      11\n",
       "Oliver      11\n",
       "Lola        10\n",
       "Tucker      10\n",
       "Penny       10\n",
       "Bo           9\n",
       "Name: name, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.name.value_counts().head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10     2333\n",
       "11        3\n",
       "50        3\n",
       "80        2\n",
       "20        2\n",
       "2         1\n",
       "16        1\n",
       "40        1\n",
       "70        1\n",
       "15        1\n",
       "90        1\n",
       "110       1\n",
       "120       1\n",
       "130       1\n",
       "150       1\n",
       "170       1\n",
       "7         1\n",
       "0         1\n",
       "Name: rating_denominator, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.rating_denominator.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12      558\n",
       "11      464\n",
       "10      461\n",
       "13      351\n",
       "9       158\n",
       "8       102\n",
       "7        55\n",
       "14       54\n",
       "5        37\n",
       "6        32\n",
       "3        19\n",
       "4        17\n",
       "1         9\n",
       "2         9\n",
       "420       2\n",
       "0         2\n",
       "15        2\n",
       "75        2\n",
       "80        1\n",
       "20        1\n",
       "24        1\n",
       "26        1\n",
       "44        1\n",
       "50        1\n",
       "60        1\n",
       "165       1\n",
       "84        1\n",
       "88        1\n",
       "144       1\n",
       "182       1\n",
       "143       1\n",
       "666       1\n",
       "960       1\n",
       "1776      1\n",
       "17        1\n",
       "27        1\n",
       "45        1\n",
       "99        1\n",
       "121       1\n",
       "204       1\n",
       "Name: rating_numerator, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.rating_numerator.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "https://twitter.com/dog_rates/status/837820167694528512/photo/1,https://twitter.com/dog_rates/status/837820167694528512/photo/1                                                                                                                                    2\n",
       "https://twitter.com/dog_rates/status/783839966405230592/photo/1,https://twitter.com/dog_rates/status/783839966405230592/photo/1,https://twitter.com/dog_rates/status/783839966405230592/photo/1                                                                    2\n",
       "https://twitter.com/dog_rates/status/740373189193256964/photo/1,https://twitter.com/dog_rates/status/740373189193256964/photo/1,https://twitter.com/dog_rates/status/740373189193256964/photo/1,https://twitter.com/dog_rates/status/740373189193256964/photo/1    2\n",
       "https://twitter.com/dog_rates/status/866334964761202691/photo/1,https://twitter.com/dog_rates/status/866334964761202691/photo/1                                                                                                                                    2\n",
       "https://twitter.com/dog_rates/status/809920764300447744/photo/1                                                                                                                                                                                                    2\n",
       "                                                                                                                                                                                                                                                                  ..\n",
       "https://twitter.com/littlewiewel/status/800852955880628224,https://twitter.com/littlewiewel/status/800852955880628224                                                                                                                                              1\n",
       "https://twitter.com/dog_rates/status/883117836046086144/photo/1,https://twitter.com/dog_rates/status/883117836046086144/photo/1                                                                                                                                    1\n",
       "https://twitter.com/dog_rates/status/809084759137812480/photo/1                                                                                                                                                                                                    1\n",
       "https://twitter.com/dog_rates/status/807106840509214720/video/1                                                                                                                                                                                                    1\n",
       "https://twitter.com/dog_rates/status/675878199931371520/photo/1                                                                                                                                                                                                    1\n",
       "Name: expanded_urls, Length: 2218, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.expanded_urls.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>     2221\n",
       "<a href=\"http://vine.co\" rel=\"nofollow\">Vine - Make a Scene</a>                          91\n",
       "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>                       33\n",
       "<a href=\"https://about.twitter.com/products/tweetdeck\" rel=\"nofollow\">TweetDeck</a>      11\n",
       "Name: source, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.source.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        179\n",
       "610        3\n",
       "345        3\n",
       "2918       3\n",
       "1691       3\n",
       "        ... \n",
       "33345      1\n",
       "814        1\n",
       "23108      1\n",
       "2630       1\n",
       "8143       1\n",
       "Name: Favorites, Length: 2007, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_df.Favorites.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1972     5\n",
       "3652     5\n",
       "83       5\n",
       "146      4\n",
       "61       4\n",
       "        ..\n",
       "8209     1\n",
       "19       1\n",
       "2068     1\n",
       "30742    1\n",
       "0        1\n",
       "Name: Retweets, Length: 1724, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_df.Retweets.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2175"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df.retweeted_status_id.isnull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df.expanded_urls.isnull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df.tweet_id.duplicated())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2356"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1914    6.658147e+17\n",
       "2298    6.670655e+17\n",
       "1339    6.671522e+17\n",
       "149     6.671522e+17\n",
       "2169    6.678065e+17\n",
       "            ...     \n",
       "2351             NaN\n",
       "2352             NaN\n",
       "2353             NaN\n",
       "2354             NaN\n",
       "2355             NaN\n",
       "Name: in_reply_to_status_id, Length: 2356, dtype: float64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.in_reply_to_status_id.sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(images_df.tweet_id.duplicated())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2075 entries, 0 to 2074\n",
      "Data columns (total 12 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   tweet_id  2075 non-null   int64  \n",
      " 1   jpg_url   2075 non-null   object \n",
      " 2   img_num   2075 non-null   int64  \n",
      " 3   p1        2075 non-null   object \n",
      " 4   p1_conf   2075 non-null   float64\n",
      " 5   p1_dog    2075 non-null   bool   \n",
      " 6   p2        2075 non-null   object \n",
      " 7   p2_conf   2075 non-null   float64\n",
      " 8   p2_dog    2075 non-null   bool   \n",
      " 9   p3        2075 non-null   object \n",
      " 10  p3_conf   2075 non-null   float64\n",
      " 11  p3_dog    2075 non-null   bool   \n",
      "dtypes: bool(3), float64(3), int64(2), object(4)\n",
      "memory usage: 119.6+ KB\n"
     ]
    }
   ],
   "source": [
    "images_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1438</th>\n",
       "      <td>696900204696625153</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-02-09 03:35:31 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Rosie. She's a Benebark Cumberpatch. S...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/696900204...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Rosie</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id  in_reply_to_status_id  in_reply_to_user_id  \\\n",
       "1438  696900204696625153                    NaN                  NaN   \n",
       "\n",
       "                      timestamp  \\\n",
       "1438  2016-02-09 03:35:31 +0000   \n",
       "\n",
       "                                                 source  \\\n",
       "1438  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                   text  retweeted_status_id  \\\n",
       "1438  This is Rosie. She's a Benebark Cumberpatch. S...                  NaN   \n",
       "\n",
       "      retweeted_status_user_id retweeted_status_timestamp  \\\n",
       "1438                       NaN                        NaN   \n",
       "\n",
       "                                          expanded_urls  rating_numerator  \\\n",
       "1438  https://twitter.com/dog_rates/status/696900204...                12   \n",
       "\n",
       "      rating_denominator   name doggo floofer pupper puppo  \n",
       "1438                  10  Rosie  None    None   None  None  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>jpg_url</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1827</th>\n",
       "      <td>835297930240217089</td>\n",
       "      <td>https://pbs.twimg.com/media/C5eTCOVUsAAWhvc.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Rottweiler</td>\n",
       "      <td>0.341276</td>\n",
       "      <td>True</td>\n",
       "      <td>Border_terrier</td>\n",
       "      <td>0.336220</td>\n",
       "      <td>True</td>\n",
       "      <td>Gordon_setter</td>\n",
       "      <td>0.045448</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>670385711116361728</td>\n",
       "      <td>https://pbs.twimg.com/media/CU2wPyWWUAAb1MJ.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>whippet</td>\n",
       "      <td>0.178027</td>\n",
       "      <td>True</td>\n",
       "      <td>Chesapeake_Bay_retriever</td>\n",
       "      <td>0.105969</td>\n",
       "      <td>True</td>\n",
       "      <td>beagle</td>\n",
       "      <td>0.078720</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>670411370698022913</td>\n",
       "      <td>https://pbs.twimg.com/media/CU3HlZtW4AAezbt.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Maltese_dog</td>\n",
       "      <td>0.584397</td>\n",
       "      <td>True</td>\n",
       "      <td>miniature_schnauzer</td>\n",
       "      <td>0.064201</td>\n",
       "      <td>True</td>\n",
       "      <td>toy_poodle</td>\n",
       "      <td>0.060868</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id                                          jpg_url  \\\n",
       "1827  835297930240217089  https://pbs.twimg.com/media/C5eTCOVUsAAWhvc.jpg   \n",
       "228   670385711116361728  https://pbs.twimg.com/media/CU2wPyWWUAAb1MJ.jpg   \n",
       "231   670411370698022913  https://pbs.twimg.com/media/CU3HlZtW4AAezbt.jpg   \n",
       "\n",
       "      img_num           p1   p1_conf  p1_dog                        p2  \\\n",
       "1827        1   Rottweiler  0.341276    True            Border_terrier   \n",
       "228         1      whippet  0.178027    True  Chesapeake_Bay_retriever   \n",
       "231         1  Maltese_dog  0.584397    True       miniature_schnauzer   \n",
       "\n",
       "       p2_conf  p2_dog             p3   p3_conf  p3_dog  \n",
       "1827  0.336220    True  Gordon_setter  0.045448    True  \n",
       "228   0.105969    True         beagle  0.078720    True  \n",
       "231   0.064201    True     toy_poodle  0.060868    True  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images_df.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(tweet_df.Tweet_id.duplicated())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tweet_id', 'Retweets', 'Favorites']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(tweet_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality issues\n",
    "\n",
    "#### df  :\n",
    "\n",
    "1. Nulls represented as nones in the doggo,floofer,pupper and puppo columns.\n",
    "\n",
    "2. After melting the four columns we will need to remove dupilcates.\n",
    "\n",
    "3. then reset the index for the table.\n",
    "\n",
    "3. convert the timestamp and retweeted_status_timestamp columns data type to datetime data type,\n",
    "\n",
    "4. change the data type for in_reply_to_status_id, in_reply_to_user_id, retweeted_status_id and retweeted_status_user_id columns from float to integer.\n",
    "\n",
    "#### Master_df :\n",
    "\n",
    "6. remove unwanted columns of the master table.\n",
    "\n",
    "7. remove the raws that doesnt have jpg_url.(You only want original ratings (no retweets) that have images).\n",
    "\n",
    "8. Nulls represented as nones, a,an and the in the dog_name columns.\n",
    "\n",
    "11. remove duplicates.\n",
    "\n",
    "12. reset index.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 7,
        "hidden": false,
        "row": 40,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "### Tidiness issues\n",
    "\n",
    "1. melt the four columns doggo,floofer,pupper and puppo into one column Type\n",
    "\n",
    "2. merge the three dataframes.\n",
    "\n",
    "4. reorder columns in master_df.\n",
    "\n",
    "5. rename columns in master_df.\n",
    "\n",
    "6. images_df table: lowercase for p1,p2 and p3 columns. \n",
    "\n",
    "7. lowercase prediction column master_df.\n",
    "\n",
    "8. columns name lowercase master_df."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 4,
        "hidden": false,
        "row": 32,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "## Cleaning Data\n",
    "In this section, clean **all** of the issues you documented while assessing. \n",
    "\n",
    "**Note:** Make a copy of the original data before cleaning. Cleaning includes merging individual pieces of data according to the rules of [tidy data](https://cran.r-project.org/web/packages/tidyr/vignettes/tidy-data.html). The result should be a high-quality and tidy master pandas DataFrame (or DataFrames, if appropriate)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make copies of original pieces of data\n",
    "df_clean = df.copy()\n",
    "tweet_df_clean = tweet_df.copy()\n",
    "images_df_clean = images_df.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Quality issue #1  :\n",
    "\n",
    "we better start with tidying issues but we need to melt the four columns first so we have to reconverted the nulls."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define:\n",
    "\n",
    "Nulls represented as nones in doggo,floofer,pupper and puppo columns."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clean.doggo = df_clean.doggo.replace('None',np.nan)\n",
    "df_clean.floofer = df_clean.floofer.replace('None',np.nan)\n",
    "df_clean.pupper = df_clean.pupper.replace('None',np.nan)\n",
    "df_clean.puppo = df_clean.puppo.replace('None',np.nan)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-08-01 16:23:56 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Phineas. He's a mystical boy. Only eve...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Phineas</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-08-01 00:17:27 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Tilly. She's just checking pup on you....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Tilly</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-31 00:18:03 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Archie. He is a rare Norwegian Pouncin...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891815181...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Archie</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-30 15:58:51 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Darla. She commenced a snooze mid meal...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891689557...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Darla</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-29 16:00:24 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Franklin. He would like you to stop ca...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891327558...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Franklin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id  in_reply_to_status_id  in_reply_to_user_id  \\\n",
       "0  892420643555336193                    NaN                  NaN   \n",
       "1  892177421306343426                    NaN                  NaN   \n",
       "2  891815181378084864                    NaN                  NaN   \n",
       "3  891689557279858688                    NaN                  NaN   \n",
       "4  891327558926688256                    NaN                  NaN   \n",
       "\n",
       "                   timestamp  \\\n",
       "0  2017-08-01 16:23:56 +0000   \n",
       "1  2017-08-01 00:17:27 +0000   \n",
       "2  2017-07-31 00:18:03 +0000   \n",
       "3  2017-07-30 15:58:51 +0000   \n",
       "4  2017-07-29 16:00:24 +0000   \n",
       "\n",
       "                                              source  \\\n",
       "0  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "3  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "4  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                text  retweeted_status_id  \\\n",
       "0  This is Phineas. He's a mystical boy. Only eve...                  NaN   \n",
       "1  This is Tilly. She's just checking pup on you....                  NaN   \n",
       "2  This is Archie. He is a rare Norwegian Pouncin...                  NaN   \n",
       "3  This is Darla. She commenced a snooze mid meal...                  NaN   \n",
       "4  This is Franklin. He would like you to stop ca...                  NaN   \n",
       "\n",
       "   retweeted_status_user_id retweeted_status_timestamp  \\\n",
       "0                       NaN                        NaN   \n",
       "1                       NaN                        NaN   \n",
       "2                       NaN                        NaN   \n",
       "3                       NaN                        NaN   \n",
       "4                       NaN                        NaN   \n",
       "\n",
       "                                       expanded_urls  rating_numerator  \\\n",
       "0  https://twitter.com/dog_rates/status/892420643...                13   \n",
       "1  https://twitter.com/dog_rates/status/892177421...                13   \n",
       "2  https://twitter.com/dog_rates/status/891815181...                12   \n",
       "3  https://twitter.com/dog_rates/status/891689557...                13   \n",
       "4  https://twitter.com/dog_rates/status/891327558...                12   \n",
       "\n",
       "   rating_denominator      name doggo floofer pupper puppo  \n",
       "0                  10   Phineas   NaN     NaN    NaN   NaN  \n",
       "1                  10     Tilly   NaN     NaN    NaN   NaN  \n",
       "2                  10    Archie   NaN     NaN    NaN   NaN  \n",
       "3                  10     Darla   NaN     NaN    NaN   NaN  \n",
       "4                  10  Franklin   NaN     NaN    NaN   NaN  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2356 entries, 0 to 2355\n",
      "Data columns (total 17 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   tweet_id                    2356 non-null   int64  \n",
      " 1   in_reply_to_status_id       78 non-null     float64\n",
      " 2   in_reply_to_user_id         78 non-null     float64\n",
      " 3   timestamp                   2356 non-null   object \n",
      " 4   source                      2356 non-null   object \n",
      " 5   text                        2356 non-null   object \n",
      " 6   retweeted_status_id         181 non-null    float64\n",
      " 7   retweeted_status_user_id    181 non-null    float64\n",
      " 8   retweeted_status_timestamp  181 non-null    object \n",
      " 9   expanded_urls               2297 non-null   object \n",
      " 10  rating_numerator            2356 non-null   int64  \n",
      " 11  rating_denominator          2356 non-null   int64  \n",
      " 12  name                        2356 non-null   object \n",
      " 13  doggo                       97 non-null     object \n",
      " 14  floofer                     10 non-null     object \n",
      " 15  pupper                      257 non-null    object \n",
      " 16  puppo                       30 non-null     object \n",
      "dtypes: float64(4), int64(3), object(10)\n",
      "memory usage: 220.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df_clean.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidiness Issues\n",
    "\n",
    "### Issue #1:\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "source": [
    "#### Define\n",
    "\n",
    "melt the four columns doggo,floofer,pupper and puppo into one column Type"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clean=pd.melt(df_clean,id_vars=['tweet_id',\n",
    " 'in_reply_to_status_id',\n",
    " 'in_reply_to_user_id',\n",
    " 'timestamp',\n",
    " 'source',\n",
    " 'text',\n",
    " 'retweeted_status_id',\n",
    " 'retweeted_status_user_id',\n",
    " 'retweeted_status_timestamp',\n",
    " 'expanded_urls',\n",
    " 'rating_numerator',\n",
    " 'rating_denominator',\n",
    " 'name'],value_vars=['doggo','floofer','pupper','puppo'],value_name='Dog_Type')\n",
    "df_clean = df_clean.drop('variable',axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9394    True\n",
       "9395    True\n",
       "9396    True\n",
       "9397    True\n",
       "9398    True\n",
       "9399    True\n",
       "9400    True\n",
       "9401    True\n",
       "9402    True\n",
       "9403    True\n",
       "9404    True\n",
       "9405    True\n",
       "9406    True\n",
       "9407    True\n",
       "9408    True\n",
       "9409    True\n",
       "9410    True\n",
       "9411    True\n",
       "9412    True\n",
       "9413    True\n",
       "9414    True\n",
       "9415    True\n",
       "9416    True\n",
       "9417    True\n",
       "9418    True\n",
       "9419    True\n",
       "9420    True\n",
       "9421    True\n",
       "9422    True\n",
       "9423    True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean.duplicated().tail(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality Issue #2 :\n",
    "\n",
    "#### Define :\n",
    "After melting the fou columns we need to remove the dupilcates\n",
    "\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clean.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df_clean.duplicated())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>Dog_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2071</th>\n",
       "      <td>671115716440031232</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-11-29 23:57:10 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Meet Phred. He isn't steering, looking at the ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/671115716...</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>Phred</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1949</th>\n",
       "      <td>673689733134946305</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-12-07 02:25:23 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>When you're having a blast and remember tomorr...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/673689733...</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1149</th>\n",
       "      <td>726828223124897792</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-01 17:38:46 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Bloop. He's a Phoenician Winnebago. To...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/726828223...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Bloop</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1792</th>\n",
       "      <td>677334615166730240</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-12-17 03:48:51 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This dog gave up mid jump. 9/10 https://t.co/K...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/677334615...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1259</th>\n",
       "      <td>710272297844797440</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-03-17 01:11:26 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>We 👏🏻 only 👏🏻 rate 👏🏻 dogs. Pls stop sending i...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/710272297...</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>infuriating</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1345</th>\n",
       "      <td>704491224099647488</td>\n",
       "      <td>7.044857e+17</td>\n",
       "      <td>28785486.0</td>\n",
       "      <td>2016-03-01 02:19:31 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>13/10 hero af\\n@ABC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1995</th>\n",
       "      <td>672594978741354496</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-12-04 01:55:13 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Meet Scott. Just trying to catch his train to ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/672594978...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Scott</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>748220828303695873</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-06-29 18:25:21 +0000</td>\n",
       "      <td>&lt;a href=\"http://vine.co\" rel=\"nofollow\"&gt;Vine -...</td>\n",
       "      <td>Stop what you're doing and watch this heckin m...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://vine.co/v/iiLjKuYJpr6</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1019</th>\n",
       "      <td>746790600704425984</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-06-25 19:42:08 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>When you just can't resist... 10/10 topnotch t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/746790600...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2255</th>\n",
       "      <td>667773195014021121</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-11-20 18:35:10 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com\" rel=\"nofollow\"&gt;Tw...</td>\n",
       "      <td>This is a rare Hungarian Pinot named Jessiga. ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/667773195...</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id  in_reply_to_status_id  in_reply_to_user_id  \\\n",
       "2071  671115716440031232                    NaN                  NaN   \n",
       "1949  673689733134946305                    NaN                  NaN   \n",
       "1149  726828223124897792                    NaN                  NaN   \n",
       "1792  677334615166730240                    NaN                  NaN   \n",
       "1259  710272297844797440                    NaN                  NaN   \n",
       "1345  704491224099647488           7.044857e+17           28785486.0   \n",
       "1995  672594978741354496                    NaN                  NaN   \n",
       "999   748220828303695873                    NaN                  NaN   \n",
       "1019  746790600704425984                    NaN                  NaN   \n",
       "2255  667773195014021121                    NaN                  NaN   \n",
       "\n",
       "                      timestamp  \\\n",
       "2071  2015-11-29 23:57:10 +0000   \n",
       "1949  2015-12-07 02:25:23 +0000   \n",
       "1149  2016-05-01 17:38:46 +0000   \n",
       "1792  2015-12-17 03:48:51 +0000   \n",
       "1259  2016-03-17 01:11:26 +0000   \n",
       "1345  2016-03-01 02:19:31 +0000   \n",
       "1995  2015-12-04 01:55:13 +0000   \n",
       "999   2016-06-29 18:25:21 +0000   \n",
       "1019  2016-06-25 19:42:08 +0000   \n",
       "2255  2015-11-20 18:35:10 +0000   \n",
       "\n",
       "                                                 source  \\\n",
       "2071  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1949  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1149  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1792  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1259  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1345  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1995  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "999   <a href=\"http://vine.co\" rel=\"nofollow\">Vine -...   \n",
       "1019  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2255  <a href=\"http://twitter.com\" rel=\"nofollow\">Tw...   \n",
       "\n",
       "                                                   text  retweeted_status_id  \\\n",
       "2071  Meet Phred. He isn't steering, looking at the ...                  NaN   \n",
       "1949  When you're having a blast and remember tomorr...                  NaN   \n",
       "1149  This is Bloop. He's a Phoenician Winnebago. To...                  NaN   \n",
       "1792  This dog gave up mid jump. 9/10 https://t.co/K...                  NaN   \n",
       "1259  We 👏🏻 only 👏🏻 rate 👏🏻 dogs. Pls stop sending i...                  NaN   \n",
       "1345                                13/10 hero af\\n@ABC                  NaN   \n",
       "1995  Meet Scott. Just trying to catch his train to ...                  NaN   \n",
       "999   Stop what you're doing and watch this heckin m...                  NaN   \n",
       "1019  When you just can't resist... 10/10 topnotch t...                  NaN   \n",
       "2255  This is a rare Hungarian Pinot named Jessiga. ...                  NaN   \n",
       "\n",
       "      retweeted_status_user_id retweeted_status_timestamp  \\\n",
       "2071                       NaN                        NaN   \n",
       "1949                       NaN                        NaN   \n",
       "1149                       NaN                        NaN   \n",
       "1792                       NaN                        NaN   \n",
       "1259                       NaN                        NaN   \n",
       "1345                       NaN                        NaN   \n",
       "1995                       NaN                        NaN   \n",
       "999                        NaN                        NaN   \n",
       "1019                       NaN                        NaN   \n",
       "2255                       NaN                        NaN   \n",
       "\n",
       "                                          expanded_urls  rating_numerator  \\\n",
       "2071  https://twitter.com/dog_rates/status/671115716...                 6   \n",
       "1949  https://twitter.com/dog_rates/status/673689733...                11   \n",
       "1149  https://twitter.com/dog_rates/status/726828223...                12   \n",
       "1792  https://twitter.com/dog_rates/status/677334615...                 9   \n",
       "1259  https://twitter.com/dog_rates/status/710272297...                11   \n",
       "1345                                                NaN                13   \n",
       "1995  https://twitter.com/dog_rates/status/672594978...                 9   \n",
       "999                       https://vine.co/v/iiLjKuYJpr6                13   \n",
       "1019  https://twitter.com/dog_rates/status/746790600...                10   \n",
       "2255  https://twitter.com/dog_rates/status/667773195...                 8   \n",
       "\n",
       "      rating_denominator         name Dog_Type  \n",
       "2071                  10        Phred      NaN  \n",
       "1949                  10         None      NaN  \n",
       "1149                  10        Bloop      NaN  \n",
       "1792                  10         None      NaN  \n",
       "1259                  10  infuriating      NaN  \n",
       "1345                  10         None      NaN  \n",
       "1995                  10        Scott      NaN  \n",
       "999                   10         None      NaN  \n",
       "1019                  10         None      NaN  \n",
       "2255                  10            a      NaN  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2750 entries, 0 to 8151\n",
      "Data columns (total 14 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   tweet_id                    2750 non-null   int64  \n",
      " 1   in_reply_to_status_id       87 non-null     float64\n",
      " 2   in_reply_to_user_id         87 non-null     float64\n",
      " 3   timestamp                   2750 non-null   object \n",
      " 4   source                      2750 non-null   object \n",
      " 5   text                        2750 non-null   object \n",
      " 6   retweeted_status_id         219 non-null    float64\n",
      " 7   retweeted_status_user_id    219 non-null    float64\n",
      " 8   retweeted_status_timestamp  219 non-null    object \n",
      " 9   expanded_urls               2685 non-null   object \n",
      " 10  rating_numerator            2750 non-null   int64  \n",
      " 11  rating_denominator          2750 non-null   int64  \n",
      " 12  name                        2750 non-null   object \n",
      " 13  Dog_Type                    394 non-null    object \n",
      "dtypes: float64(4), int64(3), object(7)\n",
      "memory usage: 247.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df_clean.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality Issue #3 :\n",
    "\n",
    "#### Define :\n",
    "\n",
    " restet the index for df_clean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clean = df_clean.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2750 entries, 0 to 2749\n",
      "Data columns (total 14 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   tweet_id                    2750 non-null   int64  \n",
      " 1   in_reply_to_status_id       87 non-null     float64\n",
      " 2   in_reply_to_user_id         87 non-null     float64\n",
      " 3   timestamp                   2750 non-null   object \n",
      " 4   source                      2750 non-null   object \n",
      " 5   text                        2750 non-null   object \n",
      " 6   retweeted_status_id         219 non-null    float64\n",
      " 7   retweeted_status_user_id    219 non-null    float64\n",
      " 8   retweeted_status_timestamp  219 non-null    object \n",
      " 9   expanded_urls               2685 non-null   object \n",
      " 10  rating_numerator            2750 non-null   int64  \n",
      " 11  rating_denominator          2750 non-null   int64  \n",
      " 12  name                        2750 non-null   object \n",
      " 13  Dog_Type                    394 non-null    object \n",
      "dtypes: float64(4), int64(3), object(7)\n",
      "memory usage: 225.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df_clean.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidiness Issue #2 :\n",
    "\n",
    "#### Define\n",
    "\n",
    "merge the three dataframes."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = pd.merge(tweet_df_clean,df_clean,left_on=['Tweet_id'],right_on=['tweet_id'],how='left')\n",
    "master_df = pd.merge(dfs,images_df_clean,left_on=['Tweet_id'],right_on=['tweet_id'],how='left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tweet_id',\n",
       " 'Retweets',\n",
       " 'Favorites',\n",
       " 'tweet_id_x',\n",
       " 'in_reply_to_status_id',\n",
       " 'in_reply_to_user_id',\n",
       " 'timestamp',\n",
       " 'source',\n",
       " 'text',\n",
       " 'retweeted_status_id',\n",
       " 'retweeted_status_user_id',\n",
       " 'retweeted_status_timestamp',\n",
       " 'expanded_urls',\n",
       " 'rating_numerator',\n",
       " 'rating_denominator',\n",
       " 'name',\n",
       " 'Dog_Type',\n",
       " 'tweet_id_y',\n",
       " 'jpg_url',\n",
       " 'img_num',\n",
       " 'p1',\n",
       " 'p1_conf',\n",
       " 'p1_dog',\n",
       " 'p2',\n",
       " 'p2_conf',\n",
       " 'p2_dog',\n",
       " 'p3',\n",
       " 'p3_conf',\n",
       " 'p3_dog']"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(master_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Assesing master_df :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet_id</th>\n",
       "      <th>Retweets</th>\n",
       "      <th>Favorites</th>\n",
       "      <th>tweet_id_x</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>...</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>585</th>\n",
       "      <td>813130366689148928</td>\n",
       "      <td>513</td>\n",
       "      <td>4968</td>\n",
       "      <td>813130366689148928</td>\n",
       "      <td>8.131273e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>2016-12-25 21:12:41 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>I've been informed by multiple sources that th...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320</th>\n",
       "      <td>838952994649550848</td>\n",
       "      <td>4505</td>\n",
       "      <td>21289</td>\n",
       "      <td>838952994649550848</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-03-07 03:22:35 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>SHE MISPLACED HER HOOMAN 13/10 MISTAKES HAPPEN...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2722</th>\n",
       "      <td>666353288456101888</td>\n",
       "      <td>77</td>\n",
       "      <td>229</td>\n",
       "      <td>666353288456101888</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-11-16 20:32:58 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here we have a mixed Asiago from the Galápagos...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>malamute</td>\n",
       "      <td>0.336874</td>\n",
       "      <td>True</td>\n",
       "      <td>Siberian_husky</td>\n",
       "      <td>0.147655</td>\n",
       "      <td>True</td>\n",
       "      <td>Eskimo_dog</td>\n",
       "      <td>0.093412</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2214</th>\n",
       "      <td>675534494439489536</td>\n",
       "      <td>470</td>\n",
       "      <td>1953</td>\n",
       "      <td>675534494439489536</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-12-12 04:35:48 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Seriously guys?! Only send in dogs. I only rat...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>chow</td>\n",
       "      <td>0.749368</td>\n",
       "      <td>True</td>\n",
       "      <td>schipperke</td>\n",
       "      <td>0.133738</td>\n",
       "      <td>True</td>\n",
       "      <td>Newfoundland</td>\n",
       "      <td>0.049914</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>875747767867523072</td>\n",
       "      <td>4497</td>\n",
       "      <td>25773</td>\n",
       "      <td>875747767867523072</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-06-16 16:11:53 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Goose. He's a womanizer. Cheeky as h*c...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Labrador_retriever</td>\n",
       "      <td>0.799551</td>\n",
       "      <td>True</td>\n",
       "      <td>Chesapeake_Bay_retriever</td>\n",
       "      <td>0.179975</td>\n",
       "      <td>True</td>\n",
       "      <td>vizsla</td>\n",
       "      <td>0.004618</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846</th>\n",
       "      <td>783466772167098368</td>\n",
       "      <td>2608</td>\n",
       "      <td>9468</td>\n",
       "      <td>783466772167098368</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-10-05 00:40:09 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Finn. He likes eavesdropping from fili...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.789000</td>\n",
       "      <td>True</td>\n",
       "      <td>miniature_pinscher</td>\n",
       "      <td>0.115916</td>\n",
       "      <td>True</td>\n",
       "      <td>toy_terrier</td>\n",
       "      <td>0.036294</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1794</th>\n",
       "      <td>691675652215414786</td>\n",
       "      <td>577</td>\n",
       "      <td>2116</td>\n",
       "      <td>691675652215414786</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-25 17:35:00 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Richie and Plip. They are the best of ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.182898</td>\n",
       "      <td>True</td>\n",
       "      <td>teddy</td>\n",
       "      <td>0.128077</td>\n",
       "      <td>False</td>\n",
       "      <td>West_Highland_white_terrier</td>\n",
       "      <td>0.097875</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>863062471531167744</td>\n",
       "      <td>2687</td>\n",
       "      <td>8945</td>\n",
       "      <td>863062471531167744</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-05-12 16:05:02 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Say hello to Quinn. She's quite the goofball. ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>French_bulldog</td>\n",
       "      <td>0.935804</td>\n",
       "      <td>True</td>\n",
       "      <td>pug</td>\n",
       "      <td>0.059576</td>\n",
       "      <td>True</td>\n",
       "      <td>boxer</td>\n",
       "      <td>0.001412</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>873</th>\n",
       "      <td>780601303617732608</td>\n",
       "      <td>3839</td>\n",
       "      <td>13525</td>\n",
       "      <td>780601303617732608</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-09-27 02:53:48 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Meet Hercules. He can have whatever he wants f...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Saint_Bernard</td>\n",
       "      <td>0.995143</td>\n",
       "      <td>True</td>\n",
       "      <td>Cardigan</td>\n",
       "      <td>0.003044</td>\n",
       "      <td>True</td>\n",
       "      <td>English_springer</td>\n",
       "      <td>0.001050</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1613</th>\n",
       "      <td>703611486317502464</td>\n",
       "      <td>1709</td>\n",
       "      <td>4229</td>\n",
       "      <td>703611486317502464</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-02-27 16:03:45 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Meet Scooter. He's experiencing the pupper equ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Pembroke</td>\n",
       "      <td>0.756441</td>\n",
       "      <td>True</td>\n",
       "      <td>basenji</td>\n",
       "      <td>0.126621</td>\n",
       "      <td>True</td>\n",
       "      <td>Cardigan</td>\n",
       "      <td>0.080117</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1352</th>\n",
       "      <td>729838605770891264</td>\n",
       "      <td>431</td>\n",
       "      <td>1856</td>\n",
       "      <td>729838605770891264</td>\n",
       "      <td>7.291135e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>2016-05-10 01:00:58 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>\"Challenge completed\" \\n(pupgraded to 12/10) h...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>stone_wall</td>\n",
       "      <td>0.758218</td>\n",
       "      <td>False</td>\n",
       "      <td>patio</td>\n",
       "      <td>0.074205</td>\n",
       "      <td>False</td>\n",
       "      <td>prison</td>\n",
       "      <td>0.013826</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>826</th>\n",
       "      <td>786051337297522688</td>\n",
       "      <td>179</td>\n",
       "      <td>1614</td>\n",
       "      <td>786051337297522688</td>\n",
       "      <td>7.727430e+17</td>\n",
       "      <td>7.305050e+17</td>\n",
       "      <td>2016-10-12 03:50:17 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>13/10 for breakdancing puppo @shibbnbot</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1353</th>\n",
       "      <td>729823566028484608</td>\n",
       "      <td>1390</td>\n",
       "      <td>4358</td>\n",
       "      <td>729823566028484608</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-10 00:01:12 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Stefan. He's a downright remarkable pu...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>kelpie</td>\n",
       "      <td>0.218408</td>\n",
       "      <td>True</td>\n",
       "      <td>Arabian_camel</td>\n",
       "      <td>0.114368</td>\n",
       "      <td>False</td>\n",
       "      <td>coyote</td>\n",
       "      <td>0.096409</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>952</th>\n",
       "      <td>772193107915964416</td>\n",
       "      <td>1612</td>\n",
       "      <td>6665</td>\n",
       "      <td>772193107915964416</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-09-03 22:02:38 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Willem. He's a Penn State pupper. Thin...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Pembroke</td>\n",
       "      <td>0.367945</td>\n",
       "      <td>True</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.223522</td>\n",
       "      <td>True</td>\n",
       "      <td>Pekinese</td>\n",
       "      <td>0.164871</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>901</th>\n",
       "      <td>778027034220126208</td>\n",
       "      <td>1885</td>\n",
       "      <td>7320</td>\n",
       "      <td>778027034220126208</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-09-20 00:24:34 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Sophie. She's a Jubilant Bush Pupper. ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>clumber</td>\n",
       "      <td>0.946718</td>\n",
       "      <td>True</td>\n",
       "      <td>cocker_spaniel</td>\n",
       "      <td>0.015950</td>\n",
       "      <td>True</td>\n",
       "      <td>Lhasa</td>\n",
       "      <td>0.006519</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2175</th>\n",
       "      <td>676582956622721024</td>\n",
       "      <td>312</td>\n",
       "      <td>1310</td>\n",
       "      <td>676582956622721024</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-12-15 02:02:01 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Meet Vinnie. He's having fun while being safe....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>seat_belt</td>\n",
       "      <td>0.790028</td>\n",
       "      <td>False</td>\n",
       "      <td>Boston_bull</td>\n",
       "      <td>0.196307</td>\n",
       "      <td>True</td>\n",
       "      <td>French_bulldog</td>\n",
       "      <td>0.012429</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1111</th>\n",
       "      <td>753039830821511168</td>\n",
       "      <td>24013</td>\n",
       "      <td>41080</td>\n",
       "      <td>753039830821511168</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-07-13 01:34:21 +0000</td>\n",
       "      <td>&lt;a href=\"http://vine.co\" rel=\"nofollow\"&gt;Vine -...</td>\n",
       "      <td>So this just changed my life. 13/10 please enj...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1755</th>\n",
       "      <td>693629975228977152</td>\n",
       "      <td>894</td>\n",
       "      <td>2685</td>\n",
       "      <td>693629975228977152</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-31 03:00:47 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This pupper is afraid of its own feet. 12/10 w...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>pug</td>\n",
       "      <td>0.841987</td>\n",
       "      <td>True</td>\n",
       "      <td>French_bulldog</td>\n",
       "      <td>0.069791</td>\n",
       "      <td>True</td>\n",
       "      <td>Boston_bull</td>\n",
       "      <td>0.038720</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1434</th>\n",
       "      <td>717421804990701568</td>\n",
       "      <td>945</td>\n",
       "      <td>3446</td>\n",
       "      <td>717421804990701568</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-04-05 18:41:02 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Ebby. She's a Zimbabwean Feta. Embarra...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>miniature_pinscher</td>\n",
       "      <td>0.286479</td>\n",
       "      <td>True</td>\n",
       "      <td>Italian_greyhound</td>\n",
       "      <td>0.084134</td>\n",
       "      <td>True</td>\n",
       "      <td>beagle</td>\n",
       "      <td>0.064697</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>758</th>\n",
       "      <td>793226087023144960</td>\n",
       "      <td>3338</td>\n",
       "      <td>11001</td>\n",
       "      <td>793226087023144960</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-10-31 23:00:11 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Juno. She spooked me up real good, but...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>wire-haired_fox_terrier</td>\n",
       "      <td>0.456047</td>\n",
       "      <td>True</td>\n",
       "      <td>Lakeland_terrier</td>\n",
       "      <td>0.273428</td>\n",
       "      <td>True</td>\n",
       "      <td>English_springer</td>\n",
       "      <td>0.083643</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                Tweet_id  Retweets  Favorites          tweet_id_x  \\\n",
       "585   813130366689148928       513       4968  813130366689148928   \n",
       "320   838952994649550848      4505      21289  838952994649550848   \n",
       "2722  666353288456101888        77        229  666353288456101888   \n",
       "2214  675534494439489536       470       1953  675534494439489536   \n",
       "96    875747767867523072      4497      25773  875747767867523072   \n",
       "846   783466772167098368      2608       9468  783466772167098368   \n",
       "1794  691675652215414786       577       2116  691675652215414786   \n",
       "173   863062471531167744      2687       8945  863062471531167744   \n",
       "873   780601303617732608      3839      13525  780601303617732608   \n",
       "1613  703611486317502464      1709       4229  703611486317502464   \n",
       "1352  729838605770891264       431       1856  729838605770891264   \n",
       "826   786051337297522688       179       1614  786051337297522688   \n",
       "1353  729823566028484608      1390       4358  729823566028484608   \n",
       "952   772193107915964416      1612       6665  772193107915964416   \n",
       "901   778027034220126208      1885       7320  778027034220126208   \n",
       "2175  676582956622721024       312       1310  676582956622721024   \n",
       "1111  753039830821511168     24013      41080  753039830821511168   \n",
       "1755  693629975228977152       894       2685  693629975228977152   \n",
       "1434  717421804990701568       945       3446  717421804990701568   \n",
       "758   793226087023144960      3338      11001  793226087023144960   \n",
       "\n",
       "      in_reply_to_status_id  in_reply_to_user_id                  timestamp  \\\n",
       "585            8.131273e+17         4.196984e+09  2016-12-25 21:12:41 +0000   \n",
       "320                     NaN                  NaN  2017-03-07 03:22:35 +0000   \n",
       "2722                    NaN                  NaN  2015-11-16 20:32:58 +0000   \n",
       "2214                    NaN                  NaN  2015-12-12 04:35:48 +0000   \n",
       "96                      NaN                  NaN  2017-06-16 16:11:53 +0000   \n",
       "846                     NaN                  NaN  2016-10-05 00:40:09 +0000   \n",
       "1794                    NaN                  NaN  2016-01-25 17:35:00 +0000   \n",
       "173                     NaN                  NaN  2017-05-12 16:05:02 +0000   \n",
       "873                     NaN                  NaN  2016-09-27 02:53:48 +0000   \n",
       "1613                    NaN                  NaN  2016-02-27 16:03:45 +0000   \n",
       "1352           7.291135e+17         4.196984e+09  2016-05-10 01:00:58 +0000   \n",
       "826            7.727430e+17         7.305050e+17  2016-10-12 03:50:17 +0000   \n",
       "1353                    NaN                  NaN  2016-05-10 00:01:12 +0000   \n",
       "952                     NaN                  NaN  2016-09-03 22:02:38 +0000   \n",
       "901                     NaN                  NaN  2016-09-20 00:24:34 +0000   \n",
       "2175                    NaN                  NaN  2015-12-15 02:02:01 +0000   \n",
       "1111                    NaN                  NaN  2016-07-13 01:34:21 +0000   \n",
       "1755                    NaN                  NaN  2016-01-31 03:00:47 +0000   \n",
       "1434                    NaN                  NaN  2016-04-05 18:41:02 +0000   \n",
       "758                     NaN                  NaN  2016-10-31 23:00:11 +0000   \n",
       "\n",
       "                                                 source  \\\n",
       "585   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "320   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2722  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2214  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "96    <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "846   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1794  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "173   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "873   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1613  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1352  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "826   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1353  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "952   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "901   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2175  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1111  <a href=\"http://vine.co\" rel=\"nofollow\">Vine -...   \n",
       "1755  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1434  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "758   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                   text  retweeted_status_id  \\\n",
       "585   I've been informed by multiple sources that th...                  NaN   \n",
       "320   SHE MISPLACED HER HOOMAN 13/10 MISTAKES HAPPEN...                  NaN   \n",
       "2722  Here we have a mixed Asiago from the Galápagos...                  NaN   \n",
       "2214  Seriously guys?! Only send in dogs. I only rat...                  NaN   \n",
       "96    This is Goose. He's a womanizer. Cheeky as h*c...                  NaN   \n",
       "846   This is Finn. He likes eavesdropping from fili...                  NaN   \n",
       "1794  This is Richie and Plip. They are the best of ...                  NaN   \n",
       "173   Say hello to Quinn. She's quite the goofball. ...                  NaN   \n",
       "873   Meet Hercules. He can have whatever he wants f...                  NaN   \n",
       "1613  Meet Scooter. He's experiencing the pupper equ...                  NaN   \n",
       "1352  \"Challenge completed\" \\n(pupgraded to 12/10) h...                  NaN   \n",
       "826             13/10 for breakdancing puppo @shibbnbot                  NaN   \n",
       "1353  This is Stefan. He's a downright remarkable pu...                  NaN   \n",
       "952   This is Willem. He's a Penn State pupper. Thin...                  NaN   \n",
       "901   This is Sophie. She's a Jubilant Bush Pupper. ...                  NaN   \n",
       "2175  Meet Vinnie. He's having fun while being safe....                  NaN   \n",
       "1111  So this just changed my life. 13/10 please enj...                  NaN   \n",
       "1755  This pupper is afraid of its own feet. 12/10 w...                  NaN   \n",
       "1434  This is Ebby. She's a Zimbabwean Feta. Embarra...                  NaN   \n",
       "758   This is Juno. She spooked me up real good, but...                  NaN   \n",
       "\n",
       "      ...  img_num                       p1   p1_conf  p1_dog  \\\n",
       "585   ...      NaN                      NaN       NaN     NaN   \n",
       "320   ...      NaN                      NaN       NaN     NaN   \n",
       "2722  ...      1.0                 malamute  0.336874    True   \n",
       "2214  ...      1.0                     chow  0.749368    True   \n",
       "96    ...      1.0       Labrador_retriever  0.799551    True   \n",
       "846   ...      1.0                Chihuahua  0.789000    True   \n",
       "1794  ...      1.0                Chihuahua  0.182898    True   \n",
       "173   ...      2.0           French_bulldog  0.935804    True   \n",
       "873   ...      1.0            Saint_Bernard  0.995143    True   \n",
       "1613  ...      1.0                 Pembroke  0.756441    True   \n",
       "1352  ...      1.0               stone_wall  0.758218   False   \n",
       "826   ...      NaN                      NaN       NaN     NaN   \n",
       "1353  ...      1.0                   kelpie  0.218408    True   \n",
       "952   ...      1.0                 Pembroke  0.367945    True   \n",
       "901   ...      1.0                  clumber  0.946718    True   \n",
       "2175  ...      1.0                seat_belt  0.790028   False   \n",
       "1111  ...      NaN                      NaN       NaN     NaN   \n",
       "1755  ...      1.0                      pug  0.841987    True   \n",
       "1434  ...      2.0       miniature_pinscher  0.286479    True   \n",
       "758   ...      1.0  wire-haired_fox_terrier  0.456047    True   \n",
       "\n",
       "                            p2   p2_conf p2_dog                           p3  \\\n",
       "585                        NaN       NaN    NaN                          NaN   \n",
       "320                        NaN       NaN    NaN                          NaN   \n",
       "2722            Siberian_husky  0.147655   True                   Eskimo_dog   \n",
       "2214                schipperke  0.133738   True                 Newfoundland   \n",
       "96    Chesapeake_Bay_retriever  0.179975   True                       vizsla   \n",
       "846         miniature_pinscher  0.115916   True                  toy_terrier   \n",
       "1794                     teddy  0.128077  False  West_Highland_white_terrier   \n",
       "173                        pug  0.059576   True                        boxer   \n",
       "873                   Cardigan  0.003044   True             English_springer   \n",
       "1613                   basenji  0.126621   True                     Cardigan   \n",
       "1352                     patio  0.074205  False                       prison   \n",
       "826                        NaN       NaN    NaN                          NaN   \n",
       "1353             Arabian_camel  0.114368  False                       coyote   \n",
       "952                  Chihuahua  0.223522   True                     Pekinese   \n",
       "901             cocker_spaniel  0.015950   True                        Lhasa   \n",
       "2175               Boston_bull  0.196307   True               French_bulldog   \n",
       "1111                       NaN       NaN    NaN                          NaN   \n",
       "1755            French_bulldog  0.069791   True                  Boston_bull   \n",
       "1434         Italian_greyhound  0.084134   True                       beagle   \n",
       "758           Lakeland_terrier  0.273428   True             English_springer   \n",
       "\n",
       "       p3_conf  p3_dog  \n",
       "585        NaN     NaN  \n",
       "320        NaN     NaN  \n",
       "2722  0.093412    True  \n",
       "2214  0.049914    True  \n",
       "96    0.004618    True  \n",
       "846   0.036294    True  \n",
       "1794  0.097875    True  \n",
       "173   0.001412    True  \n",
       "873   0.001050    True  \n",
       "1613  0.080117    True  \n",
       "1352  0.013826   False  \n",
       "826        NaN     NaN  \n",
       "1353  0.096409   False  \n",
       "952   0.164871    True  \n",
       "901   0.006519    True  \n",
       "2175  0.012429    True  \n",
       "1111       NaN     NaN  \n",
       "1755  0.038720    True  \n",
       "1434  0.064697    True  \n",
       "758   0.083643    True  \n",
       "\n",
       "[20 rows x 29 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.sample(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet_id</th>\n",
       "      <th>Retweets</th>\n",
       "      <th>Favorites</th>\n",
       "      <th>tweet_id_x</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>...</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>8853</td>\n",
       "      <td>39467</td>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-08-01 16:23:56 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Phineas. He's a mystical boy. Only eve...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>orange</td>\n",
       "      <td>0.097049</td>\n",
       "      <td>False</td>\n",
       "      <td>bagel</td>\n",
       "      <td>0.085851</td>\n",
       "      <td>False</td>\n",
       "      <td>banana</td>\n",
       "      <td>0.076110</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>6514</td>\n",
       "      <td>33819</td>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-08-01 00:17:27 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Tilly. She's just checking pup on you....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.323581</td>\n",
       "      <td>True</td>\n",
       "      <td>Pekinese</td>\n",
       "      <td>0.090647</td>\n",
       "      <td>True</td>\n",
       "      <td>papillon</td>\n",
       "      <td>0.068957</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>4328</td>\n",
       "      <td>25461</td>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-31 00:18:03 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Archie. He is a rare Norwegian Pouncin...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.716012</td>\n",
       "      <td>True</td>\n",
       "      <td>malamute</td>\n",
       "      <td>0.078253</td>\n",
       "      <td>True</td>\n",
       "      <td>kelpie</td>\n",
       "      <td>0.031379</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>8964</td>\n",
       "      <td>42908</td>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-30 15:58:51 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Darla. She commenced a snooze mid meal...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>paper_towel</td>\n",
       "      <td>0.170278</td>\n",
       "      <td>False</td>\n",
       "      <td>Labrador_retriever</td>\n",
       "      <td>0.168086</td>\n",
       "      <td>True</td>\n",
       "      <td>spatula</td>\n",
       "      <td>0.040836</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>9774</td>\n",
       "      <td>41048</td>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-29 16:00:24 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Franklin. He would like you to stop ca...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>basset</td>\n",
       "      <td>0.555712</td>\n",
       "      <td>True</td>\n",
       "      <td>English_springer</td>\n",
       "      <td>0.225770</td>\n",
       "      <td>True</td>\n",
       "      <td>German_short-haired_pointer</td>\n",
       "      <td>0.175219</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>891087950875897856</td>\n",
       "      <td>3261</td>\n",
       "      <td>20562</td>\n",
       "      <td>891087950875897856</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-29 00:08:17 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here we have a majestic great white breaching ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Chesapeake_Bay_retriever</td>\n",
       "      <td>0.425595</td>\n",
       "      <td>True</td>\n",
       "      <td>Irish_terrier</td>\n",
       "      <td>0.116317</td>\n",
       "      <td>True</td>\n",
       "      <td>Indian_elephant</td>\n",
       "      <td>0.076902</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>890971913173991426</td>\n",
       "      <td>2158</td>\n",
       "      <td>12041</td>\n",
       "      <td>890971913173991426</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-28 16:27:12 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Meet Jax. He enjoys ice cream so much he gets ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Appenzeller</td>\n",
       "      <td>0.341703</td>\n",
       "      <td>True</td>\n",
       "      <td>Border_collie</td>\n",
       "      <td>0.199287</td>\n",
       "      <td>True</td>\n",
       "      <td>ice_lolly</td>\n",
       "      <td>0.193548</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>890729181411237888</td>\n",
       "      <td>16716</td>\n",
       "      <td>56848</td>\n",
       "      <td>890729181411237888</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-28 00:22:40 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>When you watch your owner call another dog a g...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Pomeranian</td>\n",
       "      <td>0.566142</td>\n",
       "      <td>True</td>\n",
       "      <td>Eskimo_dog</td>\n",
       "      <td>0.178406</td>\n",
       "      <td>True</td>\n",
       "      <td>Pembroke</td>\n",
       "      <td>0.076507</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>890609185150312448</td>\n",
       "      <td>4429</td>\n",
       "      <td>28226</td>\n",
       "      <td>890609185150312448</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-27 16:25:51 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Zoey. She doesn't want to be one of th...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Irish_terrier</td>\n",
       "      <td>0.487574</td>\n",
       "      <td>True</td>\n",
       "      <td>Irish_setter</td>\n",
       "      <td>0.193054</td>\n",
       "      <td>True</td>\n",
       "      <td>Chesapeake_Bay_retriever</td>\n",
       "      <td>0.118184</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>890240255349198849</td>\n",
       "      <td>7711</td>\n",
       "      <td>32467</td>\n",
       "      <td>890240255349198849</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-26 15:59:51 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Cassie. She is a college pup. Studying...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Pembroke</td>\n",
       "      <td>0.511319</td>\n",
       "      <td>True</td>\n",
       "      <td>Cardigan</td>\n",
       "      <td>0.451038</td>\n",
       "      <td>True</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.029248</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Tweet_id  Retweets  Favorites          tweet_id_x  \\\n",
       "0  892420643555336193      8853      39467  892420643555336193   \n",
       "1  892177421306343426      6514      33819  892177421306343426   \n",
       "2  891815181378084864      4328      25461  891815181378084864   \n",
       "3  891689557279858688      8964      42908  891689557279858688   \n",
       "4  891327558926688256      9774      41048  891327558926688256   \n",
       "5  891087950875897856      3261      20562  891087950875897856   \n",
       "6  890971913173991426      2158      12041  890971913173991426   \n",
       "7  890729181411237888     16716      56848  890729181411237888   \n",
       "8  890609185150312448      4429      28226  890609185150312448   \n",
       "9  890240255349198849      7711      32467  890240255349198849   \n",
       "\n",
       "   in_reply_to_status_id  in_reply_to_user_id                  timestamp  \\\n",
       "0                    NaN                  NaN  2017-08-01 16:23:56 +0000   \n",
       "1                    NaN                  NaN  2017-08-01 00:17:27 +0000   \n",
       "2                    NaN                  NaN  2017-07-31 00:18:03 +0000   \n",
       "3                    NaN                  NaN  2017-07-30 15:58:51 +0000   \n",
       "4                    NaN                  NaN  2017-07-29 16:00:24 +0000   \n",
       "5                    NaN                  NaN  2017-07-29 00:08:17 +0000   \n",
       "6                    NaN                  NaN  2017-07-28 16:27:12 +0000   \n",
       "7                    NaN                  NaN  2017-07-28 00:22:40 +0000   \n",
       "8                    NaN                  NaN  2017-07-27 16:25:51 +0000   \n",
       "9                    NaN                  NaN  2017-07-26 15:59:51 +0000   \n",
       "\n",
       "                                              source  \\\n",
       "0  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "3  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "4  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "5  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "6  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "7  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "8  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "9  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                text  retweeted_status_id  \\\n",
       "0  This is Phineas. He's a mystical boy. Only eve...                  NaN   \n",
       "1  This is Tilly. She's just checking pup on you....                  NaN   \n",
       "2  This is Archie. He is a rare Norwegian Pouncin...                  NaN   \n",
       "3  This is Darla. She commenced a snooze mid meal...                  NaN   \n",
       "4  This is Franklin. He would like you to stop ca...                  NaN   \n",
       "5  Here we have a majestic great white breaching ...                  NaN   \n",
       "6  Meet Jax. He enjoys ice cream so much he gets ...                  NaN   \n",
       "7  When you watch your owner call another dog a g...                  NaN   \n",
       "8  This is Zoey. She doesn't want to be one of th...                  NaN   \n",
       "9  This is Cassie. She is a college pup. Studying...                  NaN   \n",
       "\n",
       "   ...  img_num                        p1   p1_conf  p1_dog  \\\n",
       "0  ...      1.0                    orange  0.097049   False   \n",
       "1  ...      1.0                 Chihuahua  0.323581    True   \n",
       "2  ...      1.0                 Chihuahua  0.716012    True   \n",
       "3  ...      1.0               paper_towel  0.170278   False   \n",
       "4  ...      2.0                    basset  0.555712    True   \n",
       "5  ...      1.0  Chesapeake_Bay_retriever  0.425595    True   \n",
       "6  ...      1.0               Appenzeller  0.341703    True   \n",
       "7  ...      2.0                Pomeranian  0.566142    True   \n",
       "8  ...      1.0             Irish_terrier  0.487574    True   \n",
       "9  ...      1.0                  Pembroke  0.511319    True   \n",
       "\n",
       "                   p2   p2_conf p2_dog                           p3   p3_conf  \\\n",
       "0               bagel  0.085851  False                       banana  0.076110   \n",
       "1            Pekinese  0.090647   True                     papillon  0.068957   \n",
       "2            malamute  0.078253   True                       kelpie  0.031379   \n",
       "3  Labrador_retriever  0.168086   True                      spatula  0.040836   \n",
       "4    English_springer  0.225770   True  German_short-haired_pointer  0.175219   \n",
       "5       Irish_terrier  0.116317   True              Indian_elephant  0.076902   \n",
       "6       Border_collie  0.199287   True                    ice_lolly  0.193548   \n",
       "7          Eskimo_dog  0.178406   True                     Pembroke  0.076507   \n",
       "8        Irish_setter  0.193054   True     Chesapeake_Bay_retriever  0.118184   \n",
       "9            Cardigan  0.451038   True                    Chihuahua  0.029248   \n",
       "\n",
       "   p3_dog  \n",
       "0   False  \n",
       "1    True  \n",
       "2    True  \n",
       "3   False  \n",
       "4    True  \n",
       "5   False  \n",
       "6   False  \n",
       "7    True  \n",
       "8    True  \n",
       "9    True  \n",
       "\n",
       "[10 rows x 29 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        216\n",
       "10342      5\n",
       "2706       5\n",
       "8735       4\n",
       "1691       4\n",
       "        ... \n",
       "4551       1\n",
       "459        1\n",
       "4811       1\n",
       "8655       1\n",
       "8143       1\n",
       "Name: Favorites, Length: 2007, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.Favorites.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2246    https://pbs.twimg.com/ext_tw_video_thumb/67480...\n",
       "1125    https://pbs.twimg.com/ext_tw_video_thumb/67535...\n",
       "2225    https://pbs.twimg.com/ext_tw_video_thumb/67535...\n",
       "2209    https://pbs.twimg.com/ext_tw_video_thumb/67574...\n",
       "2210    https://pbs.twimg.com/ext_tw_video_thumb/67574...\n",
       "                              ...                        \n",
       "2429                                                  NaN\n",
       "2540                                                  NaN\n",
       "2580                                                  NaN\n",
       "2603                                                  NaN\n",
       "2689                                                  NaN\n",
       "Name: jpg_url, Length: 2747, dtype: object"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.jpg_url.sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2747 entries, 0 to 2746\n",
      "Data columns (total 29 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   Tweet_id                    2747 non-null   int64  \n",
      " 1   Retweets                    2747 non-null   int64  \n",
      " 2   Favorites                   2747 non-null   int64  \n",
      " 3   tweet_id_x                  2747 non-null   int64  \n",
      " 4   in_reply_to_status_id       87 non-null     float64\n",
      " 5   in_reply_to_user_id         87 non-null     float64\n",
      " 6   timestamp                   2747 non-null   object \n",
      " 7   source                      2747 non-null   object \n",
      " 8   text                        2747 non-null   object \n",
      " 9   retweeted_status_id         216 non-null    float64\n",
      " 10  retweeted_status_user_id    216 non-null    float64\n",
      " 11  retweeted_status_timestamp  216 non-null    object \n",
      " 12  expanded_urls               2682 non-null   object \n",
      " 13  rating_numerator            2747 non-null   int64  \n",
      " 14  rating_denominator          2747 non-null   int64  \n",
      " 15  name                        2747 non-null   object \n",
      " 16  Dog_Type                    393 non-null    object \n",
      " 17  tweet_id_y                  2406 non-null   float64\n",
      " 18  jpg_url                     2406 non-null   object \n",
      " 19  img_num                     2406 non-null   float64\n",
      " 20  p1                          2406 non-null   object \n",
      " 21  p1_conf                     2406 non-null   float64\n",
      " 22  p1_dog                      2406 non-null   object \n",
      " 23  p2                          2406 non-null   object \n",
      " 24  p2_conf                     2406 non-null   float64\n",
      " 25  p2_dog                      2406 non-null   object \n",
      " 26  p3                          2406 non-null   object \n",
      " 27  p3_conf                     2406 non-null   float64\n",
      " 28  p3_dog                      2406 non-null   object \n",
      "dtypes: float64(9), int64(6), object(14)\n",
      "memory usage: 493.6+ KB\n"
     ]
    }
   ],
   "source": [
    "master_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "393"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(master_df.Tweet_id.duplicated())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10     2724\n",
       "11        3\n",
       "50        3\n",
       "80        2\n",
       "20        2\n",
       "2         1\n",
       "16        1\n",
       "40        1\n",
       "70        1\n",
       "15        1\n",
       "90        1\n",
       "110       1\n",
       "120       1\n",
       "130       1\n",
       "150       1\n",
       "170       1\n",
       "7         1\n",
       "0         1\n",
       "Name: rating_denominator, dtype: int64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.rating_denominator.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "354       0\n",
       "2726      2\n",
       "606       7\n",
       "0        10\n",
       "1824     10\n",
       "       ... \n",
       "1951    110\n",
       "2125    120\n",
       "1950    130\n",
       "1067    150\n",
       "1344    170\n",
       "Name: rating_denominator, Length: 2747, dtype: int64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.rating_denominator.sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "354    0\n",
       "Name: rating_denominator, dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.rating_denominator.loc[[354]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality Issue #6 :\n",
    "\n",
    "#### Define :\n",
    "\n",
    "remove the unwanted columns of the master table."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df.drop(['Retweets','tweet_id_x',\n",
    " 'in_reply_to_status_id',\n",
    " 'in_reply_to_user_id',\n",
    " 'timestamp',\n",
    " 'source',\n",
    " 'text',\n",
    " 'retweeted_status_id',\n",
    " 'retweeted_status_user_id',\n",
    " 'retweeted_status_timestamp','name','Dog_Type','tweet_id_y','img_num','p1_dog','p2',\n",
    " 'p2_conf',\n",
    " 'p2_dog',\n",
    " 'p3',\n",
    " 'p3_conf',\n",
    " 'p3_dog'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet_id</th>\n",
       "      <th>Favorites</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>jpg_url</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>39467</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg</td>\n",
       "      <td>orange</td>\n",
       "      <td>0.097049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>33819</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.323581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>25461</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891815181...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.716012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>42908</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891689557...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>https://pbs.twimg.com/media/DF_q7IAWsAEuuN8.jpg</td>\n",
       "      <td>paper_towel</td>\n",
       "      <td>0.170278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>41048</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891327558...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>https://pbs.twimg.com/media/DF6hr6BUMAAzZgT.jpg</td>\n",
       "      <td>basset</td>\n",
       "      <td>0.555712</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Tweet_id  Favorites  \\\n",
       "0  892420643555336193      39467   \n",
       "1  892177421306343426      33819   \n",
       "2  891815181378084864      25461   \n",
       "3  891689557279858688      42908   \n",
       "4  891327558926688256      41048   \n",
       "\n",
       "                                       expanded_urls  rating_numerator  \\\n",
       "0  https://twitter.com/dog_rates/status/892420643...                13   \n",
       "1  https://twitter.com/dog_rates/status/892177421...                13   \n",
       "2  https://twitter.com/dog_rates/status/891815181...                12   \n",
       "3  https://twitter.com/dog_rates/status/891689557...                13   \n",
       "4  https://twitter.com/dog_rates/status/891327558...                12   \n",
       "\n",
       "   rating_denominator                                          jpg_url  \\\n",
       "0                  10  https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg   \n",
       "1                  10  https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg   \n",
       "2                  10  https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg   \n",
       "3                  10  https://pbs.twimg.com/media/DF_q7IAWsAEuuN8.jpg   \n",
       "4                  10  https://pbs.twimg.com/media/DF6hr6BUMAAzZgT.jpg   \n",
       "\n",
       "            p1   p1_conf  \n",
       "0       orange  0.097049  \n",
       "1    Chihuahua  0.323581  \n",
       "2    Chihuahua  0.716012  \n",
       "3  paper_towel  0.170278  \n",
       "4       basset  0.555712  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2747 entries, 0 to 2746\n",
      "Data columns (total 8 columns):\n",
      " #   Column              Non-Null Count  Dtype  \n",
      "---  ------              --------------  -----  \n",
      " 0   Tweet_id            2747 non-null   int64  \n",
      " 1   Favorites           2747 non-null   int64  \n",
      " 2   expanded_urls       2682 non-null   object \n",
      " 3   rating_numerator    2747 non-null   int64  \n",
      " 4   rating_denominator  2747 non-null   int64  \n",
      " 5   jpg_url             2406 non-null   object \n",
      " 6   p1                  2406 non-null   object \n",
      " 7   p1_conf             2406 non-null   float64\n",
      "dtypes: float64(1), int64(4), object(3)\n",
      "memory usage: 225.0+ KB\n"
     ]
    }
   ],
   "source": [
    "master_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality Issue #7 :\n",
    "\n",
    "#### Define :\n",
    "\n",
    "in the master_df remove the raws that doesnt have jpg_url.(You only want original ratings (no retweets) that have images)\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df=master_df[master_df.jpg_url.notnull()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2406 entries, 0 to 2746\n",
      "Data columns (total 8 columns):\n",
      " #   Column              Non-Null Count  Dtype  \n",
      "---  ------              --------------  -----  \n",
      " 0   Tweet_id            2406 non-null   int64  \n",
      " 1   Favorites           2406 non-null   int64  \n",
      " 2   expanded_urls       2406 non-null   object \n",
      " 3   rating_numerator    2406 non-null   int64  \n",
      " 4   rating_denominator  2406 non-null   int64  \n",
      " 5   jpg_url             2406 non-null   object \n",
      " 6   p1                  2406 non-null   object \n",
      " 7   p1_conf             2406 non-null   float64\n",
      "dtypes: float64(1), int64(4), object(3)\n",
      "memory usage: 141.0+ KB\n"
     ]
    }
   ],
   "source": [
    "master_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidiness Issue #3 :\n",
    "\n",
    "#### Define :\n",
    "reorder columns in master_df\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df = master_df[['Tweet_id', 'jpg_url', 'expanded_urls','rating_numerator','rating_denominator','Favorites','p1','p1_conf']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2406 entries, 0 to 2746\n",
      "Data columns (total 8 columns):\n",
      " #   Column              Non-Null Count  Dtype  \n",
      "---  ------              --------------  -----  \n",
      " 0   Tweet_id            2406 non-null   int64  \n",
      " 1   jpg_url             2406 non-null   object \n",
      " 2   expanded_urls       2406 non-null   object \n",
      " 3   rating_numerator    2406 non-null   int64  \n",
      " 4   rating_denominator  2406 non-null   int64  \n",
      " 5   Favorites           2406 non-null   int64  \n",
      " 6   p1                  2406 non-null   object \n",
      " 7   p1_conf             2406 non-null   float64\n",
      "dtypes: float64(1), int64(4), object(3)\n",
      "memory usage: 141.0+ KB\n"
     ]
    }
   ],
   "source": [
    "master_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidiness Issue #4 :\n",
    "\n",
    "#### Define :\n",
    "rename columns in master_df.\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df = master_df.rename(columns = {'jpg_url':'image_url','expanded_urls':'dog_urls','p1':'prediction','p1_conf':'prediction_confident'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tweet_id',\n",
       " 'image_url',\n",
       " 'dog_urls',\n",
       " 'rating_numerator',\n",
       " 'rating_denominator',\n",
       " 'Favorites',\n",
       " 'prediction',\n",
       " 'prediction_confident']"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(master_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet_id</th>\n",
       "      <th>image_url</th>\n",
       "      <th>dog_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>Favorites</th>\n",
       "      <th>prediction</th>\n",
       "      <th>prediction_confident</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2082</th>\n",
       "      <td>679475951516934144</td>\n",
       "      <td>https://pbs.twimg.com/media/CW37xZbUoAAUXe5.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/679475951...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>2304</td>\n",
       "      <td>Maltese_dog</td>\n",
       "      <td>0.145742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1060</th>\n",
       "      <td>759099523532779520</td>\n",
       "      <td>https://pbs.twimg.com/media/Cojc_Q0WcAAqi_K.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/759099523...</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>16101</td>\n",
       "      <td>Shetland_sheepdog</td>\n",
       "      <td>0.129034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2476</th>\n",
       "      <td>670804601705242624</td>\n",
       "      <td>https://pbs.twimg.com/media/CU8tOJZWUAAlNoF.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/670804601...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>2098</td>\n",
       "      <td>Pomeranian</td>\n",
       "      <td>0.868560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>862831371563274240</td>\n",
       "      <td>https://pbs.twimg.com/media/C_lkieeVwAAm0L4.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/862831371...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>20011</td>\n",
       "      <td>Australian_terrier</td>\n",
       "      <td>0.207281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2291</th>\n",
       "      <td>674255168825880576</td>\n",
       "      <td>https://pbs.twimg.com/media/CVtvf6bWwAAd1rT.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/674255168...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>1581</td>\n",
       "      <td>Eskimo_dog</td>\n",
       "      <td>0.615741</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Tweet_id                                        image_url  \\\n",
       "2082  679475951516934144  https://pbs.twimg.com/media/CW37xZbUoAAUXe5.jpg   \n",
       "1060  759099523532779520  https://pbs.twimg.com/media/Cojc_Q0WcAAqi_K.jpg   \n",
       "2476  670804601705242624  https://pbs.twimg.com/media/CU8tOJZWUAAlNoF.jpg   \n",
       "174   862831371563274240  https://pbs.twimg.com/media/C_lkieeVwAAm0L4.jpg   \n",
       "2291  674255168825880576  https://pbs.twimg.com/media/CVtvf6bWwAAd1rT.jpg   \n",
       "\n",
       "                                               dog_urls  rating_numerator  \\\n",
       "2082  https://twitter.com/dog_rates/status/679475951...                 9   \n",
       "1060  https://twitter.com/dog_rates/status/759099523...                 7   \n",
       "2476  https://twitter.com/dog_rates/status/670804601...                10   \n",
       "174   https://twitter.com/dog_rates/status/862831371...                13   \n",
       "2291  https://twitter.com/dog_rates/status/674255168...                10   \n",
       "\n",
       "      rating_denominator  Favorites          prediction  prediction_confident  \n",
       "2082                  10       2304         Maltese_dog              0.145742  \n",
       "1060                  10      16101   Shetland_sheepdog              0.129034  \n",
       "2476                  10       2098          Pomeranian              0.868560  \n",
       "174                   10      20011  Australian_terrier              0.207281  \n",
       "2291                  10       1581          Eskimo_dog              0.615741  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2406 entries, 0 to 2746\n",
      "Data columns (total 8 columns):\n",
      " #   Column                Non-Null Count  Dtype  \n",
      "---  ------                --------------  -----  \n",
      " 0   Tweet_id              2406 non-null   int64  \n",
      " 1   image_url             2406 non-null   object \n",
      " 2   dog_urls              2406 non-null   object \n",
      " 3   rating_numerator      2406 non-null   int64  \n",
      " 4   rating_denominator    2406 non-null   int64  \n",
      " 5   Favorites             2406 non-null   int64  \n",
      " 6   prediction            2406 non-null   object \n",
      " 7   prediction_confident  2406 non-null   float64\n",
      "dtypes: float64(1), int64(4), object(3)\n",
      "memory usage: 141.0+ KB\n"
     ]
    }
   ],
   "source": [
    "master_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidiness Issue #7 :\n",
    "\n",
    "#### Define :\n",
    "\n",
    "master_df columns lowercase.\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df.columns = map(str.lower, master_df.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>image_url</th>\n",
       "      <th>dog_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>favorites</th>\n",
       "      <th>prediction</th>\n",
       "      <th>prediction_confident</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>39467</td>\n",
       "      <td>orange</td>\n",
       "      <td>0.097049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>33819</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.323581</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id                                        image_url  \\\n",
       "0  892420643555336193  https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg   \n",
       "1  892177421306343426  https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg   \n",
       "\n",
       "                                            dog_urls  rating_numerator  \\\n",
       "0  https://twitter.com/dog_rates/status/892420643...                13   \n",
       "1  https://twitter.com/dog_rates/status/892177421...                13   \n",
       "\n",
       "   rating_denominator  favorites prediction  prediction_confident  \n",
       "0                  10      39467     orange              0.097049  \n",
       "1                  10      33819  Chihuahua              0.323581  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Quality Issues :\n",
    "\n",
    "### Issue #4 :\n",
    "\n",
    "#### Define :\n",
    "\n",
    "convert the timestamp and retweeted_status_timestamp columns data type to datetime data type.\n",
    "\n",
    "\n",
    "##### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs.timestamp = pd.to_datetime(dfs.timestamp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs.retweeted_status_timestamp= pd.to_datetime(dfs.retweeted_status_timestamp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2747 entries, 0 to 2746\n",
      "Data columns (total 17 columns):\n",
      " #   Column                      Non-Null Count  Dtype              \n",
      "---  ------                      --------------  -----              \n",
      " 0   Tweet_id                    2747 non-null   int64              \n",
      " 1   Retweets                    2747 non-null   int64              \n",
      " 2   Favorites                   2747 non-null   int64              \n",
      " 3   tweet_id                    2747 non-null   int64              \n",
      " 4   in_reply_to_status_id       87 non-null     float64            \n",
      " 5   in_reply_to_user_id         87 non-null     float64            \n",
      " 6   timestamp                   2747 non-null   datetime64[ns, UTC]\n",
      " 7   source                      2747 non-null   object             \n",
      " 8   text                        2747 non-null   object             \n",
      " 9   retweeted_status_id         216 non-null    float64            \n",
      " 10  retweeted_status_user_id    216 non-null    float64            \n",
      " 11  retweeted_status_timestamp  216 non-null    datetime64[ns, UTC]\n",
      " 12  expanded_urls               2682 non-null   object             \n",
      " 13  rating_numerator            2747 non-null   int64              \n",
      " 14  rating_denominator          2747 non-null   int64              \n",
      " 15  name                        2747 non-null   object             \n",
      " 16  Dog_Type                    393 non-null    object             \n",
      "dtypes: datetime64[ns, UTC](2), float64(4), int64(6), object(5)\n",
      "memory usage: 332.6+ KB\n"
     ]
    }
   ],
   "source": [
    "dfs.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Issue #5 :\n",
    "\n",
    "#### Define :\n",
    "\n",
    "change the data type for in_reply_to_status_id, in_reply_to_user_id, retweeted_status_id and retweeted_status_user_id columns from float to integer.\n",
    "\n",
    "##### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs.in_reply_to_status_id= dfs.in_reply_to_status_id.astype(\"Int64\")\n",
    "dfs.in_reply_to_user_id= dfs.in_reply_to_user_id.astype(\"Int64\")\n",
    "dfs.retweeted_status_id= dfs.retweeted_status_id.astype(\"Int64\")\n",
    "dfs.retweeted_status_user_id= dfs.retweeted_status_user_id.astype(\"Int64\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2747 entries, 0 to 2746\n",
      "Data columns (total 17 columns):\n",
      " #   Column                      Non-Null Count  Dtype              \n",
      "---  ------                      --------------  -----              \n",
      " 0   Tweet_id                    2747 non-null   int64              \n",
      " 1   Retweets                    2747 non-null   int64              \n",
      " 2   Favorites                   2747 non-null   int64              \n",
      " 3   tweet_id                    2747 non-null   int64              \n",
      " 4   in_reply_to_status_id       87 non-null     Int64              \n",
      " 5   in_reply_to_user_id         87 non-null     Int64              \n",
      " 6   timestamp                   2747 non-null   datetime64[ns, UTC]\n",
      " 7   source                      2747 non-null   object             \n",
      " 8   text                        2747 non-null   object             \n",
      " 9   retweeted_status_id         216 non-null    Int64              \n",
      " 10  retweeted_status_user_id    216 non-null    Int64              \n",
      " 11  retweeted_status_timestamp  216 non-null    datetime64[ns, UTC]\n",
      " 12  expanded_urls               2682 non-null   object             \n",
      " 13  rating_numerator            2747 non-null   int64              \n",
      " 14  rating_denominator          2747 non-null   int64              \n",
      " 15  name                        2747 non-null   object             \n",
      " 16  Dog_Type                    393 non-null    object             \n",
      "dtypes: Int64(4), datetime64[ns, UTC](2), int64(6), object(5)\n",
      "memory usage: 343.4+ KB\n"
     ]
    }
   ],
   "source": [
    "dfs.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidiness Issue #5 :\n",
    "\n",
    "#### Define :\n",
    "in images_df table lowercase for p1,p2 and p3 columns.\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "images_df_clean.p1=images_df_clean.p1.str.lower()\n",
    "images_df_clean.p2=images_df_clean.p2.str.lower()\n",
    "images_df_clean.p3=images_df_clean.p3.str.lower()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>jpg_url</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>680130881361686529</td>\n",
       "      <td>https://pbs.twimg.com/media/CXBPbVtWAAA2Vus.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>maltese_dog</td>\n",
       "      <td>0.199121</td>\n",
       "      <td>True</td>\n",
       "      <td>west_highland_white_terrier</td>\n",
       "      <td>0.197897</td>\n",
       "      <td>True</td>\n",
       "      <td>shih-tzu</td>\n",
       "      <td>0.157130</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>670403879788544000</td>\n",
       "      <td>https://pbs.twimg.com/media/CU3AxW1WoAA3_35.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>pug</td>\n",
       "      <td>0.802223</td>\n",
       "      <td>True</td>\n",
       "      <td>french_bulldog</td>\n",
       "      <td>0.172557</td>\n",
       "      <td>True</td>\n",
       "      <td>bull_mastiff</td>\n",
       "      <td>0.007163</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1212</th>\n",
       "      <td>743210557239623680</td>\n",
       "      <td>https://pbs.twimg.com/media/ClBqDuDWkAALK2e.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.930705</td>\n",
       "      <td>True</td>\n",
       "      <td>chesapeake_bay_retriever</td>\n",
       "      <td>0.025934</td>\n",
       "      <td>True</td>\n",
       "      <td>labrador_retriever</td>\n",
       "      <td>0.007535</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id                                          jpg_url  \\\n",
       "612   680130881361686529  https://pbs.twimg.com/media/CXBPbVtWAAA2Vus.jpg   \n",
       "229   670403879788544000  https://pbs.twimg.com/media/CU3AxW1WoAA3_35.jpg   \n",
       "1212  743210557239623680  https://pbs.twimg.com/media/ClBqDuDWkAALK2e.jpg   \n",
       "\n",
       "      img_num                p1   p1_conf  p1_dog  \\\n",
       "612         1       maltese_dog  0.199121    True   \n",
       "229         1               pug  0.802223    True   \n",
       "1212        1  golden_retriever  0.930705    True   \n",
       "\n",
       "                               p2   p2_conf  p2_dog                  p3  \\\n",
       "612   west_highland_white_terrier  0.197897    True            shih-tzu   \n",
       "229                french_bulldog  0.172557    True        bull_mastiff   \n",
       "1212     chesapeake_bay_retriever  0.025934    True  labrador_retriever   \n",
       "\n",
       "       p3_conf  p3_dog  \n",
       "612   0.157130    True  \n",
       "229   0.007163    True  \n",
       "1212  0.007535    True  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images_df_clean.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "labrador_retriever    104\n",
       "golden_retriever       92\n",
       "cardigan               74\n",
       "chihuahua              44\n",
       "pomeranian             42\n",
       "                     ... \n",
       "hyena                   1\n",
       "home_theater            1\n",
       "promontory              1\n",
       "cannon                  1\n",
       "barracouta              1\n",
       "Name: p2, Length: 404, dtype: int64"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images_df_clean.p2.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidiness Issue 6 :\n",
    "\n",
    "#### Define :\n",
    "prediction column lowercase.\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df.prediction=master_df.prediction.str.lower()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>image_url</th>\n",
       "      <th>dog_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>favorites</th>\n",
       "      <th>prediction</th>\n",
       "      <th>prediction_confident</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2210</th>\n",
       "      <td>675740360753160193</td>\n",
       "      <td>https://pbs.twimg.com/ext_tw_video_thumb/67574...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/675740360...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>1257</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.800495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>962</th>\n",
       "      <td>771171053431250945</td>\n",
       "      <td>https://pbs.twimg.com/media/CVgdFjNWEAAxmbq.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/673320132...</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>samoyed</td>\n",
       "      <td>0.978833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1027</th>\n",
       "      <td>761750502866649088</td>\n",
       "      <td>https://pbs.twimg.com/media/CYLDikFWEAAIy1y.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/685325112...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.586937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1941</th>\n",
       "      <td>684800227459624960</td>\n",
       "      <td>https://pbs.twimg.com/media/CYDmK7ZVAAI_ylL.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/684800227...</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>2979</td>\n",
       "      <td>miniature_schnauzer</td>\n",
       "      <td>0.294457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2561</th>\n",
       "      <td>669351434509529089</td>\n",
       "      <td>https://pbs.twimg.com/media/CUoDk8mWsAAMyBL.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/669351434...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>474</td>\n",
       "      <td>cuirass</td>\n",
       "      <td>0.756829</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id                                          image_url  \\\n",
       "2210  675740360753160193  https://pbs.twimg.com/ext_tw_video_thumb/67574...   \n",
       "962   771171053431250945    https://pbs.twimg.com/media/CVgdFjNWEAAxmbq.jpg   \n",
       "1027  761750502866649088    https://pbs.twimg.com/media/CYLDikFWEAAIy1y.jpg   \n",
       "1941  684800227459624960    https://pbs.twimg.com/media/CYDmK7ZVAAI_ylL.jpg   \n",
       "2561  669351434509529089    https://pbs.twimg.com/media/CUoDk8mWsAAMyBL.jpg   \n",
       "\n",
       "                                               dog_urls  rating_numerator  \\\n",
       "2210  https://twitter.com/dog_rates/status/675740360...                12   \n",
       "962   https://twitter.com/dog_rates/status/673320132...                11   \n",
       "1027  https://twitter.com/dog_rates/status/685325112...                10   \n",
       "1941  https://twitter.com/dog_rates/status/684800227...                11   \n",
       "2561  https://twitter.com/dog_rates/status/669351434...                10   \n",
       "\n",
       "      rating_denominator  favorites           prediction  prediction_confident  \n",
       "2210                  10       1257     golden_retriever              0.800495  \n",
       "962                   10          0              samoyed              0.978833  \n",
       "1027                  10          0     golden_retriever              0.586937  \n",
       "1941                  10       2979  miniature_schnauzer              0.294457  \n",
       "2561                  10        474              cuirass              0.756829  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality Issue 9 :\n",
    "\n",
    "#### Define :\n",
    "remove duplicates of master_df.\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(master_df.duplicated())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality issue 10 :\n",
    "#### Define :\n",
    "reset index for master_df.\n",
    "\n",
    "#### code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df=master_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>image_url</th>\n",
       "      <th>dog_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>favorites</th>\n",
       "      <th>prediction</th>\n",
       "      <th>prediction_confident</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>889638837579907072</td>\n",
       "      <td>https://pbs.twimg.com/media/DFihzFfXsAYGDPR.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/889638837...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>27672</td>\n",
       "      <td>french_bulldog</td>\n",
       "      <td>0.991650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>889531135344209921</td>\n",
       "      <td>https://pbs.twimg.com/media/DFg_2PVW0AEHN3p.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/889531135...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>15359</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.953442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>889278841981685760</td>\n",
       "      <td>https://pbs.twimg.com/ext_tw_video_thumb/88927...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/889278841...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>25652</td>\n",
       "      <td>whippet</td>\n",
       "      <td>0.626152</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id                                          image_url  \\\n",
       "13  889638837579907072    https://pbs.twimg.com/media/DFihzFfXsAYGDPR.jpg   \n",
       "14  889531135344209921    https://pbs.twimg.com/media/DFg_2PVW0AEHN3p.jpg   \n",
       "15  889278841981685760  https://pbs.twimg.com/ext_tw_video_thumb/88927...   \n",
       "\n",
       "                                             dog_urls  rating_numerator  \\\n",
       "13  https://twitter.com/dog_rates/status/889638837...                12   \n",
       "14  https://twitter.com/dog_rates/status/889531135...                13   \n",
       "15  https://twitter.com/dog_rates/status/889278841...                13   \n",
       "\n",
       "    rating_denominator  favorites        prediction  prediction_confident  \n",
       "13                  10      27672    french_bulldog              0.991650  \n",
       "14                  10      15359  golden_retriever              0.953442  \n",
       "15                  10      25652           whippet              0.626152  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.loc[13:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>image_url</th>\n",
       "      <th>dog_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>favorites</th>\n",
       "      <th>prediction</th>\n",
       "      <th>prediction_confident</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2043</th>\n",
       "      <td>666411507551481857</td>\n",
       "      <td>https://pbs.twimg.com/media/CT-RugiWIAELEaq.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666411507...</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>459</td>\n",
       "      <td>coho</td>\n",
       "      <td>0.404640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2044</th>\n",
       "      <td>666407126856765440</td>\n",
       "      <td>https://pbs.twimg.com/media/CT-NvwmW4AAugGZ.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666407126...</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>113</td>\n",
       "      <td>black-and-tan_coonhound</td>\n",
       "      <td>0.529139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2045</th>\n",
       "      <td>666396247373291520</td>\n",
       "      <td>https://pbs.twimg.com/media/CT-D2ZHWIAA3gK1.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666396247...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>172</td>\n",
       "      <td>chihuahua</td>\n",
       "      <td>0.978108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2046</th>\n",
       "      <td>666373753744588802</td>\n",
       "      <td>https://pbs.twimg.com/media/CT9vZEYWUAAlZ05.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666373753...</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>194</td>\n",
       "      <td>soft-coated_wheaten_terrier</td>\n",
       "      <td>0.326467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2047</th>\n",
       "      <td>666362758909284353</td>\n",
       "      <td>https://pbs.twimg.com/media/CT9lXGsUcAAyUFt.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666362758...</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>804</td>\n",
       "      <td>guinea_pig</td>\n",
       "      <td>0.996496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2048</th>\n",
       "      <td>666353288456101888</td>\n",
       "      <td>https://pbs.twimg.com/media/CT9cx0tUEAAhNN_.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666353288...</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>229</td>\n",
       "      <td>malamute</td>\n",
       "      <td>0.336874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2049</th>\n",
       "      <td>666345417576210432</td>\n",
       "      <td>https://pbs.twimg.com/media/CT9Vn7PWoAA_ZCM.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666345417...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>307</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.858744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2050</th>\n",
       "      <td>666337882303524864</td>\n",
       "      <td>https://pbs.twimg.com/media/CT9OwFIWEAMuRje.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666337882...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>204</td>\n",
       "      <td>ox</td>\n",
       "      <td>0.416669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2051</th>\n",
       "      <td>666293911632134144</td>\n",
       "      <td>https://pbs.twimg.com/media/CT8mx7KW4AEQu8N.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666293911...</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>522</td>\n",
       "      <td>three-toed_sloth</td>\n",
       "      <td>0.914671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2052</th>\n",
       "      <td>666287406224695296</td>\n",
       "      <td>https://pbs.twimg.com/media/CT8g3BpUEAAuFjg.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666287406...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>152</td>\n",
       "      <td>maltese_dog</td>\n",
       "      <td>0.857531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2053</th>\n",
       "      <td>666273097616637952</td>\n",
       "      <td>https://pbs.twimg.com/media/CT8T1mtUwAA3aqm.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666273097...</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>184</td>\n",
       "      <td>italian_greyhound</td>\n",
       "      <td>0.176053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2054</th>\n",
       "      <td>666268910803644416</td>\n",
       "      <td>https://pbs.twimg.com/media/CT8QCd1WEAADXws.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666268910...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>108</td>\n",
       "      <td>desktop_computer</td>\n",
       "      <td>0.086502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2055</th>\n",
       "      <td>666104133288665088</td>\n",
       "      <td>https://pbs.twimg.com/media/CT56LSZWoAAlJj2.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666104133...</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>14765</td>\n",
       "      <td>hen</td>\n",
       "      <td>0.965932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2056</th>\n",
       "      <td>666102155909144576</td>\n",
       "      <td>https://pbs.twimg.com/media/CT54YGiWUAEZnoK.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666102155...</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>81</td>\n",
       "      <td>english_setter</td>\n",
       "      <td>0.298617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2057</th>\n",
       "      <td>666099513787052032</td>\n",
       "      <td>https://pbs.twimg.com/media/CT51-JJUEAA6hV8.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666099513...</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>164</td>\n",
       "      <td>lhasa</td>\n",
       "      <td>0.582330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2058</th>\n",
       "      <td>666094000022159362</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5w9gUW4AAsBNN.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666094000...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>169</td>\n",
       "      <td>bloodhound</td>\n",
       "      <td>0.195217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2059</th>\n",
       "      <td>666082916733198337</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5m4VGWEAAtKc8.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666082916...</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>121</td>\n",
       "      <td>pug</td>\n",
       "      <td>0.489814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2060</th>\n",
       "      <td>666073100786774016</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5d9DZXAAALcwe.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666073100...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>335</td>\n",
       "      <td>walker_hound</td>\n",
       "      <td>0.260857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2061</th>\n",
       "      <td>666071193221509120</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5cN_3WEAAlOoZ.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666071193...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>154</td>\n",
       "      <td>gordon_setter</td>\n",
       "      <td>0.503672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2062</th>\n",
       "      <td>666063827256086533</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5Vg_wXIAAXfnj.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666063827...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>496</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.775930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2063</th>\n",
       "      <td>666058600524156928</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5Qw94XAAA_2dP.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666058600...</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>115</td>\n",
       "      <td>miniature_poodle</td>\n",
       "      <td>0.201493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2064</th>\n",
       "      <td>666057090499244032</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5PY90WoAAQGLo.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666057090...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>304</td>\n",
       "      <td>shopping_cart</td>\n",
       "      <td>0.962465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2065</th>\n",
       "      <td>666055525042405380</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5N9tpXIAAifs1.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666055525...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>448</td>\n",
       "      <td>chow</td>\n",
       "      <td>0.692517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2066</th>\n",
       "      <td>666051853826850816</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5KoJ1WoAAJash.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666051853...</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>1253</td>\n",
       "      <td>box_turtle</td>\n",
       "      <td>0.933012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2067</th>\n",
       "      <td>666050758794694657</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5Jof1WUAEuVxN.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666050758...</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>136</td>\n",
       "      <td>bernese_mountain_dog</td>\n",
       "      <td>0.651137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2068</th>\n",
       "      <td>666049248165822465</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666049248...</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>111</td>\n",
       "      <td>miniature_pinscher</td>\n",
       "      <td>0.560311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2069</th>\n",
       "      <td>666044226329800704</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666044226...</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>311</td>\n",
       "      <td>rhodesian_ridgeback</td>\n",
       "      <td>0.408143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2070</th>\n",
       "      <td>666033412701032449</td>\n",
       "      <td>https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666033412...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>128</td>\n",
       "      <td>german_shepherd</td>\n",
       "      <td>0.596461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2071</th>\n",
       "      <td>666029285002620928</td>\n",
       "      <td>https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666029285...</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>132</td>\n",
       "      <td>redbone</td>\n",
       "      <td>0.506826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2072</th>\n",
       "      <td>666020888022790149</td>\n",
       "      <td>https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666020888...</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>2535</td>\n",
       "      <td>welsh_springer_spaniel</td>\n",
       "      <td>0.465074</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id                                        image_url  \\\n",
       "2043  666411507551481857  https://pbs.twimg.com/media/CT-RugiWIAELEaq.jpg   \n",
       "2044  666407126856765440  https://pbs.twimg.com/media/CT-NvwmW4AAugGZ.jpg   \n",
       "2045  666396247373291520  https://pbs.twimg.com/media/CT-D2ZHWIAA3gK1.jpg   \n",
       "2046  666373753744588802  https://pbs.twimg.com/media/CT9vZEYWUAAlZ05.jpg   \n",
       "2047  666362758909284353  https://pbs.twimg.com/media/CT9lXGsUcAAyUFt.jpg   \n",
       "2048  666353288456101888  https://pbs.twimg.com/media/CT9cx0tUEAAhNN_.jpg   \n",
       "2049  666345417576210432  https://pbs.twimg.com/media/CT9Vn7PWoAA_ZCM.jpg   \n",
       "2050  666337882303524864  https://pbs.twimg.com/media/CT9OwFIWEAMuRje.jpg   \n",
       "2051  666293911632134144  https://pbs.twimg.com/media/CT8mx7KW4AEQu8N.jpg   \n",
       "2052  666287406224695296  https://pbs.twimg.com/media/CT8g3BpUEAAuFjg.jpg   \n",
       "2053  666273097616637952  https://pbs.twimg.com/media/CT8T1mtUwAA3aqm.jpg   \n",
       "2054  666268910803644416  https://pbs.twimg.com/media/CT8QCd1WEAADXws.jpg   \n",
       "2055  666104133288665088  https://pbs.twimg.com/media/CT56LSZWoAAlJj2.jpg   \n",
       "2056  666102155909144576  https://pbs.twimg.com/media/CT54YGiWUAEZnoK.jpg   \n",
       "2057  666099513787052032  https://pbs.twimg.com/media/CT51-JJUEAA6hV8.jpg   \n",
       "2058  666094000022159362  https://pbs.twimg.com/media/CT5w9gUW4AAsBNN.jpg   \n",
       "2059  666082916733198337  https://pbs.twimg.com/media/CT5m4VGWEAAtKc8.jpg   \n",
       "2060  666073100786774016  https://pbs.twimg.com/media/CT5d9DZXAAALcwe.jpg   \n",
       "2061  666071193221509120  https://pbs.twimg.com/media/CT5cN_3WEAAlOoZ.jpg   \n",
       "2062  666063827256086533  https://pbs.twimg.com/media/CT5Vg_wXIAAXfnj.jpg   \n",
       "2063  666058600524156928  https://pbs.twimg.com/media/CT5Qw94XAAA_2dP.jpg   \n",
       "2064  666057090499244032  https://pbs.twimg.com/media/CT5PY90WoAAQGLo.jpg   \n",
       "2065  666055525042405380  https://pbs.twimg.com/media/CT5N9tpXIAAifs1.jpg   \n",
       "2066  666051853826850816  https://pbs.twimg.com/media/CT5KoJ1WoAAJash.jpg   \n",
       "2067  666050758794694657  https://pbs.twimg.com/media/CT5Jof1WUAEuVxN.jpg   \n",
       "2068  666049248165822465  https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg   \n",
       "2069  666044226329800704  https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg   \n",
       "2070  666033412701032449  https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg   \n",
       "2071  666029285002620928  https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg   \n",
       "2072  666020888022790149  https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg   \n",
       "\n",
       "                                               dog_urls  rating_numerator  \\\n",
       "2043  https://twitter.com/dog_rates/status/666411507...                 2   \n",
       "2044  https://twitter.com/dog_rates/status/666407126...                 7   \n",
       "2045  https://twitter.com/dog_rates/status/666396247...                 9   \n",
       "2046  https://twitter.com/dog_rates/status/666373753...                11   \n",
       "2047  https://twitter.com/dog_rates/status/666362758...                 6   \n",
       "2048  https://twitter.com/dog_rates/status/666353288...                 8   \n",
       "2049  https://twitter.com/dog_rates/status/666345417...                10   \n",
       "2050  https://twitter.com/dog_rates/status/666337882...                 9   \n",
       "2051  https://twitter.com/dog_rates/status/666293911...                 3   \n",
       "2052  https://twitter.com/dog_rates/status/666287406...                 1   \n",
       "2053  https://twitter.com/dog_rates/status/666273097...                11   \n",
       "2054  https://twitter.com/dog_rates/status/666268910...                10   \n",
       "2055  https://twitter.com/dog_rates/status/666104133...                 1   \n",
       "2056  https://twitter.com/dog_rates/status/666102155...                11   \n",
       "2057  https://twitter.com/dog_rates/status/666099513...                 8   \n",
       "2058  https://twitter.com/dog_rates/status/666094000...                 9   \n",
       "2059  https://twitter.com/dog_rates/status/666082916...                 6   \n",
       "2060  https://twitter.com/dog_rates/status/666073100...                10   \n",
       "2061  https://twitter.com/dog_rates/status/666071193...                 9   \n",
       "2062  https://twitter.com/dog_rates/status/666063827...                10   \n",
       "2063  https://twitter.com/dog_rates/status/666058600...                 8   \n",
       "2064  https://twitter.com/dog_rates/status/666057090...                 9   \n",
       "2065  https://twitter.com/dog_rates/status/666055525...                10   \n",
       "2066  https://twitter.com/dog_rates/status/666051853...                 2   \n",
       "2067  https://twitter.com/dog_rates/status/666050758...                10   \n",
       "2068  https://twitter.com/dog_rates/status/666049248...                 5   \n",
       "2069  https://twitter.com/dog_rates/status/666044226...                 6   \n",
       "2070  https://twitter.com/dog_rates/status/666033412...                 9   \n",
       "2071  https://twitter.com/dog_rates/status/666029285...                 7   \n",
       "2072  https://twitter.com/dog_rates/status/666020888...                 8   \n",
       "\n",
       "      rating_denominator  favorites                   prediction  \\\n",
       "2043                  10        459                         coho   \n",
       "2044                  10        113      black-and-tan_coonhound   \n",
       "2045                  10        172                    chihuahua   \n",
       "2046                  10        194  soft-coated_wheaten_terrier   \n",
       "2047                  10        804                   guinea_pig   \n",
       "2048                  10        229                     malamute   \n",
       "2049                  10        307             golden_retriever   \n",
       "2050                  10        204                           ox   \n",
       "2051                  10        522             three-toed_sloth   \n",
       "2052                   2        152                  maltese_dog   \n",
       "2053                  10        184            italian_greyhound   \n",
       "2054                  10        108             desktop_computer   \n",
       "2055                  10      14765                          hen   \n",
       "2056                  10         81               english_setter   \n",
       "2057                  10        164                        lhasa   \n",
       "2058                  10        169                   bloodhound   \n",
       "2059                  10        121                          pug   \n",
       "2060                  10        335                 walker_hound   \n",
       "2061                  10        154                gordon_setter   \n",
       "2062                  10        496             golden_retriever   \n",
       "2063                  10        115             miniature_poodle   \n",
       "2064                  10        304                shopping_cart   \n",
       "2065                  10        448                         chow   \n",
       "2066                  10       1253                   box_turtle   \n",
       "2067                  10        136         bernese_mountain_dog   \n",
       "2068                  10        111           miniature_pinscher   \n",
       "2069                  10        311          rhodesian_ridgeback   \n",
       "2070                  10        128              german_shepherd   \n",
       "2071                  10        132                      redbone   \n",
       "2072                  10       2535       welsh_springer_spaniel   \n",
       "\n",
       "      prediction_confident  \n",
       "2043              0.404640  \n",
       "2044              0.529139  \n",
       "2045              0.978108  \n",
       "2046              0.326467  \n",
       "2047              0.996496  \n",
       "2048              0.336874  \n",
       "2049              0.858744  \n",
       "2050              0.416669  \n",
       "2051              0.914671  \n",
       "2052              0.857531  \n",
       "2053              0.176053  \n",
       "2054              0.086502  \n",
       "2055              0.965932  \n",
       "2056              0.298617  \n",
       "2057              0.582330  \n",
       "2058              0.195217  \n",
       "2059              0.489814  \n",
       "2060              0.260857  \n",
       "2061              0.503672  \n",
       "2062              0.775930  \n",
       "2063              0.201493  \n",
       "2064              0.962465  \n",
       "2065              0.692517  \n",
       "2066              0.933012  \n",
       "2067              0.651137  \n",
       "2068              0.560311  \n",
       "2069              0.408143  \n",
       "2070              0.596461  \n",
       "2071              0.506826  \n",
       "2072              0.465074  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.tail(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Storing Data\n",
    "Save gathered, assessed, and cleaned master dataset to a CSV file named \"twitter_archive_master.csv\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_archive_master = master_df.to_csv('twitter_archive_master.csv',index= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df_saved = pd.read_csv('twitter_archive_master.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>image_url</th>\n",
       "      <th>dog_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>favorites</th>\n",
       "      <th>prediction</th>\n",
       "      <th>prediction_confident</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>39467</td>\n",
       "      <td>orange</td>\n",
       "      <td>0.097049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>33819</td>\n",
       "      <td>chihuahua</td>\n",
       "      <td>0.323581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891815181...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>25461</td>\n",
       "      <td>chihuahua</td>\n",
       "      <td>0.716012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>https://pbs.twimg.com/media/DF_q7IAWsAEuuN8.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891689557...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>42908</td>\n",
       "      <td>paper_towel</td>\n",
       "      <td>0.170278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>https://pbs.twimg.com/media/DF6hr6BUMAAzZgT.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891327558...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>41048</td>\n",
       "      <td>basset</td>\n",
       "      <td>0.555712</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id                                        image_url  \\\n",
       "0  892420643555336193  https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg   \n",
       "1  892177421306343426  https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg   \n",
       "2  891815181378084864  https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg   \n",
       "3  891689557279858688  https://pbs.twimg.com/media/DF_q7IAWsAEuuN8.jpg   \n",
       "4  891327558926688256  https://pbs.twimg.com/media/DF6hr6BUMAAzZgT.jpg   \n",
       "\n",
       "                                            dog_urls  rating_numerator  \\\n",
       "0  https://twitter.com/dog_rates/status/892420643...                13   \n",
       "1  https://twitter.com/dog_rates/status/892177421...                13   \n",
       "2  https://twitter.com/dog_rates/status/891815181...                12   \n",
       "3  https://twitter.com/dog_rates/status/891689557...                13   \n",
       "4  https://twitter.com/dog_rates/status/891327558...                12   \n",
       "\n",
       "   rating_denominator  favorites   prediction  prediction_confident  \n",
       "0                  10      39467       orange              0.097049  \n",
       "1                  10      33819    chihuahua              0.323581  \n",
       "2                  10      25461    chihuahua              0.716012  \n",
       "3                  10      42908  paper_towel              0.170278  \n",
       "4                  10      41048       basset              0.555712  "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df_saved.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>image_url</th>\n",
       "      <th>dog_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>favorites</th>\n",
       "      <th>prediction</th>\n",
       "      <th>prediction_confident</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>39467</td>\n",
       "      <td>orange</td>\n",
       "      <td>0.097049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>33819</td>\n",
       "      <td>chihuahua</td>\n",
       "      <td>0.323581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891815181...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>25461</td>\n",
       "      <td>chihuahua</td>\n",
       "      <td>0.716012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>https://pbs.twimg.com/media/DF_q7IAWsAEuuN8.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891689557...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>42908</td>\n",
       "      <td>paper_towel</td>\n",
       "      <td>0.170278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>https://pbs.twimg.com/media/DF6hr6BUMAAzZgT.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891327558...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>41048</td>\n",
       "      <td>basset</td>\n",
       "      <td>0.555712</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id                                        image_url  \\\n",
       "0  892420643555336193  https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg   \n",
       "1  892177421306343426  https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg   \n",
       "2  891815181378084864  https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg   \n",
       "3  891689557279858688  https://pbs.twimg.com/media/DF_q7IAWsAEuuN8.jpg   \n",
       "4  891327558926688256  https://pbs.twimg.com/media/DF6hr6BUMAAzZgT.jpg   \n",
       "\n",
       "                                            dog_urls  rating_numerator  \\\n",
       "0  https://twitter.com/dog_rates/status/892420643...                13   \n",
       "1  https://twitter.com/dog_rates/status/892177421...                13   \n",
       "2  https://twitter.com/dog_rates/status/891815181...                12   \n",
       "3  https://twitter.com/dog_rates/status/891689557...                13   \n",
       "4  https://twitter.com/dog_rates/status/891327558...                12   \n",
       "\n",
       "   rating_denominator  favorites   prediction  prediction_confident  \n",
       "0                  10      39467       orange              0.097049  \n",
       "1                  10      33819    chihuahua              0.323581  \n",
       "2                  10      25461    chihuahua              0.716012  \n",
       "3                  10      42908  paper_towel              0.170278  \n",
       "4                  10      41048       basset              0.555712  "
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyzing and Visualizing Data\n",
    "In this section, analyze and visualize your wrangled data. You must produce at least **three (3) insights and one (1) visualization.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Insight #1 :\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "golden_retriever      150\n",
       "labrador_retriever    100\n",
       "pembroke               88\n",
       "chihuahua              83\n",
       "pug                    57\n",
       "chow                   44\n",
       "samoyed                43\n",
       "toy_poodle             39\n",
       "pomeranian             38\n",
       "cocker_spaniel         30\n",
       "Name: prediction, dtype: int64"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.prediction.value_counts().head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 421 entries, 1 to 2062\n",
      "Data columns (total 8 columns):\n",
      " #   Column                Non-Null Count  Dtype  \n",
      "---  ------                --------------  -----  \n",
      " 0   tweet_id              421 non-null    int64  \n",
      " 1   image_url             421 non-null    object \n",
      " 2   dog_urls              421 non-null    object \n",
      " 3   rating_numerator      421 non-null    int64  \n",
      " 4   rating_denominator    421 non-null    int64  \n",
      " 5   favorites             421 non-null    int64  \n",
      " 6   prediction            421 non-null    object \n",
      " 7   prediction_confident  421 non-null    float64\n",
      "dtypes: float64(1), int64(4), object(3)\n",
      "memory usage: 24.7+ KB\n"
     ]
    }
   ],
   "source": [
    "top_pred = ['golden_retriever','labrador_retriever','pembroke','chihuahua']\n",
    "predm_df = master_df[master_df['prediction'].isin(top_pred)]\n",
    "predm_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "prediction\n",
       "chihuahua             0.586888\n",
       "labrador_retriever    0.666523\n",
       "pembroke              0.717223\n",
       "golden_retriever      0.723555\n",
       "Name: prediction_confident, dtype: float64"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_confident = predm_df.groupby('prediction').mean()['prediction_confident']\n",
    "best_confident.sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Insight #2 :\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a dataframe that only contains rating_denominator equal 10.\n",
    "\n",
    "denom10 = master_df['rating_denominator']==10\n",
    "denom10_df = master_df[denom10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "golden_retriever      146\n",
       "labrador_retriever     97\n",
       "pembroke               88\n",
       "chihuahua              83\n",
       "pug                    57\n",
       "chow                   44\n",
       "samoyed                43\n",
       "toy_poodle             39\n",
       "pomeranian             38\n",
       "malamute               30\n",
       "Name: prediction, dtype: int64"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# to get the most popular three type of dogs.\n",
    "denom10_df.prediction.value_counts().head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 331 entries, 9 to 2062\n",
      "Data columns (total 8 columns):\n",
      " #   Column                Non-Null Count  Dtype  \n",
      "---  ------                --------------  -----  \n",
      " 0   tweet_id              331 non-null    int64  \n",
      " 1   image_url             331 non-null    object \n",
      " 2   dog_urls              331 non-null    object \n",
      " 3   rating_numerator      331 non-null    int64  \n",
      " 4   rating_denominator    331 non-null    int64  \n",
      " 5   favorites             331 non-null    int64  \n",
      " 6   prediction            331 non-null    object \n",
      " 7   prediction_confident  331 non-null    float64\n",
      "dtypes: float64(1), int64(4), object(3)\n",
      "memory usage: 19.4+ KB\n"
     ]
    }
   ],
   "source": [
    "# crrating a dataframe that only contains the three most popular with rating_denominator of 10.\n",
    "top_pred = ['golden_retriever','labrador_retriever','pembroke']\n",
    "pred_df = denom10_df[denom10_df['prediction'].isin(top_pred)]\n",
    "pred_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>image_url</th>\n",
       "      <th>dog_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>favorites</th>\n",
       "      <th>prediction</th>\n",
       "      <th>prediction_confident</th>\n",
       "      <th>golden_retriever</th>\n",
       "      <th>labrador_retriever</th>\n",
       "      <th>pembroke</th>\n",
       "      <th>intercept</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>890240255349198849</td>\n",
       "      <td>https://pbs.twimg.com/media/DFrEyVuW0AAO3t9.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/890240255...</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>32467</td>\n",
       "      <td>pembroke</td>\n",
       "      <td>0.511319</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>889665388333682689</td>\n",
       "      <td>https://pbs.twimg.com/media/DFi579UWsAAatzw.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/889665388...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>38818</td>\n",
       "      <td>pembroke</td>\n",
       "      <td>0.966327</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>889531135344209921</td>\n",
       "      <td>https://pbs.twimg.com/media/DFg_2PVW0AEHN3p.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/889531135...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>15359</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.953442</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>888917238123831296</td>\n",
       "      <td>https://pbs.twimg.com/media/DFYRgsOUQAARGhO.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/888917238...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>29611</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.714719</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>888804989199671297</td>\n",
       "      <td>https://pbs.twimg.com/media/DFWra-3VYAA2piG.jpg</td>\n",
       "      <td>https://twitter.com/dog_rates/status/888804989...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>26080</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.469760</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id                                        image_url  \\\n",
       "9   890240255349198849  https://pbs.twimg.com/media/DFrEyVuW0AAO3t9.jpg   \n",
       "12  889665388333682689  https://pbs.twimg.com/media/DFi579UWsAAatzw.jpg   \n",
       "14  889531135344209921  https://pbs.twimg.com/media/DFg_2PVW0AEHN3p.jpg   \n",
       "16  888917238123831296  https://pbs.twimg.com/media/DFYRgsOUQAARGhO.jpg   \n",
       "17  888804989199671297  https://pbs.twimg.com/media/DFWra-3VYAA2piG.jpg   \n",
       "\n",
       "                                             dog_urls  rating_numerator  \\\n",
       "9   https://twitter.com/dog_rates/status/890240255...                14   \n",
       "12  https://twitter.com/dog_rates/status/889665388...                13   \n",
       "14  https://twitter.com/dog_rates/status/889531135...                13   \n",
       "16  https://twitter.com/dog_rates/status/888917238...                12   \n",
       "17  https://twitter.com/dog_rates/status/888804989...                13   \n",
       "\n",
       "    rating_denominator  favorites        prediction  prediction_confident  \\\n",
       "9                   10      32467          pembroke              0.511319   \n",
       "12                  10      38818          pembroke              0.966327   \n",
       "14                  10      15359  golden_retriever              0.953442   \n",
       "16                  10      29611  golden_retriever              0.714719   \n",
       "17                  10      26080  golden_retriever              0.469760   \n",
       "\n",
       "    golden_retriever  labrador_retriever  pembroke  intercept  \n",
       "9                  0                   0         1          1  \n",
       "12                 0                   0         1          1  \n",
       "14                 1                   0         0          1  \n",
       "16                 1                   0         0          1  \n",
       "17                 1                   0         0          1  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import statsmodels.api as sm\n",
    "\n",
    "# making multipule linear regression model \n",
    "dummies = pd.get_dummies(pred_df['prediction'])\n",
    "new_pred_df = pred_df.join(dummies)\n",
    "new_pred_df['intercept']=1\n",
    "new_pred_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>    <td>rating_numerator</td> <th>  R-squared:         </th> <td>   0.017</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.011</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   2.815</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Tue, 21 Sep 2021</td> <th>  Prob (F-statistic):</th>  <td>0.0614</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>14:38:17</td>     <th>  Log-Likelihood:    </th> <td> -586.04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   331</td>      <th>  AIC:               </th> <td>   1178.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   328</td>      <th>  BIC:               </th> <td>   1189.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     2</td>      <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "           <td></td>             <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>intercept</th>          <td>   11.4432</td> <td>    0.152</td> <td>   75.184</td> <td> 0.000</td> <td>   11.144</td> <td>   11.743</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>labrador_retriever</th> <td>   -0.3092</td> <td>    0.210</td> <td>   -1.471</td> <td> 0.142</td> <td>   -0.723</td> <td>    0.104</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>golden_retriever</th>   <td>    0.1322</td> <td>    0.193</td> <td>    0.686</td> <td> 0.493</td> <td>   -0.247</td> <td>    0.511</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>111.863</td> <th>  Durbin-Watson:     </th> <td>   1.411</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td> 397.050</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td>-1.464</td>  <th>  Prob(JB):          </th> <td>6.05e-87</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td> 7.497</td>  <th>  Cond. No.          </th> <td>    4.18</td>\n",
       "</tr>\n",
       "</table><br/><br/>Notes:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:       rating_numerator   R-squared:                       0.017\n",
       "Model:                            OLS   Adj. R-squared:                  0.011\n",
       "Method:                 Least Squares   F-statistic:                     2.815\n",
       "Date:                Tue, 21 Sep 2021   Prob (F-statistic):             0.0614\n",
       "Time:                        14:38:17   Log-Likelihood:                -586.04\n",
       "No. Observations:                 331   AIC:                             1178.\n",
       "Df Residuals:                     328   BIC:                             1189.\n",
       "Df Model:                           2                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "======================================================================================\n",
       "                         coef    std err          t      P>|t|      [0.025      0.975]\n",
       "--------------------------------------------------------------------------------------\n",
       "intercept             11.4432      0.152     75.184      0.000      11.144      11.743\n",
       "labrador_retriever    -0.3092      0.210     -1.471      0.142      -0.723       0.104\n",
       "golden_retriever       0.1322      0.193      0.686      0.493      -0.247       0.511\n",
       "==============================================================================\n",
       "Omnibus:                      111.863   Durbin-Watson:                   1.411\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              397.050\n",
       "Skew:                          -1.464   Prob(JB):                     6.05e-87\n",
       "Kurtosis:                       7.497   Cond. No.                         4.18\n",
       "==============================================================================\n",
       "\n",
       "Notes:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm = sm.OLS(new_pred_df['rating_numerator'],new_pred_df[['intercept','labrador_retriever','golden_retriever']])\n",
    "result = lm.fit()\n",
    "result.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Insight #3\n",
    "\n",
    "#### Code :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-96-2bb6f9c7d932>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  denom10_df['intercept']=1\n"
     ]
    }
   ],
   "source": [
    "#making a linear regression to see the relationsheip between favorites and rating_numerator for the tweets that have rating_denominator of 10\n",
    "denom10_df['intercept']=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>        <td>favorites</td>    <th>  R-squared:         </th> <td>   0.000</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>  -0.000</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>  0.5038</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Tue, 21 Sep 2021</td> <th>  Prob (F-statistic):</th>  <td> 0.478</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>14:38:18</td>     <th>  Log-Likelihood:    </th> <td> -22240.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>  2055</td>      <th>  AIC:               </th> <td>4.448e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>  2053</td>      <th>  BIC:               </th> <td>4.449e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     1</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "          <td></td>            <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>intercept</th>        <td> 8522.5673</td> <td>  278.878</td> <td>   30.560</td> <td> 0.000</td> <td> 7975.655</td> <td> 9069.480</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>rating_numerator</th> <td>    4.7441</td> <td>    6.684</td> <td>    0.710</td> <td> 0.478</td> <td>   -8.363</td> <td>   17.852</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>1671.609</td> <th>  Durbin-Watson:     </th> <td>   1.183</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th>  <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>46369.150</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>           <td> 3.691</td>  <th>  Prob(JB):          </th> <td>    0.00</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>       <td>25.069</td>  <th>  Cond. No.          </th> <td>    43.5</td> \n",
       "</tr>\n",
       "</table><br/><br/>Notes:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:              favorites   R-squared:                       0.000\n",
       "Model:                            OLS   Adj. R-squared:                 -0.000\n",
       "Method:                 Least Squares   F-statistic:                    0.5038\n",
       "Date:                Tue, 21 Sep 2021   Prob (F-statistic):              0.478\n",
       "Time:                        14:38:18   Log-Likelihood:                -22240.\n",
       "No. Observations:                2055   AIC:                         4.448e+04\n",
       "Df Residuals:                    2053   BIC:                         4.449e+04\n",
       "Df Model:                           1                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "====================================================================================\n",
       "                       coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------------\n",
       "intercept         8522.5673    278.878     30.560      0.000    7975.655    9069.480\n",
       "rating_numerator     4.7441      6.684      0.710      0.478      -8.363      17.852\n",
       "==============================================================================\n",
       "Omnibus:                     1671.609   Durbin-Watson:                   1.183\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):            46369.150\n",
       "Skew:                           3.691   Prob(JB):                         0.00\n",
       "Kurtosis:                      25.069   Cond. No.                         43.5\n",
       "==============================================================================\n",
       "\n",
       "Notes:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm=sm.OLS(denom10_df['favorites'],denom10_df[['intercept','rating_numerator']])\n",
    "results = lm.fit()\n",
    "results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Insights:\n",
    "\n",
    "1. the prediction model has been most confidene in predicting golden_retriever by 0.72 confident ratio \n",
    "\n",
    "2. the multipule linear regression predictes that the numerator rate for pembroke is 11.53 and the predicted numerator rate for the labrador retriever is less than pembroke's rate by 0.3504 and for golden retriever is higher by 0.0910.\n",
    "\n",
    "3.  from the linear model we predict that for each number increase in the rating_numerator the favorites number would increase by 5.7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 665 entries, 1 to 2065\n",
      "Data columns (total 9 columns):\n",
      " #   Column                Non-Null Count  Dtype  \n",
      "---  ------                --------------  -----  \n",
      " 0   tweet_id              665 non-null    int64  \n",
      " 1   image_url             665 non-null    object \n",
      " 2   dog_urls              665 non-null    object \n",
      " 3   rating_numerator      665 non-null    int64  \n",
      " 4   rating_denominator    665 non-null    int64  \n",
      " 5   favorites             665 non-null    int64  \n",
      " 6   prediction            665 non-null    object \n",
      " 7   prediction_confident  665 non-null    float64\n",
      " 8   intercept             665 non-null    int64  \n",
      "dtypes: float64(1), int64(5), object(3)\n",
      "memory usage: 44.2+ KB\n"
     ]
    }
   ],
   "source": [
    "top_ten = ['golden_retriever','labrador_retriever','pembroke','chihuahua','pug','chow','samoyed','toy_poodle','pomeranian','cocker_spaniel']\n",
    "ten_df = denom10_df[denom10_df['prediction'].isin(top_ten)]\n",
    "ten_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take 30 sample of each breed\n",
    "ten_of = ten_df.groupby('prediction').apply(lambda s: s.sample(30))\n",
    "ten_new = ten_of.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "prediction\n",
       "pug                   10.366667\n",
       "chihuahua             10.733333\n",
       "toy_poodle            11.066667\n",
       "labrador_retriever    11.200000\n",
       "cocker_spaniel        11.400000\n",
       "pembroke              11.400000\n",
       "golden_retriever      11.666667\n",
       "samoyed               11.733333\n",
       "chow                  11.766667\n",
       "pomeranian            13.366667\n",
       "Name: rating_numerator, dtype: float64"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ten_new.groupby('prediction').mean()['rating_numerator'].sort_values().tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcMAAAEWCAYAAAAadfxCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqS0lEQVR4nO3deZxcVZn/8c+XgISwhCWAYW0EZIcAHSAQkG0QZVckMKBB/ZlRcYAZUaM4srgFUcEVySAmAgISQNCMbBESdtJJSDoJIAqJIUYWhQYSCBKe3x/3NN4UVdVd3Z2urr7f9+tVr751zrnnPLdS6afPvVX3KCIwMzMrstXqHYCZmVm9ORmamVnhORmamVnhORmamVnhORmamVnhORmamVnhORma9TJJEyR9o05jHyzpmS7sV7eYG5mkTSVNk/SKpO/VOx6rzMnQGp6kV3OPtyS9lnt+ag+NcZKkByQtk3RPmfphkmak+hmShvXEuAaSzpd0dZ3G7u4fAWOAF4D1IuLzZfo/RNLdktokLShT35Tql0l6XNLh3YjFqnAytIYXEeu0P4C/AMfkyq7poWH+AVwKjCutkPQu4BbgamADYCJwSyrvFkmrd7ePvqy/Hx+wNTA/Kt/dZClwJfCFCvXXArOAjYBzgUmSNu7xKM3J0PovSWtKulTSX9PjUklrprqDJT0j6SuSXpC0oNosMiLuiohfA38tU30wsDpwaUQsj4gfAgIOrRLeEEl3ptNnUyVtnYs7JJ0h6UngyVR2tKRHJb2UZqi759pvJulGSc9LelrSmbm6tdLs5kVJ84HhJa/RlyQtTnE8IemwWmOW9JPSU4CSfivp7HKdVDi+H0haJOnlNLM+MJUfCXwFGJVm+rNT+WBJP5e0JMX/DUkDKoxX7X1wuqT7ysS3naQxwKnAF9PYv63Q//6SpqfZ3XRJ+6fyCcDo3P7vmNVFxCMRcRXwVJl+3wvsBZwXEa9FxI1AK/DhcnFY9zgZWn92LrAfMAzYA9gH+Gqu/t3AEGBzsl9a4yXt0IVxdgHmlPz1PyeVV3Iq8PU0/qNA6Qz2eGBfYGdJe5HNHv6DbIZwOXBr+iW/GvBbYHY6jsOAsyW9P/VzHrBterw/HScA6Vg/BwyPiHVT/YIuxDwROCXFgqQhKY5rq/T19vGl59PJ/p02BH4F3CBpYETcBnwLuD7N9PfIjfkmsB2wJ3AE8P8qjNXR+6CsiBifjvE7aexjSttI2hCYDPyQ7N/m+8BkSRtFxOkl+9/V0ZgldgGeiohXcmWzqf6+si5yMrT+7FTgwoh4LiKeBy4APlrS5n/SbG4q2S+1k7owzjpAW0lZG7BulX0mR8S0iFhO9st6hKQtc/Xfjoh/RMRrwKeAyyPi4YhYERETgeVkv+CHAxtHxIUR8UZEPAX8L3By6uck4Jupr0Vkv7TbrQDWJEu4a0TEgoj4c60xR8Qj6XjbZ5UnA/dExLNV+sofHxFxdUT8PSLejIjvpbjK/mEiaVPgA8DZEbE0Ip4DLskdc6nOvA+66ijgyYi4KsV+LfA48I7E2QVdeV9ZFzkZWn+2GbAw93xhKmv3YkQsrVLfWa8C65WUrQe8UqZtu0XtGxHxKtk1yc3K1ZNdd/p8OkX6kqSXgC1T+62BzUrqvgJsmvbdrKSvt1+PiPgTcDZwPvCcpOskVTv+ajFPBE5L26cBV1Xpp/T4kPR5SY+lU40vAYPJZqDlbA2sASzJHfPlwCYV2nf0PuiO0r7b+9+8B/ruyvvKusjJ0Pqzv5L94my3FStf89tA0tpV6jtrHrC7JOXKdk/llbw9C5S0DtnpwfzY+VOui8hmd+vnHoPSLGQR8HRJ3boR8cG075L8WGTH+K9BIn4VESPJXqcALupizFcDx0naA9gJ+E2VflY6vnR98Etks9gNImJ9shmQStsmi8hmxkNyx7xeRFQ6fVjtfbAUGJSL5d2V4uxk3+39L+5gv86YB7xHUn4muAfV31fWRU6G1p9dC3xV0sbpOtbXyH5p510g6V3pF/LRwA3lOpI0QNJAsg/KrCZpoKQ1UvU9ZKccz0zX8T6Xyv9QJbYPShqp7BOnXwceTqcxy/lf4NOS9lVmbUlHpV+SjwAvpw/CrJXi3FVS+wdlfg18WdIGkrYA/jN3TDtIOjR9mOR14LV0HDXHHBHPkF33uwq4sf30ZyetS3b973lgdUlfY+UZ0bNAU/s1yYhYAtwBfE/SepJWk7StpPdV6L/a+2A2sIuyr8YMJJsl5z0LvKdK7P8HvFfSv0taXdIosuugv+vMgafYB5LNdJXeV+9Kx/lHsmuz56XyE8j+yLqxM31bjSLCDz/6zYPsAyCHp+2BZNfIlqTHD4GBqe5g4Bmya18vkH0l46NV+j2dbJaQf0zI1e8JzCBLKDOBPav0NQH4GXAn2amwacA2ufoAtivZ50iyZPNSOpYbgHVT3WZkv/D/BrwIPJR7DQYBv0z7zSf7CP8zqW53smT6Ctkpz98Bm3Ul5tTmtBT7IR38G610fMAA4OfAy+nYvljy77gRcF86tpmpbDBwWfo3bCP7+sHJFcar+D5I9e3vgUW5Y9gu1W1PlpBeAn5Tof+R6d++Lf0cWfK6faPKa3FwmffVPbn6JrI/tl4Dnmh/Tfzo+YfSC25WKJIOBq6OiC3qHEq/IekgshlXU0S8Ve94zGrh06Rm1m3plPFZwBVOhNaInAzNrFsk7UR2GnEo2V16zBqOT5OamVnheWZoZmaF199vktvwhgwZEk1NTfUOw8ysocyYMeOFiOj0Tc2dDPu4pqYmWlpa6h2GmVlDkVR6Z6CqfJrUzMwKz8nQzMwKz8nQzMwKz8nQzMwKzx+g6eNaF7fRNHZyvcMwM+tVC8Yd1avjeWZoZmaF52TYQyQdK2lsveMwM7Pa+TRpibRAq2q92XBE3ArcumqiMjOzVanPzgwlNUl6XNJESXMkTZI0SNJhkmZJapV0ZVqYFEkLJH1L0oOSWiTtJel2SX+W9Olcv1+QND31eUFurMck/ZRsLbotJV2W+pnX3i43zgWSZqYYdkzlp0v6cdo+RtLDKc67JG2ays9PMd8j6SlJZ/beK2pmZpX02WSY7ACMj4jdyRb+/G+yxTJHRcRuZDPbz+TaL4qIEcC9qd2JwH7AhQCSjiBbrHMfYBiwd1qDrX2sX0bEnhGxEDg3IprJFkB9n6Tdc+O8EBF7kS0uek6ZuO8D9ouIPYHryBYrbbcj8P4Uw3m51dLfJmlMSsQtK5a1dfwqmZlZt/T1ZLgoIu5P21cDhwFPR8QfU9lE4KBc+/bTlK3AwxHxSkQ8D7wuaX3giPSYRTYD3JEsOQIsjIiHcn2dJGlmarsLsHOu7qb0cwbZStSltgBul9RKtrL4Lrm6yRGxPCJeAJ4DNi3dOSLGR0RzRDQPGDS4TPdmZtaT+noyrHV9qeXp51u57fbnqwMCvh0Rw9Jju4j4eWqztL2xpG3IZnyHpVnpZGBgmXFWUP6664+AH6fZ639U2Lfa/mZm1ov6ejLcStKItH0KcBfQJGm7VPZRYGoN/d0OfELSOgCSNpe0SZl265Elx7Z0ve8DNcY9GFictkfXuK+ZmfWyvj4reQwYLely4EngLOAh4AZJqwPTgZ91trOIuCOtyv1g9qFRXgVOI5uh5dvNljQLmAc8Bdxf2lcHzk8xLk7xblPj/mZm1ov67Er3kpqA30XErvWOpZ7WHLp9DB19ab3DMDPrVd29A42kGelDkJ3S12eGhbfb5oNp6eXbEpmZFU2fTYYRsQAo9KzQzMx6R1//AI2Zmdkq52RoZmaF52RoZmaF52RoZmaF52RoZmaF52RoZmaF52RoZmaF52RoZmaF52RoZmaF12fvQGOZ1sVtNI2dXO8wzKyf6u49QPsLzwzNzKzwnAy7QNIESSfWOw4zM+sZToZmZlZ4ToadIOljkuZImi3pqlR8kKQHJD3VPktU5mJJcyW1ShqVyn8q6di0fbOkK9P2JyV9oy4HZWZmb3My7ICkXYBzgUMjYg/grFQ1FBgJHA2MS2UfAoYBewCHAxdLGgpMAw5MbTYHdk7bI4F7y4w5RlKLpJYVy9p6/JjMzGxlToYdOxSYFBEvAETEP1L5byLirYiYD2yaykYC10bEioh4FpgKDCdLeAdK2hmYDzybkuQI4IHSASNifEQ0R0TzgEGDV+nBmZmZv1rRGQKiTPnykjb5nyuJiMWSNgCOJJslbgicBLwaEa/0YKxmZtYFnhl2bApwkqSNACRtWKXtNGCUpAGSNgYOAh5JdQ8CZ6c29wLnUOYUqZmZ9T7PDDsQEfMkfROYKmkFMKtK85vJTn3OJptNfjEi/pbq7gWOiIg/SVpINjt0MjQz6wMUUe4MoPUVzc3N0dLSUu8wzMwaiqQZEdHc2fY+TWpmZoXnZGhmZoXnZGhmZoXnZGhmZoXnZGhmZoXnZGhmZoXnZGhmZoXnZGhmZoXnZGhmZoXnZGhmZoXne5P2ca2L22gaO7neYZhZP7Rg3FH1DqHP8MzQzMwKz8nQzMwKz8mwjiSdL+mcesdhZlZ0ToZmZlZ4/TIZSlpb0mRJsyXNlTRK0tckTU/Px0tSanuPpEskTZP0mKThkm6S9KSkb+T6/O+071xJZ6eyr0s6K9fmm5LOTNtfSOPNkXRBrs25kp6QdBewQ2+9JmZmVll//TTpkcBfI+IoAEmDgTsj4sL0/CrgaOC3qf0bEXFQSmy3AHsD/wD+LOkSoAn4OLAvIOBhSVOBnwM3AT+QtBpwMrCPpCOA7YF9UvtbJR0ELE1t9iR77WcCM0qDlzQGGAMwYL2Ne/BlMTOzcvrlzBBoBQ6XdJGkAyOiDThE0sOSWoFDgV1y7W/N7TcvIpZExHLgKWBLYCRwc0QsjYhXyRLggRGxAPi7pD2BI4BZEfH3tH0EMIss4e1IlhwPTP0si4iXc+OuJCLGR0RzRDQPGDS4514VMzMrq1/ODCPij5L2Bj4IfFvSHcAZQHNELJJ0PjAwt8vy9POt3Hb789XJZneVXAGcDrwbuDKVCfh2RFyeb5hOr0YXDsnMzFahfjkzlLQZsCwirga+C+yVql6QtA5wYo1dTgOOlzRI0trACcC9qe5mstOyw4HbU9ntwCfSWEjaXNImqZ8TJK0laV3gmK4doZmZ9aR+OTMEdgMulvQW8E/gM8DxZKdBFwDTa+ksImZKmgA8koquiIhZqe4NSXcDL0XEilR2h6SdgAfT53ReBU5L/VwPPAos5F8J1czM6kgRPmvXHemDMzOBj0TEkz3df3Nzc7S0tPR0t2Zm/ZqkGRHR3Nn2/fI0aW+RtDPwJ2DKqkiEZmbWO/rradJeERHzgffUOw4zM+sezwzNzKzwnAzNzKzwnAzNzKzwnAzNzKzwnAzNzKzwnAzNzKzwnAzNzKzwnAzNzKzw/KX7Pq51cRtNYyfXOwwz62cWjDuq3iH0KZ4ZmplZ4TkZmplZ4fV6MpQ0QdI71hOUdLCk3/V2PGnsr3RQ/3+S1u+lcMzMrJcVZmYoaUCV6rLJUJnVIuKDEfHSqomsw9jMzGwV65FkKOl/JD0u6U5J10o6R9IwSQ9JmiPpZkkblNnvyLTffcCHcuVrS7pS0nRJsyQdl8pPl3STpNskPSnpOx3E9aqkCyU9DIyQdJqkRyQ9KulySQMkjQPWSmXXSGqS9Jikn5KtU7ilpAWShqQ+y/XxmXwsKc4fVWpfLraSuMdIapHUsmJZW9f+UczMrNO6nQwlNQMfBvYkS2jtiyn+EvhSROxOtsL8eSX7DQT+FzgGOBB4d676XOAPETEcOIRs1fq1U90wYBTZavajJG1ZJby1gbkRsS/w97TfARExDFgBnBoRY4HXImJYRJya9tsB+GVE7BkRC3Mx71SuD2ASuWSe2lxfpf1KsUXEffmgI2J8RDRHRPOAQYOrHJ6ZmfWEnvhqxUjgloh4DUDSb8l+0a8fEVNTm4nADSX77Qg83b4orqSrgTGp7gjgWEnnpOcDga3S9pSIaEv7zAe2BhZViG0FcGPaPgzYG5guCWAt4LkK+y2MiIfKlJftIyKel/SUpP2AJ8mS6f3AGVXGzMdmZmZ11BPJUN3YN6r0+eGIeGKlQmlfYHmuaAXVj+H1iFiR63NiRHy5E3EtrRJXpT6uB04CHgdujohQlgErtc/HZmZmddQT1wzvA46RNFDSOsBRZMnkRUkHpjYfBaaW7Pc4sI2kbdPzU3J1twP/mZIJkvbsgTinACdK2iT1uaGkrVPdPyWt0c0+bgKOJzuO6zvR3szM+ohuJ8OImA7cCswmSwgtQBswmuxa3xyy63wXluz3Otlp0cnpAzQLc9VfB9YA5kiam553N875wFeBO1JMdwJDU/X4NNY1Xe0jIl4E5gNbR8QjnRjTzMz6CEVUOlNZQyfSOhHxqqRBwDRgTETM7HbHRnNzc7S0tNQ7DDOzhiJpRkQ0d9wy01P3Jh0vaWeyD7pMdCI0M7NG0iPJMCL+vSf66ar0Xb01S4o/GhGt9YjHzMwaS79YtSJ9j9DMzKxLCnM7NjMzs0qcDM3MrPCcDM3MrPCcDM3MrPCcDM3MrPCcDM3MrPCcDM3MrPD6xfcM+7PWxW00jZ1c7zDMrAEtGHdUvUNoGJ4ZmplZ4TkZViCpKa2Y0Z0+Xu2peMzMbNVxMuwmSQPqHYOZmXVPv0iGaRb3uKSJkuZImiRpkKS9JU2VNEPS7ZKGpvb3SLpE0jRJj0kaLukmSU9K+kau69VL+0z7L5D0tbQO40cknSKpVdJcSReViW+IpAclHSVpY0k3SpqeHgf0zqtkZmaV9ItkmOwAjI+I3YGXgTOAHwEnRsTewJXAN3Pt34iIg4CfAbek9rsCp0vaqEKfn83t/3pEjCRbv/Ei4FCyRYyHSzq+vZGkTYHJwNciYjLwA+CSiBgOfBi4ovRAJI2R1CKpZcWytm68JGZm1hn96dOkiyLi/rR9NfAVsuR2pySAAcCSXPtb089WYF5ELAGQ9BSwJfBSmT7PBL6bnl+ffg4H7omI59P+1wAHAb8B1gCmAGdExNTU/nBg5xQTwHqS1o2IV9oLImI8MB5gzaHbd3/1ZTMzq6o/JcPSpPEKWZIbUaH98vTzrdx2+/P216W0z/zzpemnqOxNYAbwfqA9Ga4GjIiI16rsZ2Zmvag/nSbdSlJ74jsFeAjYuL1M0hqSdulmn/eVafMw8L50XXBAatee+AL4BLCjpLGp7A7gc+07SxpWY0xmZtbD+lMyfAwYLWkOsCHpeiFwkaTZwKPA/t3s87LSBun06peBu4HZwMyIuCVXvwI4GThE0mfJTrU2pw/lzAc+XWNMZmbWwxTR+JekJDUBv4uIXesdS09bc+j2MXT0pfUOw8waUJHvQCNpRkQ0d7Z9f7pm2C/ttvlgWgr8hjYz6w39IhlGxAKyT46amZnVrD9dMzQzM+sSJ0MzMys8J0MzMys8J0MzMys8J0MzMys8J0MzMys8J0MzMys8J0MzMys8J0MzMyu8fnEHmv6sdXEbTWMn1zsMM2tARb43aa08MzQzs8JzMjQzs8Lr88lQUpOkufWOoxaSjs0t5lupzemSftxbMZmZWWX9+pqhpNUj4s3eHjcibgVu7e1xzcysa3plZijpY2ll99mSrpK0taQpqWyKpK1Su00l3ZzazZa0f0k/75E0S9JwSdtKuk3SDEn3StoxtZkg6fuS7gYuqhDP+yQ9mh6zJK0r6WBJ09L48yX9TNJqqf1lklokzZN0Qa6fBZIukDRTUmsuhrdnfZI2lnSjpOnpcUAnXq8xabyWFcvauviqm5lZZ63ymaGkXYBzgQMi4gVJGwITgV9GxERJnwB+CByffk6NiBMkDQDWATZI/ewAXAd8PCIelTQF+HREPClpX+CnwKFp2PcCh0fEigphnQOcERH3S1oHeD2V7wPsDCwEbgM+BEwCzo2If6SYpkjaPSLmpH1eiIi9JH029fv/Ssb6AXBJRNyXkv7twE7VXrOIGA+Mh2yl+2ptzcys+3rjNOmhwKSIeAEgJZURZIkG4CrgO7m2H0vtVgBtkjYANgZuAT4cEfNSAtsfuEFS+zhr5sa8oUoiBLgf+L6ka4CbIuKZ1M8jEfEUgKRrgZFkyfAkSWPIXq+hZAmzPRnelH7OyB1T3uHAzrk415O0bpXYzMysl/VGMhTQ0eymo/o2YBFwADCP7PTuSxExrEL7pVUHixgnaTLwQeAhSYdXiCMkbUM24xseES9KmgAMzLVZnn6uoPzruRowIiJeyxfmkqOZmdVZb1wznEI2s9oIIJ0mfQA4OdWfCtyXa/uZ1G6ApPVS+Rtkp1E/JunfI+Jl4GlJH0ltJWmPzgYkaduIaI2Ii4AWYMdUtY+kbdK1wlEprvXIkmubpE2BD9R4/HcAn8uNPazG/c3MbBVb5TPDdFrzm8BUSSuAWcCZwJWSvgA8D3w8NT8LGC/pk2Qzrc8AS1I/SyUdDdwpaSlZEr1M0leBNciuJ87uZFhnSzokjTEf+D0wAngQGAfsBkwDbo6ItyTNIpuRPkV2irUWZwI/kTSH7PWeBny6szvvtvlgWnwXCTOzVUoR/nwGgKSDgXMi4ug6h7KS5ubmaGlpqXcYZmYNRdKMiGjubPs+/6V7MzOzVa2/f+n+42SnXvPuj4gzSttGxD3APb0QlpmZ9TH9OhlGxC+AX9Q7DjMz69t8mtTMzArPydDMzArPydDMzArPydDMzArPydDMzArPydDMzAqvX3+1oj9oXdxG09jJ9Q7DzPq4Bb5tY7d4ZmhmZoXnZGhmZoXXYTKU9GoH9U2S5vZcSB2P2cNjrZ9Wqa/W5oHeisfMzHpfr80MJQ2oV79pvcNKx7o+UDYZtvcdEft3OcAOdBCbmZn1gk7/Epa0jqQpkmZKapV0XK56dUkTJc2RNEnSoLTPAklfk3Qf8BFJn5I0XdJsSTfm2m0j6cFU9/XcmJJ0saS5acxRqfxgSXdL+hXQWiHeJkmPSfopMBPYUtIX0hhzJF2Qmo4DtpX0aBrrHX3nZ6rl+pB0UX52Kel8SZ+v0v4dsXX238HMzHpeLTOS14ETImIv4BDge5KU6nYAxkfE7sDLrDzTej0iRkbEdcBNETE8IvYAHgM+mdr8ALgsIoYDf8vt+yFgGLAHcDhwsaShqW4f4NyI2LlKzDsAv4yIPdP29mm/YcDekg4CxgJ/johhEfGFan1LOqJCH9cBo3JNTwJuqNJ+pdgiYmHJOGMktUhqWbGsrcrhmZlZT6glGQr4Vlqx/S5gc2DTVLcoItpXgL8aGJnb7/rc9q6S7pXUSrZS/S6p/ADg2rR9Va79SODaiFgREc8CU4Hhqe6RiHi6g5gXRsRDafuI9JhFNhvbkSxRlVOp77J9RMQsYBNJm0naA3gxIv7SwZj52FYSEeMjojkimgcMGtzBIZqZWXfV8j3DU4GNgb0j4p+SFgADU12UtM0/X5rbngAcHxGzJZ0OHFxhn3YqU1au3860EfDtiLh8pQGkphr6LttHMgk4EXg32UyxozE7E7+ZmfWCWmaGg4HnUiI8BNg6V7eVpBFp+xTgvgp9rAsskbQGWXJtdz9wctrOl08DRkkaIGlj4CDgkRpizrsd+ISkdQAkbS5pE+CVFFd3+oAsAZ5MlhAndaK9mZn1EbXMDK8BfiupBXgUeDxX9xgwWtLlwJPAZRX6+B/gYWAh2YdT2pPQWcCvJJ0F3JhrfzMwAphNNnP8YkT8TdKONcQNQETcIWkn4MF0qfNV4LSI+LOk+9PXQ34PVLzdS6U+yP5ImCdpXWBxRCzpoP2KWuM3M7NVRxHlzk5aX9Hc3BwtLS31DsPMrKFImhERzZ1t7++3mZlZ4TX8jbolbQRMKVN1WET8vbfjMTOzxtPwyTAlvGH1jsPMzBqXT5OamVnhORmamVnhORmamVnhORmamVnhORmamVnhORmamVnhORmamVnhNfz3DPu71sVtNI2teLtUMyuwBeOOqncI/YZnhmZmVnhOhmZmVngNlQwlrS/ps/WOoyOSzpd0TtqeIOnEesdkZmaVNVQyBNYH+nwyNDOzxtJoyXAcsK2kRyVdnB5zJbVKGgUg6SpJx7XvIOkaSceW60zS6ZJukXSbpCcknZer++/U91xJZ3ei/NzUx13ADhXG21vSVEkzJN0uaWiFdmMktUhqWbGsrbZXyMzMatZonyYdC+waEcMkfRj4NLAHMASYLmkacAXwX8AtkgYD+wOjq/S5D7ArsCz1MRkI4OPAvoCAhyVNJfvjoVL5ycCeZK/pTGBGfhBJawA/Ao6LiOdT8v4m8InSgCJiPDAeYM2h23v1ZTOzVazRkmHeSODaiFgBPJuS0vCIuFXSTyRtAnwIuDEi3qzSz53t6x5Kuin1G8DNEbE0V34gWQIsV75aKl+Wym8tM84OZEn3TkkAA4Al3XoFzMysRzRyMlSVuquAU8lma++YeZUonXlFlb6rjdnRDE7AvIgY0UE7MzPrZY12zfAVYN20PQ0YJWmApI2Bg4BHUt0E4GyAiJjXQZ//JmlDSWsBxwP3p76PlzRI0trACcC9HZSfIGktSesCx5QZ5wlgY0kjIDttKmmXWl8AMzPreQ01M4yIv0u6X9Jc4PfAHGA22azsixHxt9TuWUmPAb/pRLf3kc0ktwN+FREtkH0lgn8l1ysiYlYH5dcDjwILyRJkaexvpK9Y/DBdy1wduBToKFmbmdkqpoj+9/kMSYOAVmCviKj4cUxJpwPNEfG53oqtVs3NzdHS0lLvMMzMGoqkGRHR3Nn2jXaatEOSDgceB35ULRGamZm1a6jTpJ0REXcBW+XLJL0fuKik6dMRcQLZ9UUzMyuwfpcMy4mI24Hb6x2HmZn1Tf3uNKmZmVmtnAzNzKzwnAzNzKzwnAzNzKzwnAzNzKzwnAzNzKzwnAzNzKzwCvE9w0bWuriNprGT6x2GmdXBgnFH1TuEwvDM0MzMCq/fJENJE9KqEKXlm0malLZPl/TjHhrvfEnn9ERfZmZWX/3+NGlE/BV4R5I0MzNr17AzQ0kfkzRH0mxJV6XigyQ9IOmp9lmipKa0/mG7zSTdJulJSd/J9fdqbvvEtG4hko6R9LCkWZLukrRprq+dJd2Txjuz3HiSzpF0ftr+lKTpKeYb01JTZmZWZw2ZDNMK8ecCh0bEHsBZqWooMBI4GhhXYfdhwChgN2CUpC07GO4+YL+I2BO4Dvhirm5H4P3APsB5ktbooK+bImJ4ivkx4JPlGkkaI6lFUsuKZV6FysxsVWvU06SHApMi4gWAiPiHJIDfRMRbwPySGVzelPZ1DiXNB7YGFlUZawvgeklDgXcBT+fqJkfEcmC5pOeASmO221XSN4D1gXWosJJGRIwHxgOsOXT7/rf6splZH9OQM0NAQLkksbykTTn5Niv41x8E+f4G5rZ/BPw4InYD/qOkrlxfb7Ly65pvPwH4XOrrgpI6MzOrk0ZNhlOAkyRtBCBpwx7o81lJO0laDTghVz4YWJy2R3emH2ATSRtJWpPslG27dYEl6XTqqT0Qs5mZ9YCGPE0aEfMkfROYKmkFMKsHuh0L/I7slOlcstOYAOcDN0haDDwEbNNBbP+UdCHwMNkp1cdz1f+TyhcCrWTJ0czM6kwRviTVl605dPsYOvrSeodhZnXgO9B0naQZEdHc2fYNOTMskt02H0yL/0OYma1SjXrN0MzMrMc4GZqZWeE5GZqZWeE5GZqZWeE5GZqZWeE5GZqZWeE5GZqZWeE5GZqZWeE5GZqZWeH5DjR9XOviNprGTq53GGa2Cvh2a32HZ4ZmZlZ4ToZmZlZ4ToZmZlZ4ToZmZlZ4TobdIKlJ0uOSJkqaI2mSpEGSFkgakto0S7onbW8s6U5JMyVdLmlhezszM6sfJ8Pu2wEYHxG7Ay8Dn63S9jzgDxGxF3AzsFW5RpLGSGqR1LJiWVuPB2xmZitzMuy+RRFxf9q+GhhZpe1I4DqAiLgNeLFco4gYHxHNEdE8YNDgHg3WzMzeycmw+6LM8zf512s7MFenXonIzMxq4mTYfVtJGpG2TwHuAxYAe6eyD+fa3gecBCDpCGCDXorRzMyqcDLsvseA0ZLmABsClwEXAD+QdC+wItf2AuAISTOBDwBLgFd6OV4zMyvh27F131sR8emSsnuB95Zp2wa8PyLeTLPJQyJiebXOd9t8MC2+ZZOZ2SrlZNi7tgJ+LWk14A3gU3WOx8zMcDLslohYAOxaQ/sngT1XWUBmZtYlvmZoZmaF52RoZmaF52RoZmaFp4jS74xbXyLpFeCJesfRDUOAF+odRDc0cvyNHDs4/npr5PiHAGtHxMad3cEfoOn7noiI5noH0VWSWhx/fTRy7OD4662R40+xN9Wyj0+TmplZ4TkZmplZ4TkZ9n3j6x1ANzn++mnk2MHx11sjx19z7P4AjZmZFZ5nhmZmVnhOhmZmVnhOhn2YpCMlPSHpT5LG1jueWkjaUtLdkh6TNE/SWfWOqVaSBkiaJel39Y6lVpLWlzRJ0uPp32BEx3v1HZL+K71v5kq6VtLAjveqH0lXSnpO0txc2YaS7pT0ZPrZJ9cvrRD7xem9M0fSzZLWr2OIVZWLP1d3jqSQNKSjfpwM+yhJA4CfkK17uDNwiqSd6xtVTd4EPh8ROwH7AWc0WPwAZ5GtV9mIfgDcFhE7AnvQQMchaXPgTKA5InYFBgAn1zeqDk0AjiwpGwtMiYjtgSnpeV80gXfGfiewa0TsDvwR+HJvB1WDCbwzfiRtCfwb8JfOdOJk2HftA/wpIp6KiDeA64Dj6hxTp0XEkoiYmbZfIftlvHl9o+o8SVsARwFX1DuWWklaDzgI+DlARLwRES/VNajarQ6sJWl1YBDw1zrHU1VETAP+UVJ8HDAxbU8Eju/NmDqrXOwRcUdEvJmePgRs0euBdVKF1x7gEuCLQKc+Jepk2HdtDizKPX+GBkomeZKayJauerjOodTiUrL/SG/VOY6ueA/wPPCLdJr3Cklr1zuozoqIxcB3yf6iXwK0RcQd9Y2qSzaNiCWQ/XEIbFLneLrqE8Dv6x1ELSQdCyyOiNmd3cfJsO9SmbKG+x6MpHWAG4GzI+LlesfTGZKOBp6LiBn1jqWLVgf2Ai6LiD2BpfTdU3TvkK6tHQdsA2wGrC3ptPpGVUySziW75HFNvWPpLEmDgHOBr9Wyn5Nh3/UMsGXu+Rb08VNFpSStQZYIr4mIm+odTw0OAI6VtIDs9PShkq6ub0g1eQZ4JiLaZ+KTyJJjozgceDoino+IfwI3AfvXOaaueFbSUID087k6x1MTSaOBo4FTo7G+kL4t2R9Ss9P/4S2AmZLeXW0nJ8O+azqwvaRtJL2L7AMEt9Y5pk6TJLJrVo9FxPfrHU8tIuLLEbFFutHvycAfIqJhZiYR8TdgkaQdUtFhwPw6hlSrvwD7SRqU3keH0UAfAMq5FRidtkcDt9QxlppIOhL4EnBsRCyrdzy1iIjWiNgkIprS/+FngL3S/4uKnAz7qHTx+nPA7WS/CH4dEfPqG1VNDgA+SjarejQ9PljvoArkP4FrJM0BhgHfqm84nZdmtJOAmUAr2e+pPn1rMEnXAg8CO0h6RtIngXHAv0l6kuxTjePqGWMlFWL/MbAucGf6v/uzugZZRYX4a++nsWa/ZmZmPc8zQzMzKzwnQzMzKzwnQzMzKzwnQzMzKzwnQzMzKzwnQ7N+StLZ6W4cq3KMYV35ykxaiWKOpP8qKT++AW/obv2Ak6FZg1Km2v/hs8lucr0qDQNqSobpTiD7R8TuEXFJSfXxZKu0mPUqJ0OzBiKpKa1P+FOyL6VvKekySS1p/b8LUrszye7rebeku1PZEZIelDRT0g3pvrGl/Q+T9FBuHbsNUvk9kprT9hBJC9KdkS4ERqUvZo8q6WugpF9Iak03DD8kVd0BbJL2OTDXfn/gWODiVLdtetwmaYakeyXtmNpOkPRDSQ9IekrSiT35OlsBRYQffvjRIA+giWwljf1yZRumnwOAe4Dd0/MFwJC0PQSYBqydnn8J+FqZ/ucA70vbFwKXpu17yNYXbO9rQdo+HfhxhVg/D/wibe9Idpu1gekY5lbYZwJwYu75FGD7tL0v2a3x2tvdQPYH/c5ky53V/d/Hj8Z9rN7NXGpmvW9hRDyUe36SpDFkq1UMJUsOc0r22S+V35/d7pN3kd3C6m2SBgPrR8TUVDSRLOF01UjgRwAR8bikhcB7gU6tXpJmrvsDN6SYAdbMNflNRLwFzJe0aTfiNHMyNGtAS9s3JG0DnAMMj4gXJU0gm32VEnBnRJzSxTHf5F+XVcr1X065ZchqsRrwUkQMq1C/vAfHsoLzNUOzxrYeWXJsS7OjD+TqXiG72TJkq5UfIGk7yNZ8k/TefEcR0Qa8mLuO91GgfZa4ANg7beevz+XHKDUNODWN915gK+CJDo7n7f4iW//yaUkfSX1I0h4d7G/WJU6GZg0sspW8ZwHzgCuB+3PV44HfS7o7Ip4nu753bVrJ4iGy63ilRpN9gKV9tYsLU/l3gc9IeoDsmmG7u4Gdy32ABvgpMEBSK3A9cHpELKe664AvpA/cbEuWTD8paXY6xuM62N+sS7xqhZmZFZ5nhmZmVnhOhmZmVnhOhmZmVnhOhmZmVnhOhmZmVnhOhmZmVnhOhmZmVnj/Hx442CGUYKz1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def display_barh(df, title, xlabel):\n",
    "    ax = df.plot(kind='barh')\n",
    "    plt.title(title)\n",
    "    plt.xlabel(xlabel)\n",
    "    plt.ylabel(None)\n",
    "\n",
    "display_barh(ten_new.groupby('prediction').mean()['rating_numerator'].sort_values().tail(10),'Top 10 breeds by rate out of 10', 'rate out of ten')"
   ]
  }
 ],
 "metadata": {
  "extensions": {
   "jupyter_dashboards": {
    "activeView": "report_default",
    "version": 1,
    "views": {
     "grid_default": {
      "cellMargin": 10,
      "defaultCellHeight": 20,
      "maxColumns": 12,
      "name": "grid",
      "type": "grid"
     },
     "report_default": {
      "name": "report",
      "type": "report"
     }
    }
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}