Team_Swift_Recommender_sys.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Team_Swift_Recommender_sys",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Vincelogs/Team_Swift/blob/master/Team_Swift_Recommender_sys.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_9H9SzaWAH9t",
        "colab_type": "text"
      },
      "source": [
        "**Team_Swift** *Recommender System for Lucid.blog*\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b1LKnxoDyGct",
        "colab_type": "text"
      },
      "source": [
        "**DATA IMPORTATION**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HUQq29e7E6k4",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "*   importing data from google drive to colab\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h8CQ-m-k6Sbd",
        "colab_type": "code",
        "outputId": "d0881fe7-3370-4cfd-c870-c18fe410b89c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 129
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I36bwCPDNF-2",
        "colab_type": "text"
      },
      "source": [
        "**DATA_PREPARATION**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vvzOnz5JFWtX",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "*   importing libaries\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kHLeRrJriVTz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#importing needed modules\n",
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4W2bxjkufRb-",
        "colab_type": "text"
      },
      "source": [
        "**EXPLORATION OF DATA ANALYSIS**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-G_igQk3f9Vr",
        "colab_type": "code",
        "outputId": "2a5b0135-b5b4-410c-f2e8-61c581ee18d7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        }
      },
      "source": [
        "#Exploring the contact_settings table\n",
        "contact_settings = pd.read_csv('/content/drive/My Drive/lucid/contact_settings.csv')\n",
        "contact_settings.describe()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>created_at</th>\n",
              "      <th>updated_at</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>249.000000</td>\n",
              "      <td>249.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>125.000000</td>\n",
              "      <td>846.518072</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>72.024301</td>\n",
              "      <td>571.215252</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>19.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>63.000000</td>\n",
              "      <td>365.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>125.000000</td>\n",
              "      <td>726.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>187.000000</td>\n",
              "      <td>1308.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>249.000000</td>\n",
              "      <td>2263.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               id      user_id  created_at  updated_at\n",
              "count  249.000000   249.000000         0.0         0.0\n",
              "mean   125.000000   846.518072         NaN         NaN\n",
              "std     72.024301   571.215252         NaN         NaN\n",
              "min      1.000000    19.000000         NaN         NaN\n",
              "25%     63.000000   365.000000         NaN         NaN\n",
              "50%    125.000000   726.000000         NaN         NaN\n",
              "75%    187.000000  1308.000000         NaN         NaN\n",
              "max    249.000000  2263.000000         NaN         NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t3aT5N6xg-u6",
        "colab_type": "code",
        "outputId": "cfa4a58b-0386-475b-8437-1250aef3bd35",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 495
        }
      },
      "source": [
        "#Exploring the extfeeds table\n",
        "extfeeds = pd.read_csv('/content/drive/My Drive/lucid/extfeeds.csv')\n",
        "extfeeds.describe(),extfeeds.info()"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 45 entries, 0 to 44\n",
            "Data columns (total 12 columns):\n",
            "id            45 non-null int64\n",
            "user_id       45 non-null int64\n",
            "site          45 non-null object\n",
            "site_image    45 non-null object\n",
            "title         45 non-null object\n",
            "des           43 non-null object\n",
            "link          45 non-null object\n",
            "date          45 non-null object\n",
            "image         4 non-null object\n",
            "created_at    0 non-null float64\n",
            "updated_at    0 non-null float64\n",
            "tags          0 non-null float64\n",
            "dtypes: float64(3), int64(2), object(7)\n",
            "memory usage: 4.3+ KB\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(                id    user_id  created_at  updated_at  tags\n",
              " count    45.000000  45.000000         0.0         0.0   0.0\n",
              " mean   2861.000000   6.044444         NaN         NaN   NaN\n",
              " std      13.133926   7.728211         NaN         NaN   NaN\n",
              " min    2839.000000   2.000000         NaN         NaN   NaN\n",
              " 25%    2850.000000   2.000000         NaN         NaN   NaN\n",
              " 50%    2861.000000   3.000000         NaN         NaN   NaN\n",
              " 75%    2872.000000   7.000000         NaN         NaN   NaN\n",
              " max    2883.000000  45.000000         NaN         NaN   NaN, None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XSIWvW15h-Iz",
        "colab_type": "code",
        "outputId": "5d5ee422-45dc-41b7-95a4-49d1140ad7a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        }
      },
      "source": [
        "#Exploring the following table\n",
        "following = pd.read_csv('/content/drive/My Drive/lucid/following.csv')\n",
        "following.describe(),following.info()"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 5296 entries, 0 to 5295\n",
            "Data columns (total 3 columns):\n",
            "my_id          5296 non-null int64\n",
            "follower_id    5296 non-null int64\n",
            "status         5296 non-null int64\n",
            "dtypes: int64(3)\n",
            "memory usage: 124.2 KB\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(             my_id  follower_id       status\n",
              " count  5296.000000  5296.000000  5296.000000\n",
              " mean    793.865370   695.724887     0.976020\n",
              " std     518.974621   520.907520     0.153002\n",
              " min       1.000000     1.000000     0.000000\n",
              " 25%     353.000000   218.000000     1.000000\n",
              " 50%     722.000000   642.000000     1.000000\n",
              " 75%    1161.000000  1075.000000     1.000000\n",
              " max    2287.000000  2218.000000     1.000000, None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AtvsX_ZziWXA",
        "colab_type": "code",
        "outputId": "51791ee6-9a7d-44a7-de04-f234d5aa1548",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 513
        }
      },
      "source": [
        "#Exploring the notifications table\n",
        "notifications = pd.read_csv('/content/drive/My Drive/lucid/notifications.csv')\n",
        "notifications.describe(),notifications.info()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 190 entries, 0 to 189\n",
            "Data columns (total 11 columns):\n",
            "id                   190 non-null int64\n",
            "post_id              60 non-null float64\n",
            "parent_comment_id    6 non-null float64\n",
            "comment              31 non-null object\n",
            "sender_id            190 non-null int64\n",
            "user_id              190 non-null int64\n",
            "status               190 non-null int64\n",
            "action               190 non-null object\n",
            "type                 190 non-null object\n",
            "created_at           190 non-null object\n",
            "updated_at           190 non-null object\n",
            "dtypes: float64(2), int64(4), object(5)\n",
            "memory usage: 16.4+ KB\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(               id      post_id  ...      user_id      status\n",
              " count  190.000000    60.000000  ...   190.000000  190.000000\n",
              " mean    99.368421   804.616667  ...    91.600000    0.336842\n",
              " std     57.993259   295.624194  ...   345.721832    0.473879\n",
              " min      1.000000   143.000000  ...     1.000000    0.000000\n",
              " 25%     50.250000   423.000000  ...     4.000000    0.000000\n",
              " 50%     97.500000   992.000000  ...     6.000000    0.000000\n",
              " 75%    144.750000   998.000000  ...     6.000000    1.000000\n",
              " max    222.000000  1007.000000  ...  2218.000000    1.000000\n",
              " \n",
              " [8 rows x 6 columns], None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A_o0RqJeiizD",
        "colab_type": "code",
        "outputId": "2b29f7fe-8eb7-4609-8fed-86478f8bff3b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 495
        }
      },
      "source": [
        "#Exploring the posts table\n",
        "posts = pd.read_csv('/content/drive/My Drive/lucid/posts.csv')\n",
        "posts.describe(),posts.info()"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 876 entries, 0 to 875\n",
            "Data columns (total 12 columns):\n",
            "id            876 non-null int64\n",
            "user_id       876 non-null int64\n",
            "title         876 non-null object\n",
            "content       876 non-null object\n",
            "tags          182 non-null object\n",
            "slug          876 non-null object\n",
            "created_at    876 non-null object\n",
            "updated_at    876 non-null object\n",
            "image         400 non-null object\n",
            "status_id     0 non-null float64\n",
            "action        5 non-null object\n",
            "post_id       0 non-null float64\n",
            "dtypes: float64(2), int64(2), object(8)\n",
            "memory usage: 82.2+ KB\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(                id      user_id  status_id  post_id\n",
              " count   876.000000   876.000000        0.0      0.0\n",
              " mean    516.808219  1061.639269        NaN      NaN\n",
              " std     278.648058   788.070779        NaN      NaN\n",
              " min       1.000000     1.000000        NaN      NaN\n",
              " 25%     278.750000   338.000000        NaN      NaN\n",
              " 50%     523.500000   944.000000        NaN      NaN\n",
              " 75%     754.250000  1836.250000        NaN      NaN\n",
              " max    1007.000000  2297.000000        NaN      NaN, None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "crpkdmF2ixdr",
        "colab_type": "code",
        "outputId": "b96d68ab-c920-4fd6-92d2-15ecc7866a8c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 495
        }
      },
      "source": [
        "#Exploring the users table\n",
        "users = pd.read_csv('/content/drive/My Drive/lucid/users.csv')\n",
        "users.describe(),users.info()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 2293 entries, 0 to 2292\n",
            "Data columns (total 12 columns):\n",
            "id                2293 non-null int64\n",
            "name              2293 non-null object\n",
            "username          2293 non-null object\n",
            "email             2293 non-null object\n",
            "image             2293 non-null object\n",
            "provider          2293 non-null object\n",
            "provider_id       2293 non-null object\n",
            "password          0 non-null float64\n",
            "remember_token    2293 non-null object\n",
            "created_at        2293 non-null object\n",
            "updated_at        2293 non-null object\n",
            "short_bio         1443 non-null object\n",
            "dtypes: float64(1), int64(1), object(10)\n",
            "memory usage: 215.0+ KB\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(                id  password\n",
              " count  2293.000000       0.0\n",
              " mean   1147.753598       NaN\n",
              " std     663.105813       NaN\n",
              " min       1.000000       NaN\n",
              " 25%     574.000000       NaN\n",
              " 50%    1147.000000       NaN\n",
              " 75%    1720.000000       NaN\n",
              " max    2297.000000       NaN, None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HUQx1LbajGjt",
        "colab_type": "code",
        "outputId": "61781199-4f29-4bc4-aae5-46066a6e283e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        }
      },
      "source": [
        "#Exploring the user_settings table\n",
        "user_settings = pd.read_csv('/content/drive/My Drive/lucid/user_settings.csv')\n",
        "user_settings.describe()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>setting_path</th>\n",
              "      <th>tabs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>1.500000</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.707107</td>\n",
              "      <td>1.414214</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>1.250000</td>\n",
              "      <td>2.500000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>1.500000</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>1.750000</td>\n",
              "      <td>3.500000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>2.000000</td>\n",
              "      <td>4.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             id   user_id  setting_path  tabs\n",
              "count  2.000000  2.000000           0.0   0.0\n",
              "mean   1.500000  3.000000           NaN   NaN\n",
              "std    0.707107  1.414214           NaN   NaN\n",
              "min    1.000000  2.000000           NaN   NaN\n",
              "25%    1.250000  2.500000           NaN   NaN\n",
              "50%    1.500000  3.000000           NaN   NaN\n",
              "75%    1.750000  3.500000           NaN   NaN\n",
              "max    2.000000  4.000000           NaN   NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2x6-rhYxlLbg",
        "colab_type": "code",
        "outputId": "84dad65c-8b1d-4b91-a425-648a1fe23ce3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        }
      },
      "source": [
        "#Exploring the thoughts table\n",
        "thoughts = pd.read_csv('/content/drive/My Drive/lucid/thoughts.csv')\n",
        "thoughts.describe()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>user_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>16.000000</td>\n",
              "      <td>16.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>8.500000</td>\n",
              "      <td>990.250000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>4.760952</td>\n",
              "      <td>1137.363472</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>4.750000</td>\n",
              "      <td>4.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>8.500000</td>\n",
              "      <td>77.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>12.250000</td>\n",
              "      <td>2223.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>16.000000</td>\n",
              "      <td>2278.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              id      user_id\n",
              "count  16.000000    16.000000\n",
              "mean    8.500000   990.250000\n",
              "std     4.760952  1137.363472\n",
              "min     1.000000     1.000000\n",
              "25%     4.750000     4.000000\n",
              "50%     8.500000    77.000000\n",
              "75%    12.250000  2223.000000\n",
              "max    16.000000  2278.000000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tmgz1w8rlbMh",
        "colab_type": "code",
        "outputId": "e62f025f-f711-4810-cf89-c3324ce1e52d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#relevant keys in the thoughts dataframe\n",
        "thoughts.keys()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['id', 'user_id', 'content', 'created_at', 'updated_at'], dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sheh7Z1vkIbP",
        "colab_type": "code",
        "outputId": "962df3d3-4890-403d-a580-7eaf305207ea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "#relevant keys in the users dataframe\n",
        "users.keys()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['id', 'name', 'username', 'email', 'image', 'provider', 'provider_id',\n",
              "       'password', 'remember_token', 'created_at', 'updated_at', 'short_bio'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kcRDV39XkRxn",
        "colab_type": "code",
        "outputId": "0092348a-4a3b-4760-a074-ce6c6cd0d5d7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "#relevant keys in the posts dataframe\n",
        "posts.keys()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['id', 'user_id', 'title', 'content', 'tags', 'slug', 'created_at',\n",
              "       'updated_at', 'image', 'status_id', 'action', 'post_id'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OE5zRzG3kgvy",
        "colab_type": "code",
        "outputId": "f8fd8c59-2ac8-4e80-eca2-2ebd7aa5187f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#relevant keys in the following dataframe\n",
        "following.keys()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['my_id', 'follower_id', 'status'], dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lqmlBkrqk06A",
        "colab_type": "code",
        "outputId": "49cd9eb3-83be-4696-e20c-80b007fd7d6d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "#relevant keys in the notifications dataframe\n",
        "notifications.keys()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['id', 'post_id', 'parent_comment_id', 'comment', 'sender_id', 'user_id',\n",
              "       'status', 'action', 'type', 'created_at', 'updated_at'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lMppIpwwl_xK",
        "colab_type": "code",
        "outputId": "c2c9cfad-6345-420f-897f-3e1741b1f4ec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#knowing the table size of our users,posts and following dataframe using columns and rows\n",
        "users.shape,posts.shape,following.shape"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((2293, 12), (876, 12), (5296, 3))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HwpO-sgToFFE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Remove html tags\n",
        "posts['content'] = posts['content'].str.replace(r'<[^>]*>', '')\n",
        "\n",
        "# Remove white spaces including new lines\n",
        "posts['content'] = posts['content'].str.replace(r'\\s', ' ')\n",
        "\n",
        "# Remove square brackets\n",
        "posts['content'] = posts['content'].str.replace(r'\\[.*?\\]', '')\n",
        "\n",
        "# Remove image files\n",
        "posts['content'] = posts['content'].str.replace(r'\\(.*?\\)', '')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mX5p1yeooNXC",
        "colab_type": "text"
      },
      "source": [
        "**BUILDING THE \"WHO TO FOLLOW\" RECOMMENDER SYSTEM**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6MzkSu1toVHV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#importing needed modules\n",
        "import pandas as pd\n",
        "from sklearn.metrics.pairwise import linear_kernel\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yuZ_83cUokAn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tfidf = TfidfVectorizer(stop_words='english')\n",
        "# filling the missing values with empty string\n",
        "users['short_bio'] = users['short_bio'].fillna('')\n",
        "# computing TF-IDF matrix required for calculating cosine similarity\n",
        "users_short_bio_matrix = tfidf.fit_transform(users['short_bio'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TTouMx_kosW4",
        "colab_type": "code",
        "outputId": "7e9b926c-dd0c-4371-b734-3721d5f3216b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#knowing the matrix size of our users_short_bio_matrix dataframe\n",
        "print(\"The users matrix shape is \",users_short_bio_matrix.shape)"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The users matrix shape is  (2293, 2242)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uNjTRn11pONF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#calculating the cosine similarity for the users_short_bio_matrix \n",
        "cos_sim = linear_kernel(users_short_bio_matrix, users_short_bio_matrix)\n",
        "#getting the user indices\n",
        "user_indices = pd.Series(users['name'].index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mjf4tkZ3pneH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#function to recommend user to follow based on similarities in their short_bio\n",
        "def who_to_follow(index, cosine_sim=cos_sim):\n",
        "    if index<users_short_bio_matrix.shape[0]:\n",
        "        id = user_indices[index]\n",
        "        # Get the pairwsie similarity scores of all names\n",
        "        # sorting them and getting top 5\n",
        "        similarity_scores = list(enumerate(cosine_sim[id]))\n",
        "        similarity_scores = sorted(similarity_scores, key=lambda x: x[1], reverse=True)\n",
        "        similarity_scores = similarity_scores[1:6]\n",
        "        \n",
        "        # Get the names index\n",
        "        lucid_index = [i[0] for i in similarity_scores]\n",
        "        # Return the top 5 most similar names\n",
        "        return users['name'].iloc[lucid_index]\n",
        "    else: return \"No recommedations for this user\"\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tc5XTXiDq3J9",
        "colab_type": "text"
      },
      "source": [
        "**TESTING THE RECOMMENDER SYSTEM**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sKl78uSrq6UB",
        "colab_type": "code",
        "outputId": "250fa02b-3fae-4dc5-c891-0ad9dcc02ca6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "source": [
        "who_to_follow(50)"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "105     Damilare Olabimtan\n",
              "279        UDENKWOR NKECHI\n",
              "438         Angela Egerega\n",
              "604    chukwuemeka anyanwu\n",
              "641          Deborah Ajayi\n",
              "Name: name, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UJZ_uBSnrOXK",
        "colab_type": "code",
        "outputId": "e22bdaf5-399f-4f52-eadc-a4e66eab6e9e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "source": [
        "\n",
        "\n",
        "who_to_follow(2020)\n",
        "\n"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1            Elijah Okokon\n",
              "2             Jeffrey Ogah\n",
              "3    Oluwaseyi Oluwapelumi\n",
              "4                     PoRH\n",
              "5             Seyi Onifade\n",
              "Name: name, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fr1pqrt_rgp_",
        "colab_type": "text"
      },
      "source": [
        "**BUILDING THE \"WHAT ARTICLES TO READ\" RECOMMENDER SYSTEM**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xfNqHzCZrffm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tfidf = TfidfVectorizer(stop_words='english')\n",
        "# filling the missing values with empty string\n",
        "posts['content'] = posts['content'].fillna('')\n",
        "# computing TF-IDF matrix required for calculating cosine similarity\n",
        "posts_matrix = tfidf.fit_transform(posts['content'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rH70-mTqrxQ-",
        "colab_type": "code",
        "outputId": "0f0099bd-dea1-4752-9a47-07104cf9d39c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#knowing the matrix size of our posts_matrix dataframe\n",
        "print(\"The post matrix shape is \",posts_matrix.shape)"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The post matrix shape is  (876, 5122)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "02F6TovMsC_t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#calculating the cosine similarity for our posts_matrix\n",
        "cos_sim = linear_kernel(posts_matrix, posts_matrix)\n",
        "#getting our post indices\n",
        "posts_indices = pd.Series(posts['title'].index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cotK-sjEsN3m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#function to recommend user to follow based on similarities in their short_bio\n",
        "def article_to_read(index, cosine_sim=cos_sim):\n",
        "    if index<posts_matrix.shape[0]:\n",
        "        id = posts_indices[index]\n",
        "        # Get the pairwsie similarity scores of all names\n",
        "        # sorting them and getting top 5\n",
        "        similarity_scores = list(enumerate(cosine_sim[id]))\n",
        "        similarity_scores = sorted(similarity_scores, key=lambda x: x[1], reverse=True)\n",
        "        similarity_scores = similarity_scores[1:6]\n",
        "        # Get the names index\n",
        "        lucid_index = [i[0] for i in similarity_scores]\n",
        "        # Return the top 5 most similar names\n",
        "        return posts['title'].iloc[lucid_index]\n",
        "    else: return \"No recommedations for this user\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4hqdr4bquI2j",
        "colab_type": "text"
      },
      "source": [
        "**TESTING THE RECOMMENDER SYSTEM**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ovPfHm7uXIL",
        "colab_type": "code",
        "outputId": "608e2fca-7067-49a5-987c-8d2d876f7fe8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "source": [
        "article_to_read(750)"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "745                      CSS Rules\n",
              "746                      CSS Rules\n",
              "749                      CSS Rules\n",
              "751                      CSS Rules\n",
              "839    CSS (cascading style sheet)\n",
              "Name: title, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "siIErVlkLQfG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        },
        "outputId": "8b04406c-e38d-464a-e66a-6f427cdd6a4a"
      },
      "source": [
        "article_to_read(298)"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "25                                      StartNg Task One\n",
              "154                 \\r\\nwhat you wish to still learn\\r\\n\n",
              "458                                             Start NG\n",
              "312                                              StartNG\n",
              "62     OVERVIEW OF MY HNG PRE-INTERNSHIP EXPERIENCE: ...\n",
              "Name: title, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    }
  ]
}