poc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\" style=\"margin-top: 1em;\"><ul class=\"toc-item\"><li><span><a href=\"#Pipeline-part\" data-toc-modified-id=\"Pipeline-part-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Pipeline part</a></span></li><li><span><a href=\"#Backend-part\" data-toc-modified-id=\"Backend-part-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Backend part</a></span><ul class=\"toc-item\"><li><span><a href=\"#sort-just-based-of-recency\" data-toc-modified-id=\"sort-just-based-of-recency-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>sort just based of recency</a></span></li><li><span><a href=\"#sort-based-on-frecency\" data-toc-modified-id=\"sort-based-on-frecency-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>sort based on frecency</a></span></li></ul></li><li><span><a href=\"#sort-based-on-modified-frecency\" data-toc-modified-id=\"sort-based-on-modified-frecency-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>sort based on modified frecency</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pipeline part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "article_snapshots = glob.glob(\"../articles/articles-*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(article_snapshots)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pipeline.text_processing import load_article_snapshots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = load_article_snapshots(article_snapshots)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pipeline.text_processing import add_lemmatized_texts, fit_tf_idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_lemmatized_texts(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, words = fit_tf_idf(df[\"lemmatized_texts\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Backend part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from backend.recommender.recommend import (\n",
    "    recommend,\n",
    "    get_daily_google_trends,\n",
    "    get_relevant_words,\n",
    "    estimate_popularity,\n",
    "    calculate_frecency\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_trends = get_daily_google_trends()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "top_trends"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pop = estimate_popularity(top_trends, X, words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(pd.Series(pop.squeeze()) == 0).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sort just based of recency "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sort_values(\"published\", ascending=False).title.values[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sort based on frecency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recommend(df, X, words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sort based on modified frecency\n",
    "(mitigate impact of popularity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "from math import log, e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_frecency(popularity, age):\n",
    "    \"\"\"\n",
    "    see https://wiki.mozilla.org/User:Jesse/NewFrecency\n",
    "    \"\"\"\n",
    "    # how much will be older articles penalized,\n",
    "    # interpretation: the denomintaor is number of seconds after which the score halves\n",
    "    lambda_const = log(2) / (7*24*60*60)  # 7 days\n",
    "    return np.multiply(np.log(popularity), np.exp(-lambda_const * age))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "daily_trends = get_daily_google_trends()\n",
    "popularity = estimate_popularity(daily_trends, X, words) + 1\n",
    "age = time.time() - df.published.map(time.mktime)\n",
    "\n",
    "frecency = np.squeeze(np.asarray(calculate_frecency(popularity.T, age.values)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(pd.Series(frecency) == 0).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_ids = frecency.argsort()[::-1][:10]\n",
    "df.iloc[top_ids[:10], 0].values"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3 (audiozpravy)",
   "language": "python",
   "name": "audiozpravy"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}