TestV0.3.0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# docker run -p 9200:9200 -p 9300:9300 -e \"discovery.type=single-node\" docker.elastic.co/elasticsearch/elasticsearch:7.7.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crawl "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scrapy\n",
    "from scrapy.crawler import CrawlerProcess\n",
    "\n",
    "from paper_crawler.paper_crawler.spiders.semanticscholar import (\n",
    "    SemanticscholarSpider,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "process = CrawlerProcess(\n",
    "    settings={\n",
    "        \"FEEDS\": {\"papers.json\": {\"format\": \"json\"}},\n",
    "        \"LOG_ENABLED\": False\n",
    "        #     \"LOG_LEVEL\": 'INFO',\n",
    "    }\n",
    ")\n",
    "process.crawl(SemanticscholarSpider, max_papers=2000)\n",
    "process.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Crawled Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open(\"papers.json\") as f:\n",
    "    items = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2006"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(items)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Elasticsearch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from elasticsearch import Elasticsearch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Connect "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "es = Elasticsearch(hosts=[{\"host\": \"localhost\", \"port\": 9200}])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clear previous index "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'error': {'root_cause': [{'type': 'index_not_found_exception',\n",
       "    'reason': 'no such index [paper-index]',\n",
       "    'resource.type': 'index_or_alias',\n",
       "    'resource.id': 'paper-index',\n",
       "    'index_uuid': '_na_',\n",
       "    'index': 'paper-index'}],\n",
       "  'type': 'index_not_found_exception',\n",
       "  'reason': 'no such index [paper-index]',\n",
       "  'resource.type': 'index_or_alias',\n",
       "  'resource.id': 'paper-index',\n",
       "  'index_uuid': '_na_',\n",
       "  'index': 'paper-index'},\n",
       " 'status': 404}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.delete(\"paper-index\", ignore=404)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create new index "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acknowledged': True, 'shards_acknowledged': True, 'index': 'paper-index'}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.create(index=\"paper-index\", ignore=400)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create persistent layer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from elasticsearch_dsl import (\n",
    "    Document,\n",
    "    Date,\n",
    "    Nested,\n",
    "    Boolean,\n",
    "    analyzer,\n",
    "    InnerDoc,\n",
    "    Completion,\n",
    "    Keyword,\n",
    "    Text,\n",
    "    Integer,\n",
    "    Float,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Paper(Document):\n",
    "    title = Text(fields={\"raw\": Keyword()})\n",
    "    date = Integer()\n",
    "    abstract = Text()\n",
    "    authors = Text()\n",
    "    references = Text()\n",
    "    page_rank = Float()\n",
    "\n",
    "    class Index:\n",
    "        name = \"paper-index\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the mappings in Elasticsearch\n",
    "Paper.init(using=es)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Insert items "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from elasticsearch import TransportError\n",
    "from elasticsearch.helpers import bulk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Solution #1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2006, [])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://elasticsearch-py.readthedocs.io/en/master/helpers.html#bulk-helpers\n",
    "# https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-bulk.html\n",
    "\n",
    "\n",
    "def gendata():\n",
    "    for idx, item in enumerate(items):\n",
    "        if item[\"date\"] == \"\" or not item[\"date\"].isdigit():\n",
    "            del item[\"date\"]\n",
    "        yield {\n",
    "            \"_index\": \"paper-index\",\n",
    "            \"_id\": item[\"id\"],\n",
    "            \"page_rank\": 1.0,\n",
    "            **item,\n",
    "        }\n",
    "\n",
    "\n",
    "bulk(es, gendata())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Solution #2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for item in items:\n",
    "    paper = Paper(meta={\"id\": item[\"id\"]}, page_rank=1.0, **item)\n",
    "    paper.save(using=es)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Funcions for inserting items and clearing index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_utils(host=None):\n",
    "    if host is None:\n",
    "        host = {\"host\": \"localhost\", \"port\": 9200}\n",
    "    es = Elasticsearch(hosts=[{\"host\": \"localhost\", \"port\": 9200}])\n",
    "\n",
    "    def clear_index():\n",
    "        es.indices.delete(\"paper-index\", ignore=404)\n",
    "        es.indices.create(index=\"paper-index\", ignore=400)\n",
    "\n",
    "    def insert_items(items):\n",
    "        for item in items:\n",
    "            paper = Paper(meta={\"id\": item[\"id\"]}, **item)\n",
    "            paper.save(using=es)\n",
    "\n",
    "    return clear_index, insert_items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "clear, insert = gen_utils()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "insert(items)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating page rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_papers = list(items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_ids = sorted(list(map(lambda x: x[\"id\"], all_papers)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_matrix = np.zeros((len(all_ids), len(all_ids)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_loc = dict()\n",
    "for index, paper_id in enumerate(all_ids):\n",
    "    id_loc[paper_id] = index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, paper_id in enumerate(all_ids):\n",
    "    paper = Paper.get(id=paper_id, using=es)\n",
    "    if paper.references is not None:\n",
    "        for reference_id in paper.references:\n",
    "            try:\n",
    "                p_matrix[index, id_loc[reference_id]] = 1\n",
    "            except KeyError:\n",
    "                continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.1\n",
    "N = len(all_ids)\n",
    "v = np.ones((1, N))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "row_sums = np.sum(p_matrix, axis=1, keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first part is for rows having nonzero elements\n",
    "# second part is for dead-ends\n",
    "p_matrix = ((row_sums > 0) * 1) * (\n",
    "    (1 - alpha) * p_matrix / (row_sums + np.logical_not(row_sums > 0) * 1)\n",
    "    + alpha * v / N\n",
    ") + (np.logical_not(row_sums > 0) * 1) * v / N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "x0 = np.ones((1, N)) / N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "while True:\n",
    "    next_state = x0 @ p_matrix\n",
    "    if np.allclose(next_state, x0, rtol=0.0001):\n",
    "        break\n",
    "    x0 = next_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "next_state = next_state * 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, paper_id in enumerate(all_ids):\n",
    "    paper = Paper.get(id=paper_id, using=es)\n",
    "    paper.update(page_rank=next_state[0, index], using=es)\n",
    "    paper.save(using=es)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Solution #2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2006, [])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def gendata():\n",
    "    for index, paper_id in enumerate(all_ids):\n",
    "        yield {\n",
    "            \"_index\": \"paper-index\",\n",
    "            \"_id\": items[index][\"id\"],\n",
    "            \"_source\": {\"doc\": {\"page_rank\": next_state[0, index]}},\n",
    "            \"_op_type\": \"update\",\n",
    "        }\n",
    "\n",
    "\n",
    "bulk(es, gendata())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Solution #3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 429\n",
    "body = \"\"\n",
    "for index, paper_id in enumerate(all_ids):\n",
    "    body += f\"\"\"{{ \"update\" : {{\"_id\" : \"{paper_id}\", \"_index\" : \"paper-index\"}} }}\n",
    "{{ \"doc\" : {{\"page_rank\" : {next_state[0, index]} }}}}\n",
    "\"\"\"\n",
    "    break\n",
    "body += \"\"\n",
    "res = es.bulk(body)\n",
    "res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Search "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from elasticsearch_dsl import Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_search = \"\"\n",
    "# title_search = \"Recurrent Models with Fast-Forward\"\n",
    "# abstract_search = 'different \"thinned\" networks'\n",
    "abstract_search = \"\"\n",
    "year_search = 0\n",
    "title_weight = 1\n",
    "abstract_weight = 100\n",
    "year_weight = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search(\n",
    "    title_search: str,\n",
    "    abstract_search: str,\n",
    "    year_search: int,\n",
    "    title_weight: float,\n",
    "    abstract_weight: float,\n",
    "    year_weight: float,\n",
    "    apply_page_rank: bool = True,\n",
    "):\n",
    "    if apply_page_rank:\n",
    "        return es.search(\n",
    "            index=\"paper-index\",\n",
    "            body={\n",
    "                \"query\": {\n",
    "                    # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html#function-weight\n",
    "                    \"function_score\": {\n",
    "                        \"query\": {\n",
    "                            \"bool\": {\n",
    "                                \"should\": [\n",
    "                                    {\"match\": {\"title\": title_search}},\n",
    "                                    {\"match\": {\"abstract\": abstract_search}},\n",
    "                                    {\"range\": {\"date\": {\"gte\": year_search}}},\n",
    "                                ]\n",
    "                            }\n",
    "                        },\n",
    "                        \"functions\": [\n",
    "                            {\n",
    "                                \"filter\": {\"match\": {\"title\": title_search}},\n",
    "                                \"weight\": title_weight,\n",
    "                            },\n",
    "                            {\n",
    "                                \"filter\": {\n",
    "                                    \"match\": {\"abstract\": abstract_search}\n",
    "                                },\n",
    "                                \"weight\": abstract_weight,\n",
    "                            },\n",
    "                            # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-range-query.html\n",
    "                            {\n",
    "                                \"filter\": {\n",
    "                                    \"range\": {\"date\": {\"gte\": year_search}}\n",
    "                                },\n",
    "                                \"weight\": year_weight,\n",
    "                            },\n",
    "                            # https://www.elastic.co/guide/en/elasticsearch/reference/current/static-scoring-signals.html\n",
    "                            # https://www.elastic.co/guide/en/elasticsearch/reference/7.x/query-dsl-rank-feature-query.html#rank-feature-query-saturation\n",
    "                            {\n",
    "                                \"script_score\": {\n",
    "                                    \"script\": {\n",
    "                                        #                                 \"source\": \"_score * saturation(doc['page_rank'].value, 10)\"\n",
    "                                        \"source\": \"_score * doc['page_rank'].value\"\n",
    "                                    }\n",
    "                                }\n",
    "                            },\n",
    "                        ],\n",
    "                    }\n",
    "                }\n",
    "            },\n",
    "        )\n",
    "    return es.search(\n",
    "        index=\"paper-index\",\n",
    "        body={\n",
    "            \"query\": {\n",
    "                # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html#function-weight\n",
    "                \"function_score\": {\n",
    "                    \"query\": {\n",
    "                        \"bool\": {\n",
    "                            \"should\": [\n",
    "                                {\"match\": {\"title\": title_search}},\n",
    "                                {\"match\": {\"abstract\": abstract_search}},\n",
    "                                {\"range\": {\"date\": {\"gte\": year_search}}},\n",
    "                            ]\n",
    "                        }\n",
    "                    },\n",
    "                    \"functions\": [\n",
    "                        {\n",
    "                            \"filter\": {\"match\": {\"title\": title_search}},\n",
    "                            \"weight\": title_weight,\n",
    "                        },\n",
    "                        {\n",
    "                            \"filter\": {\"match\": {\"abstract\": abstract_search}},\n",
    "                            \"weight\": abstract_weight,\n",
    "                        },\n",
    "                        # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-range-query.html\n",
    "                        {\n",
    "                            \"filter\": {\n",
    "                                \"range\": {\"date\": {\"gte\": year_search}}\n",
    "                            },\n",
    "                            \"weight\": year_weight,\n",
    "                        },\n",
    "                    ],\n",
    "                }\n",
    "            }\n",
    "        },\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Improving Neural Networks with Dropout\n",
      "\t Score: 9083.909\n",
      "\t Year: 2013\n",
      "Dropout: a simple way to prevent neural networks from overfitting\n",
      "\t Score: 6437.26\n",
      "\t Year: 2014\n",
      "Learning Networks of Neurons with Boolean Logic\n",
      "\t Score: 6280.1772\n",
      "\t Year: 1987\n",
      "Neural Network Classifiers Estimate Bayesian a posteriori Probabilities\n",
      "\t Score: 4665.441\n",
      "\t Year: 1991\n",
      "Improving the speed of neural networks on CPUs\n",
      "\t Score: 3927.0295\n",
      "\t Year: 2011\n",
      "Exploring Strategies for Training Deep Neural Networks\n",
      "\t Score: 3711.072\n",
      "\t Year: 2009\n",
      "The Aim of Inductive Logic\n",
      "\t Score: 3299.706\n",
      "\t Year: 1966\n",
      "Speech recognition with deep recurrent neural networks\n",
      "\t Score: 2925.1804\n",
      "\t Year: 2013\n",
      "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition\n",
      "\t Score: 2786.2263\n",
      "\t Year: 2015\n",
      "BPS: a learning algorithm for capturing the dynamic nature of speech\n",
      "\t Score: 2398.877\n",
      "\t Year: 1989\n"
     ]
    }
   ],
   "source": [
    "res = search(\n",
    "    title_search,\n",
    "    abstract_search,\n",
    "    year_search,\n",
    "    title_weight,\n",
    "    abstract_weight,\n",
    "    year_weight,\n",
    "    apply_page_rank=True,\n",
    ")\n",
    "for hit in res[\"hits\"][\"hits\"]:\n",
    "    print(\n",
    "        f'{hit[\"_source\"][\"title\"]}\\n\\t Score: {hit[\"_score\"]}\\n\\t Year: {hit[\"_source\"][\"date\"]}'\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Improving Neural Networks with Dropout\n",
      "\t Score: 1603.5023\n",
      "\t Year: 2013\n",
      "Dropout: a simple way to prevent neural networks from overfitting\n",
      "\t Score: 1566.7781\n",
      "\t Year: 2014\n",
      "Task Decomposition Through Competition in a Modular Connectionist Architecture: The What and Where Vision Tasks\n",
      "\t Score: 806.23944\n",
      "\t Year: 1991\n",
      "Actively Searching for an Effective Neural Network Ensemble\n",
      "\t Score: 649.9195\n",
      "\t Year: 1996\n",
      "Sluice networks: Learning what to share between loosely related tasks\n",
      "\t Score: 634.2689\n",
      "\t Year: 2017\n",
      "Structured Attention Networks\n",
      "\t Score: 634.2689\n",
      "\t Year: 2017\n",
      "Best practices for convolutional neural networks applied to visual document analysis\n",
      "\t Score: 632.9023\n",
      "\t Year: 2003\n",
      "Convolutional neural networks applied to house numbers digit classification\n",
      "\t Score: 632.8181\n",
      "\t Year: 2012\n",
      "Training and Analysing Deep Recurrent Neural Networks\n",
      "\t Score: 626.65784\n",
      "\t Year: 2013\n",
      "Distributed Deep Learning Using Synchronous Stochastic Gradient Descent\n",
      "\t Score: 598.5747\n",
      "\t Year: 2016\n"
     ]
    }
   ],
   "source": [
    "res = search(\n",
    "    title_search,\n",
    "    abstract_search,\n",
    "    year_search,\n",
    "    title_weight,\n",
    "    abstract_weight,\n",
    "    year_weight,\n",
    "    apply_page_rank=False,\n",
    ")\n",
    "for hit in res[\"hits\"][\"hits\"]:\n",
    "    print(\n",
    "        f'{hit[\"_source\"][\"title\"]}\\n\\t Score: {hit[\"_score\"]}\\n\\t Year: {hit[\"_source\"][\"date\"]}'\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# With page rank\n",
    "res = es.search(\n",
    "    index=\"paper-index\",\n",
    "    body={\n",
    "        \"query\": {\n",
    "            # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html#function-weight\n",
    "            \"function_score\": {\n",
    "                \"functions\": [\n",
    "                    {\n",
    "                        \"filter\": {\"match\": {\"title\": title_search}},\n",
    "                        \"weight\": title_weight,\n",
    "                    },\n",
    "                    {\n",
    "                        \"filter\": {\"match\": {\"abstract\": abstract_search}},\n",
    "                        \"weight\": abstract_weight,\n",
    "                    },\n",
    "                    # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-range-query.html\n",
    "                    {\n",
    "                        \"filter\": {\"range\": {\"date\": {\"gte\": year_search}}},\n",
    "                        \"weight\": year_weight,\n",
    "                    },\n",
    "                    # https://www.elastic.co/guide/en/elasticsearch/reference/current/static-scoring-signals.html\n",
    "                    # https://www.elastic.co/guide/en/elasticsearch/reference/7.x/query-dsl-rank-feature-query.html#rank-feature-query-saturation\n",
    "                    {\n",
    "                        \"script_score\": {\n",
    "                            \"script\": {\n",
    "                                #                                 \"source\": \"_score * saturation(doc['page_rank'].value, 10)\"\n",
    "                                \"source\": \"_score * doc['page_rank'].value\"\n",
    "                            }\n",
    "                        }\n",
    "                    },\n",
    "                ]\n",
    "            }\n",
    "        }\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Lottery Ticket Hypothesis: Training Pruned Neural Networks: 0.0\n",
      "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding: 0.0\n",
      "Optimal Brain Surgeon and general network pruning: 0.0\n",
      "Understanding Dropout: 0.0\n",
      "Diversity Networks: Neural Network Compression Using Determinantal Point Processes: 0.0\n",
      "Data-free Parameter Pruning for Deep Neural Networks: 0.0\n",
      "A Deep Neural Network Compression Pipeline: Pruning, Quantization, Huffman Encoding: 0.0\n",
      "Pruning Filters for Efficient ConvNets: 0.0\n",
      "Deep Compression: Compressing Deep Neural Network with Pruning, Trained Quantization and Huffman Coding: 0.0\n",
      "ThiNet: A Filter Level Pruning Method for Deep Neural Network Compression: 0.0\n"
     ]
    }
   ],
   "source": [
    "for hit in res[\"hits\"][\"hits\"]:\n",
    "    print(f'{hit[\"_source\"][\"title\"]}: {hit[\"_score\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Without page rank\n",
    "res = es.search(\n",
    "    index=\"paper-index\",\n",
    "    body={\n",
    "        \"query\": {\n",
    "            # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html#function-weight\n",
    "            \"function_score\": {\n",
    "                \"functions\": [\n",
    "                    {\n",
    "                        \"filter\": {\"match\": {\"title\": title_search}},\n",
    "                        \"weight\": title_weight,\n",
    "                    },\n",
    "                    {\n",
    "                        \"filter\": {\"match\": {\"abstract\": abstract_search}},\n",
    "                        \"weight\": abstract_weight,\n",
    "                    },\n",
    "                    # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-range-query.html\n",
    "                    {\n",
    "                        \"filter\": {\"range\": {\"date\": {\"gte\": year_search}}},\n",
    "                        \"weight\": year_weight,\n",
    "                    },\n",
    "                ]\n",
    "            }\n",
    "        }\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Lottery Ticket Hypothesis: Training Pruned Neural Networks: 100.0\n",
      "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding: 50.0\n",
      "Character-Level Language Modeling with Deeper Self-Attention: 50.0\n",
      "U-Net: Machine Reading Comprehension with Unanswerable Questions: 50.0\n",
      "GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding: 50.0\n",
      "Memory Architectures in Recurrent Neural Network Language Models: 50.0\n",
      "Annotation Artifacts in Natural Language Inference Data: 50.0\n",
      "Transforming Question Answering Datasets Into Natural Language Inference Datasets: 50.0\n",
      "Visual Referring Expression Recognition: What Do Systems Actually Learn?: 50.0\n",
      "Visual Dialog: 50.0\n"
     ]
    }
   ],
   "source": [
    "for hit in res[\"hits\"][\"hits\"]:\n",
    "    print(f'{hit[\"_source\"][\"title\"]}: {hit[\"_score\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiments "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = Search(using=es, index=\"paper-index\").query(\n",
    "    \"match\", abstrat=\"different thinned networks\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = Search(using=es, index=\"paper-index\").query(\"match\", id=\"323694313\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = s.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [],
   "source": [
    "for hit in response:\n",
    "    print(hit.meta.score, hit.title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# by calling .search we get back a standard Search object\n",
    "s = Paper.search(using=es)\n",
    "# the search is already limited to the index and doc_type of our document\n",
    "s = s.query(\"match\", title=\"the lottery\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = s.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.071335 The Lottery Ticket Hypothesis: Training Pruned Neural Networks 0.0002729904488451375\n",
      "2.595128 The structure of the “THE”-multiprogramming system 0.0003314600978953064\n",
      "2.3790498 The Perception of the Visual World 0.00040618841508893987\n",
      "2.3790498 Computing the Maximum and the Median 0.0003331176564811347\n",
      "2.2939515 The Stanford Cart and the CMU Rover 0.0002858594703635938\n",
      "2.2866845 The Randomization Bases of the Problem of the Amalgamation of Weighted Means 0.0002637217548452573\n",
      "2.2830682 For Valid Generalization the Size of the Weights is More Important than the Size of the Network 0.0029864949725032395\n",
      "2.24891 The Sample Complexity of Pattern Classification with Neural Networks: The Size of the Weights is More Important than the Size of the Network 0.0002745209411868165\n",
      "2.2335901 Modeling the Shape of the Scene: A Holistic Representation of the Spatial Envelope 0.00035578959729973584\n",
      "2.2147312 The CNN is universal as the Turing machine 0.00035804376339981716\n"
     ]
    }
   ],
   "source": [
    "for hit in response:\n",
    "    print(hit.meta.score, hit.title, hit.page_rank)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HITS "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bidict import bidict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "papers = items\n",
    "# Give each author an id and store them in this dict\n",
    "author_ids = bidict()\n",
    "id_counter = 0\n",
    "for paper in papers:\n",
    "    paper_authors = paper[\"authors\"]\n",
    "    for author in paper_authors:\n",
    "        if author not in author_ids:\n",
    "            author_ids[author] = id_counter\n",
    "            id_counter += 1\n",
    "\n",
    "# Map each paper to it's authors' ids\n",
    "paper_authors_dict = dict()\n",
    "for paper in papers:\n",
    "    paper_id = paper[\"id\"]\n",
    "    paper_authors = paper[\"authors\"]\n",
    "    paper_authors_dict[paper_id] = []\n",
    "    for author in paper_authors:\n",
    "        author_id = author_ids[author]\n",
    "        paper_authors_dict[paper_id].append(author_id)\n",
    "\n",
    "# Map each author to the authors he/she has referenced\n",
    "author_references = dict()\n",
    "for paper in papers:\n",
    "    paper_authors = paper[\"authors\"]\n",
    "    paper_references = paper[\"references\"]\n",
    "    references = []\n",
    "    for reference_id in paper_references:\n",
    "        if reference_id in paper_authors_dict:\n",
    "            references += paper_authors_dict[reference_id]\n",
    "    for author in paper_authors:\n",
    "        author_id = author_ids[author]\n",
    "        if author_id not in author_references:\n",
    "            author_references[author_id] = []\n",
    "        author_references[author_id] += references"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_authors = len(author_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "connectivity_matrix = np.zeros((num_authors, num_authors))\n",
    "for author, references in author_references.items():\n",
    "    for reference in references:\n",
    "        connectivity_matrix[author, reference] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.ones(num_authors)\n",
    "h = np.ones(num_authors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "for _ in range(5):\n",
    "    for i in range(num_authors):\n",
    "        h[i] = np.sum(connectivity_matrix[i, :] * a)\n",
    "    for i in range(num_authors):\n",
    "        a[i] = np.sum(connectivity_matrix[:, i].T * h)\n",
    "    a /= np.sum(a)\n",
    "    h /= np.sum(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_authors = []\n",
    "for i in np.argpartition(a, -10)[-10:]:\n",
    "    best_authors.append((author_ids.inverse[i], a[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Authors:\n",
      "Ilya Sutskever : 0.01332247665339998\n",
      "Geoffrey E. Hinton : 0.012897100909998679\n",
      "Quoc V. Le : 0.010406961290190133\n",
      "Yoshua Bengio : 0.01031043642665406\n",
      "Yann LeCun : 0.010073318403498548\n",
      "Rob Fergus : 0.009878935870591975\n",
      "Alex Krizhevsky : 0.009769722710715257\n",
      "Andrew Y. Ng : 0.009080287367886264\n",
      "Oriol Vinyals : 0.008076150300655045\n",
      "Marc'Aurelio Ranzato : 0.007608169735770792\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Authors:\")\n",
    "best_authors = sorted(best_authors, key=lambda x: -x[1])\n",
    "for author, authority in best_authors:\n",
    "    print(f\"{author} : {authority}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ranking SVM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./MIR_Phase3/data/train.txt\", \"r\") as f:\n",
    "    train_contents = f.read()\n",
    "with open(\"./MIR_Phase3/data/vali.txt\", \"r\") as f:\n",
    "    val_contents = f.read()\n",
    "with open(\"./MIR_Phase3/data/test.txt\", \"r\") as f:\n",
    "    test_contents = f.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parse Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataclasses import dataclass\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class QueryResult:\n",
    "    qid: int\n",
    "    doc_id: str\n",
    "    relevance: int\n",
    "    feature_vector: np.array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Dict, List\n",
    "\n",
    "\n",
    "def parse_data(data: str) -> Dict[int, List[QueryResult]]:\n",
    "    query_to_result_dict = dict()\n",
    "    for line in data.split(\"\\n\"):\n",
    "        splitted_line = line.split()\n",
    "        try:\n",
    "            relevance = int(splitted_line[0])\n",
    "            qid = int(line.split()[1][4:])\n",
    "            doc_id = line.split()[50]\n",
    "            feature_vector = []\n",
    "            for feature in line.split()[2:48]:\n",
    "                feature_vector.append(float(feature.split(\":\")[1]))\n",
    "            if qid not in query_to_result_dict:\n",
    "                query_to_result_dict[qid] = []\n",
    "            query_to_result_dict[qid].append(\n",
    "                QueryResult(qid, doc_id, relevance, np.array(feature_vector))\n",
    "            )\n",
    "        except IndexError:\n",
    "            pass\n",
    "    return query_to_result_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_query_to_result_dict = parse_data(train_contents)\n",
    "val_query_to_result_dict = parse_data(val_contents)\n",
    "test_query_to_result_dict = parse_data(test_contents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Data for Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "POSITIVE_CLASS = 1\n",
    "NEGATIVE_CLASS = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_data_for_model(query_to_result_dict: Dict[int, List[QueryResult]]):\n",
    "    X = []\n",
    "    Y = []\n",
    "    for qid, results in query_to_result_dict.items():\n",
    "        # Sort in decresing order\n",
    "        results = sorted(results, key=lambda x: -x.relevance)\n",
    "        num_results = len(results)\n",
    "        for idx, result in enumerate(results):\n",
    "            for other_result in results[idx + 1 : -1]:\n",
    "                if result.relevance > other_result.relevance:\n",
    "                    positive_vector = (\n",
    "                        result.feature_vector - other_result.feature_vector\n",
    "                    )\n",
    "                    X.append(positive_vector)\n",
    "                    X.append(-positive_vector)\n",
    "                    Y.append(POSITIVE_CLASS)\n",
    "                    Y.append(NEGATIVE_CLASS)\n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = prepare_data_for_model(train_query_to_result_dict)\n",
    "X_val, y_val = prepare_data_for_model(val_query_to_result_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check Hyperparameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# My laptop melts with all the data\n",
    "num_train = len(X_train) // 10\n",
    "X_train, y_train = X_train[:num_train], y_train[:num_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'kernel': 'rbf', 'C': 1.0}\n",
      "0.7668227033352838\n",
      "{'kernel': 'linear', 'C': 1.0}\n",
      "0.7753803393797543\n",
      "{'kernel': 'linear', 'C': 10.0}\n",
      "0.7746123464014043\n",
      "{'kernel': 'linear', 'C': 0.1}\n",
      "0.7777208894090111\n"
     ]
    }
   ],
   "source": [
    "settings = [\n",
    "    {\"kernel\": \"rbf\", \"C\": 1.0},\n",
    "    {\"kernel\": \"linear\", \"C\": 1.0},\n",
    "    {\"kernel\": \"linear\", \"C\": 10.0},\n",
    "    {\"kernel\": \"linear\", \"C\": 0.1},\n",
    "]\n",
    "for setting in settings:\n",
    "    print(setting)\n",
    "    clf = SVC(**setting)\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred_val = clf.predict(X_val)\n",
    "    print(accuracy_score(y_val, y_pred_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, break_ties=False, cache_size=200, class_weight=None, coef0=0.0,\n",
       "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
       "    max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, y_train = prepare_data_for_model(train_query_to_result_dict)\n",
    "X_val, y_val = prepare_data_for_model(val_query_to_result_dict)\n",
    "clf = SVC(kernel=\"linear\")\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare(doc1, doc2):\n",
    "    return (\n",
    "        1\n",
    "        if clf.predict([doc1.feature_vector - doc2.feature_vector])[0]\n",
    "        == POSITIVE_CLASS\n",
    "        else -1\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python program for implementation of Quicksort Sort\n",
    "\n",
    "# This function takes last element as pivot, places\n",
    "# the pivot element at its correct position in sorted\n",
    "# array, and places all smaller (smaller than pivot)\n",
    "# to left of pivot and all greater elements to right\n",
    "# of pivot\n",
    "def partition(arr, low, high):\n",
    "    i = low - 1  # index of smaller element\n",
    "    pivot = arr[high]  # pivot\n",
    "\n",
    "    for j in range(low, high):\n",
    "\n",
    "        # If current element is smaller than or\n",
    "        # equal to pivot\n",
    "        if compare(pivot, arr[j]) == -1:\n",
    "\n",
    "            # increment index of smaller element\n",
    "            i = i + 1\n",
    "            arr[i], arr[j] = arr[j], arr[i]\n",
    "\n",
    "    arr[i + 1], arr[high] = arr[high], arr[i + 1]\n",
    "    return i + 1\n",
    "\n",
    "\n",
    "# The main function that implements QuickSort\n",
    "# arr[] --> Array to be sorted,\n",
    "# low  --> Starting index,\n",
    "# high  --> Ending index\n",
    "\n",
    "# Function to do Quick sort\n",
    "def quickSort(arr, low, high):\n",
    "    if low < high:\n",
    "\n",
    "        # pi is partitioning index, arr[p] is now\n",
    "        # at right place\n",
    "        pi = partition(arr, low, high)\n",
    "\n",
    "        # Separately sort elements before\n",
    "        # partition and after partition\n",
    "        quickSort(arr, low, pi - 1)\n",
    "        quickSort(arr, pi + 1, high)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.metrics import ndcg_at_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "evals = []\n",
    "for qid, results in test_query_to_result_dict.items():\n",
    "    gt = [x.relevance for x in sorted(results, key=lambda x: -x.relevance)]\n",
    "    quickSort(results, 0, len(results) - 1)\n",
    "    pred = [x.relevance for x in results]\n",
    "    evals.append(ndcg_at_k(pred, gt, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4403820991960565"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.average(evals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}