topic_batch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2594f27f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"downloads/pos_100k.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "05471dac",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting bertopic\n",
      "  Downloading bertopic-0.9.3-py2.py3-none-any.whl (57 kB)\n",
      "     |████████████████████████████████| 57 kB 3.3 MB/s             \n",
      "\u001b[?25hRequirement already satisfied: pandas>=1.1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from bertopic) (1.1.5)\n",
      "Requirement already satisfied: scikit-learn>=0.22.2.post1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from bertopic) (0.24.1)\n",
      "Collecting plotly<4.14.3,>=4.7.0\n",
      "  Downloading plotly-4.14.2-py2.py3-none-any.whl (13.2 MB)\n",
      "     |████████████████████████████████| 13.2 MB 34.8 MB/s            \n",
      "\u001b[?25hCollecting sentence-transformers>=0.4.1\n",
      "  Downloading sentence-transformers-2.2.0.tar.gz (79 kB)\n",
      "     |████████████████████████████████| 79 kB 12.6 MB/s             \n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: pyyaml<6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from bertopic) (5.4.1)\n",
      "Requirement already satisfied: tqdm>=4.41.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from bertopic) (4.62.3)\n",
      "Collecting bertopic\n",
      "  Downloading bertopic-0.9.2-py2.py3-none-any.whl (57 kB)\n",
      "     |████████████████████████████████| 57 kB 1.1 MB/s             \n",
      "\u001b[?25h  Downloading bertopic-0.9.1-py2.py3-none-any.whl (55 kB)\n",
      "     |████████████████████████████████| 55 kB 7.6 MB/s             \n",
      "\u001b[?25h  Downloading bertopic-0.9.0-py2.py3-none-any.whl (55 kB)\n",
      "     |████████████████████████████████| 55 kB 7.2 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.8.1-py2.py3-none-any.whl (53 kB)\n",
      "     |████████████████████████████████| 53 kB 4.2 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.8.0-py2.py3-none-any.whl (53 kB)\n",
      "     |████████████████████████████████| 53 kB 3.3 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.7.0-py2.py3-none-any.whl (40 kB)\n",
      "     |████████████████████████████████| 40 kB 1.4 MB/s             \n",
      "\u001b[?25h  Downloading bertopic-0.6.0-py2.py3-none-any.whl (25 kB)\n",
      "Requirement already satisfied: numpy>=1.19.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from bertopic) (1.19.5)\n",
      "Collecting torch>=1.4.0\n",
      "  Downloading torch-1.10.2-cp36-cp36m-manylinux1_x86_64.whl (881.9 MB)\n",
      "     |████████████████████████████████| 881.9 MB 7.7 kB/s              \n",
      "\u001b[?25hCollecting umap-learn>=0.5.0\n",
      "  Downloading umap-learn-0.5.3.tar.gz (88 kB)\n",
      "     |████████████████████████████████| 88 kB 12.5 MB/s             \n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting hdbscan>=0.8.27\n",
      "  Downloading hdbscan-0.8.28.tar.gz (5.2 MB)\n",
      "     |████████████████████████████████| 5.2 MB 67.0 MB/s            \n",
      "\u001b[?25h  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: joblib>=1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from hdbscan>=0.8.27->bertopic) (1.0.1)\n",
      "Requirement already satisfied: cython>=0.27 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from hdbscan>=0.8.27->bertopic) (0.29.22)\n",
      "Requirement already satisfied: scipy>=1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from hdbscan>=0.8.27->bertopic) (1.5.3)\n",
      "  Downloading hdbscan-0.8.27.tar.gz (6.4 MB)\n",
      "     |████████████████████████████████| 6.4 MB 8.4 MB/s            \n",
      "\u001b[?25h  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: six in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from hdbscan>=0.8.27->bertopic) (1.15.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas>=1.1.5->bertopic) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas>=1.1.5->bertopic) (2021.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from scikit-learn>=0.22.2.post1->bertopic) (2.1.0)\n",
      "Collecting transformers<5.0.0,>=4.6.0\n",
      "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
      "     |████████████████████████████████| 4.0 MB 60.9 MB/s            \n",
      "\u001b[?25hCollecting torchvision\n",
      "  Downloading torchvision-0.11.2-cp36-cp36m-manylinux1_x86_64.whl (23.3 MB)\n",
      "     |████████████████████████████████| 23.3 MB 83.5 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: nltk in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from sentence-transformers>=0.4.1->bertopic) (3.4.4)\n",
      "Collecting sentencepiece\n",
      "  Downloading sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "     |████████████████████████████████| 1.2 MB 33.9 MB/s            \n",
      "\u001b[?25hCollecting huggingface-hub\n",
      "  Downloading huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\n",
      "     |████████████████████████████████| 67 kB 9.5 MB/s              \n",
      "\u001b[?25hRequirement already satisfied: dataclasses in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from torch>=1.4.0->bertopic) (0.8)\n",
      "Requirement already satisfied: typing-extensions in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from torch>=1.4.0->bertopic) (4.0.1)\n",
      "Requirement already satisfied: numba>=0.49 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from umap-learn>=0.5.0->bertopic) (0.52.0)\n",
      "Collecting pynndescent>=0.5\n",
      "  Downloading pynndescent-0.5.7.tar.gz (1.1 MB)\n",
      "     |████████████████████████████████| 1.1 MB 67.8 MB/s            \n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from numba>=0.49->umap-learn>=0.5.0->bertopic) (49.6.0.post20210108)\n",
      "Requirement already satisfied: llvmlite<0.36,>=0.35.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from numba>=0.49->umap-learn>=0.5.0->bertopic) (0.35.0)\n",
      "Collecting sacremoses\n",
      "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\n",
      "     |████████████████████████████████| 880 kB 107.3 MB/s            \n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
      "  Downloading tokenizers-0.12.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
      "     |████████████████████████████████| 6.6 MB 25.3 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: requests in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (2.26.0)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (3.0.12)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (2020.11.13)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (21.3)\n",
      "Requirement already satisfied: importlib-metadata in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (3.7.0)\n",
      "Collecting torch>=1.4.0\n",
      "  Downloading torch-1.10.1-cp36-cp36m-manylinux1_x86_64.whl (881.9 MB)\n",
      "     |████████████████████████████████| 881.9 MB 7.0 kB/s              \n",
      "\u001b[?25hRequirement already satisfied: pillow!=8.3.0,>=5.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from torchvision->sentence-transformers>=0.4.1->bertopic) (8.4.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from packaging>=20.0->transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (2.4.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from importlib-metadata->transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (3.4.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (1.26.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (3.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (2021.5.30)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (2.0.9)\n",
      "Requirement already satisfied: click in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from sacremoses->transformers<5.0.0,>=4.6.0->sentence-transformers>=0.4.1->bertopic) (7.1.2)\n",
      "Building wheels for collected packages: hdbscan, sentence-transformers, umap-learn, pynndescent, sacremoses\n",
      "  Building wheel for hdbscan (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for hdbscan: filename=hdbscan-0.8.27-cp36-cp36m-linux_x86_64.whl size=2667453 sha256=0c7560f123cca66ebdf160fc81298bfd3f007257ccca8f188d23ad7a6dd1bfe0\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/70/52/fe/83ef5a0e879c48df9968d931d41046226e4cb19a166f0223d2\n",
      "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.0-py3-none-any.whl size=120747 sha256=3e23255b049b46f334f5c3c810bc6cc7d5f437b393c339c0f99bae410e01ea34\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/ce/2d/86/18ade2fbe66e17694cf8f5667a68d2bd06a65bce42546ea675\n",
      "  Building wheel for umap-learn (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for umap-learn: filename=umap_learn-0.5.3-py3-none-any.whl size=82820 sha256=6f6169867c5f093bafafbc88e21438949e9aa25586562cd2ea839d849883b5d9\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/54/ca/cf/35a4f9a744b0bce9549e4e492261bef3903434d0118f3ee858\n",
      "  Building wheel for pynndescent (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pynndescent: filename=pynndescent-0.5.7-py3-none-any.whl size=54271 sha256=d2e98651602bccf330b448112b8a249132331858a65a4f6079d49bae085c2337\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/fd/8a/9e/58bdbe4211581dd2aeffe2ce46020a000bf67127afac24e772\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895254 sha256=213a6c2a51c9a339fc61a0595d97b3c2b4b7a91334d82cd24f11085ffa377b7a\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/4c/64/31/e9900a234b23fb3e9dc565d6114a9d6ff84a72dbdd356502b4\n",
      "Successfully built hdbscan sentence-transformers umap-learn pynndescent sacremoses\n",
      "Installing collected packages: torch, tokenizers, sacremoses, huggingface-hub, transformers, torchvision, sentencepiece, pynndescent, umap-learn, sentence-transformers, hdbscan, bertopic\n",
      "Successfully installed bertopic-0.6.0 hdbscan-0.8.27 huggingface-hub-0.4.0 pynndescent-0.5.7 sacremoses-0.0.53 sentence-transformers-2.2.0 sentencepiece-0.1.96 tokenizers-0.12.1 torch-1.10.1 torchvision-0.11.2 transformers-4.18.0 umap-learn-0.5.3\n",
      "\u001b[33mWARNING: Skipping umap as it is not installed.\u001b[0m\n",
      "Requirement already satisfied: umap-learn in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (0.5.3)\n",
      "Requirement already satisfied: scipy>=1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from umap-learn) (1.5.3)\n",
      "Requirement already satisfied: numba>=0.49 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from umap-learn) (0.52.0)\n",
      "Requirement already satisfied: scikit-learn>=0.22 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from umap-learn) (0.24.1)\n",
      "Requirement already satisfied: pynndescent>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from umap-learn) (0.5.7)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from umap-learn) (1.19.5)\n",
      "Requirement already satisfied: tqdm in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from umap-learn) (4.62.3)\n",
      "Requirement already satisfied: llvmlite<0.36,>=0.35.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from numba>=0.49->umap-learn) (0.35.0)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from numba>=0.49->umap-learn) (49.6.0.post20210108)\n",
      "Requirement already satisfied: joblib>=0.11 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pynndescent>=0.5->umap-learn) (1.0.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from scikit-learn>=0.22->umap-learn) (2.1.0)\n",
      "Collecting stop-words\n",
      "  Downloading stop-words-2018.7.23.tar.gz (31 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hBuilding wheels for collected packages: stop-words\n",
      "  Building wheel for stop-words (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for stop-words: filename=stop_words-2018.7.23-py3-none-any.whl size=32916 sha256=141c84dd40935cc0da64963692ca32f272d88a534918ec16a76f17d99dc84260\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/61/13/fb/bde87253355995b963f6355ca563a614453668bfc1b44fdc54\n",
      "Successfully built stop-words\n",
      "Installing collected packages: stop-words\n",
      "Successfully installed stop-words-2018.7.23\n"
     ]
    }
   ],
   "source": [
    "!pip install bertopic\n",
    "!pip uninstall umap\n",
    "!pip install umap-learn\n",
    "!pip install stop-words"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82df5ec1",
   "metadata": {},
   "source": [
    "## BERTOPIC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c7de78bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bertopic import BERTopic\n",
    "m = BERTopic(embedding_model='sentence-transformers/all-mpnet-base-v2', calculate_probabilities=True, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5c87a878",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_topics(texts, distance_thresh=0.035, model=m):\n",
    "    topic_labels = model.transform(texts)[0]\n",
    "    topic_dict = {}\n",
    "    for idx, ID in enumerate(topic_labels):\n",
    "        topics = model.get_topic(ID)\n",
    "        for t in topics:\n",
    "            topic = t[0]\n",
    "            dist = t[1]\n",
    "            if dist > distance_thresh:\n",
    "                if not topic_dict.get(topic):\n",
    "                    topic_dict[topic] = [ texts[idx] ] \n",
    "                else:\n",
    "                    sents = topic_dict[topic] + [texts[idx]]\n",
    "                    topic_dict[topic] = sents\n",
    "    return topic_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2c0de2b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from stop_words import get_stop_words\n",
    "def get_topic_name(topic_id, m, take=3,):\n",
    "    try:\n",
    "        ts = m.get_topic(topic_id)\n",
    "        ts = [word[0] for word in ts if word not in get_stop_words('english')]\n",
    "\n",
    "        name = \"\"\n",
    "        for t in ts[:take]:\n",
    "            name += t\n",
    "            name += \" \"\n",
    "        return name.strip()\n",
    "    except Exception as e:\n",
    "        print(\"Except\", topic_id, e)\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1e0653b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df=df.sample(frac=1)\n",
    "# df1 = df[40000:]\n",
    "# sents1= list(df1[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "152f6379",
   "metadata": {},
   "outputs": [],
   "source": [
    "sents= list(df[\"text\"])\n",
    "import time\n",
    "#9:50am"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71923802",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "673d75ae9e9646939396823f13c1e564",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.18k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4f44d9238ee42cebd309580f9d8077c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/190 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "84c6453a590943afb9e9ba107677f2d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/10.1k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5429360c29964433bdcc881564b6c5f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/571 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9272e2ce0c0e49c29b74ac67346f7e98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/116 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4654d1f9e765451dabe7e5eda84af043",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/39.3k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d65e50553084822baee003fb34a6962",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/349 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24a4b821192e4fa98c3407c97e35feef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/438M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ae9acc1fcc1414fb07ff208c790fcb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/53.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30b4e2f6dac441058cad161b30fa2ef1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/239 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93ea175d0ebc4e5abfc33112605f1596",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/466k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9709339932f040b8af5544765524f431",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/363 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d80d703dfed499fbe0648c3201a84ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/13.1k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "541c160aee994179b9a36dddbe1c8017",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6992011c665d4703a355241816baf7ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/3125 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-16 17:59:04,838 - BERTopic - Transformed documents to Embeddings\n",
      "2022-05-16 18:01:08,046 - BERTopic - Reduced dimensionality with UMAP\n"
     ]
    }
   ],
   "source": [
    "topics, probs = m.fit_transform(sents)\n",
    "#4:27"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aaa289ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "topics[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60506ba1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa1fa2ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_topic_name(22, m, 3)\n",
    "#tuberculosis tb mycobacterium'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbc6f4f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "df[\"probs\"] = [list(reversed(list(np.argpartition(ps, -3)[-3:]))) for ps in probs]\n",
    "df[\"topic_id\"] = topics\n",
    "df[\"topic_name\"] = df1[\"topic_id\"].apply(lambda t: get_topic_name(t,m, 3))\n",
    "df.to_csv (\"downloads/topic200k.csv\", index = False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "ed90fedf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv(\"downloads/df2.csv\")\n",
    "df0 = pd.read_csv(\"downloads/df1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "03e07090",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df0.append(df1).append(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "03c67736",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv (\"downloads/65k_enriched_topic.csv\", index = False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d3f819b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def stepwise(step, df, f, key):\n",
    "    l = []\n",
    "    tn=[]\n",
    "    for idx in range(step, len(df)+step-1, step):\n",
    "        m = BERTopic(embedding_model='sentence-transformers/all-mpnet-base-v2', calculate_probabilities=True, verbose=True)\n",
    "        print(idx-step, idx)# dataframe[idx-step:idx])\n",
    "        sl = df[idx-step:idx][key]\n",
    "        batch = m.fit_transform(list(sl))\n",
    "        l = l + list([batch])\n",
    "#         print(batch[1])\n",
    "        probs = [list(reversed(list(np.argpartition(ps, -3)[-3:]))) for ps in batch[1]]\n",
    "        t_names = [get_topic_name(p[0], m, 3) for p in probs]\n",
    "        tn += [t_names]\n",
    "    return l, tn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a1662593",
   "metadata": {},
   "outputs": [],
   "source": [
    "#batches, t_names = stepwise(100, df[:300], m.fit_transform, \"text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "c822b38c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten(l):\n",
    "    return [item for sublist in l for item in sublist]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6b7565b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 66547)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t =[]\n",
    "ps = []\n",
    "for batch in batches:\n",
    "    #print(batch[0])\n",
    "    t += [batch[0]]\n",
    "    ps += [batch[1]]\n",
    "#        a+=[t]\n",
    "\n",
    "a = [flatten(t), flatten(ps)]\n",
    "len(a), len(a[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f14a4777",
   "metadata": {},
   "outputs": [],
   "source": [
    "topics,probs = a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dbff178",
   "metadata": {},
   "outputs": [],
   "source": [
    "# td = extract_topics(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fee5d951",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "466c5cec",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "2566eb56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([<class 'list'>], dtype=object)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"probs\"].map(type).unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "00d52aa1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.get_topic()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "d76276ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['amyotrophic',\n",
       " 'als',\n",
       " 'sclerosis',\n",
       " 'neuron',\n",
       " 'neurons',\n",
       " 'neurodegenerative',\n",
       " 'spinal',\n",
       " 'degeneration',\n",
       " 'death',\n",
       " 'neurone']"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "[word[0] for word in m.get_topic(14) if word not in get_stop_words('english')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25d9e25c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "d26286d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns={\"sentence_id\": \"problem_id\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "36a24854",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_sent</th>\n",
       "      <th>doi</th>\n",
       "      <th>coreId</th>\n",
       "      <th>oai</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Liu and Mertz (Genes Dev.</td>\n",
       "      <td>10.1128/MCB.25.15.6303-6313.2005</td>\n",
       "      <td>3717551</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:1190326</td>\n",
       "      <td>Binding of hnRNP L to the Pre-mRNA Processing ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Rett syndrome (RTT), a neurodevelopmental diso...</td>\n",
       "      <td>10.1086/382228</td>\n",
       "      <td>3710391</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:1182264</td>\n",
       "      <td>X-Chromosome Inactivation Patterns Are Unbalan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Estimating optimal sample size for microbiolog...</td>\n",
       "      <td>10.1128/AEM.69.10.6174-6178.2003</td>\n",
       "      <td>3286476</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:201250</td>\n",
       "      <td>A Model To Estimate the Optimal Sample Size fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The antiphospholipid syndrome is a thrombotic ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8064782</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:2398067</td>\n",
       "      <td>Antiphospholipid syndrome in a patient with ra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alcohol dependence (AD) is a complex disorder ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8674979</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:3266269</td>\n",
       "      <td>Genome-Wide Association Study of Copy Number V...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42196</th>\n",
       "      <td>Autism spectrum disorder (ASD) is a common neu...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8273706</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:2662567</td>\n",
       "      <td>The genetic and neurobiologic compass points t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42197</th>\n",
       "      <td>This paper revisits the problem of the readine...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8689613</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:3291942</td>\n",
       "      <td>On the Relevance of Using Open Wireless Sensor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42198</th>\n",
       "      <td>Thyroid cancer is a rare disease.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8126095</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:2466601</td>\n",
       "      <td>Thyroid carcinoma and its treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42199</th>\n",
       "      <td>Hyperosmolar hyperglycaemic state (HHS) is a m...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8696563</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:3302717</td>\n",
       "      <td>A case of hyperosmolar hyperglycaemic state wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42200</th>\n",
       "      <td>Craniopharyngiomas are difficult to treat.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8729638</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:3356029</td>\n",
       "      <td>Outcome of Endoscopy-Assisted Microscopic Exte...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>42201 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              first_sent  \\\n",
       "0                              Liu and Mertz (Genes Dev.   \n",
       "1      Rett syndrome (RTT), a neurodevelopmental diso...   \n",
       "2      Estimating optimal sample size for microbiolog...   \n",
       "3      The antiphospholipid syndrome is a thrombotic ...   \n",
       "4      Alcohol dependence (AD) is a complex disorder ...   \n",
       "...                                                  ...   \n",
       "42196  Autism spectrum disorder (ASD) is a common neu...   \n",
       "42197  This paper revisits the problem of the readine...   \n",
       "42198                  Thyroid cancer is a rare disease.   \n",
       "42199  Hyperosmolar hyperglycaemic state (HHS) is a m...   \n",
       "42200         Craniopharyngiomas are difficult to treat.   \n",
       "\n",
       "                                    doi   coreId  \\\n",
       "0      10.1128/MCB.25.15.6303-6313.2005  3717551   \n",
       "1                        10.1086/382228  3710391   \n",
       "2      10.1128/AEM.69.10.6174-6178.2003  3286476   \n",
       "3                                   NaN  8064782   \n",
       "4                                   NaN  8674979   \n",
       "...                                 ...      ...   \n",
       "42196                               NaN  8273706   \n",
       "42197                               NaN  8689613   \n",
       "42198                               NaN  8126095   \n",
       "42199                               NaN  8696563   \n",
       "42200                               NaN  8729638   \n",
       "\n",
       "                                     oai  \\\n",
       "0      oai:pubmedcentral.nih.gov:1190326   \n",
       "1      oai:pubmedcentral.nih.gov:1182264   \n",
       "2       oai:pubmedcentral.nih.gov:201250   \n",
       "3      oai:pubmedcentral.nih.gov:2398067   \n",
       "4      oai:pubmedcentral.nih.gov:3266269   \n",
       "...                                  ...   \n",
       "42196  oai:pubmedcentral.nih.gov:2662567   \n",
       "42197  oai:pubmedcentral.nih.gov:3291942   \n",
       "42198  oai:pubmedcentral.nih.gov:2466601   \n",
       "42199  oai:pubmedcentral.nih.gov:3302717   \n",
       "42200  oai:pubmedcentral.nih.gov:3356029   \n",
       "\n",
       "                                                   title  \n",
       "0      Binding of hnRNP L to the Pre-mRNA Processing ...  \n",
       "1      X-Chromosome Inactivation Patterns Are Unbalan...  \n",
       "2      A Model To Estimate the Optimal Sample Size fo...  \n",
       "3      Antiphospholipid syndrome in a patient with ra...  \n",
       "4      Genome-Wide Association Study of Copy Number V...  \n",
       "...                                                  ...  \n",
       "42196  The genetic and neurobiologic compass points t...  \n",
       "42197  On the Relevance of Using Open Wireless Sensor...  \n",
       "42198                Thyroid carcinoma and its treatment  \n",
       "42199  A case of hyperosmolar hyperglycaemic state wi...  \n",
       "42200  Outcome of Endoscopy-Assisted Microscopic Exte...  \n",
       "\n",
       "[42201 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e662388a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}