06_Topic_modeling_dementia_teach.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "e985762c",
      "metadata": {
        "id": "e985762c"
      },
      "source": [
        "Ref:<br>\n",
        "https://github.com/MaartenGr/BERTopic<br>\n",
        "https://scikit-learn.org/0.19/modules/generated/sklearn.datasets.fetch_20newsgroups.html#sklearn.datasets.fetch_20newsgroups<br>\n",
        "https://towardsdatascience.com/meet-bertopic-berts-cousin-for-advanced-topic-modeling-ea5bf0b7faa3<br>\n",
        "https://www.youtube.com/watch?v=sZcGuYHWN_w<br>\n",
        "https://maartengr.github.io/BERTopic/index.html<br>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "da3dcc19",
      "metadata": {
        "scrolled": false,
        "id": "da3dcc19"
      },
      "outputs": [],
      "source": [
        "!pip install bertopic\n",
        "!pip install ipywidgets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1d0bc3ae",
      "metadata": {
        "scrolled": true,
        "id": "1d0bc3ae"
      },
      "outputs": [],
      "source": [
        "from bertopic import BERTopic\n",
        "from sklearn.datasets import fetch_20newsgroups\n",
        "from umap import UMAP\n",
        "import IPython"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a9d6c454",
      "metadata": {
        "id": "a9d6c454"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv('file_name.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e3839c4a",
      "metadata": {
        "id": "e3839c4a"
      },
      "outputs": [],
      "source": [
        "df.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f2d396a3",
      "metadata": {
        "id": "f2d396a3"
      },
      "outputs": [],
      "source": [
        "df.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8e3bab0e",
      "metadata": {
        "id": "8e3bab0e"
      },
      "outputs": [],
      "source": [
        "#for thai language\n",
        "doc = list(df.tr_processed)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#for eng language\n",
        "doc = list(df.processed)"
      ],
      "metadata": {
        "id": "ixAWLjdOPXYv"
      },
      "id": "ixAWLjdOPXYv",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1b65caba",
      "metadata": {
        "scrolled": false,
        "id": "1b65caba"
      },
      "outputs": [],
      "source": [
        "topic_model = BERTopic()\n",
        "umap_model = UMAP(n_neighbors=15, n_components=5, \n",
        "                  min_dist=0.0, metric='cosine', random_state=42)\n",
        "topic_model = BERTopic(umap_model=umap_model, top_n_words = 15)\n",
        "topics, probs = topic_model.fit_transform(doc)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "023c5df0",
      "metadata": {
        "scrolled": true,
        "id": "023c5df0"
      },
      "outputs": [],
      "source": [
        "topic_info = topic_model.get_topic_info()\n",
        "topic_info.to_csv('topic_info.csv',index=True)\n",
        "topic_info_result = topic_model.get_topics()\n",
        "topic_info_result_df = pd.DataFrame.from_dict(topic_info_result)\n",
        "topic_info_result_df.to_csv('topic_info_result.csv',index=True)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fig_vis_hie = topic_model.visualize_hierarchy()\n",
        "fig_vis_hie.write_html('/content/vis_hie.html')\n",
        "IPython.display.HTML(open('/content/vis_hie.html').read())"
      ],
      "metadata": {
        "id": "2xgDtu9CLond"
      },
      "id": "2xgDtu9CLond",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig_vis_top = topic_model.visualize_topics()\n",
        "fig_vis_top.write_html('/content/vis_top.html')\n",
        "IPython.display.HTML(open('/content/vis_top.html').read())"
      ],
      "metadata": {
        "id": "kbQW-XdzMME7"
      },
      "id": "kbQW-XdzMME7",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8b998fdb",
      "metadata": {
        "id": "8b998fdb"
      },
      "outputs": [],
      "source": [
        "topic_model.get_representative_docs(6)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python [conda env:dementia]",
      "language": "python",
      "name": "conda-env-dementia-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.9"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}