evaluate_models_using_octis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3ed6c1a7",
   "metadata": {},
   "source": [
    "Evaluate non BEERTopic topic models using coherence and topic diversity scores. Evaluate LDA, NMF, LSI and HDP."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e46b0ea7",
   "metadata": {},
   "source": [
    "Preliminaries\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2270bd82",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from octis.dataset.dataset import Dataset\n",
    "from octis.models.LDA import LDA\n",
    "from octis.models.HDP import HDP\n",
    "from octis.models.LSI import LSI\n",
    "from octis.models.NMF import NMF\n",
    "from octis.models.NMF_scikit import NMF_scikit\n",
    "from octis.evaluation_metrics.coherence_metrics import Coherence\n",
    "from octis.evaluation_metrics.diversity_metrics import TopicDiversity\n",
    "import time\n",
    "current_iteration = 0\n",
    "\n",
    "from octis.dataset.dataset import Dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58a09144",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the dataset\n",
    "dataset = Dataset()\n",
    "dataset.load_custom_dataset_from_folder('./travels_tales')\n",
    "\n",
    "# Perform a basic operation to verify dataset loading\n",
    "print(\"Number of documents:\", len(dataset.get_corpus()))\n",
    "\n",
    "# Define the k values\n",
    "k_values = [200, 218, 236]\n",
    "\n",
    "# Initialize the DataFrame to store results\n",
    "results_df = pd.DataFrame(columns=[\"Model\", \"k\", \"Topic Coherence_npmi\", \"Topic Coherence_c_v\", \"Topic Diversity\"])\n",
    "\n",
    "# Initialize metrics\n",
    "topic_diversity_metric = TopicDiversity(topk=10)\n",
    "coherence_metric = Coherence(texts=dataset.get_corpus(), topk=10, measure='c_npmi')\n",
    "cv_metric = Coherence(texts=dataset.get_corpus(), topk=10, measure='c_v')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd7e1b6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Models excluding HDP\n",
    "for k in k_values:\n",
    "    for model_name, Model in [(\"LDA\", LDA), (\"NMF\", NMF), (\"NMF-Scikit\", NMF_scikit), (\"LSI\", LSI)]:\n",
    "        start_time = time.time()  # Start time for this iteration\n",
    "        \n",
    "        # Training the model\n",
    "        print(f\"Training {model_name} with k={k}...\")\n",
    "        model = Model(num_topics=k)\n",
    "        output = model.train_model(dataset)\n",
    "        \n",
    "        # Calculate scores\n",
    "        topic_diversity_score = topic_diversity_metric.score(output)\n",
    "        coherence_score = coherence_metric.score(output)\n",
    "        cv_score = cv_metric.score(output)\n",
    "        end_time = time.time()  # End time for this iteration\n",
    "        elapsed_time = time.strftime(\"%H:%M:%S\", time.gmtime(end_time - start_time))\n",
    "        \n",
    "        \n",
    "        # Append results to DataFrame using concat\n",
    "        new_row = pd.DataFrame([{\n",
    "            \"Model\": model_name,\n",
    "            \"k\": k,\n",
    "            \"Topic Coherence_npmi\": coherence_score,\n",
    "            \"Topic Coherence_c_v\": cv_score,\n",
    "            \"Topic Diversity\": topic_diversity_score,\n",
    "            \"Time\": elapsed_time\n",
    "        }])\n",
    "        current_iteration += 1\n",
    "        results_df = pd.concat([results_df, new_row], ignore_index=True)\n",
    "\n",
    "results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ae79b08",
   "metadata": {},
   "source": [
    "We run HDP independently."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19a3bb57",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# For HDP, run without specifying k\n",
    "hdp_model = HDP()\n",
    "start_time = time.time()  # Start time for this iteration\n",
    "hdp_output = hdp_model.train_model(dataset)\n",
    "end_time = time.time()  # End time for this iteration\n",
    "elapsed_time = time.strftime(\"%H:%M:%S\", time.gmtime(end_time - start_time))\n",
    "\n",
    "hdp_topics = hdp_output['topics']\n",
    "hdp_k=len(hdp_topics)\n",
    "\n",
    "    \n",
    "hdp_coherence_score = coherence_metric.score(hdp_output)\n",
    "hdp_cv_score = cv_metric.score(hdp_output)\n",
    "hdp_topic_diversity_score = topic_diversity_metric.score(hdp_output)\n",
    "\n",
    "# Append HDP results to DataFrame\n",
    "hdp_row = pd.DataFrame([{\n",
    "    \"Model\": \"HDP\",\n",
    "    \"k\": hdp_k,  # HDP does not use a fixed k\n",
    "    \"Topic Coherence_npmi\": hdp_coherence_score,\n",
    "    \"Topic Coherence_c_v\": hdp_cv_score,\n",
    "    \"Topic Diversity\": hdp_topic_diversity_score,\n",
    "    \"Time\": elapsed_time\n",
    "}])\n",
    "\n",
    "results_df = pd.concat([results_df, hdp_row], ignore_index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a506f755",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Print or save the DataFrame\n",
    "results_df\n",
    "\n",
    "results_df.to_csv('./results.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:octis]",
   "language": "python",
   "name": "conda-env-octis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}