WorkSpace.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sf30I_qklK1h"
      },
      "source": [
        "# Basic"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/home/abpal/miniconda3/envs/bertopic_env/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "from bertopic import BERTopic\n",
        "from sentence_transformers import SentenceTransformer\n",
        "import joblib\n",
        "from typing import List, Dict\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Hello this is an example text with an email address Please remove punctuation and escape sequences including\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 18846/18846 [00:01<00:00, 12742.90it/s]\n"
          ]
        }
      ],
      "source": [
        "from sklearn.datasets import fetch_20newsgroups\n",
        "newsgroups_train = fetch_20newsgroups(subset='all')\n",
        "\n",
        "import re\n",
        "\n",
        "def clean_text(text):\n",
        "    # Remove escape sequences\n",
        "    text = re.sub(r'\\\\[a-zA-Z]', ' ', text)\n",
        "\n",
        "    # Remove email addresses\n",
        "    text = re.sub(r'\\S+@\\S+', ' ', text)\n",
        "\n",
        "    # Remove punctuation\n",
        "    text = re.sub(r'[^\\w\\s]', ' ', text)\n",
        "\n",
        "    # Remove extra whitespace\n",
        "    text = ' '.join(text.split())\n",
        "\n",
        "    return text\n",
        "\n",
        "# Example usage:\n",
        "text = \"Hello, this is an example text with an email address john@example.com. \\nPlease remove punctuation and escape sequences, including \\\\n.\"\n",
        "cleaned_text = clean_text(text)\n",
        "print(cleaned_text)\n",
        "\n",
        "\n",
        "from tqdm import tqdm\n",
        "\n",
        "docs = []\n",
        "for text in tqdm(newsgroups_train[\"data\"]):\n",
        "  docs.append(clean_text(text))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Coherence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "import gensim.corpora as corpora\n",
        "from gensim.models.coherencemodel import CoherenceModel\n",
        "from bertopic.representation import PartOfSpeech,MaximalMarginalRelevance,KeyBERTInspired\n",
        "\n",
        "def load_model(model:int) :\n",
        "    \"\"\"Load the corresponding model based on the model number\n",
        "\n",
        "    Args:\n",
        "        model (int): ranging from 1 - 4 \n",
        "    \"\"\"\n",
        "    if model == 1 :\n",
        "        return BERTopic()\n",
        "    elif model == 2 :\n",
        "        representation_model = KeyBERTInspired()\n",
        "        return BERTopic(representation_model=representation_model)\n",
        "    elif model == 3 :\n",
        "        representation_model = PartOfSpeech(\"en_core_web_sm\")\n",
        "        return BERTopic(representation_model=representation_model)\n",
        "    else : \n",
        "        representation_model = MaximalMarginalRelevance(diversity=0.3)\n",
        "        return BERTopic(representation_model=representation_model)\n",
        "    \n",
        "def find_coherence(model_num,topk):\n",
        "    filtered_text = docs\n",
        "    topic_model = load_model(model_num)   \n",
        "    topics, probabilities = topic_model.fit_transform(filtered_text)\n",
        "\n",
        "    documents = pd.DataFrame({\"Document\": filtered_text,\n",
        "                            \"ID\": range(len(filtered_text)),\n",
        "                            \"Topic\": topics})\n",
        "    documents_per_topic = documents.groupby(['Topic'], as_index=False).agg({'Document': ' '.join})\n",
        "    cleaned_docs = topic_model._preprocess_text(documents_per_topic.Document.values)\n",
        "\n",
        "    # Extract vectorizer and analyzer from BERTopic\n",
        "    vectorizer = topic_model.vectorizer_model\n",
        "    analyzer = vectorizer.build_analyzer()\n",
        "\n",
        "    # Extract features for Topic Coherence evaluation\n",
        "    words = vectorizer.get_feature_names_out()\n",
        "    tokens = [analyzer(doc) for doc in cleaned_docs]\n",
        "    dictionary = corpora.Dictionary(tokens)\n",
        "    corpus = [dictionary.doc2bow(token) for token in tokens]\n",
        "    topic_words = [[words for words, _ in topic_model.get_topic(topic)[:topk]] \n",
        "                for topic in range(len(set(topics))-1)]\n",
        "\n",
        "    # Evaluate\n",
        "    coherence_model = CoherenceModel(topics=topic_words, \n",
        "                                    texts=tokens, \n",
        "                                    corpus=corpus,\n",
        "                                    dictionary=dictionary, \n",
        "                                    coherence='c_v') # {'u_mass', 'c_v', 'c_uci', 'c_npmi'}\n",
        "    coherence = coherence_model.get_coherence()\n",
        "    return coherence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.8091945454235786"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "find_coherence(1,5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.7746149460417646"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "find_coherence(2,5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.800246994577821"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "find_coherence(3,5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.7830908607497608"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "find_coherence(4,5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.744357408230734"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "find_coherence(1,10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.6712307973519218"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "find_coherence(2,10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.6868834009743978"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "find_coherence(3,10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.6611768459510198"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "find_coherence(4,10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'1.3.2'"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import sklearn\n",
        "sklearn.__version__"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "-1.1535041838398743"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "coherence_model = CoherenceModel(topics=topic_words, \n",
        "                                 texts=tokens, \n",
        "                                 corpus=corpus,\n",
        "                                 dictionary=dictionary, \n",
        "                                 coherence='u_mass')\n",
        "coherence = coherence_model.get_coherence()\n",
        "coherence"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Comprehensiveness Test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Hello this is an example text with an email address Please remove punctuation and escape sequences including\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 0/11314 [00:00<?, ?it/s]"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 11314/11314 [00:00<00:00, 12015.80it/s]\n"
          ]
        }
      ],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 18846/18846 [00:01<00:00, 12332.34it/s]\n",
            "/home/abpal/miniconda3/envs/bertopic_env/lib/python3.11/site-packages/pynndescent/pynndescent_.py:346: NumbaPendingDeprecationWarning: \u001b[1m\u001b[1mCode using Numba extension API maybe depending on 'old_style' error-capturing, which is deprecated and will be replaced by 'new_style' in a future release. See details at https://numba.readthedocs.io/en/latest/reference/deprecation.html#deprecation-of-old-style-numba-captured-errors\n",
            "Exception origin:\n",
            "  File \"/home/abpal/miniconda3/envs/bertopic_env/lib/python3.11/site-packages/numba/core/types/functions.py\", line 486, in __getnewargs__\n",
            "    raise ReferenceError(\"underlying object has vanished\")\n",
            "\u001b[0m\u001b[0m\n",
            "  init_rp_tree(data, dist, current_graph, leaf_array)\n",
            "/home/abpal/miniconda3/envs/bertopic_env/lib/python3.11/site-packages/pynndescent/pynndescent_.py:348: NumbaPendingDeprecationWarning: \u001b[1m\u001b[1mCode using Numba extension API maybe depending on 'old_style' error-capturing, which is deprecated and will be replaced by 'new_style' in a future release. See details at https://numba.readthedocs.io/en/latest/reference/deprecation.html#deprecation-of-old-style-numba-captured-errors\n",
            "Exception origin:\n",
            "  File \"/home/abpal/miniconda3/envs/bertopic_env/lib/python3.11/site-packages/numba/core/types/functions.py\", line 486, in __getnewargs__\n",
            "    raise ReferenceError(\"underlying object has vanished\")\n",
            "\u001b[0m\u001b[0m\n",
            "  init_random(n_neighbors, data, current_graph, dist, rng_state)\n",
            "/home/abpal/miniconda3/envs/bertopic_env/lib/python3.11/site-packages/pynndescent/pynndescent_.py:358: NumbaPendingDeprecationWarning: \u001b[1m\u001b[1m\u001b[1mCode using Numba extension API maybe depending on 'old_style' error-capturing, which is deprecated and will be replaced by 'new_style' in a future release. See details at https://numba.readthedocs.io/en/latest/reference/deprecation.html#deprecation-of-old-style-numba-captured-errors\n",
            "Exception origin:\n",
            "  File \"/home/abpal/miniconda3/envs/bertopic_env/lib/python3.11/site-packages/numba/core/types/functions.py\", line 486, in __getnewargs__\n",
            "    raise ReferenceError(\"underlying object has vanished\")\n",
            "\u001b[0m\u001b[0m\u001b[0m\n",
            "  nn_descent_internal_low_memory_parallel(\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "      <th>Count</th>\n",
              "      <th>Name</th>\n",
              "      <th>Representation</th>\n",
              "      <th>Representative_Docs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1</td>\n",
              "      <td>6878</td>\n",
              "      <td>-1_to_the_for_of</td>\n",
              "      <td>[to, the, for, of, and, in, from, is, you, this]</td>\n",
              "      <td>[From Subject God morality and massacres Lines...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>222</td>\n",
              "      <td>0_scsi_drive_ide_drives</td>\n",
              "      <td>[scsi, drive, ide, drives, controller, disk, h...</td>\n",
              "      <td>[From Darryl Okahata Subject Re Help adding a ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>165</td>\n",
              "      <td>1_ncsl_nist_mail_internet</td>\n",
              "      <td>[ncsl, nist, mail, internet, enviroleague, jap...</td>\n",
              "      <td>[Subject Re What is going on From Scott Doty D...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>152</td>\n",
              "      <td>2_espn_game_abc_games</td>\n",
              "      <td>[espn, game, abc, games, coverage, hockey, bas...</td>\n",
              "      <td>[From KING COBRA Subject RE ESPN UP YOURS Nntp...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>148</td>\n",
              "      <td>3_clipper_encryption_chip_crypto</td>\n",
              "      <td>[clipper, encryption, chip, crypto, government...</td>\n",
              "      <td>[From Jim Gillogly Subject Clipper Chip and cr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>379</th>\n",
              "      <td>378</td>\n",
              "      <td>10</td>\n",
              "      <td>378_date_clock_bios_dos</td>\n",
              "      <td>[date, clock, bios, dos, menu, 448, midnight, ...</td>\n",
              "      <td>[From Barry Sardis Subject Re Date is stuck Or...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>380</th>\n",
              "      <td>379</td>\n",
              "      <td>10</td>\n",
              "      <td>379_alarm_alarms_viper_sensor</td>\n",
              "      <td>[alarm, alarms, viper, sensor, car, shock, gar...</td>\n",
              "      <td>[From ROBERT WILLIAM FUSI Subject Re New Alarm...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>381</th>\n",
              "      <td>380</td>\n",
              "      <td>10</td>\n",
              "      <td>380_reston_nasa_ssf_overhead</td>\n",
              "      <td>[reston, nasa, ssf, overhead, allen, funds, de...</td>\n",
              "      <td>[From David B Mckissock Subject NASA Wraps New...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>382</th>\n",
              "      <td>381</td>\n",
              "      <td>10</td>\n",
              "      <td>381_ether_twist_mcaloon_planets</td>\n",
              "      <td>[ether, twist, mcaloon, planets, orbit, moon, ...</td>\n",
              "      <td>[From Phil G Fraering Subject Re PLANETS STILL...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>383</th>\n",
              "      <td>382</td>\n",
              "      <td>10</td>\n",
              "      <td>382_bd_fbi_fire_atf</td>\n",
              "      <td>[bd, fbi, fire, atf, outcome, actions, survivo...</td>\n",
              "      <td>[From Jim De Arras Subject Re ATF BURNS DIVIDI...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>384 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     Topic  Count                              Name  \\\n",
              "0       -1   6878                  -1_to_the_for_of   \n",
              "1        0    222           0_scsi_drive_ide_drives   \n",
              "2        1    165         1_ncsl_nist_mail_internet   \n",
              "3        2    152             2_espn_game_abc_games   \n",
              "4        3    148  3_clipper_encryption_chip_crypto   \n",
              "..     ...    ...                               ...   \n",
              "379    378     10           378_date_clock_bios_dos   \n",
              "380    379     10     379_alarm_alarms_viper_sensor   \n",
              "381    380     10      380_reston_nasa_ssf_overhead   \n",
              "382    381     10   381_ether_twist_mcaloon_planets   \n",
              "383    382     10               382_bd_fbi_fire_atf   \n",
              "\n",
              "                                        Representation  \\\n",
              "0     [to, the, for, of, and, in, from, is, you, this]   \n",
              "1    [scsi, drive, ide, drives, controller, disk, h...   \n",
              "2    [ncsl, nist, mail, internet, enviroleague, jap...   \n",
              "3    [espn, game, abc, games, coverage, hockey, bas...   \n",
              "4    [clipper, encryption, chip, crypto, government...   \n",
              "..                                                 ...   \n",
              "379  [date, clock, bios, dos, menu, 448, midnight, ...   \n",
              "380  [alarm, alarms, viper, sensor, car, shock, gar...   \n",
              "381  [reston, nasa, ssf, overhead, allen, funds, de...   \n",
              "382  [ether, twist, mcaloon, planets, orbit, moon, ...   \n",
              "383  [bd, fbi, fire, atf, outcome, actions, survivo...   \n",
              "\n",
              "                                   Representative_Docs  \n",
              "0    [From Subject God morality and massacres Lines...  \n",
              "1    [From Darryl Okahata Subject Re Help adding a ...  \n",
              "2    [Subject Re What is going on From Scott Doty D...  \n",
              "3    [From KING COBRA Subject RE ESPN UP YOURS Nntp...  \n",
              "4    [From Jim Gillogly Subject Clipper Chip and cr...  \n",
              "..                                                 ...  \n",
              "379  [From Barry Sardis Subject Re Date is stuck Or...  \n",
              "380  [From ROBERT WILLIAM FUSI Subject Re New Alarm...  \n",
              "381  [From David B Mckissock Subject NASA Wraps New...  \n",
              "382  [From Phil G Fraering Subject Re PLANETS STILL...  \n",
              "383  [From Jim De Arras Subject Re ATF BURNS DIVIDI...  \n",
              "\n",
              "[384 rows x 5 columns]"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "docs = []\n",
        "count = 0\n",
        "for text in tqdm(newsgroups_train[\"data\"]):\n",
        "  count+=1\n",
        "  docs.append(clean_text(text) + f\" c_{count}\")\n",
        "\n",
        "topic_model = BERTopic()\n",
        "topics, probs = topic_model.fit_transform(docs)\n",
        "topic_model.get_topic_info()\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['From Subject God morality and massacres Lines 258 A listmember D Andrew Killie I think wrote in response to the suggestion that genocide may sometimes be the will of God Any God who works that way is indescribably evil and unworthy of my worship or faith Nobuya Higgy Higashiyama replied as in substance did others Where is your source of moral standards by which you judge God s behavior It is often argued that we have no standing by which to judge God s actions Who is the clay to talk back to the potter But we find a contrary view in Scripture When God proposes to destroy the city of Sodom Genesis 18 Abraham says Suppose that there are some good men in the city Will you destroy the righteous along with the wicked Far be it from you Lord to do such a thing Shall not the Judge of all the earth do right I am told that the Hebrew is actually a bit stronger than this and can perhaps be better rendered dynamic equivalence as Shame on you Lord if you do such a thing There are those who say that the definition of good is whatever God happens to want But if that is so then the statement that God is good has no meaning It simply says that God does what He wants That being the case no one can either love or obey God because He is good The only motive left for obeying Him is that He is powerful Just as it makes sense to obey a dictator even when he tells you to round up all Jews and exterminate them because if you defy him you might end up in the gas chamber yourself so it makes sense to obey God because He has the power to punish you if you don t This ethical theory I take to be in radical contradiction to Genesis 18 and to Christianity in general Any theory that makes our moral judgements worthless makes any further discussion of morality or of the goodness of God meaningless However it does not follow that our moral judgements are always infallible in particular cases still less our judgements in particular cases about the course of action most likely to achieve a good result When I read the Scriptural accounts of the actions of God in history those actions often seem to me very different from what I might expect of a God who loves us and desires what is best for us Moreover leaving the Scriptures aside and considering the natural world I find that Nature is often very different from what I might expect if it were the work of a benevolent deity Origen said Those who believe that the Author of Nature is also the Author of the Scriptures must expect to find in the Scriptures the same sort of difficulties that they find in Nature Now that some such difficulties should exist is not in itself an argument against the existence power wisdom and goodness of God On the contrary their ABSENCE would be such an argument Suppose that I am watching Bobby Fisher play chess and suppose that every time he makes a move I find myself nodding and saying Good move Just what I was expecting him to do Same move I would have made if I were playing That would be a sign that Fisher is no better a chess player than myself Given that he is better I expect that at least some of his moves will have me thinking Now what do suppose induced him to do that or even Boy that was a real slip he s just thrown the game away Similarly if God understands the workings of the universe better than I do it is to be expected that sometimes it will look to me as if He has made a mistake One difference between Fisher at the chessboard and God at the controls of the universe is that I can see the end of the chess game If Fisher wins I revise my earlier inference that it was carelessness that made him lose his queen 23 moves earlier However if he loses and particularly if I can see that there was a time when he had an opportunity for a checkmate in two moves and did not take it then I know that he is not as good a player as I had thought With God on the other hand I shall not in this life see the total result of some of His actions Therefore my grounds for judging that I have seen a bad move on His part must always be far shakier than my grounds for making a similar judgement about Fisher In the book of Genesis we read that Joseph s ten older brothers who with good reason found him insufferable conspired to sell him into slavery in Egypt There he eventually became Viceroy and when there was a famine in Canaan he was able to provide for his family When his brothers nervously apologized he told them Do not worry You meant to do me evil but God turned it into good I once heard a rabbi speak on this text He said The history of the Jews is largely a history of events that look like catastrophes that threaten the continued survival of the religion or the people or both But amazingly those events turn out to be the saving of the Jews and of Judaism The sale of Joseph by his brothers looked like the breakup of the family But in fact it ended with a reconciliation of the quarrel between them The famine that drove the family out of Canaan looked like a misfortune for them But in fact if they had stayed in Canaan they would almost certainly have intermarried with the Canaanites and been assimilated into their culture Their oppression by the Egyptians a few generations after their arrival in Egypt again looked like a disaster But God used it to bring them out of Egypt and into the Promised Land Here the people built a Temple and regularly offered sacrifices But the Babylonians captured Jerusalem and Judea destroyed Temple and city and countryside and deported most of the people to Babylon You might have thought that that would be the end of the people and the religion But it was not Living in Canaan the people had been under constant danger of assimilation Again and again they had turned from the worship of the LORD to the worship of the Canaanite fertility cults with their ritual prostitution and ritual human sacrifice The Babylonian captivity put a stop to that Never again did the Jews show any interest in polytheism or idolatry Neither the worship of the Canaanites mor that of the Babylonians ever again had a foothold among them Nor is that all Judaism had been in danger of becoming simply a system of sacrifices and Temple observances The only prescribed acts of worship consisted of coming to Jerusalem every so often and offering a sacrifice During the Captivity with the Temple gone the Jews invented the synagogue a place of meeting for reading and study and discussion of the Scriptures They came to realize clearly what they were in danger of forgetting while they continued to live in Judea that God is not simply a local or tribal deity not just the controller of the land of Canaan or the patron of the Jewish people but the Creator of the world and the Ruler and Judge of all humans everywhere Time passed and the Babylonian Empire was replaced by that of the Persians and then that of the Greeks or rather the Macedonians The ruler Antiochus Epiphanes was determined to stamp out Judaism and to this end he made the reading and the study of the Torah punishable by death Again one might think that this would be the end of Judaism But it was not The people met for worship and instead of reading the Torah portion appointed for the day they would read some passage from the prophets that had a similar theme and then discuss that Before this time the Torah the so called Five Books of Moses Genesis Exodus Leviticus Numbers and Deuteronomy were the only books read and studied in the synagogue If it had not been for Antiochus the books of the prophets would probably have been forgotten altogether His hatred for Judaism saved them Other examples here omitted for brevity s sake Some of you may remember that Julie Andrews first became famous as Eliza Doolittle in the stage production of MY FAIR LADY When Warner Brothers undertook to make a movie of it everyone expected that Julie who had been so magnificent on stage would play the same role in the movie Instead the studio decided to go with an established screen star and cast Audrey Hepburn Julie Andrews was naturally crushed But she later realized that if she had played the screen role she would have been type cast for life as an Elize Doolittle type It would have been a disaster for her As it was Walt Disney offered her the role of Mary Poppins and she won an Oscar for it At the presentation she stood there smiling and looking at Walt Disney she said And now my special thanks to the man who made all this possible JACK WARNER It was the most memorable line of the evening In a similar spirit we Jews might thank the men who in the providence of God have preserved Judaism and kept it alive to this day beginning with Joseph s brothers and continuing with two Pharaohs with Nebuchadnezzar with Antiochus Epiphanes After the formal meeting had broken up one woman came up to him privately and said You were talking about the Holocaust weren t you He answered If that is an example that came to your mind then you are right I was talking about it to you But I would not talk about it to everyone for not everyone can bear it I assume that he meant that without the Holocaust there would have been no state of Israel Someone hearing the rabbi s lecture might leap to the conclusion that God is dependent on the wickedness of men to accomplish His purposes or at least that the rabbi thought so He might then go on to suppose that the wickedness is in fact God s doing that He stirred up Joseph s brothers to a murderous hate against him and that when the Israelites were in Egypt God hardened Pharaoh s heart so that he oppressed the people and would not let them go And this raises questions about how an action can be considered wicked and at the same time be considered something that God has brought about I suggest another way of looking at it Consider a sculptor who has a log of wood from which he proposes to carve a statue But the log instead of having a smooth even grain throughout has a large knot that spoils the appearance of the surface The sculptor considers the wood for a while and then carves a statue that features the knot that makes that particular interruption in the grain and color of the wood correspond to some feature of the statue so that observers will say How fortunate the sculptor was in finding a piece of wood with a knot like that in just the right place Its presence is the crowning touch the thing that makes the statue a great work of art In reality the knot far from being what the sculptor was looking for was a challenge to his skill If the wood had not contained that flaw he would still have made a great work of art but a different one So if Joseph s brothers had not sold him God would still have brought about His purposes for the Jewish people but He would have done so in another manner If Judas had not betrayed Jesus if Caiaphas and his fellow leaders had not rejected Jesus but had rather acknowledged Him as the Annointed of God if Pilate had followed his conscience rather than his fears and had set Jesus free it might appear that there would have been no Crucifixion and therefore no Redemption and therefore no Salvation Not so God did not need Judas sin to redeem us If Judas had done right then God in Christ would still have reconciled the world to Himself We do not know how just as we do not know how Michelangelo would have painted the Sistine Chapel if its interior had instead been shaped like Grand Central Station and just as we do not know how Bobby Fisher would have won his fourth game agianst Spassky if Spassky had refused the exchange of bishops and had attacked Fisher s knight instead don t bother to look up the game in question I am making up this example but the point is none the less valid Thus we may say both 1 that God used say the cowardice of Pilate to accomplish His purposes and 2 that the said cowardice was not God s doing and that Pilate would not have thwarted God s plans by behaving justly and courageously What then are we to make of the place where God says to Moses I will harden Pharaoh s heart so that he will not let the people go Some Christians have taken this to mean that Pharaoh was a puppet with God pulling the strings and that his stubbornness and cruelty were not his own work but the work of God in him I suppose rather that what God was telling Moses was something like this If you see that Pharaoh is not willing to let the people go do not be discouraged or suppose that the situation is out of my control My purposes will not be thwarted If Pharaoh chooses to hear you and let the people go well and good If he does not I will fit his resistance into my plans and fit it so perfectly that future historians and theologians will suppose that I would have been thrown for a loss if Pharaoh had obeyed me To return to the question that started this all off Is it possible that the Serbs in slaughtering the Moslems of Bosnia are instruments of God s will First point What they are doing is wrong just as what Joseph s brothers did was wrong just as what Judas did was wrong They intend it for evil If God somehow brings good out of it that does not make them any less subject to just condemnation and punishment Second point Of course God will bring good out of it But not the same good that He would have brought if the Serbians had refrained from the sins of robbery and rape and murder Nor does the good He purposes excuse us from the duty of doing what is right Yours James Kiefer c_13328',\n",
              " 'From Tom Lane Subject JPEG image compression Frequently Asked Questions Summary Useful info about JPEG JPG image files and programs Keywords JPEG image compression FAQ Supersedes Nntp Posting Host g gp cs cmu edu Reply To Organization School of Computer Science Carnegie Mellon Expires Mon 31 May 1993 03 14 50 GMT Lines 1034 Archive name jpeg faq Last modified 2 May 1993 This FAQ article discusses JPEG image compression Suggestions for additions and clarifications are welcome New since version of 18 April 1993 New version of XV supports 24 bit viewing for X Windows New versions of DVPEG Image Alchemy for DOS New versions of Image Archiver PMView for OS 2 New listing MGIF for monochrome display Ataris This article includes the following sections 1 What is JPEG 2 Why use JPEG 3 When should I use JPEG and when should I stick with GIF 4 How well does JPEG compress images 5 What are good quality settings for JPEG 6 Where can I get JPEG software 6A canned software viewers etc 6B source code 7 What s all this hoopla about color quantization 8 How does JPEG work 9 What about lossless JPEG 10 Why all the argument about file formats 11 How do I recognize which file format I have and what do I do about it 12 What about arithmetic coding 13 Does loss accumulate with repeated compression decompression 14 What are some rules of thumb for converting GIF images to JPEG Sections 1 6 are basic info that every JPEG user needs to know sections 7 14 are advanced info for the curious This article is posted every 2 weeks You can always find the latest version in the news answers archive at rtfm mit edu 18 70 0 226 By FTP fetch pub usenet news answers jpeg faq or if you don t have FTP send e mail to with body send usenet news answers jpeg faq Many other FAQ articles are also stored in this archive For more instructions on use of the archive send e mail to the same address with the words help and index no quotes on separate lines If you don t get a reply the server may be misreading your return address add a line such as path to specify your correct e mail address to reply to 1 What is JPEG JPEG pronounced jay peg is a standardized image compression mechanism JPEG stands for Joint Photographic Experts Group the original name of the committee that wrote the standard JPEG is designed for compressing either full color or gray scale digital images of natural real world scenes It does not work so well on non realistic images such as cartoons or line drawings JPEG does not handle black and white 1 bit per pixel images nor does it handle motion picture compression Standards for compressing those types of images are being worked on by other committees named JBIG and MPEG respectively JPEG is lossy meaning that the image you get out of decompression isn t quite identical to what you originally put in The algorithm achieves much of its compression by exploiting known limitations of the human eye notably the fact that small color details aren t perceived as well as small details of light and dark Thus JPEG is intended for compressing images that will be looked at by humans If you plan to machine analyze your images the small errors introduced by JPEG may be a problem for you even if they are invisible to the eye A useful property of JPEG is that the degree of lossiness can be varied by adjusting compression parameters This means that the image maker can trade off file size against output image quality You can make extremely small files if you don t mind poor quality this is useful for indexing image archives making thumbnail views or icons etc etc Conversely if you aren t happy with the output quality at the default compression setting you can jack up the quality until you are satisfied and accept lesser compression 2 Why use JPEG There are two good reasons to make your image files smaller and to store 24 bit per pixel color data instead of 8 bit per pixel data Making image files smaller is a big win for transmitting files across networks and for archiving libraries of images Being able to compress a 2 Mbyte full color file down to 100 Kbytes or so makes a big difference in disk space and transmission time If you are comparing GIF and JPEG the size ratio is more like four to one More details below If your viewing software doesn t support JPEG directly you ll have to convert JPEG to some other format for viewing or manipulating images Even with a JPEG capable viewer it takes longer to decode and view a JPEG image than to view an image of a simpler format GIF for instance Thus using JPEG is essentially a time space tradeoff you give up some time in order to store or transmit an image more cheaply It s worth noting that when network or phone transmission is involved the time savings from transferring a shorter file can be much greater than the extra time to decompress the file I ll let you do the arithmetic yourself The other reason why JPEG will gradually replace GIF as a standard Usenet posting format is that JPEG can store full color information 24 bits pixel 16 million colors instead of 8 or less 256 or fewer colors If you have only 8 bit display hardware then this may not seem like much of an advantage to you Within a couple of years though 8 bit GIF will look as obsolete as black and white MacPaint format does today Furthermore for reasons detailed in section 7 JPEG is far more useful than GIF for exchanging images among people with widely varying color display hardware Hence JPEG is considerably more appropriate than GIF for use as a Usenet posting standard 3 When should I use JPEG and when should I stick with GIF JPEG is not going to displace GIF entirely for some types of images GIF is superior in image quality file size or both One of the first things to learn about JPEG is which kinds of images to apply it to As a rule of thumb JPEG is superior to GIF for storing full color or gray scale images of realistic scenes that means scanned photographs and similar material JPEG is superior even if you don t have 24 bit display hardware and it is a LOT superior if you do See section 7 for details GIF does significantly better on images with only a few distinct colors such as cartoons and line drawings In particular large areas of pixels that are all exactly the same color are compressed very efficiently indeed by GIF JPEG can t squeeze these files as much as GIF does without introducing visible defects This sort of image is best kept in GIF form In particular single color borders are quite cheap in GIF files but they should be avoided in JPEG files JPEG also has a hard time with very sharp edges a row of pure black pixels adjacent to a row of pure white pixels for example Sharp edges tend to come out blurred unless you use a very high quality setting Again this sort of thing is not found in scanned photographs but it shows up fairly often in GIF files borders overlaid text etc The blurriness is particularly objectionable with text that s only a few pixels high If you have a GIF with a lot of small size overlaid text don t JPEG it Computer drawn images ray traced scenes for instance usually fall between scanned images and cartoons in terms of complexity The more complex and subtly rendered the image the more likely that JPEG will do well on it The same goes for semi realistic artwork fantasy drawings and such Plain black and white two level images should never be converted to JPEG You need at least about 16 gray levels before JPEG is useful for gray scale images It should also be noted that GIF is lossless for gray scale images of up to 256 levels while JPEG is not If you have an existing library of GIF images you may wonder whether you should convert them to JPEG You will lose a little image quality if you do Section 7 which argues that JPEG image quality is superior to GIF only applies if both formats start from a full color original If you start from a GIF you ve already irretrievably lost a great deal of information JPEG can only make things worse However the disk space savings may justify converting anyway This is a decision you ll have to make for yourself If you do convert a GIF library to JPEG see section 14 for hints Be prepared to leave some images in GIF format since some GIFs will not convert well 4 How well does JPEG compress images Pretty darn well Here are some sample file sizes for an image I have handy a 727x525 full color image of a ship in a harbor The first three files are for comparison purposes the rest were created with the free JPEG software described in section 6B File Size in bytes Comments ship ppm 1145040 Original file in PPM format no compression 24 bits or 3 bytes per pixel plus a few bytes overhead ship ppm Z 963829 PPM file passed through Unix compress compress doesn t accomplish a lot you ll note Other text oriented compressors give similar results ship gif 240438 Converted to GIF with ppmquant fs 256 ppmtogif Most of the savings is the result of losing color info GIF saves 8 bits pixel not 24 See sec 7 ship jpg95 155622 cjpeg Q 95 highest useful quality setting This is indistinguishable from the 24 bit original at least to my nonprofessional eyeballs ship jpg75 58009 cjpeg Q 75 default setting You have to look mighty darn close to distinguish this from the original even with both on screen at once ship jpg50 38406 cjpeg Q 50 This has slight defects if you know what to look for you could tell it s been JPEGed without seeing the original Still as good image quality as many recent postings in Usenet pictures groups ship jpg25 25192 cjpeg Q 25 JPEG s characteristic blockiness becomes apparent at this setting djpeg blocksmooth helps some Still I ve seen plenty of Usenet postings that were of poorer image quality than this ship jpg5o 6587 cjpeg Q 5 optimize optimize cuts table overhead Blocky but perfectly satisfactory for preview or indexing purposes Note that this file is TINY the compression ratio from the original is 173 1 In this case JPEG can make a file that s a factor of four or five smaller than a GIF of comparable quality the Q 75 file is every bit as good as the GIF better if you have a full color display This seems to be a typical ratio for real world scenes 5 What are good quality settings for JPEG Most JPEG compressors let you pick a file size vs image quality tradeoff by selecting a quality setting There seems to be widespread confusion about the meaning of these settings Quality 95 does NOT mean keep 95 of the information as some have claimed The quality scale is purely arbitrary it s not a percentage of anything The name of the game in using JPEG is to pick the lowest quality setting smallest file size that decompresses into an image indistinguishable from the original This setting will vary from one image to another and from one observer to another but here are some rules of thumb The default quality setting Q 75 is very often the best choice This setting is about the lowest you can go without expecting to see defects in a typical image Try Q 75 first if you see defects then go up Except for experimental purposes never go above Q 95 saying Q 100 will produce a file two or three times as large as Q 95 but of hardly any better quality If the image was less than perfect quality to begin with you might be able to go down to Q 50 without objectionable degradation On the other hand you might need to go to a HIGHER quality setting to avoid further degradation The second case seems to apply much of the time when converting GIFs to JPEG The default Q 75 is about right for compressing 24 bit images but Q 85 to 95 is usually better for converting GIFs see section 14 for more info If you want a very small file say for preview or indexing purposes and are prepared to tolerate large defects a Q setting in the range of 5 to 10 is about right Q 2 or so may be amusing as op art Note the quality settings discussed in this article apply to the free JPEG software described in section 6B and to many programs based on it Other JPEG implementations such as Image Alchemy may use a completely different quality scale Some programs don t even provide a numeric scale just high medium low style choices 6 Where can I get JPEG software Most of the programs described in this section are available by FTP If you don t know how to use FTP see the FAQ article How to find sources If you don t have direct access to FTP read about ftpmail servers in the same article That article appears regularly in news answers or you can get it by sending e mail to with send usenet news answers finding sources in the body The Anonymous FTP List FAQ may also be helpful it s usenet news answers ftp list faq in the news answers archive NOTE this list changes constantly If you have a copy more than a couple months old get the latest JPEG FAQ from the news answers archive 6A If you are looking for canned software viewers etc The first part of this list is system specific programs that only run on one kind of system If you don t see what you want for your machine check out the portable JPEG software described at the end of the list Note that this list concentrates on free and shareware programs that you can obtain over Internet but some commercial programs are listed too X Windows XV shareware 25 is an excellent viewer for JPEG GIF and many other image formats It can also do format conversion and some simple image manipulations It s available for FTP from export lcs mit edu 18 24 0 12 file contrib xv 3 00 tar Z Version 3 00 is a major upgrade with support for 24 bit displays and many other improvements however it is brand new and still has some bugs lurking If you prefer not to be on the bleeding edge stick with version 2 21 also available from export Note that version 2 21 is not a good choice if you have a 24 bit display you ll get only 8 bit color nor for converting 24 bit images to JPEG But 2 21 works fine for converting GIF and other 8 bit images to JPEG CAUTION there is a glitch in version 2 21 be sure to check the save at normal size checkbox when saving a JPEG file or the file will be blurry Another good choice for X Windows is John Cristy s free ImageMagick package also available from export lcs mit edu file contrib ImageMagick tar Z This package handles many image processing and conversion tasks The ImageMagick viewer handles 24 bit displays correctly for colormapped displays it does better though slower color quantization than XV or the basic free JPEG software Both of the above are large complex packages If you just want a simple image viewer try xloadimage or xli xloadimage supports JPEG in its latest release 3 03 xloadimage is free and available from export lcs mit edu file contrib xloadimage 3 03 tar Z xli is a variant version of xloadimage said by its fans to be somewhat faster and more robust than the original The current xli is indeed faster and more robust than the current xloadimage at least with respect to JPEG files because it has the IJG v4 decoder while xloadimage 3 03 is using a hacked over v1 The next xloadimage release will fix this xli is also free and available from export lcs mit edu file contrib xli 1 14 tar Z Both programs are said to do the right thing with 24 bit displays MS DOS This covers plain DOS for Windows or OS 2 programs see the next headings One good choice is Eric Praetzel s free DVPEG which views JPEG and GIF files The current version 2 5 is available by FTP from sunee uwaterloo ca 129 97 50 50 file pub jpeg viewers dvpeg25 zip This is a good basic viewer that works on either 286 or 386 486 machines The user interface is not flashy but it s functional Another freeware JPEG GIF TGA viewer is Mohammad Rezaei s Hiview The current version 1 2 is available from Simtel20 and mirror sites see NOTE below file msdos graphics hv12 zip Hiview requires a 386 or better CPU and a VCPI compatible memory manager QEMM386 and 386MAX work Windows and OS 2 do not Hiview is currently the fastest viewer for images that are no bigger than your screen For larger images it scales the image down to fit on the screen rather than using panning scrolling as most viewers do You may or may not prefer this approach but there s no denying that it slows down loading of large images considerably Note installation is a bit tricky read the directions carefully A shareware alternative is ColorView for DOS 30 This is easier to install than either of the two freeware alternatives Its user interface is also much spiffier looking although personally I find it harder to use more keystrokes inconsistent behavior It is faster than DVPEG but a little slower than Hiview at least on my hardware For images larger than screen size DVPEG and ColorView seem to be about the same speed and both are faster than Hiview The current version is 2 1 available from Simtel20 and mirror sites see NOTE below file msdos graphics dcview21 zip Requires a VESA graphics driver if you don t have one look in vesadrv2 zip or vesa tsr zip from the same directory Many recent PCs have a built in VESA driver so don t try to load a VESA driver unless ColorView complains that the driver is missing A second shareware alternative is Fullview which has been kicking around the net for a while but I don t know any stable archive location for it The current rather old version is inferior to the above viewers anyway The author tells me that a new version of Fullview will be out shortly and it will be submitted to the Simtel20 archives at that time The well known GIF viewer CompuShow CSHOW supports JPEG in its latest revision 8 60a However CSHOW s JPEG implementation isn t very good it s slow about half the speed of the above viewers and image quality is poor except on hi color displays Too bad it d have been nice to see a good JPEG capability in CSHOW Shareware 25 Available from Simtel20 and mirror sites see NOTE below file msdos gif cshw860a zip Due to the remarkable variety of PC graphics hardware any one of these viewers might not work on your particular machine If you can t get any of them to work you ll need to use one of the following conversion programs to convert JPEG to GIF then view with your favorite GIF viewer If you have hi color hardware don t use GIF as the intermediate format try to find a TARGA capable viewer instead VPIC5 0 is reputed to do the right thing with hi color displays The Independent JPEG Group s free JPEG converters are FTPable from Simtel20 and mirror sites see NOTE below file msdos graphics jpeg4 zip or jpeg4386 zip if you have a 386 and extended memory These files are DOS compilations of the free source code described in section 6B they will convert JPEG to and from GIF Targa and PPM formats Handmade Software offers free JPEG GIF conversion tools GIF2JPG JPG2GIF These are slow and are limited to conversion to and from GIF format in particular you can t get 24 bit color output from a JPEG The major advantage of these tools is that they will read and write HSI s proprietary JPEG format as well as the Usenet standard JFIF format Since HSI format files are rather widespread on BBSes this is a useful capability Version 2 0 of these tools is free prior versions were shareware Get it from Simtel20 and mirror sites see NOTE below file msdos graphics gif2jpg2 zip NOTE do not use HSI format for files to be posted on Internet since it is not readable on non PC platforms Handmade Software also has a shareware image conversion and manipulation package Image Alchemy This will translate JPEG files both JFIF and HSI formats to and from many other image formats It can also display images A demo version of Image Alchemy version 1 6 2 is available from Simtel20 and mirror sites see NOTE below file msdos graphics alch162 zip NOTE ABOUT SIMTEL20 The Internet s key archive site for PC related programs is Simtel20 full name wsmr simtel20 army mil 192 88 110 20 Simtel20 runs a non Unix system with weird directory names where this document refers to directory eg msdos graphics at Simtel20 that really means pd1 msdos graphics If you are not physically on MILnet you should expect rather slow FTP transfer rates from Simtel20 There are several Internet sites that maintain copies mirrors of the Simtel20 archives most FTP users should go to one of the mirror sites instead A popular USA mirror site is oak oakland edu 141 210 10 117 which keeps Simtel20 files in eg pub msdos graphics If you have no FTP capability you can retrieve files from Simtel20 by e mail see informational postings in comp archives msdos announce to find out how If you are outside the USA consult the same newsgroup to learn where your nearest Simtel20 mirror is Microsoft Windows There are several Windows programs capable of displaying JPEG images Windows viewers are generally slower than DOS viewers on the same hardware due to Windows system overhead Note that you can run the DOS conversion programs described above inside a Windows DOS window The newest entry is WinECJ which is free and EXTREMELY fast Version 1 0 is available from ftp rahul net file pub bryanw pc jpeg wecj zip Requires Windows 3 1 and 256 or more colors mode This is a no frills viewer with the bad habit of hogging the machine completely while it decodes and the image quality is noticeably worse than other viewers But it s so fast you ll use it anyway at least for previewing JView is freeware fairly fast has good on line help and can write out the decompressed image in Windows BMP format but it can t create new JPEG files and it doesn t view GIFs JView also lacks some other useful features of the shareware viewers such as brightness adjustment but it s an excellent basic viewer The current version 0 9 is available from ftp cica indiana edu 129 79 20 84 file pub pc win3 desktop jview090 zip Mirrors of this archive can be found at some other Internet sites including wuarchive wustl edu WinJPEG shareware 20 displays JPEG GIF Targa TIFF and BMP image files it can write all of these formats too so it can be used as a converter It has some other nifty features including color balance adjustment and slideshow The current version is 2 1 available from Simtel20 and mirror sites see NOTE above file msdos windows3 winjp210 zip This is a slow 286 compatible version if you register you ll get the 386 only version which is roughly 25 faster ColorView is another shareware entry 30 This was an early and promising contender but it has not been updated in some time and at this point it has no real advantages over WinJPEG If you want to try it anyway the current version is 0 97 available from ftp cica indiana edu file pub pc win3 desktop cview097 zip I understand that a new version will be appearing once the authors are finished with ColorView for DOS DVPEG see DOS heading also works under Windows but only in full screen mode not in a window OS 2 The following files are available from hobbes nmsu edu 128 123 35 151 Note check pub uploads for more recent versions the hobbes moderator is not very fast about moving uploads into their permanent directories pub os2 2 x graphics jpegv4 zip 32 bit version of free IJG conversion programs version 4 pub os2 all graphics jpeg4 16 zip 16 bit version of same for OS 2 1 x pub os2 2 x graphics imgarc12 zip Image Archiver 1 02 image conversion viewing with PM graphical interface Strong on conversion functions viewing is a bit weaker Shareware 15 pub os2 2 x graphics pmjpeg11 zip PMJPEG 1 1 OS 2 2 x port of WinJPEG a popular viewer for Windows see description in Windows section Shareware 20 pub os2 2 x graphics pmview85 zip PMView 0 85 JPEG GIF BMP viewer GIF viewing very fast JPEG viewing fast if you have huge amounts of RAM otherwise about the same speed as the above programs Strong 24 bit display support Shareware 20 Macintosh Most Mac JPEG programs rely on Apple s JPEG implementation which is part of the QuickTime system extension so you need to have QuickTime installed To use QuickTime you need a 68020 or better CPU and you need to be running System 6 0 7 or later If you re running System 6 you must also install the 32 bit QuickDraw extension this is built in on System 7 You can get QuickTime by FTP from ftp apple com file dts mac quicktime quicktime hqx As of 11 92 this file contains QuickTime 1 5 which is better than QT 1 0 in several ways With respect to JPEG it is marginally faster and considerably less prone to crash when fed a corrupt JPEG file However some applications seem to have compatibility problems with QT 1 5 Mac users should keep in mind that QuickTime s JPEG format PICT JPEG is not the same as the Usenet standard JFIF JPEG format See section 10 for details If you post images on Usenet make sure they are in JFIF format Most of the programs mentioned below can generate either format The first choice is probably JPEGView a free program for viewing images that are in JFIF format PICT JPEG format or GIF format It also can convert between the two JPEG formats The current version 2 0 is a big improvement over prior versions Get it from sumex aim stanford edu 36 44 0 6 file info mac app jpeg view 20 hqx Requires System 7 and QuickTime On 8 bit displays JPEGView usually produces the best color image quality of all the currently available Mac JPEG viewers JPEGView can view large images in much less memory than other Mac viewers in fact it s the only one that can deal with JPEG images much over 640x480 pixels on a typical 4MB Mac Given a large image JPEGView automatically scales it down to fit on the screen rather than presenting scroll bars like most other viewers You can zoom in on any desired portion though Some people like this behavior some don t Overall JPEGView s user interface is very well thought out GIFConverter a shareware 40 image viewer converter supports JFIF and PICT JPEG as well as GIF and several other image formats The latest version is 2 3 2 Get it from sumex aim stanford edu file info mac art gif gif converter 232 hqx Requires System 6 0 5 or later GIFConverter is not better than JPEGView as a plain JPEG GIF viewer but it has much more extensive image manipulation and format conversion capabilities so you may find it worth its shareware fee if you do a lot of playing around with images Also the newest version of GIFConverter can load and save JFIF images without QuickTime so it is your best bet if your machine is too old to run QuickTime But it s faster with QuickTime Note If GIFConverter runs out of memory trying to load a large JPEG try converting the file to GIF with JPEG Convert then viewing the GIF version JPEG Convert a Mac version of the free IJG JPEG conversion utilities is available from sumex aim stanford edu file info mac app jpeg convert 10 hqx This will run on any Mac but it only does file conversion not viewing You can use it in conjunction with any GIF viewer Previous versions of this FAQ recommended Imagery JPEG v0 6 a JPEG GIF converter based on an old version of the IJG code If you are using this program you definitely should replace it with JPEG Convert Apple s free program PictPixie can view images in JFIF QuickTime JPEG and GIF format and can convert between these formats You can get PictPixie from ftp apple com file dts mac quicktime qt 1 0 stuff pictpixie hqx Requires QuickTime PictPixie was intended as a developer s tool and it s really not the best choice unless you like to fool around with QuickTime Some of its drawbacks are that it requires lots of memory it produces relatively poor color image quality on anything less than a 24 bit display and it has a relatively unfriendly user interface Worse PictPixie is an unsupported program meaning it has some minor bugs that Apple does not intend to fix There is an old version of PictPixie called PICTCompressor floating around the net If you have this you should trash it as it s even buggier Also the QuickTime Starter Kit includes a much cleaned up descendant of PictPixie called Picture Compressor Note that Picture Compressor is NOT free and may not be distributed on the net Storm Technology s Picture Decompress is a free JPEG viewer converter This rather old program is inferior to the above programs in many ways but it will run without System 7 or QuickTime so you may be forced to use it on older systems It does need 32 bit QuickDraw so really old machines can t use it You can get it from sumex aim stanford edu file info mac app picture decompress 201 hqx You must set the file type of a downloaded image file to JPEG to allow Picture Decompress to open it If your machine is too old to run 32 bit QuickDraw a Mac Plus for instance GIFConverter is your only choice for single program JPEG viewing If you don t want to pay for GIFConverter use JPEG Convert and a free GIF viewer More and more commercial Mac applications are supporting JPEG although not all can deal with the Usenet standard JFIF format Adobe Photoshop version 2 0 1 or later can read and write JFIF format JPEG files use the JPEG plug in from the Acquire menu You must set the file type of a downloaded JPEG file to JPEG to allow Photoshop to recognize it Amiga Most programs listed in this section are stored in the AmiNet archive at amiga physik unizh ch 130 60 80 80 There are many mirror sites of this archive and you should try to use the closest one In the USA a good choice is wuarchive wustl edu look under mirrors amiga physik unizh ch HamLab Plus is an excellent JPEG viewer converter as well as being a general image manipulation tool It s cheap shareware 20 and can read several formats besides JPEG The current version is 2 0 8 A demo version is available from amiga physik unizh ch and mirror sites file amiga gfx edit hamlab208d lha The demo version will crop images larger than 512x512 but it is otherwise fully functional Rend24 shareware 30 is an image renderer that can display JPEG ILBM and GIF images The program can be used to create animations even capturing frames on the fly from rendering packages like Lightwave The current version is 1 05 available from amiga physik unizh ch and mirror sites file amiga os30 gfx rend105 lha Note although this directory is supposedly for AmigaDOS 3 0 programs the program will also run under AmigaDOS 1 3 2 04 or 2 1 Viewtek is a free JPEG ILBM GIF ANIM viewer The current version is 1 04 available from amiga physik unizh ch and mirror sites file amiga gfx show ViewTek104 lha If you re willing to spend real money there are several commercial packages that support JPEG Two are written by Thomas Krehbiel the author of Rend24 and Viewtek These are CineMorph a standalone image morphing package and ImageFX an impressive 24 bit image capture conversion editing painting effects and prepress package that also includes CineMorph Both are distributed by Great Valley Products Art Department Professional ADPro from ASDG Inc is the most widely used commercial image manipulation software for Amigas ImageMaster from Black Belt Systems is another well regarded commercial graphics package with JPEG support The free IJG JPEG software is available compiled for Amigas from amiga physik unizh ch and mirror sites in directory amiga gfx conv file AmigaJPEGV4 lha These programs convert JPEG to from PPM GIF Targa formats The Amiga world is heavily infested with quick and dirty JPEG programs many based on an ancient beta test version of the free IJG JPEG software thanks to a certain magazine that published same on its disk of the month without so much as notifying the authors Among these are AugJPEG NewAmyJPEG VJPEG and probably others I have not even heard of In my opinion anything older than IJG version 3 March 1992 is not worth the disk space it s stored on if you have such a program trash it and get something newer Atari ST The free IJG JPEG software is available compiled for Atari ST TT etc from atari archive umich edu file atari Graphics jpeg4bin zoo These programs convert JPEG to from PPM GIF Targa formats For monochrome ST monitors try MGIF which manages to achieve four level grayscale effect by flickering Version 4 1 reads JPEG files Available from atari archive umich edu file atari Graphics mgif41b zoo I have not heard of any other free or shareware JPEG capable viewers for Ataris but surely there must be some by now Pointers appreciated Acorn Archimedes ChangeFSI supplied with RISC OS 3 version 3 10 can convert from and view JPEG JFIF format Provision is also made to convert images to JPEG although this must be done from the CLI rather than by double clicking Recent versions since 7 11 of the shareware program Translator can handle JPEG along with about 30 other image formats While older versions can be found on some Archimedes bboards the current version is only available by registering with the author John Kortink Nutterbrink 31 7544 WJ Enschede The Netherlands Price 35 Dutch guilders about 22 or 10 pounds There s also a commercial product called JPEG which provides JPEG read write functionality and direct JPEG viewing as well as a host of other image format conversion and processing options This is more expensive but not necessarily better than the above programs Contact DT Software FREEPOST Cambridge UK Tel 0223 841099 Portable software for almost any system If none of the above fits your situation you can obtain and compile the free JPEG conversion software described in 6B You ll also need a viewer program If your display is 8 bits or less any GIF viewer will do fine if you have a display with more color capability try to find a viewer that can read Targa or PPM 24 bit image files There are numerous commercial JPEG offerings with more popping up every day I recommend that you not spend money on one of these unless you find the available free or shareware software vastly too slow In that case purchase a hardware assisted product Ask pointed questions about whether the product complies with the final JPEG standard and about whether it can handle the JFIF file format many of the earliest commercial releases are not and never will be compatible with anyone else s files 6B If you are looking for source code to work with Free portable C code for JPEG compression is available from the Independent JPEG Group which I lead A package containing our source code documentation and some small test files is available from several places The official archive site for this source code is ftp uu net 137 39 1 9 or 192 48 96 9 Look under directory graphics jpeg the current release is jpegsrc v4 tar Z This is a compressed TAR file don t forget to retrieve in binary mode You can retrieve this file by FTP or UUCP If you are on a PC and don t know how to cope with tar Z format you may prefer ZIP format which you can find at Simtel20 and mirror sites see NOTE above file msdos graphics jpegsrc4 zip This file will also be available on CompuServe in the GRAPHSUPPORT forum GO PICS library 15 as jpsrc4 zip If you have no FTP access you can retrieve the source from your nearest comp sources misc archive version 4 appeared as issues 55 72 of volume 34 If you don t know how to retrieve comp sources misc postings see the FAQ article How to find sources referred to at the top of section 6 The free JPEG code provides conversion between JPEG JFIF format and image files in GIF PBMPLUS PPM PGM Utah RLE and Truevision Targa file formats The core compression and decompression modules can easily be reused in other programs such as image viewers The package is highly portable we have tested it on many machines ranging from PCs to Crays We have released this software for both noncommercial and commercial use Companies are welcome to use it as the basis for JPEG related products We do not ask a royalty although we do ask for an acknowledgement in product literature see the README file in the distribution for details We hope to make this software industrial quality although as with anything that s free we offer no warranty and accept no liability The Independent JPEG Group is a volunteer organization if you d like to contribute to improving our software you are welcome to join 7 What s all this hoopla about color quantization Most people don t have full color 24 bit per pixel display hardware Typical display hardware stores 8 or fewer bits per pixel so it can display 256 or fewer distinct colors at a time To display a full color image the computer must map the image into an appropriate set of representative colors This process is called color quantization This is something of a misnomer color selection would be a better term We re stuck with the standard usage though Clearly color quantization is a lossy process It turns out that for most images the details of the color quantization algorithm have MUCH more impact on the final image quality than do any errors introduced by JPEG except at the very lowest JPEG quality settings Since JPEG is a full color format converting a color JPEG image for display on 8 bit or less hardware requires color quantization This is true for all color JPEGs even if you feed a 256 or less color GIF into JPEG what comes out of the decompressor is not 256 colors but thousands of colors This happens because JPEG s lossiness affects each pixel a little differently so two pixels that started with identical colors will probably come out with slightly different colors Each original color gets smeared into a group of nearby colors Therefore quantization is always required to display a color JPEG on a colormapped display regardless of the image source The only way to avoid quantization is to ask for gray scale output Incidentally because of this effect it s nearly meaningless to talk about the number of colors used by a JPEG image Even if you attempted to count the number of distinct pixel values different JPEG decoders would give you different results because of roundoff error differences I occasionally see posted images described as 256 color JPEG This tells me that the poster a hasn t read this FAQ and b probably converted the JPEG from a GIF JPEGs can be classified as color or gray scale just like photographs but number of colors just isn t a useful concept for JPEG On the other hand a GIF image by definition has already been quantized to 256 or fewer colors A GIF does have a definite number of colors in its palette and the format doesn t allow more than 256 palette entries For purposes of Usenet picture distribution GIF has the advantage that the sender precomputes the color quantization so recipients don t have to This is also the disadvantage of GIF you re stuck with the sender s quantization If the sender quantized to a different number of colors than what you can display you have to re quantize resulting in much poorer image quality than if you had quantized once from a full color image Furthermore if the sender didn t use a high quality color quantization algorithm you re out of luck For this reason JPEG offers the promise of significantly better image quality for all users whose machines don t match the sender s display hardware JPEG s full color image can be quantized to precisely match the user s display hardware Furthermore you will be able to take advantage of future improvements in quantization algorithms there is a lot of active research in this area or purchase better display hardware to get a better view of JPEG images you already have With a GIF you re stuck forevermore with what was sent It s also worth mentioning that many GIF viewing programs include rather shoddy quantization routines If you view a 256 color GIF on a 16 color EGA display for example you are probably getting a much worse image than you need to This is partly an inevitable consequence of doing two color quantizations one to create the GIF one to display it but often it s also due to sloppiness JPEG conversion programs will be forced to use high quality quantizers in order to get acceptable results at all and in normal use they will quantize directly to the number of colors to be displayed Thus JPEG is likely to provide better results than the average GIF program for low color resolution displays as well as high resolution ones Finally an ever growing number of people have better than 8 bit display hardware already 15 bit hi color PC displays true 24 bit displays on workstations and Macintoshes etc For these people GIF is already obsolete as it cannot represent an image to the full capabilities of their display JPEG images can drive these displays much more effectively Thus JPEG is an all around better choice than GIF for representing images in a machine independent fashion 8 How does JPEG work The buzz words to know are chrominance subsampling discrete cosine transforms coefficient quantization and Huffman or arithmetic entropy coding This article s long enough already so I m not going to say more than that here For technical information see the comp compression FAQ This is available from the news answers archive at rtfm mit edu in files pub usenet news answers compression faq part 1 3 If you need help in using the news answers archive see the top of this article 9 What about lossless JPEG There s a great deal of confusion on this subject The JPEG committee did define a truly lossless compression algorithm i e one that guarantees the final output is bit for bit identical to the original input However this lossless mode has almost nothing in common with the regular lossy JPEG algorithm and it offers much less compression At present very few implementations of lossless JPEG exist and all of them are commercial Saying Q 100 to the free JPEG software DOES NOT get you a lossless image What it does get rid of is deliberate information loss in the coefficient quantization step There is still a good deal of information loss in the color subsampling step With the V4 free JPEG code you can also say sample 1x1 to turn off subsampling Keep in mind that many commercial JPEG implementations cannot cope with the resulting file Even with both quantization and subsampling turned off the regular JPEG algorithm is not lossless because it is subject to roundoff errors in various calculations The maximum error is a few counts in any one pixel value it s highly unlikely that this could be perceived by the human eye but it might be a concern if you are doing machine processing of an image At this minimum loss setting regular JPEG produces files that are perhaps half the size of an uncompressed 24 bit per pixel image True lossless JPEG provides roughly the same amount of compression but it guarantees bit for bit accuracy If you have an application requiring lossless storage of images with less than 6 bits per pixel per color component you may want to look into the JBIG bilevel image compression standard This performs better than JPEG lossless on such images JPEG lossless is superior to JBIG on images with 6 or more bits per pixel furthermore JPEG is public domain at least with a Huffman back end while the JBIG techniques are heavily covered by patents 10 Why all the argument about file formats Strictly speaking JPEG refers only to a family of compression algorithms it does not refer to a specific image file format The JPEG committee was prevented from defining a file format by turf wars within the international standards organizations Since we can t actually exchange images with anyone else unless we agree on a common file format this leaves us with a problem In the absence of official standards a number of JPEG program writers have just gone off to do their own thing and as a result their programs aren t compatible with anybody else s The closest thing we have to a de facto standard JPEG format is some work that s been coordinated by people at C Cube Microsystems They have defined two JPEG based file formats JFIF JPEG File Interchange Format a low end format that transports pixels and not much else TIFF JPEG aka TIFF 6 0 an extension of the Aldus TIFF format TIFF is a high end format that will let you record just about everything you ever wanted to know about an image and a lot more besides TIFF is a lot more complex than JFIF and may well prove less transportable because different vendors have historically implemented slightly different and incompatible subsets of TIFF It s not likely that adding JPEG to the mix will do anything to improve this situation Both of these formats were developed with input from all the major vendors of JPEG related products it s reasonably likely that future commercial products will adhere to one or both standards I believe that Usenet should adopt JFIF as the replacement for GIF in picture postings JFIF is simpler than TIFF and is available now the TIFF 6 0 spec has only recently been officially adopted and it is still unusably vague on some crucial details Even when TIFF JPEG is well defined the JFIF format is likely to be a widely supported lowest common denominator TIFF JPEG files may never be as transportable A particular case that people may be interested in is Apple s QuickTime software for the Macintosh QuickTime uses a JFIF compatible format wrapped inside the Mac specific PICT structure Conversion between JFIF and QuickTime JPEG is pretty straightforward and several Mac programs are available to do it see Mac portion of section 6A If you have an editor that handles binary files you can strip a QuickTime JPEG PICT down to JFIF by hand see section 11 for details Another particular case is Handmade Software s programs GIF2JPG JPG2GIF and Image Alchemy These programs are capable of reading and writing JFIF format By default though they write a proprietary format developed by HSI This format is NOT readable by any non HSI programs and should not be used for Usenet postings Use the j switch to get JFIF output This applies to old versions of these programs the current releases emit JFIF format by default You still should be careful not to post HSI format files unless you want to get flamed by people on non PC platforms 11 How do I recognize which file format I have and what do I do about it If you have an alleged JPEG file that your software won t read it s likely to be HSI format or some other proprietary JPEG based format You can tell what you have by inspecting the first few bytes of the file 1 A JFIF standard file will start with the characters hex FF D8 FF E0 followed by two variable bytes often hex 00 10 followed by JFIF 2 If you see FF D8 at the start but not the rest of it you may have a raw JPEG file This is probably decodable as is by JFIF software it s worth a try anyway 3 HSI files start with hsi1 You re out of luck unless you have HSI software Portions of the file may look like plain JPEG data but they won t decompress properly with non HSI programs 4 A Macintosh PICT file if JPEG compressed will have a couple hundred bytes of header followed by a JFIF header scan for JFIF Strip off everything before the FF D8 and you should be able to read it 5 Anything else it s a proprietary format or not JPEG at all If you are lucky the file may consist of a header and a raw JPEG data stream If you can identify the start of the JPEG data stream look for FF D8 try stripping off everything before that In uuencoded Usenet postings the characteristic JFIF pattern is begin line M_ C_X whereas uuencoded HSI files will start with begin line M I If you learn to check for the former you can save yourself the trouble of downloading non JFIF files 12 What about arithmetic coding The JPEG spec defines two different back end modules for the final output of compressed data either Huffman coding or arithmetic coding is allowed The choice has no impact on image quality but arithmetic coding usually produces a smaller compressed file On typical images arithmetic coding produces a file 5 or 10 percent smaller than Huffman coding All the file size numbers previously cited are for Huffman coding Unfortunately the particular variant of arithmetic coding specified by the JPEG standard is subject to patents owned by IBM AT T and Mitsubishi Thus you cannot legally use arithmetic coding unless you obtain licenses from these companies The fair use doctrine allows people to implement and test the algorithm but actually storing any images with it is dubious at best At least in the short run I recommend that people not worry about arithmetic coding the space savings isn t great enough to justify the potential legal hassles In particular arithmetic coding should not be used for any images to be exchanged on Usenet There is some small chance that the legal situation may change in the future Stay tuned for further details 13 Does loss accumulate with repeated compression decompression It would be nice if having compressed an image with JPEG you could decompress it manipulate it crop off a border say and recompress it without any further image degradation beyond what you lost initially Unfortunately THIS IS NOT THE CASE In general recompressing an altered image loses more information though usually not as much as was lost the first time around The next best thing would be that if you decompress an image and recompress it without changing it then there is no further loss i e you get an identical JPEG file Even this is not true at least not with the current free JPEG software It s essentially a problem of accumulation of roundoff error If you repeatedly compress and decompress the image will eventually degrade to where you can see visible changes from the first generation output It usually takes many such cycles to get visible change One of the things on our to do list is to see if accumulation of error can be avoided or limited but I am not optimistic about it In any case the most that could possibly be guaranteed would be that compressing the unmodified full color output of djpeg at the original quality setting would introduce no further loss Even such simple changes as cropping off a border could cause further roundoff error degradation If you re wondering why it s because the pixel block boundaries move If you cropped off only multiples of 16 pixels you might be safe but that s a mighty limited capability The bottom line is that JPEG is a useful format for archival storage and transmission of images but you don t want to use it as an intermediate format for sequences of image manipulation steps Use a lossless format PPM RLE TIFF etc while working on the image then JPEG it when you are ready to file it away Aside from avoiding degradation you will save a lot of compression decompression time this way 14 What are some rules of thumb for converting GIF images to JPEG As stated earlier you will lose some amount of image information if you convert an existing GIF image to JPEG If you can obtain the original full color data the GIF was made from it s far better to make a JPEG from that But if you need to save space and have only the GIF to work from here are some suggestions for getting maximum space savings with minimum loss of quality The first rule when converting a GIF library is to look at each JPEG to make sure you are happy with it before throwing away the corresponding GIF that will give you a chance to re do the conversion with a higher quality setting if necessary Some GIFs may be better left as GIFs as explained in section 3 in particular cartoon type GIFs with sixteen or fewer colors don t convert well You may find that a JPEG file of reasonable quality will be larger than the GIF So check the sizes too Experience to date suggests that large high visual quality GIFs are the best candidates for conversion to JPEG They chew up the most storage so offer the most potential savings and they convert to JPEG with least degradation Don t waste your time converting any GIF much under 100 Kbytes Also don t expect JPEG files converted from GIFs to be as small as those created directly from full color originals To maintain image quality you may have to let the converted files be as much as twice as big as straight through JPEG files would be i e shoot for 1 2 or 1 3rd the size of the GIF file not 1 4th as suggested in earlier comparisons Many people have developed an odd habit of putting a large constant color border around a GIF image While useless this was nearly free in terms of storage cost in GIF files It is NOT free in JPEG files and the sharp border boundary can create visible artifacts ghost edges Do yourself a favor and crop off any border before JPEGing If you are on an X Windows system XV s manual and automatic cropping functions are a very painless way to do this cjpeg s default Q setting of 75 is appropriate for full color input but for GIF inputs Q settings of 85 to 95 often seem to be necessary to avoid image degradation If you apply smoothing as suggested below the higher Q setting may not be necessary Color GIFs of photographs or complex artwork are usually dithered to fool your eye into seeing more than the 256 colors that GIF can actually store If you enlarge the image you will see that adjacent pixels are often of significantly different colors at normal size the eye averages these pixels together to produce the illusion of an intermediate color value The trouble with dithering is that to JPEG it looks like high spatial frequency color noise and JPEG can t compress noise very well The resulting JPEG file is both larger and of lower image quality than what you would have gotten from JPEGing the original full color image if you had it To get around this you want to smooth the GIF image before compression Smoothing averages together nearby pixels thus approximating the color that you thought you saw anyway and in the process getting rid of the rapid color changes that give JPEG trouble Appropriate use of smoothing will often let you avoid using a high Q factor thus further reducing the size of the compressed file while still obtaining a better looking output image than you d get without smoothing With the V4 free JPEG software or products based on it a simple smoothing capability is built in Try smooth 10 or so when converting GIFs Values of 10 to 25 seem to work well for high quality GIFs Heavy handed dithering may require larger smoothing factors If you can see regular fine scale patterns on the GIF image even without enlargement then strong smoothing is definitely called for Too large a smoothing factor will blur the output image which you don t want If you are an image processing wizard you can also do smoothing with a separate filtering program such as pnmconvol from the PBMPLUS package However cjpeg s built in smoother is a LOT faster than pnmconvol The upshot of all this is that cjpeg quality 85 smooth 10 is probably a good starting point for converting GIFs But if you really care about the image you ll want to check the results and maybe try a few other settings For more information about JPEG in general or the free JPEG software in particular contact the Independent JPEG Group at tom lane organizer Independent JPEG Group Internet BITNET c_1949',\n",
              " 'From David Veal Subject Re re fillibuster Lines 323 Organization University of Tennessee Computing Center In article Phill Hallam Baker writes In article David Veal writes In article Phill Hallam Baker writes In article David Veal writes Well yes and no The Federalist papers are propaganda and it is therefore difficult to determine precisely what Maddison etc were up to from them There are a couple of ways to look at them One is We want you to support this Constitution so we ll say anything that we think will appeal to you or the more straightforward This is why we think what we ve suggested in this Constitution is a good idea You clearly consider the former to be the primary situation The point is that they did not make pains to point out where the consitution may have been aginst the new yorker s interests Also they did not want to raise opposition by basing their advocacy on unpopular principles Horrors appealing to popular principles Can we perhaps as the question of whether the Constitution might have been written to appeal to the principles rather than as you appear to believe it was written with something else in mind and propoganda put out by its supporters But let s be honest about something here When was the last time you brought up all the valid points against your own arguments Or are they simply propogranda We can t know what Phill really means because he s obviously using arguments designed to convince Well I know Hamilton was a dyed in the wool monarchist and probably the authoritarian extreme to Jefferson s democratic impules But what would you suggest as a means of determining their opinions on the government if we don t consider what they wrote about the government I don t propose that any means exists for determining their true opinions Thus their true opinions died with them and are of little help today Their opinions have not the slightest bearing on the matter though only their arguments These are true or false regardless of who said them or why If they re true or false regardles of why they were said why on earth did you make a point of calling them propogranda That would seem to be irrelevent The difficulty that most US posters seem to have is in considering that their arguments may have been flawed or no longer apply to modern societies Oh I have no argument with questioning them I don t believe they no longer apply but that s because I think most of them were good arguments I m not entirely happy about the situation because they were obviously only applied to a minority of the time but I don t think that alone is sufficient to invalidate them If they were alive today the one opinion we could count upon these men to express is that a careful study of the mechanisms of government is necessary and that an ongoing improvement of the same is required They gave their opinions in certain areas and have been proved right In other areas they got it wrong They ensured that there was a mechanism to adapt and improve the consititution this can only happen if there is a willingness to accept that the structural problems within the US political system may require constitutional change as a solution Since the U S constitution is the basis for the U S political system most changes in it would require Constitutional change In this particular case however the fillibuster is a matter of procedure and tradition It only should have been made part of the Constitution If the Senate was less powerful than the House of Lords than we d almost have to state that the House of Representatives was also In fact they both were because the British government had much greater power than did the American system In principle no in practice yes In principle no That they had less power of that they should have had less power The British government today is theoreticaly dependent on the will of the Monarch By convention any monarch seeking to exercise that power is deposed The subtly is that the Prime Minister is not able to identify their politics with the national interest in the same manner that US Presidents regularly do Phill we re discusing the power of legislative houses While the Prime Minister is member of Parliament he is more analgous although badly to the U S President Now please explain to me how the U S House of Representatives is in principle more powerful than the House of Lords or the Senate but in practice is less Are you suggesting that the writers of the Constitution really intended for them to be more powerful but gosh darn the thing was ratified before they realized they d forgotten to put those extra restrictions on the Senate in I disagree The system is not too slow it was simply designed to handle less than it has demanded that it handle As somebody in Washington put it whose name I forget Congress has become everybody s city council One reason for that is that at every level the government is rendered unable to come to decisions These decisions are pushed up to the next higher level instead Not at all As any entry level political science course will tell you people who want laws implemented will always choose the level of government to attack which presents them with the best chance of getting what they want With national interest groups it is simply a very rational thing to do to want the Federal government to enact a law rather than the states Less people to persuade and less contributions to make Why do those concerned about abortion primarily concentrate at the Federal level Simply because if they win that battle all the little state battlefields are won by extension The same extends to insurance medicine and most other questions Local government has not failed in that it hasn t done what it should but that it is dominated by local interests Thus non local interests who want localities to abide by their rules can t get their rules past the local government Thus since they ve got more clout only in the wrong place they appeal to the next higher level because it can impose its will on the lower I mean let s get real here Do we really need the Congres of the United States deciding that x traffice light should be on thus and such pattern Or that carjacking needs to be a federal as opposed to a local crime The more people want the more Congress will take power to sell it to them for their votes I don t think the rise of special interests is coincidence with the increased power of Congress Congress is more than capable of quick action and has more than enough power and time on its hands if it confined itself to what its original jurisidiction was and allowed more local autonomy If they were to start from a social welfare model instead of the current no state subsidy motto they would be better placed As it is there is plenty of state money being handed out The problem is that it is distributed on the basis of power in congress and not on the basis of actual need Bingo The higher up the governmental ladder the less actual need matters because political power can be concentrated at higher levels while people with less cloud only find themselves reduced to in effectiveness In order to set up a school project in New York state you have to pay off the other 49 states with pork defense contracts agricultural subsidies etc Or to be precise 30 of the states since you need 60 to beat the filibuster Then why not simply leave New York s education to New York I remain unconcinved that there is any state in the Union which is not capable of educating its own children if that s what they want to do And if you leave it to them you only have to worry about the pork in that state And since industries can t concentrate their political power and wealth rather they must divide it among the states to try and get what they want individual voices have more relative impact The problem with the fillibuster is not that you must buy off states but that the Congress has acquired too much power to sell pork It is not a case of the system of government they created failing but that it is operating under a set of conditions they specifically wanted to avoid Namely a concentration of power It would seem then that the proper thing to do is not to reduce the power of either House in some attempt to grease the wheels All you ll get then is a system which moves quicker to do stupid things It would make more sense to make more decisions at a local level No you have to break the machine free of seizure before you can redirect it But why on earth should we want to redirect it You said yourself that you have to sell pork to get things through Congress If Congres has less authority to sell pork and retains its authority to enact national legislation within its granted jurisdiction the pork problem is significantly reduced The current blocks on power simply absolve congress of any responsibility to come to a decision The current blocks essentially state that inaction is preferable to action thus it the system is weighted against action Considering the government the usually the institution with the sole power to enforce its decisions by force I consider bias against making those decisions a good thing Pushing the decisions lower in the pyramid won t work unless the lower levels are less corrupt In most cases they are worse not better The difference with the lower pyramid is that a they have more legal legitimate authority in most matters under our Constitution than the federal government and b at those lower levels power is harder to concentrate And c you get the benefit of not imposing new deicisons on everybody at once You get to see them tried out without a national decision Congressional action usually treats the entire country as a whole yet even with similar problems in different areas different solutions may be called for And while I often don t agree with the decisions my local and state reps make at least I have a better option of going to the city council and shooting my mouth off I d much rather the majority of laws be made by accessible people who hang around and end up having to put up with them rather than somebody far off in Washington with half a million or more constituents I m curious what you base your assumption that lower levels are more corrupt I fail to see where any restrictions implied or otherwise were placed on the veto It could just as easily have been read as a means to put a check on democratically popular but unwise in the executive s opinion policies Since we were arguing from the Federalist papers I would point to them Phill you brought up the Federalist papers We were arguing the fillibuster and whether or not a minority of Senators should be allowed to hold up a bill You claimed the Senate was suppose to be a far less powerful House and I contended there was nothing in the Constitution or other writings which indicated this Which was when you brought up that we can t decide what the founders wanted based on the Federalist papers You argued against them I never argued from them I have primarily referred to the Constitution which places only very small restrictions on the Senate than for the House The US constitution gives almost no reasoning as to how it should work The only part where a reason is given is the right to bear arms ammendment where the well regulated militia justification is ambiguous The U S Constitution is a nuts and bolts document The Delcaration of Independence was the high brow reasoning There are a couple of other examples though such as the reasoning for the power to tax and the reasoning for the power to grant permits both in Article I Section 8 That the veto was meant to be an exceptional measure follows from the fact of the senate If the President was meant to revise legislation then there would be three chambers of the legislature not two Furthermore the separation of powers would have been much less distinct To a certain extend I do believe the veto has become something it wasn t intended However I also believe it is inevitable considering the Congress own abuse of their power to make bills say whatever they want them to say Unlike most people I think we shouldn t be worrying about the veto which is fine but of the problem in Congress which almost necessitates its abuse There is no limit in the Constitution to the President s veto power regarding what a bill is for Previous Presidents have used the veto for any number of reasons most usually having something to do with their agenda I am really curious how you single Bush out as the President who abused vetos He has the record for vetos BUSH Phill that s absurd Bush had 37 vetos one of which was over ridden Go read up on FDR if you think that s anything resembling a record Why is it not a reasonable restriction Because 51 Senators is the magic holy number upon which Laws must be based If 41 Senators feel safe enough with their state constituencies to stand up and fillibuster isn t that enough to indicate there s a sufficient question as to whether a law is a good idea or not to re evaluate it Up to a point the fact is though that when the majority are opposed by a minority the minority should not be allowed to win by default Why not What is inherently wrong with biasing the system against action Historically governemnt action in the U S when dealing with issues with a bare minority and a large minority have not been successful When you re in a position of imposing federal power on diverse people why should the federal government not have to got through something more than a bare majority Why one earth should 51 be sufficient to enact a law which covers 250 million people in very very diverse places and living in radically different conditions Why shouldn t a super majority be required Because the bill at issue is a money bill relating to a short term proposal Now we re switching from a general question of a fillibuster to a specific bill I don t see how it make a difference It is not a change in the law where a presupposition in favour of the status quo is arguable Sure it s arguable Theyr e arguing it However requiring 60 to bring it to a vote ensures that they ll have to have a good argument Something that isn t based solely on party lines Any system in which the simple majority is given absolute power to ignore the minority then the minority will be ignored I do not see this as a positive thing And for all that I m sure the Republicans are looking for pork as much as the Democrats they ve got some legitimate objections to the legislation in question So instead you consider a system under which the minority automatically win to be superior No I am completely happy with a system which requires a minority for action Since U S history is a history of carving up population groups and implementing piece meal on minorities I feel minorities should have sufficent clout to prevent action they feel strongly enough about And 41 is hardly a tiny minority I don t advocate the minority being capable of initiating actionm but I see no problem with biasing the federal system against action David Veal Univ of Tenn Div of Cont Education Info Services Group I still remember the way you laughed the day your pushed me down the elevator shaft I m beginning to think you don t love me anymore Weird Al c_4610']"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic_info()[\"Representative_Docs\"][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "topic_model = BERTopic()\n",
        "topics, probs = topic_model.fit_transform(docs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "      <th>Count</th>\n",
              "      <th>Name</th>\n",
              "      <th>Representation</th>\n",
              "      <th>Representative_Docs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1</td>\n",
              "      <td>3680</td>\n",
              "      <td>-1_the_to_for_and</td>\n",
              "      <td>[the, to, for, and, of, in, is, from, it, you]</td>\n",
              "      <td>[From vera shanti noyes Subject Re Serbian gen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>512</td>\n",
              "      <td>0_he_year_baseball_team</td>\n",
              "      <td>[he, year, baseball, team, game, runs, braves,...</td>\n",
              "      <td>[From David M Tate Subject Re Young Catchers A...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>362</td>\n",
              "      <td>1_gun_guns_firearms_weapons</td>\n",
              "      <td>[gun, guns, firearms, weapons, militia, contro...</td>\n",
              "      <td>[From David Veal Subject Re My Gun is like my ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>286</td>\n",
              "      <td>2_clipper_encryption_chip_key</td>\n",
              "      <td>[clipper, encryption, chip, key, escrow, gover...</td>\n",
              "      <td>[From Jim Gillogly Subject Clipper Chip and cr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>246</td>\n",
              "      <td>3_mail_list_address_mailing</td>\n",
              "      <td>[mail, list, address, mailing, tobacco, health...</td>\n",
              "      <td>[From richard welty Subject rec autos Automoti...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>212</td>\n",
              "      <td>11</td>\n",
              "      <td>212_octopus_detroit_ice_zazula</td>\n",
              "      <td>[octopus, detroit, ice, zazula, tradition, oct...</td>\n",
              "      <td>[From David R Jaracz Subject Re Octopus in Det...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>214</th>\n",
              "      <td>213</td>\n",
              "      <td>10</td>\n",
              "      <td>213_god_malcolm_jews_law</td>\n",
              "      <td>[god, malcolm, jews, law, chosen, jesus, israe...</td>\n",
              "      <td>[From Malcolm Lee Subject Re A KIND and LOVING...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>215</th>\n",
              "      <td>214</td>\n",
              "      <td>10</td>\n",
              "      <td>214_130_road_autobahns_texas</td>\n",
              "      <td>[130, road, autobahns, texas, driving, autobah...</td>\n",
              "      <td>[From Craig Boyle Subject Re Too fast Article ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>216</th>\n",
              "      <td>215</td>\n",
              "      <td>10</td>\n",
              "      <td>215_ashok_trumpet_wintrumpet_wim</td>\n",
              "      <td>[ashok, trumpet, wintrumpet, wim, aiyar, ndis,...</td>\n",
              "      <td>[From Subject Re Trumpet for Windows other new...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>217</th>\n",
              "      <td>216</td>\n",
              "      <td>10</td>\n",
              "      <td>216_her_denning_respectability_she</td>\n",
              "      <td>[her, denning, respectability, she, dorothy, t...</td>\n",
              "      <td>[From Thomas Grant Edwards Subject Re Once tap...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>218 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     Topic  Count                                Name  \\\n",
              "0       -1   3680                   -1_the_to_for_and   \n",
              "1        0    512             0_he_year_baseball_team   \n",
              "2        1    362         1_gun_guns_firearms_weapons   \n",
              "3        2    286       2_clipper_encryption_chip_key   \n",
              "4        3    246         3_mail_list_address_mailing   \n",
              "..     ...    ...                                 ...   \n",
              "213    212     11      212_octopus_detroit_ice_zazula   \n",
              "214    213     10            213_god_malcolm_jews_law   \n",
              "215    214     10        214_130_road_autobahns_texas   \n",
              "216    215     10    215_ashok_trumpet_wintrumpet_wim   \n",
              "217    216     10  216_her_denning_respectability_she   \n",
              "\n",
              "                                        Representation  \\\n",
              "0       [the, to, for, and, of, in, is, from, it, you]   \n",
              "1    [he, year, baseball, team, game, runs, braves,...   \n",
              "2    [gun, guns, firearms, weapons, militia, contro...   \n",
              "3    [clipper, encryption, chip, key, escrow, gover...   \n",
              "4    [mail, list, address, mailing, tobacco, health...   \n",
              "..                                                 ...   \n",
              "213  [octopus, detroit, ice, zazula, tradition, oct...   \n",
              "214  [god, malcolm, jews, law, chosen, jesus, israe...   \n",
              "215  [130, road, autobahns, texas, driving, autobah...   \n",
              "216  [ashok, trumpet, wintrumpet, wim, aiyar, ndis,...   \n",
              "217  [her, denning, respectability, she, dorothy, t...   \n",
              "\n",
              "                                   Representative_Docs  \n",
              "0    [From vera shanti noyes Subject Re Serbian gen...  \n",
              "1    [From David M Tate Subject Re Young Catchers A...  \n",
              "2    [From David Veal Subject Re My Gun is like my ...  \n",
              "3    [From Jim Gillogly Subject Clipper Chip and cr...  \n",
              "4    [From richard welty Subject rec autos Automoti...  \n",
              "..                                                 ...  \n",
              "213  [From David R Jaracz Subject Re Octopus in Det...  \n",
              "214  [From Malcolm Lee Subject Re A KIND and LOVING...  \n",
              "215  [From Craig Boyle Subject Re Too fast Article ...  \n",
              "216  [From Subject Re Trumpet for Windows other new...  \n",
              "217  [From Thomas Grant Edwards Subject Re Once tap...  \n",
              "\n",
              "[218 rows x 5 columns]"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.get_topic_info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{-1: [('the', 0.004435851293820019),\n",
              "  ('to', 0.004306697260717246),\n",
              "  ('for', 0.004170222032990103),\n",
              "  ('and', 0.004139216193339582),\n",
              "  ('of', 0.004127582952899307),\n",
              "  ('in', 0.004059938265242132),\n",
              "  ('is', 0.0040128897426359895),\n",
              "  ('from', 0.003995195630813896),\n",
              "  ('it', 0.0039713982882120785),\n",
              "  ('you', 0.003848332457738762)],\n",
              " 0: [('he', 0.009524914670512264),\n",
              "  ('year', 0.008733899214707205),\n",
              "  ('baseball', 0.008268781970731446),\n",
              "  ('team', 0.007391859995143984),\n",
              "  ('game', 0.007211154782534197),\n",
              "  ('runs', 0.007022716864435164),\n",
              "  ('braves', 0.006419315708481003),\n",
              "  ('pitching', 0.006350243836276345),\n",
              "  ('games', 0.006039906190242213),\n",
              "  ('his', 0.005948185092608325)],\n",
              " 1: [('gun', 0.017723091522972037),\n",
              "  ('guns', 0.010075815431080897),\n",
              "  ('firearms', 0.007756770338103917),\n",
              "  ('weapons', 0.006836784802652345),\n",
              "  ('militia', 0.006378108439892725),\n",
              "  ('control', 0.005611490266868494),\n",
              "  ('amendment', 0.005528962246213336),\n",
              "  ('of', 0.005339235329466024),\n",
              "  ('crime', 0.0050613445780270236),\n",
              "  ('you', 0.005014919180534072)],\n",
              " 2: [('clipper', 0.019388378815325936),\n",
              "  ('encryption', 0.017872445939134347),\n",
              "  ('chip', 0.016116619252304624),\n",
              "  ('key', 0.01596990792429967),\n",
              "  ('escrow', 0.01147453198893397),\n",
              "  ('government', 0.009435504707379971),\n",
              "  ('keys', 0.00914269700073626),\n",
              "  ('enforcement', 0.008116154305479026),\n",
              "  ('secure', 0.007675651761563494),\n",
              "  ('algorithm', 0.007454391540977454)],\n",
              " 3: [('mail', 0.010211086941537629),\n",
              "  ('list', 0.008972561869211354),\n",
              "  ('address', 0.007998159062822821),\n",
              "  ('mailing', 0.007868069896011257),\n",
              "  ('tobacco', 0.007601266386725081),\n",
              "  ('health', 0.007002872335127216),\n",
              "  ('nist', 0.006395942630964236),\n",
              "  ('smokeless', 0.0059997135890823405),\n",
              "  ('send', 0.005912699732026394),\n",
              "  ('internet', 0.005846598334245431)],\n",
              " 4: [('israel', 0.022420128648215572),\n",
              "  ('israeli', 0.021493410300821487),\n",
              "  ('jews', 0.01310896923212174),\n",
              "  ('arab', 0.011010377972844074),\n",
              "  ('arabs', 0.009772506628150721),\n",
              "  ('gaza', 0.009339458596541271),\n",
              "  ('palestinian', 0.007895589935135354),\n",
              "  ('israelis', 0.007035674199625564),\n",
              "  ('palestinians', 0.006591279895681179),\n",
              "  ('palestine', 0.0065760654301236605)],\n",
              " 5: [('car', 0.02222612063096313),\n",
              "  ('cars', 0.01419074160027127),\n",
              "  ('engine', 0.00791152844960128),\n",
              "  ('toyota', 0.007509597367099748),\n",
              "  ('wagon', 0.007053330707160193),\n",
              "  ('models', 0.007042044008915126),\n",
              "  ('integra', 0.006955281014334726),\n",
              "  ('convertible', 0.006809777121977576),\n",
              "  ('ford', 0.00637284138393873),\n",
              "  ('nissan', 0.006010181344188042)],\n",
              " 6: [('bike', 0.02462621271255774),\n",
              "  ('dod', 0.02312344667290229),\n",
              "  ('bikes', 0.012173446634220052),\n",
              "  ('motorcycle', 0.011748475588590252),\n",
              "  ('motorcycles', 0.009244769374690365),\n",
              "  ('ride', 0.009043150213629456),\n",
              "  ('honda', 0.00800445806085702),\n",
              "  ('riding', 0.006779175170340776),\n",
              "  ('miles', 0.006139195680544893),\n",
              "  ('denizens', 0.006084321273739118)],\n",
              " 7: [('amp', 0.02351669830175659),\n",
              "  ('sale', 0.01500086534568814),\n",
              "  ('stereo', 0.013360787036649396),\n",
              "  ('adcom', 0.013221564572247557),\n",
              "  ('offer', 0.013028225769785086),\n",
              "  ('shipping', 0.011660023465477686),\n",
              "  ('amplifier', 0.01146504469026831),\n",
              "  ('condition', 0.010766425367558912),\n",
              "  ('sony', 0.010575078036113014),\n",
              "  ('cd', 0.010076068798727215)],\n",
              " 8: [('moon', 0.028634192461607894),\n",
              "  ('lunar', 0.018238612459366206),\n",
              "  ('billion', 0.013525263172114263),\n",
              "  ('mining', 0.012386533120510176),\n",
              "  ('prize', 0.012205986384393578),\n",
              "  ('space', 0.011765731157471268),\n",
              "  ('cost', 0.008827956176632377),\n",
              "  ('ssto', 0.007550966768469531),\n",
              "  ('henry', 0.0074557852478963765),\n",
              "  ('race', 0.007251883094390558)],\n",
              " 9: [('polygon', 0.032623190816556),\n",
              "  ('points', 0.022295227275939012),\n",
              "  ('sphere', 0.020061846835310426),\n",
              "  ('p3', 0.019107113566713196),\n",
              "  ('p2', 0.01885012728834905),\n",
              "  ('den', 0.018725018296616665),\n",
              "  ('p1', 0.017559244194514788),\n",
              "  ('radius', 0.016045777170833386),\n",
              "  ('algorithm', 0.014452585899186248),\n",
              "  ('routine', 0.012528957872752126)],\n",
              " 10: [('db', 0.05752459627076633),\n",
              "  ('0d', 0.03888479824182422),\n",
              "  ('cx', 0.031205580204365806),\n",
              "  ('mov', 0.021124144640461303),\n",
              "  ('bh', 0.019612319787812423),\n",
              "  ('145', 0.017832625083484484),\n",
              "  ('s6', 0.015321882937870304),\n",
              "  ('si', 0.013324003484337389),\n",
              "  ('6um', 0.012797864970918919),\n",
              "  ('sc', 0.011790166855538232)],\n",
              " 11: [('jews', 0.015653388482558876),\n",
              "  ('nazi', 0.014888217134034338),\n",
              "  ('holocaust', 0.012420103122248014),\n",
              "  ('zionism', 0.010125451436877044),\n",
              "  ('hitler', 0.009199814268107093),\n",
              "  ('nazis', 0.008717170459722447),\n",
              "  ('anti', 0.00847715042646757),\n",
              "  ('german', 0.008041970631559339),\n",
              "  ('germany', 0.007531929370011663),\n",
              "  ('jewish', 0.0074501491790341665)],\n",
              " 12: [('image', 0.021520089013744952),\n",
              "  ('3d', 0.01970191733156244),\n",
              "  ('graphics', 0.018342828425115757),\n",
              "  ('package', 0.012682998294029996),\n",
              "  ('library', 0.011593171240172189),\n",
              "  ('plplot', 0.010666298971977916),\n",
              "  ('processing', 0.009263824782925156),\n",
              "  ('software', 0.007664126048839988),\n",
              "  ('user', 0.00746428397766298),\n",
              "  ('analysis', 0.007303998629725716)],\n",
              " 13: [('armenian', 0.02584689015440968),\n",
              "  ('turkish', 0.022023132176340634),\n",
              "  ('armenians', 0.019767806738875456),\n",
              "  ('argic', 0.014808801500075726),\n",
              "  ('serdar', 0.014685831229295244),\n",
              "  ('genocide', 0.014041488054879932),\n",
              "  ('turks', 0.012866128356158482),\n",
              "  ('armenia', 0.011223721270894898),\n",
              "  ('soviet', 0.010952711540891263),\n",
              "  ('turkey', 0.009511462966635726)],\n",
              " 14: [('espn', 0.03492372001360173),\n",
              "  ('abc', 0.027048980920967695),\n",
              "  ('coverage', 0.020367519486290786),\n",
              "  ('game', 0.019269521970372815),\n",
              "  ('games', 0.018480989329439944),\n",
              "  ('sports', 0.01575680188071436),\n",
              "  ('hockey', 0.015421920964158115),\n",
              "  ('wip', 0.014886287077256682),\n",
              "  ('tv', 0.013012317961360174),\n",
              "  ('baseball', 0.01262349401053561)],\n",
              " 15: [('cancer', 0.019731343563769627),\n",
              "  ('migraine', 0.015901015292776266),\n",
              "  ('medical', 0.00975746050046462),\n",
              "  ('banks', 0.009547759970721976),\n",
              "  ('gordon', 0.00935193751321454),\n",
              "  ('centers', 0.00861587051159391),\n",
              "  ('research', 0.00808933118301145),\n",
              "  ('doctor', 0.008033986208578541),\n",
              "  ('dyer', 0.007969045887057236),\n",
              "  ('center', 0.007781046535005129)],\n",
              " 16: [('32', 0.013367825369469377),\n",
              "  ('motherboard', 0.01239506421300277),\n",
              "  ('ibm', 0.011330559842532117),\n",
              "  ('bit', 0.011151743261328175),\n",
              "  ('os', 0.011131917362373924),\n",
              "  ('mac', 0.009405966788022772),\n",
              "  ('486', 0.008881846931618151),\n",
              "  ('ram', 0.008438044535989305),\n",
              "  ('cpu', 0.00770788051259899),\n",
              "  ('scsi', 0.00761255695555371)],\n",
              " 17: [('sale', 0.029309906376462632),\n",
              "  ('shipping', 0.01708672407782507),\n",
              "  ('drive', 0.017073056589711408),\n",
              "  ('offer', 0.014135202800182012),\n",
              "  ('floppy', 0.01337930161265224),\n",
              "  ('card', 0.013005707548000572),\n",
              "  ('items', 0.012530522285688827),\n",
              "  ('meg', 0.012147880265896836),\n",
              "  ('monitor', 0.012065137133905052),\n",
              "  ('pc', 0.01205806328303999)],\n",
              " 18: [('drive', 0.047548074305885694),\n",
              "  ('drives', 0.0286684938336195),\n",
              "  ('disk', 0.026986800801659466),\n",
              "  ('controller', 0.024073157121421756),\n",
              "  ('bios', 0.021893221912735097),\n",
              "  ('hard', 0.021143745762587797),\n",
              "  ('slave', 0.021063339422209306),\n",
              "  ('jumper', 0.020114360771196167),\n",
              "  ('master', 0.018061791171272396),\n",
              "  ('floppy', 0.014777650407829348)],\n",
              " 19: [('clock', 0.025958132095281628),\n",
              "  ('speed', 0.025898954693017066),\n",
              "  ('mhz', 0.01790131279691458),\n",
              "  ('processor', 0.0173164484778819),\n",
              "  ('040', 0.015010205235389859),\n",
              "  ('68040', 0.014223244999596242),\n",
              "  ('68070', 0.013907556090259458),\n",
              "  ('quicktime', 0.012394000453879573),\n",
              "  ('3do', 0.012317449832387223),\n",
              "  ('680x0', 0.011589630075216215)],\n",
              " 20: [('msg', 0.06609719328462355),\n",
              "  ('food', 0.030198114038450907),\n",
              "  ('sensitivity', 0.02116231405385479),\n",
              "  ('chinese', 0.018223103768597176),\n",
              "  ('superstition', 0.017994163762886515),\n",
              "  ('glutamate', 0.0163389376415504),\n",
              "  ('foods', 0.013200211609467485),\n",
              "  ('olney', 0.010940035848249915),\n",
              "  ('restaurant', 0.010597965814785542),\n",
              "  ('effects', 0.010229309523303183)],\n",
              " 21: [('israel', 0.03463907589632724),\n",
              "  ('lebanese', 0.029238716385344973),\n",
              "  ('lebanon', 0.022360988410865666),\n",
              "  ('israeli', 0.02079589136814107),\n",
              "  ('hezbollah', 0.015766776542902473),\n",
              "  ('civilians', 0.015080060744743846),\n",
              "  ('attacks', 0.015019690709891923),\n",
              "  ('villages', 0.014524070750547707),\n",
              "  ('soldiers', 0.011004084248243732),\n",
              "  ('peace', 0.01085588873635621)],\n",
              " 22: [('modem', 0.07651461842461875),\n",
              "  ('fax', 0.03483199876687063),\n",
              "  ('baud', 0.01839371220405999),\n",
              "  ('modems', 0.01817840551698623),\n",
              "  ('9600', 0.013293635074587699),\n",
              "  ('4k', 0.012742697431471921),\n",
              "  ('2400', 0.011001304289916707),\n",
              "  ('bps', 0.010225873134298229),\n",
              "  ('internal', 0.010084530870259845),\n",
              "  ('14', 0.009836927432782567)],\n",
              " 23: [('church', 0.01729929300368967),\n",
              "  ('lds', 0.016637600565653814),\n",
              "  ('mormons', 0.016375714125672736),\n",
              "  ('mormon', 0.014982657081223),\n",
              "  ('bible', 0.010669356419596715),\n",
              "  ('christian', 0.009388682017304757),\n",
              "  ('jesus', 0.008810166837342082),\n",
              "  ('temple', 0.008546729538359445),\n",
              "  ('mcconkie', 0.007808717090028294),\n",
              "  ('zoroastrians', 0.0076876281210413065)],\n",
              " 24: [('islam', 0.03269933105148507),\n",
              "  ('rushdie', 0.030040242613714234),\n",
              "  ('islamic', 0.027751739361622355),\n",
              "  ('gregg', 0.02262243741103092),\n",
              "  ('muslim', 0.01996324346413134),\n",
              "  ('jaeger', 0.01629046194349961),\n",
              "  ('muslims', 0.015107048099123875),\n",
              "  ('khomeini', 0.011579402593543243),\n",
              "  ('law', 0.011264425040554742),\n",
              "  ('fatwa', 0.011015688461453491)],\n",
              " 25: [('atf', 0.018767739303868467),\n",
              "  ('gas', 0.017640604369128166),\n",
              "  ('fbi', 0.01479234396263025),\n",
              "  ('survivors', 0.012484972324177286),\n",
              "  ('fire', 0.012305667315203463),\n",
              "  ('tear', 0.011746837681870972),\n",
              "  ('children', 0.01131564634617487),\n",
              "  ('ranch', 0.010535599021875878),\n",
              "  ('dividian', 0.010119339729804971),\n",
              "  ('koresh', 0.009137036070082372)],\n",
              " 26: [('atheism', 0.023439243633471475),\n",
              "  ('atheists', 0.020378045155309343),\n",
              "  ('argument', 0.010433937620097932),\n",
              "  ('atheist', 0.010400384275803675),\n",
              "  ('god', 0.00989997102845456),\n",
              "  ('religion', 0.009099946461232347),\n",
              "  ('fallacy', 0.009027496024456311),\n",
              "  ('alt', 0.00881905591041217),\n",
              "  ('religious', 0.008363546599877295),\n",
              "  ('is', 0.00676210252373365)],\n",
              " 27: [('card', 0.04326671195138094),\n",
              "  ('video', 0.02998241803800602),\n",
              "  ('diamond', 0.0273493759933703),\n",
              "  ('stealth', 0.024629377454574247),\n",
              "  ('winbench', 0.02430320002165266),\n",
              "  ('cards', 0.020748163566417177),\n",
              "  ('vlb', 0.019528679066170763),\n",
              "  ('s3', 0.018734892667539003),\n",
              "  ('winmarks', 0.017686538676006263),\n",
              "  ('windows', 0.014271963512038436)],\n",
              " 28: [('sky', 0.022196633976597383),\n",
              "  ('rockets', 0.01798114252129288),\n",
              "  ('billboard', 0.0172349670751682),\n",
              "  ('advertising', 0.016620378933398315),\n",
              "  ('space', 0.016559871422538754),\n",
              "  ('inflatable', 0.014213819835217762),\n",
              "  ('vandalizing', 0.012326027328541022),\n",
              "  ('nasa', 0.01210884133396737),\n",
              "  ('launch', 0.011536605596526198),\n",
              "  ('rocketry', 0.011366568371080165)],\n",
              " 29: [('radar', 0.06383376560953195),\n",
              "  ('detector', 0.040682225705538685),\n",
              "  ('detectors', 0.025966689260125308),\n",
              "  ('car', 0.012460697033697512),\n",
              "  ('traffic', 0.011441542563624738),\n",
              "  ('police', 0.009849725388757733),\n",
              "  ('alarm', 0.009456414150785524),\n",
              "  ('pavement', 0.007161885970453065),\n",
              "  ('beam', 0.00697791020305177),\n",
              "  ('illegal', 0.006845707759171882)],\n",
              " 30: [('oil', 0.061931225048713526),\n",
              "  ('drain', 0.01649482701070013),\n",
              "  ('injector', 0.015073283914468471),\n",
              "  ('changing', 0.01357241215053111),\n",
              "  ('plug', 0.012574479821149995),\n",
              "  ('fuel', 0.01157434038780261),\n",
              "  ('bolt', 0.011014977971118861),\n",
              "  ('engine', 0.010427256230782283),\n",
              "  ('orlin', 0.009390211394350699),\n",
              "  ('miles', 0.009341075877531442)],\n",
              " 31: [('jesus', 0.018066933104876077),\n",
              "  ('judas', 0.016288995227480108),\n",
              "  ('luke', 0.012704143687559661),\n",
              "  ('resurrection', 0.012266729670653774),\n",
              "  ('matthew', 0.011018687240364793),\n",
              "  ('he', 0.009814066201912053),\n",
              "  ('tomb', 0.009518274730538753),\n",
              "  ('christ', 0.008584956551946766),\n",
              "  ('rise', 0.007739738485368016),\n",
              "  ('his', 0.007594615907973463)],\n",
              " 32: [('hell', 0.04737936310499159),\n",
              "  ('eternal', 0.02174643287529527),\n",
              "  ('god', 0.018194347586301885),\n",
              "  ('heaven', 0.01736446241980484),\n",
              "  ('believe', 0.011026119765792769),\n",
              "  ('atheists', 0.010808048154843854),\n",
              "  ('death', 0.010479371712617019),\n",
              "  ('die', 0.01041996131374149),\n",
              "  ('jesus', 0.009845330815952857),\n",
              "  ('he', 0.008648124488220029)],\n",
              " 33: [('colormap', 0.03977906729152105),\n",
              "  ('visual', 0.037184175713736854),\n",
              "  ('color', 0.03661322422892283),\n",
              "  ('colormaps', 0.02301492614412844),\n",
              "  ('colors', 0.01845267714415286),\n",
              "  ('window', 0.01749153333663532),\n",
              "  ('dpy', 0.01690981962863336),\n",
              "  ('depth', 0.015944599374560307),\n",
              "  ('default', 0.013225421485717868),\n",
              "  ('clients', 0.012722502866152216)],\n",
              " 34: [('helmet', 0.05232986835934859),\n",
              "  ('helmets', 0.01811291512632558),\n",
              "  ('shoei', 0.01689268434637797),\n",
              "  ('leather', 0.014078527315791617),\n",
              "  ('piece', 0.013387992928806604),\n",
              "  ('jacket', 0.012944663117845888),\n",
              "  ('my', 0.011905347043403161),\n",
              "  ('bike', 0.011897350023280873),\n",
              "  ('passenger', 0.011373993351281433),\n",
              "  ('hamer', 0.010746915589367962)],\n",
              " 35: [('swap', 0.04367996113513644),\n",
              "  ('windows', 0.03700075630982135),\n",
              "  ('memory', 0.026972939518380954),\n",
              "  ('file', 0.02138314044047644),\n",
              "  ('mathcad', 0.01753852823842711),\n",
              "  ('ram', 0.017488805040197734),\n",
              "  ('permanent', 0.015094302692474323),\n",
              "  ('disk', 0.013159835135248776),\n",
              "  ('emm386', 0.012695354822920486),\n",
              "  ('dos', 0.011039635726811867)],\n",
              " 36: [('space', 0.028631217412070604),\n",
              "  ('launch', 0.024808020422128846),\n",
              "  ('station', 0.015438736861417942),\n",
              "  ('redesign', 0.01372304259138659),\n",
              "  ('shuttle', 0.012654828810108367),\n",
              "  ('satellite', 0.012046097499035315),\n",
              "  ('commercial', 0.009554693013489542),\n",
              "  ('nasa', 0.009119489173976564),\n",
              "  ('satellites', 0.008651450277856397),\n",
              "  ('orbit', 0.007939586025356797)],\n",
              " 37: [('government', 0.020610091379741853),\n",
              "  ('hendricks', 0.014150975155599231),\n",
              "  ('libertarians', 0.013600023963511933),\n",
              "  ('libertarian', 0.011714893695317075),\n",
              "  ('regulation', 0.009415111960060526),\n",
              "  ('libertarianism', 0.008984296221507659),\n",
              "  ('steve', 0.008346019201876673),\n",
              "  ('limiting', 0.008061647972312687),\n",
              "  ('gritz', 0.007918413486034535),\n",
              "  ('govt', 0.007612104450270848)],\n",
              " 38: [('fbi', 0.030866590221264704),\n",
              "  ('batf', 0.01757434817624363),\n",
              "  ('koresh', 0.01436026330390633),\n",
              "  ('roby', 0.013171817680758525),\n",
              "  ('compound', 0.012629412571657636),\n",
              "  ('they', 0.010590950986422794),\n",
              "  ('children', 0.009624561458762783),\n",
              "  ('waco', 0.009029632516150573),\n",
              "  ('were', 0.008992608684409544),\n",
              "  ('fire', 0.008711007005027973)],\n",
              " 39: [('space', 0.03200190437500336),\n",
              "  ('nasa', 0.014992861771642935),\n",
              "  ('shuttle', 0.012317932623359859),\n",
              "  ('satellite', 0.011604053063338364),\n",
              "  ('astronaut', 0.00883786429219556),\n",
              "  ('flight', 0.008610217547387562),\n",
              "  ('mission', 0.007536632756094615),\n",
              "  ('propulsion', 0.007183814896688501),\n",
              "  ('telescope', 0.007002959777064442),\n",
              "  ('satellites', 0.006375719385444813)],\n",
              " 40: [('gordon', 0.021081121837315184),\n",
              "  ('banks', 0.02001219049272715),\n",
              "  ('doctor', 0.016097953973046756),\n",
              "  ('cancer', 0.015489976650861954),\n",
              "  ('homeopathy', 0.013293384894342153),\n",
              "  ('gr', 0.012703633613427547),\n",
              "  ('patients', 0.01068602885926702),\n",
              "  ('clinic', 0.010246670279137142),\n",
              "  ('treatment', 0.009629383081865553),\n",
              "  ('medicine', 0.00939812977833476)],\n",
              " 41: [('tax', 0.02786945365090536),\n",
              "  ('taxes', 0.01850662079141587),\n",
              "  ('clinton', 0.015377730983615947),\n",
              "  ('deficit', 0.014973796701215769),\n",
              "  ('vat', 0.013489354318771492),\n",
              "  ('reagan', 0.012206580537677519),\n",
              "  ('broward', 0.012152310181974125),\n",
              "  ('sales', 0.010624381544914489),\n",
              "  ('horne', 0.009628304896627865),\n",
              "  ('spending', 0.009181532256743617)],\n",
              " 42: [('jewish', 0.08177776921539447),\n",
              "  ('baseball', 0.04609760903799509),\n",
              "  ('players', 0.037784135728070024),\n",
              "  ('lowenstein', 0.02879276338797875),\n",
              "  ('koufax', 0.027381244848251135),\n",
              "  ('stankowitz', 0.023234567028398892),\n",
              "  ('hank', 0.021723674272779128),\n",
              "  ('vb30', 0.02157495509779897),\n",
              "  ('berg', 0.021432919285883353),\n",
              "  ('moe', 0.020972197040201267)],\n",
              " 43: [('lib', 0.07252663258201605),\n",
              "  ('libxmu', 0.04665724165975849),\n",
              "  ('xmu', 0.041446407410231256),\n",
              "  ('symbol', 0.029060913697074007),\n",
              "  ('x11r5', 0.025885874304294156),\n",
              "  ('undefined', 0.025151003972854923),\n",
              "  ('doug', 0.023574728925934898),\n",
              "  ('usr', 0.022951781596950296),\n",
              "  ('gcc', 0.019628983999579144),\n",
              "  ('libx11', 0.018478788266970546)],\n",
              " 44: [('gay', 0.027068597891534617),\n",
              "  ('men', 0.02636977733663955),\n",
              "  ('homosexual', 0.022937245462243166),\n",
              "  ('male', 0.017623111934504044),\n",
              "  ('promiscuous', 0.015773192655212317),\n",
              "  ('partners', 0.015205032150229838),\n",
              "  ('sexual', 0.014928757087948758),\n",
              "  ('study', 0.014579849274501688),\n",
              "  ('bi', 0.011957657863509046),\n",
              "  ('gonorrhea', 0.011898612894394563)],\n",
              " 45: [('simms', 0.07011476998072941),\n",
              "  ('simm', 0.042541741573350604),\n",
              "  ('ram', 0.024520262799062824),\n",
              "  ('memory', 0.019932994477900583),\n",
              "  ('vram', 0.01855499099508336),\n",
              "  ('meg', 0.018198948157571056),\n",
              "  ('dram', 0.015881556762666515),\n",
              "  ('256k', 0.01527366789482675),\n",
              "  ('pin', 0.013240517427503339),\n",
              "  ('60ns', 0.012726925898616265)],\n",
              " 46: [('objective', 0.06070724122015324),\n",
              "  ('morality', 0.041026892448818714),\n",
              "  ('values', 0.036101668229317166),\n",
              "  ('dwyer', 0.023232806786080664),\n",
              "  ('frank', 0.017071604589368593),\n",
              "  ('2000', 0.017030639965249398),\n",
              "  ('subjective', 0.013923563994695859),\n",
              "  ('say', 0.013153131606456115),\n",
              "  ('christian', 0.012245126345282468),\n",
              "  ('science', 0.012118784534245076)],\n",
              " 47: [('cd', 0.03879021483204363),\n",
              "  ('adaptec', 0.03244905177606661),\n",
              "  ('rom', 0.025516627778886587),\n",
              "  ('toshiba', 0.02253624293551307),\n",
              "  ('tape', 0.022526501947990673),\n",
              "  ('cdrom', 0.021311534854943415),\n",
              "  ('drive', 0.019493229260823728),\n",
              "  ('1542', 0.01931567443892611),\n",
              "  ('sys', 0.019214892352730856),\n",
              "  ('aspi4dos', 0.019071893048556682)],\n",
              " 48: [('air', 0.013554812641460426),\n",
              "  ('aftermarket', 0.012635450890490124),\n",
              "  ('heat', 0.01180054847137725),\n",
              "  ('timing', 0.011639568480320639),\n",
              "  ('substitutes', 0.01070921094448406),\n",
              "  ('fan', 0.010378703142152307),\n",
              "  ('r12', 0.01035867212293032),\n",
              "  ('snow', 0.009901720930379531),\n",
              "  ('freon', 0.009566793865554769),\n",
              "  ('cold', 0.009046825853901463)],\n",
              " 49: [('games', 0.04354391841027354),\n",
              "  ('genesis', 0.03482891794664618),\n",
              "  ('game', 0.02951118094338538),\n",
              "  ('sega', 0.02884085802624527),\n",
              "  ('sale', 0.021637424521712047),\n",
              "  ('snes', 0.021114910059987217),\n",
              "  ('trade', 0.017157706249359157),\n",
              "  ('super', 0.015475115902729672),\n",
              "  ('nintendo', 0.014940053251043035),\n",
              "  ('docs', 0.012731973458111214)],\n",
              " 50: [('biggest', 0.02063515938317999),\n",
              "  ('gainey', 0.016142643689141003),\n",
              "  ('he', 0.013703149194572379),\n",
              "  ('player', 0.012846638634203421),\n",
              "  ('team', 0.011975548511832631),\n",
              "  ('hockey', 0.011500766425437804),\n",
              "  ('selanne', 0.011464010751172947),\n",
              "  ('game', 0.011147458178857091),\n",
              "  ('roussel', 0.010263850115349152),\n",
              "  ('disappointment', 0.010201407600969442)],\n",
              " 51: [('sleeve', 0.04705238616165106),\n",
              "  ('cds', 0.039062043588550525),\n",
              "  ('sale', 0.03840658403327203),\n",
              "  ('picture', 0.035825676514886086),\n",
              "  ('cd', 0.029630140522496143),\n",
              "  ('45', 0.028595791237095658),\n",
              "  ('promo', 0.026258096592324497),\n",
              "  ('hiram', 0.02289373745796139),\n",
              "  ('jazz', 0.021457421588597342),\n",
              "  ('kou', 0.01839207564736915)],\n",
              " 52: [('science', 0.027944034913057057),\n",
              "  ('methodology', 0.021562783225545338),\n",
              "  ('scientific', 0.01628310258842001),\n",
              "  ('homeopathy', 0.01214029716598881),\n",
              "  ('rational', 0.011907489815855751),\n",
              "  ('sas', 0.010267992012164445),\n",
              "  ('turpin', 0.009559228098099282),\n",
              "  ('lady', 0.009188980412946204),\n",
              "  ('theory', 0.008677931528722985),\n",
              "  ('merrill', 0.008665118191339107)],\n",
              " 53: [('moral', 0.03842472835795522),\n",
              "  ('morality', 0.03747553588186956),\n",
              "  ('keith', 0.022454477206701157),\n",
              "  ('animals', 0.019604721788289987),\n",
              "  ('natural', 0.019411898587414524),\n",
              "  ('objective', 0.01854502042789782),\n",
              "  ('schneider', 0.017506061478853634),\n",
              "  ('species', 0.016999046893010798),\n",
              "  ('livesey', 0.01696103910930539),\n",
              "  ('immoral', 0.01645325651624946)],\n",
              " 54: [('receiver', 0.02044582947051801),\n",
              "  ('antenna', 0.020193628838852583),\n",
              "  ('rf', 0.015534504001605868),\n",
              "  ('td', 0.015151123358454532),\n",
              "  ('mjm', 0.014575331583137214),\n",
              "  ('circuits', 0.012608791770613732),\n",
              "  ('fm', 0.012233733503328868),\n",
              "  ('channel', 0.011994928946265757),\n",
              "  ('frequencies', 0.011872342718817888),\n",
              "  ('radio', 0.01150098322751251)],\n",
              " 55: [('drugs', 0.055240666768650325),\n",
              "  ('drug', 0.02463265626920238),\n",
              "  ('legalization', 0.013243261601924),\n",
              "  ('alcohol', 0.013142680746736425),\n",
              "  ('marijuana', 0.01280930080532373),\n",
              "  ('cocaine', 0.0121908741699975),\n",
              "  ('cigarettes', 0.011579795069334745),\n",
              "  ('hypocrisy', 0.011196211173729865),\n",
              "  ('lsd', 0.010564885131795758),\n",
              "  ('heroin', 0.010356825180746796)],\n",
              " 56: [('os', 0.048049714313809805),\n",
              "  ('nt', 0.047204716117866447),\n",
              "  ('windows', 0.03162322091716366),\n",
              "  ('os2', 0.02642833723185951),\n",
              "  ('comp', 0.019980025849674583),\n",
              "  ('chicogo', 0.016892580085695268),\n",
              "  ('dos', 0.013352769249145725),\n",
              "  ('programmer', 0.012756572504717213),\n",
              "  ('unix', 0.011368757237743474),\n",
              "  ('yap', 0.010658041242072416)],\n",
              " 57: [('atheism', 0.01932570747228459),\n",
              "  ('religion', 0.01625633900530546),\n",
              "  ('motto', 0.014571565232028818),\n",
              "  ('genocide', 0.013269544530777725),\n",
              "  ('caused', 0.011034248802106756),\n",
              "  ('qur', 0.009385312629782904),\n",
              "  ('bobby', 0.008325885048980509),\n",
              "  ('mozumder', 0.008280956127462793),\n",
              "  ('god', 0.008021333370698646),\n",
              "  ('atheist', 0.00777422361351623)],\n",
              " 58: [('liar', 0.019034384102827685),\n",
              "  ('christian', 0.01661174088817055),\n",
              "  ('christianity', 0.015092945286940494),\n",
              "  ('lunatic', 0.013547696464006446),\n",
              "  ('he', 0.012977951283107358),\n",
              "  ('convenient', 0.012108969347380516),\n",
              "  ('bible', 0.009601412453463513),\n",
              "  ('bissell', 0.00938293836099607),\n",
              "  ('people', 0.00923531924175977),\n",
              "  ('dan', 0.007933229657688117)],\n",
              " 59: [('sho', 0.031029630471277058),\n",
              "  ('automatic', 0.019323011004933673),\n",
              "  ('shift', 0.01814453223549476),\n",
              "  ('manual', 0.0172617409157551),\n",
              "  ('brakes', 0.015483665573584741),\n",
              "  ('speed', 0.014602060610059042),\n",
              "  ('transmission', 0.014039622924770172),\n",
              "  ('auto', 0.013860832709469771),\n",
              "  ('clutch', 0.012004445121280231),\n",
              "  ('shifting', 0.011756021732416359)],\n",
              " 60: [('duo', 0.04237876169979818),\n",
              "  ('230', 0.021625692871483464),\n",
              "  ('apple', 0.014992750336477716),\n",
              "  ('machine', 0.0143327864588398),\n",
              "  ('problem', 0.013891588625729347),\n",
              "  ('crashes', 0.013831945179487926),\n",
              "  ('date', 0.0135516128794278),\n",
              "  ('sleep', 0.01333003900384697),\n",
              "  ('problems', 0.012331788623672322),\n",
              "  ('system', 0.011619627227712563)],\n",
              " 61: [('cruel', 0.02775562220373549),\n",
              "  ('punishment', 0.0244702912489585),\n",
              "  ('penalty', 0.02335995292159089),\n",
              "  ('death', 0.023008080375026965),\n",
              "  ('innocent', 0.018113019807733836),\n",
              "  ('keith', 0.017424524565381842),\n",
              "  ('murder', 0.016828966390205185),\n",
              "  ('we', 0.013963203004063774),\n",
              "  ('atheists', 0.01341575365250499),\n",
              "  ('system', 0.012345343835224282)],\n",
              " 62: [('drivers', 0.08067375688001413),\n",
              "  ('driver', 0.04455558656939253),\n",
              "  ('card', 0.03685700512737878),\n",
              "  ('speedstar', 0.030303549000708724),\n",
              "  ('trident', 0.030218062109637327),\n",
              "  ('video', 0.027837806280637836),\n",
              "  ('ati', 0.025504088034020347),\n",
              "  ('ftp', 0.02395286972045904),\n",
              "  ('diamond', 0.023439787426843967),\n",
              "  ('latest', 0.01704013707343407)],\n",
              " 63: [('mouse', 0.0765052259656308),\n",
              "  ('irq', 0.050675129640173414),\n",
              "  ('com3', 0.03464356530516214),\n",
              "  ('com1', 0.03064217526205329),\n",
              "  ('port', 0.027607418035328065),\n",
              "  ('interrupt', 0.024919412032918024),\n",
              "  ('windows', 0.022954460794727405),\n",
              "  ('com2', 0.018906087030243372),\n",
              "  ('driver', 0.017547949766827012),\n",
              "  ('irq5', 0.017466105426813986)],\n",
              " 64: [('truth', 0.024776929254624647),\n",
              "  ('arrogance', 0.023317425445970357),\n",
              "  ('christians', 0.01913412992555919),\n",
              "  ('darren', 0.01605259589199787),\n",
              "  ('belief', 0.01596903270676264),\n",
              "  ('arrogant', 0.015197549081295931),\n",
              "  ('believe', 0.013076193977947072),\n",
              "  ('beliefs', 0.011838809556378126),\n",
              "  ('authority', 0.011468683576484891),\n",
              "  ('we', 0.010966866931726889)],\n",
              " 65: [('ftp', 0.033648702633188296),\n",
              "  ('mpeg', 0.022785181342542773),\n",
              "  ('xavier', 0.01793189753132703),\n",
              "  ('funet', 0.014299324621350655),\n",
              "  ('interviews', 0.013806920081051745),\n",
              "  ('files', 0.013694453515099798),\n",
              "  ('mono', 0.013692301414031379),\n",
              "  ('mpegplay', 0.013673638952252454),\n",
              "  ('version', 0.01306022501330074),\n",
              "  ('site', 0.013001393713806823)],\n",
              " 66: [('leafs', 0.04542218855547748),\n",
              "  ('wings', 0.026596506628014285),\n",
              "  ('toronto', 0.017824636162530362),\n",
              "  ('goal', 0.017372756098426614),\n",
              "  ('detroit', 0.016889878225111465),\n",
              "  ('game', 0.016877610166137032),\n",
              "  ('habs', 0.016347098978389353),\n",
              "  ('nords', 0.01606135021070593),\n",
              "  ('roy', 0.01593535370301895),\n",
              "  ('red', 0.012583802468039397)],\n",
              " 67: [('allocation', 0.04538414401108726),\n",
              "  ('linked', 0.03490335224076774),\n",
              "  ('pic46', 0.03180400979949627),\n",
              "  ('dos', 0.031174042492658562),\n",
              "  ('fx', 0.028968998443372036),\n",
              "  ('cross', 0.02809240560993877),\n",
              "  ('unit', 0.023863871912616897),\n",
              "  ('cfg', 0.021528104592048118),\n",
              "  ('disk', 0.01889442227090405),\n",
              "  ('windows', 0.017118659138336333)],\n",
              " 68: [('phone', 0.05014659570307878),\n",
              "  ('hook', 0.043068264671417),\n",
              "  ('number', 0.035966551126500386),\n",
              "  ('line', 0.03352707556345628),\n",
              "  ('led', 0.019877865536784656),\n",
              "  ('dial', 0.019757335418269743),\n",
              "  ('telephone', 0.016659605199103967),\n",
              "  ('find', 0.016024469211985623),\n",
              "  ('operator', 0.01539029816432841),\n",
              "  ('al', 0.014625729176386435)],\n",
              " 69: [('polio', 0.026108198171667416),\n",
              "  ('disease', 0.018476917128045858),\n",
              "  ('disorder', 0.01730013049934665),\n",
              "  ('syndrome', 0.01615790883106488),\n",
              "  ('patients', 0.014470799689760712),\n",
              "  ('she', 0.014001833217849308),\n",
              "  ('recurrent', 0.011520750134983396),\n",
              "  ('chronic', 0.011188252150438501),\n",
              "  ('seizures', 0.011011860605773626),\n",
              "  ('cirrhosis', 0.010221934932441945)],\n",
              " 70: [('insurance', 0.058341857939558275),\n",
              "  ('car', 0.016714114540660893),\n",
              "  ('fault', 0.015819170251969057),\n",
              "  ('deductible', 0.014619879236131733),\n",
              "  ('rates', 0.011303155655970535),\n",
              "  ('lawyers', 0.010655763647167943),\n",
              "  ('pay', 0.010506543782746403),\n",
              "  ('alamo', 0.010327157398672447),\n",
              "  ('farm', 0.010202799083831591),\n",
              "  ('company', 0.009785090141253469)],\n",
              " 71: [('clinton', 0.036042658668834694),\n",
              "  ('administration', 0.02513004037670331),\n",
              "  ('bush', 0.013505229146473504),\n",
              "  ('privacy', 0.009294089480192764),\n",
              "  ('chip', 0.009166718582915722),\n",
              "  ('been', 0.008989849707383377),\n",
              "  ('qualcomm', 0.008388746276968239),\n",
              "  ('has', 0.008357777866262805),\n",
              "  ('freedoms', 0.008274575950051074),\n",
              "  ('na', 0.008217137759957825)],\n",
              " 72: [('universe', 0.01648198826896743),\n",
              "  ('larson', 0.01461397968106104),\n",
              "  ('existence', 0.013644705829027758),\n",
              "  ('why', 0.012666633007441395),\n",
              "  ('theory', 0.01179118811278132),\n",
              "  ('science', 0.011161853523103782),\n",
              "  ('god', 0.011092980579108227),\n",
              "  ('exist', 0.010612019072434116),\n",
              "  ('exists', 0.009495616738626337),\n",
              "  ('question', 0.007821536898211159)],\n",
              " 73: [('entry', 0.016765228729978047),\n",
              "  ('output', 0.013127466325757602),\n",
              "  ('file', 0.011684687528651844),\n",
              "  ('anonymous', 0.011287685199452266),\n",
              "  ('privacy', 0.009780796167046597),\n",
              "  ('ripem', 0.00978069193577836),\n",
              "  ('key', 0.009173639317600242),\n",
              "  ('oname', 0.009124172542695095),\n",
              "  ('printf', 0.008361490735093947),\n",
              "  ('entries', 0.008300536415979807)],\n",
              " 74: [('wave', 0.036536272980022544),\n",
              "  ('riders', 0.03498984788996173),\n",
              "  ('motorcycling', 0.02723340169295404),\n",
              "  ('harley', 0.024717635417266865),\n",
              "  ('bike', 0.023726251623244342),\n",
              "  ('cruiser', 0.02045331253832962),\n",
              "  ('bikers', 0.01798048131259823),\n",
              "  ('riceburner', 0.01464571243329153),\n",
              "  ('bikes', 0.014318117201736771),\n",
              "  ('waves', 0.013997592973383237)],\n",
              " 75: [('des', 0.036386292311788436),\n",
              "  ('key', 0.025823161404338015),\n",
              "  ('keys', 0.022672650701737296),\n",
              "  ('machine', 0.02241063109839902),\n",
              "  ('keyseach', 0.02113317071014923),\n",
              "  ('skipjack', 0.01841535125665072),\n",
              "  ('shelf', 0.018125118283249923),\n",
              "  ('bits', 0.01792216678456689),\n",
              "  ('80', 0.015326680356242897),\n",
              "  ('bellovin', 0.014978652488575887)],\n",
              " 76: [('monitor', 0.04798010238986198),\n",
              "  ('vga', 0.03576640094493703),\n",
              "  ('lc', 0.029559489192041582),\n",
              "  ('video', 0.02811064489283298),\n",
              "  ('monitors', 0.025641981097577055),\n",
              "  ('nec', 0.02422496799841668),\n",
              "  ('gnd', 0.02342526729805694),\n",
              "  ('adapter', 0.019461632546351748),\n",
              "  ('multisync', 0.019301219497037742),\n",
              "  ('mac', 0.01747518883814862)],\n",
              " 77: [('dog', 0.07378682407814625),\n",
              "  ('dogs', 0.02332893383151737),\n",
              "  ('springer', 0.01542171927188345),\n",
              "  ('spagthorpe', 0.014045432758606334),\n",
              "  ('chase', 0.01199431473399499),\n",
              "  ('bike', 0.011950520197554425),\n",
              "  ('riding', 0.010759126940852196),\n",
              "  ('viking', 0.010238753325753469),\n",
              "  ('dod', 0.010181946276984943),\n",
              "  ('ryan', 0.00972055664267139)],\n",
              " 78: [('vs', 0.04763007115323175),\n",
              "  ('pens', 0.03271793682178543),\n",
              "  ('cup', 0.024815752688104865),\n",
              "  ('pitt', 0.02206293425262249),\n",
              "  ('predictions', 0.02192982902010034),\n",
              "  ('bos', 0.020931534332271053),\n",
              "  ('win', 0.01959157076691825),\n",
              "  ('stanley', 0.019194220096740315),\n",
              "  ('cal', 0.019051421959842187),\n",
              "  ('chi', 0.01896456812669453)],\n",
              " 79: [('keyboard', 0.04194446906569598),\n",
              "  ('key', 0.03410758889531026),\n",
              "  ('accelerators', 0.030120798835613086),\n",
              "  ('ctrl', 0.027523591483224797),\n",
              "  ('keycode', 0.024926281550410552),\n",
              "  ('numlock', 0.024613728652305576),\n",
              "  ('keys', 0.023957080612814992),\n",
              "  ('emacs', 0.019307114527942572),\n",
              "  ('shift', 0.019029927190836418),\n",
              "  ('map', 0.01744409992493338)],\n",
              " 80: [('henrik', 0.04294310448315325),\n",
              "  ('armenia', 0.04095106210333802),\n",
              "  ('planes', 0.030356292813381248),\n",
              "  ('turkish', 0.02852931838039495),\n",
              "  ('armenians', 0.027057888719933837),\n",
              "  ('turkey', 0.02443552519116733),\n",
              "  ('karabakh', 0.02152634956460066),\n",
              "  ('azeris', 0.02108588884892774),\n",
              "  ('yalcin', 0.02054046823857599),\n",
              "  ('onur', 0.02054046823857599)],\n",
              " 81: [('europeans', 0.04310642588640173),\n",
              "  ('nhl', 0.04293879408408432),\n",
              "  ('players', 0.02886381587006655),\n",
              "  ('european', 0.026352954939902695),\n",
              "  ('league', 0.02379383784791651),\n",
              "  ('teams', 0.022633031888515592),\n",
              "  ('canadian', 0.022229449525490916),\n",
              "  ('hockey', 0.02149196077459709),\n",
              "  ('canadians', 0.020143914199745823),\n",
              "  ('euros', 0.017366655044997487)],\n",
              " 82: [('greek', 0.040253719998349216),\n",
              "  ('greece', 0.031758035354911104),\n",
              "  ('turkish', 0.028759178879371898),\n",
              "  ('greeks', 0.027893992937606797),\n",
              "  ('turks', 0.02191663256139346),\n",
              "  ('cyprus', 0.02084066132305781),\n",
              "  ('turkey', 0.01949697029391169),\n",
              "  ('minority', 0.011316852753707004),\n",
              "  ('cypriot', 0.009820714167549673),\n",
              "  ('were', 0.008483254166298033)],\n",
              " 83: [('spacecraft', 0.03483681867319195),\n",
              "  ('probe', 0.020903911800807945),\n",
              "  ('venus', 0.020052439548618274),\n",
              "  ('galileo', 0.01919384566680738),\n",
              "  ('command', 0.01717219810902346),\n",
              "  ('mars', 0.017079433115504104),\n",
              "  ('mission', 0.01687866205285214),\n",
              "  ('magellan', 0.01684707300321312),\n",
              "  ('probes', 0.016258657126528953),\n",
              "  ('timer', 0.01601727528387408)],\n",
              " 84: [('monitors', 0.041384336411503055),\n",
              "  ('nanao', 0.033631447959329576),\n",
              "  ('17', 0.027879088101892012),\n",
              "  ('monitor', 0.02732133791076259),\n",
              "  ('t560i', 0.023189717213629897),\n",
              "  ('nec', 0.02248099229536457),\n",
              "  ('f550iw', 0.020895886279649348),\n",
              "  ('trinitron', 0.020470109274746455),\n",
              "  ('viewsonic', 0.018895485484381344),\n",
              "  ('mag', 0.018290241198392256)],\n",
              " 85: [('countersteering', 0.040753610091598585),\n",
              "  ('bike', 0.023775223957555665),\n",
              "  ('countersteeringfaq', 0.020966341958063565),\n",
              "  ('motorcycle', 0.01996674098766627),\n",
              "  ('technique', 0.01840200318321019),\n",
              "  ('mjs', 0.01795092970556541),\n",
              "  ('rider', 0.015560218247157965),\n",
              "  ('left', 0.014924973104250084),\n",
              "  ('neck', 0.01471710315790149),\n",
              "  ('horse', 0.014710673395805632)],\n",
              " 86: [('16550', 0.030926265237105715),\n",
              "  ('uart', 0.0288170200716232),\n",
              "  ('modem', 0.027496787377096977),\n",
              "  ('serial', 0.023565161128892807),\n",
              "  ('ports', 0.016723769903110135),\n",
              "  ('procomm', 0.014133412884247926),\n",
              "  ('windows', 0.01266057509001012),\n",
              "  ('card', 0.012357017833909098),\n",
              "  ('uarts', 0.011887978400017953),\n",
              "  ('16450', 0.011388617572668793)],\n",
              " 87: [('window', 0.1117742690217814),\n",
              "  ('manager', 0.03699587857858775),\n",
              "  ('mainwin', 0.0338228269778248),\n",
              "  ('xsizehints', 0.03177861965783948),\n",
              "  ('detailwin', 0.0275987471509577),\n",
              "  ('myhint', 0.02545765657026969),\n",
              "  ('mydisplay', 0.023790047858523615),\n",
              "  ('win', 0.01805423452196873),\n",
              "  ('pposition', 0.01762863231748539),\n",
              "  ('position', 0.017006315560529863)],\n",
              " 88: [('rosicrucian', 0.035750689885533905),\n",
              "  ('order', 0.03499005868089706),\n",
              "  ('alicea', 0.024760717484390418),\n",
              "  ('tony', 0.023892818260624143),\n",
              "  ('reuss', 0.02303292806969808),\n",
              "  ('oto', 0.021998197123637465),\n",
              "  ('amorc', 0.020953339005630525),\n",
              "  ('ancient', 0.017592213459223113),\n",
              "  ('templars', 0.01665946002021359),\n",
              "  ('crucis', 0.01665946002021359)],\n",
              " 89: [('mp', 0.00923832884435735),\n",
              "  ('mv', 0.009153345580799036),\n",
              "  ('printer', 0.008828123565697288),\n",
              "  ('m8', 0.007477393661866451),\n",
              "  ('m5', 0.0073295949614911576),\n",
              "  ('m0', 0.00713212975241463),\n",
              "  ('mh', 0.006972104918353441),\n",
              "  ('mz', 0.006930170781050254),\n",
              "  ('m9', 0.006923269552564154),\n",
              "  ('mw', 0.006752573561138059)],\n",
              " 90: [('printer', 0.055126016355688666),\n",
              "  ('postscript', 0.028317184197226678),\n",
              "  ('laserwriter', 0.022926235373324836),\n",
              "  ('printers', 0.016944314354888836),\n",
              "  ('print', 0.016727463152071114),\n",
              "  ('toner', 0.015707830928425592),\n",
              "  ('laser', 0.014920462583480162),\n",
              "  ('upgrade', 0.013767265841191246),\n",
              "  ('laserjet', 0.012423693886194962),\n",
              "  ('apple', 0.012077701455186322)],\n",
              " 91: [('gif', 0.04793870847223857),\n",
              "  ('images', 0.04247140420837378),\n",
              "  ('otis', 0.038399271353821846),\n",
              "  ('ics', 0.026560612230881638),\n",
              "  ('uci', 0.02643644731438005),\n",
              "  ('incoming', 0.025059282680715618),\n",
              "  ('image', 0.020108395232805137),\n",
              "  ('earth', 0.015865795490662915),\n",
              "  ('jpg', 0.013200697770479271),\n",
              "  ('files', 0.012820438905091722)],\n",
              " 92: [('candida', 0.041602234858385524),\n",
              "  ('yeast', 0.033147785131485336),\n",
              "  ('albicans', 0.014900356228055541),\n",
              "  ('bloom', 0.013369410209176676),\n",
              "  ('vitamin', 0.012717302714317478),\n",
              "  ('sinus', 0.012715385375726241),\n",
              "  ('symptoms', 0.012127700306356064),\n",
              "  ('nystatin', 0.01121017973124399),\n",
              "  ('infections', 0.010720687321824374),\n",
              "  ('thrush', 0.01065210208357295)],\n",
              " 93: [('font', 0.06602237572974362),\n",
              "  ('fonts', 0.056634840671676984),\n",
              "  ('truetype', 0.051069471071517894),\n",
              "  ('atm', 0.03484566255560286),\n",
              "  ('characters', 0.02290935488282653),\n",
              "  ('printer', 0.021855083333737237),\n",
              "  ('character', 0.020293032502696956),\n",
              "  ('print', 0.019031644169816606),\n",
              "  ('tt', 0.01868709094107554),\n",
              "  ('windows', 0.018464684766646894)],\n",
              " 94: [('scope', 0.04745649139712722),\n",
              "  ('scopes', 0.02643032116589795),\n",
              "  ('fluke', 0.021279806118278538),\n",
              "  ('meter', 0.017152415546820194),\n",
              "  ('oscilloscope', 0.014791698515163425),\n",
              "  ('generator', 0.014416690983837101),\n",
              "  ('medin', 0.014402434682721376),\n",
              "  ('dmm', 0.013298045162380416),\n",
              "  ('huntsville', 0.01306326163525),\n",
              "  ('205', 0.012160642075175034)],\n",
              " 95: [('ticket', 0.04312699519336934),\n",
              "  ('hotel', 0.032022315201974835),\n",
              "  ('airline', 0.028675103580026777),\n",
              "  ('tickets', 0.027445772115492536),\n",
              "  ('package', 0.02715776885598122),\n",
              "  ('vacation', 0.021445583644695725),\n",
              "  ('hiram', 0.01959475718643194),\n",
              "  ('coupons', 0.015937730161201907),\n",
              "  ('nights', 0.014489491656136657),\n",
              "  ('kou', 0.013992696362738179)],\n",
              " 96: [('copy', 0.03184680128299566),\n",
              "  ('protection', 0.018246129824238115),\n",
              "  ('protected', 0.0167642428156443),\n",
              "  ('bits', 0.015690824112412127),\n",
              "  ('random', 0.015081104636668764),\n",
              "  ('babak', 0.013083773397510648),\n",
              "  ('sehari', 0.013083773397510648),\n",
              "  ('disks', 0.01194173584615076),\n",
              "  ('md5', 0.011709481751202684),\n",
              "  ('media', 0.011646455613752812)],\n",
              " 97: [('bible', 0.02605009662905651),\n",
              "  ('texts', 0.016680667373184847),\n",
              "  ('hebrew', 0.014928417115673442),\n",
              "  ('translations', 0.014457801524749209),\n",
              "  ('text', 0.012984764442438066),\n",
              "  ('ancient', 0.011772792273082918),\n",
              "  ('manuscripts', 0.011602929363842465),\n",
              "  ('testament', 0.010659495727574464),\n",
              "  ('versions', 0.009526326851207571),\n",
              "  ('translation', 0.009520295581006045)],\n",
              " 98: [('monitor', 0.0400962286197575),\n",
              "  ('screen', 0.03054709953283952),\n",
              "  ('colors', 0.024191561779763292),\n",
              "  ('video', 0.02377928327822163),\n",
              "  ('610', 0.02140002383669883),\n",
              "  ('problem', 0.021295431718208004),\n",
              "  ('centris', 0.016594653822568174),\n",
              "  ('apple', 0.016250947577597964),\n",
              "  ('display', 0.01299590431927849),\n",
              "  ('scrolling', 0.012077375480414834)],\n",
              " 99: [('marriage', 0.06045964504685616),\n",
              "  ('married', 0.05496303372690036),\n",
              "  ('ceremony', 0.022588914673480297),\n",
              "  ('eyes', 0.020452733686394383),\n",
              "  ('church', 0.017044927008814784),\n",
              "  ('marry', 0.01640221246088724),\n",
              "  ('god', 0.016291947396692076),\n",
              "  ('commitment', 0.015567540576648827),\n",
              "  ('wedding', 0.015207579482902025),\n",
              "  ('priest', 0.014455765241796913)],\n",
              " 100: [('wire', 0.035809985988070114),\n",
              "  ('wiring', 0.034900127114926995),\n",
              "  ('ground', 0.03398410108181514),\n",
              "  ('neutral', 0.02536321087885411),\n",
              "  ('outlets', 0.021409131959087547),\n",
              "  ('outlet', 0.02050678672807652),\n",
              "  ('circuit', 0.018576786417634755),\n",
              "  ('grounding', 0.018316725248733973),\n",
              "  ('gfci', 0.01803196052703675),\n",
              "  ('conductor', 0.01471542590910466)],\n",
              " 101: [('grass', 0.048058839128199714),\n",
              "  ('valley', 0.04754932669179242),\n",
              "  ('verse', 0.03611647931267603),\n",
              "  ('lord', 0.03478952968863954),\n",
              "  ('buffalo', 0.03416757972219307),\n",
              "  ('chuck', 0.03396503926076128),\n",
              "  ('daily', 0.03242062374446204),\n",
              "  ('promise', 0.03181113055811102),\n",
              "  ('petch', 0.03178066887295378),\n",
              "  ('god', 0.027084450489804658)],\n",
              " 102: [('white', 0.021371318453347533),\n",
              "  ('ted', 0.020017740111884),\n",
              "  ('black', 0.01635475848416673),\n",
              "  ('kyle', 0.012652107993151163),\n",
              "  ('males', 0.011701833144274846),\n",
              "  ('adjective', 0.011542911698570647),\n",
              "  ('frank', 0.010550057103541977),\n",
              "  ('blacks', 0.009734050973885541),\n",
              "  ('cramm', 0.00916736624867451),\n",
              "  ('koppenhoefer', 0.00916736624867451)],\n",
              " 103: [('moa', 0.0644025104553105),\n",
              "  ('bmw', 0.0503233087638336),\n",
              "  ('bmwmoa', 0.028427316986118566),\n",
              "  ('vechorik', 0.018395772779715488),\n",
              "  ('members', 0.017020150785451882),\n",
              "  ('senner', 0.013796829584786617),\n",
              "  ('ra', 0.013494845855886383),\n",
              "  ('rathole', 0.012619738024498884),\n",
              "  ('karr', 0.012533390948640397),\n",
              "  ('ranck', 0.012222178750918836)],\n",
              " 104: [('mary', 0.04874626513196658),\n",
              "  ('maria', 0.02485702253919844),\n",
              "  ('her', 0.020043613429678725),\n",
              "  ('immaculate', 0.019444143335620463),\n",
              "  ('conception', 0.018675638720643505),\n",
              "  ('virgin', 0.018133170214694787),\n",
              "  ('she', 0.0171259270170736),\n",
              "  ('church', 0.014150012249106817),\n",
              "  ('sin', 0.0140876197255451),\n",
              "  ('god', 0.013299503571982516)],\n",
              " 105: [('card', 0.03983393722489909),\n",
              "  ('pin', 0.019469490367645136),\n",
              "  ('board', 0.01819008918325971),\n",
              "  ('3com', 0.014753949682491873),\n",
              "  ('switches', 0.014396252080163853),\n",
              "  ('cards', 0.014376773871676516),\n",
              "  ('slots', 0.014327630456004294),\n",
              "  ('dip', 0.014171575999412248),\n",
              "  ('motherboard', 0.014011123131911597),\n",
              "  ('chip', 0.011310398310401624)],\n",
              " 106: [('period', 0.04042110632333569),\n",
              "  ('pp', 0.03590879379747431),\n",
              "  ('11', 0.03438882115411042),\n",
              "  ('10', 0.03131409362828155),\n",
              "  ('12', 0.03064634664228182),\n",
              "  ('14', 0.030112106847080208),\n",
              "  ('scorer', 0.027336130327581892),\n",
              "  ('13', 0.026997262396813736),\n",
              "  ('18', 0.02628690668454579),\n",
              "  ('26', 0.026252745519652104)],\n",
              " 107: [('ink', 0.060805315183684094),\n",
              "  ('deskjet', 0.05413596995486371),\n",
              "  ('bubblejet', 0.045801267163882275),\n",
              "  ('printer', 0.045106532881233966),\n",
              "  ('hp', 0.040314480550820854),\n",
              "  ('printers', 0.034497381552489936),\n",
              "  ('laser', 0.030376967748680657),\n",
              "  ('canon', 0.028666043683329864),\n",
              "  ('bj200', 0.024843834270559403),\n",
              "  ('500', 0.023956763527785675)],\n",
              " 108: [('stephanopoulos', 0.02632086684530625),\n",
              "  ('president', 0.025540521028823107),\n",
              "  ('mr', 0.019786394655055595),\n",
              "  ('myers', 0.01711401876770428),\n",
              "  ('jobs', 0.01578939143273691),\n",
              "  ('ms', 0.012052404081151918),\n",
              "  ('we', 0.01205047409993875),\n",
              "  ('secretary', 0.010899274638345077),\n",
              "  ('going', 0.009115066449517914),\n",
              "  ('that', 0.008495355214373265)],\n",
              " 109: [('summer', 0.032805594944765516),\n",
              "  ('room', 0.0298513256086524),\n",
              "  ('sublet', 0.023251013806139316),\n",
              "  ('bedroom', 0.020824075742904007),\n",
              "  ('condo', 0.02070660393034436),\n",
              "  ('washer', 0.016855741986045412),\n",
              "  ('dryer', 0.016698438671235834),\n",
              "  ('kitchen', 0.016589320876877745),\n",
              "  ('parking', 0.016091059595171552),\n",
              "  ('furnished', 0.0154419891745341)],\n",
              " 110: [('battery', 0.06519802500917089),\n",
              "  ('batteries', 0.03898745781086948),\n",
              "  ('ups', 0.02128992931638886),\n",
              "  ('copper', 0.021118047567963108),\n",
              "  ('zinc', 0.020765670385155525),\n",
              "  ('switchover', 0.018384321423999057),\n",
              "  ('inverter', 0.016785248546553444),\n",
              "  ('electrolyte', 0.016785248546553444),\n",
              "  ('dc', 0.015645167005437298),\n",
              "  ('anneser', 0.015158087050680873)],\n",
              " 111: [('nmm', 0.03409298787906984),\n",
              "  ('cop', 0.026052935778245724),\n",
              "  ('ticket', 0.021901835980052155),\n",
              "  ('traffic', 0.019682217190104507),\n",
              "  ('jerks', 0.017703122829346406),\n",
              "  ('mikkel', 0.016589525072531267),\n",
              "  ('speeding', 0.015755628547563967),\n",
              "  ('meters', 0.015469202004740218),\n",
              "  ('niels', 0.015174105282296919),\n",
              "  ('michelsen', 0.014179530200197207)],\n",
              " 112: [('keenan', 0.06411717375716296),\n",
              "  ('ottawa', 0.03587986581813323),\n",
              "  ('rangers', 0.03263202846843915),\n",
              "  ('lindros', 0.031390619140483515),\n",
              "  ('dare', 0.027492020883536722),\n",
              "  ('gary', 0.02370534600557325),\n",
              "  ('flyers', 0.02247820872745028),\n",
              "  ('phillie', 0.022293358787804158),\n",
              "  ('coach', 0.021560683351330195),\n",
              "  ('montreal', 0.020463172136845426)],\n",
              " 113: [('scsi', 0.16083492939730515),\n",
              "  ('ide', 0.04518565279889544),\n",
              "  ('ranges', 0.01861032426232893),\n",
              "  ('mac', 0.018527150603026336),\n",
              "  ('bit', 0.018316277363860926),\n",
              "  ('controller', 0.01782761593097694),\n",
              "  ('chip', 0.01712783598837212),\n",
              "  ('burst', 0.016841377724832088),\n",
              "  ('grubb', 0.01675015613556721),\n",
              "  ('esdi', 0.016490019144693373)],\n",
              " 114: [('tiff', 0.08705897152296994),\n",
              "  ('42', 0.05933160314500781),\n",
              "  ('philosophical', 0.04389054968083025),\n",
              "  ('significance', 0.04115130519825896),\n",
              "  ('spec', 0.021257841301790703),\n",
              "  ('alice', 0.018838745002606114),\n",
              "  ('question', 0.01671453544812359),\n",
              "  ('gripe', 0.014319165782913116),\n",
              "  ('joachim', 0.013380337675684442),\n",
              "  ('cartwright', 0.012940140786658309)],\n",
              " 115: [('pantheism', 0.02005550915864571),\n",
              "  ('environmentalism', 0.02005550915864571),\n",
              "  ('disasters', 0.018518908138038172),\n",
              "  ('paganism', 0.01784228667201666),\n",
              "  ('environmental', 0.013791763560335757),\n",
              "  ('watt', 0.01186703001659862),\n",
              "  ('terrance', 0.010852671126335455),\n",
              "  ('disaster', 0.010827839994141164),\n",
              "  ('environment', 0.010557667554827673),\n",
              "  ('shk', 0.0100932473549044)],\n",
              " 116: [('mouse', 0.11206558661060574),\n",
              "  ('apple', 0.023182235598875618),\n",
              "  ('ball', 0.020030547854563603),\n",
              "  ('motion', 0.019952989376397334),\n",
              "  ('smoothly', 0.018340856128276916),\n",
              "  ('trackball', 0.016468250985786144),\n",
              "  ('jumpy', 0.01482142588720753),\n",
              "  ('adb', 0.014236196405887786),\n",
              "  ('jump', 0.013901907155490657),\n",
              "  ('eckton', 0.013336016155476037)],\n",
              " 117: [('ra', 0.045352501627398026),\n",
              "  ('satan', 0.032876032679317074),\n",
              "  ('god', 0.02824537289139002),\n",
              "  ('lucifer', 0.021139585223623015),\n",
              "  ('evil', 0.01763523376034793),\n",
              "  ('bible', 0.013551007895118818),\n",
              "  ('created', 0.011755854188012203),\n",
              "  ('free', 0.01118992046506768),\n",
              "  ('ye', 0.010443315399473869),\n",
              "  ('angels', 0.009726775669766206)],\n",
              " 118: [('pain', 0.05575331186869879),\n",
              "  ('hernia', 0.030349791653337826),\n",
              "  ('clubbing', 0.02606760320396382),\n",
              "  ('bone', 0.016774803054464476),\n",
              "  ('banks', 0.016193463929525073),\n",
              "  ('she', 0.015897956483465463),\n",
              "  ('gordon', 0.015861339545170883),\n",
              "  ('compartment', 0.013846774098341405),\n",
              "  ('arm', 0.011849123495371343),\n",
              "  ('shoulder', 0.0114699596851341)],\n",
              " 119: [('coach', 0.030474935760359638),\n",
              "  ('kariya', 0.02977314037559044),\n",
              "  ('jokerit', 0.028543913687659083),\n",
              "  ('tps', 0.026876702006221564),\n",
              "  ('suhonen', 0.024486436679693753),\n",
              "  ('jets', 0.023739862143657336),\n",
              "  ('tommi', 0.02230780031597292),\n",
              "  ('daigle', 0.021145227732221075),\n",
              "  ('assat', 0.020092071364420223),\n",
              "  ('finnish', 0.019783786324698915)],\n",
              " 120: [('plastic', 0.046891608499752525),\n",
              "  ('paint', 0.02747244852213807),\n",
              "  ('detailing', 0.020446801473756587),\n",
              "  ('scratches', 0.019921423803706862),\n",
              "  ('wax', 0.019688406433439044),\n",
              "  ('tip', 0.01951944834472602),\n",
              "  ('polish', 0.01712767125685143),\n",
              "  ('compound', 0.01698198659737699),\n",
              "  ('scuffed', 0.016033925119521616),\n",
              "  ('coat', 0.014153594913308688)],\n",
              " 121: [('law', 0.03832264201423967),\n",
              "  ('sabbath', 0.03808167352234645),\n",
              "  ('worship', 0.028322282547589075),\n",
              "  ('paul', 0.024950360359406773),\n",
              "  ('jesus', 0.02280903732805763),\n",
              "  ('ceremonial', 0.018695057633989806),\n",
              "  ('christians', 0.017787055862500352),\n",
              "  ('day', 0.01730231866043801),\n",
              "  ('sunday', 0.01369365468387221),\n",
              "  ('gentiles', 0.012744040803802033)],\n",
              " 122: [('simms', 0.07371255766235264),\n",
              "  ('256k', 0.05114098695700618),\n",
              "  ('jh', 0.031140783152590217),\n",
              "  ('old', 0.025996184784232712),\n",
              "  ('sipps', 0.022664222259582048),\n",
              "  ('simm', 0.022553470594147494),\n",
              "  ('them', 0.022411483749906008),\n",
              "  ('inovative', 0.02156595009336607),\n",
              "  ('bunch', 0.019579418063118578),\n",
              "  ('haines', 0.016610647625008235)],\n",
              " 123: [('faith', 0.029777521672115604),\n",
              "  ('god', 0.026419593997648025),\n",
              "  ('saved', 0.0194926287562126),\n",
              "  ('our', 0.01474651296003785),\n",
              "  ('christ', 0.014321714911872584),\n",
              "  ('we', 0.01395373831303231),\n",
              "  ('romans', 0.011907853373130821),\n",
              "  ('shall', 0.011423569100598637),\n",
              "  ('salvation', 0.010776346894459539),\n",
              "  ('lord', 0.010503855296494202)],\n",
              " 124: [('cooling', 0.07194375984711195),\n",
              "  ('towers', 0.06114717565632201),\n",
              "  ('water', 0.05196756677049633),\n",
              "  ('nuclear', 0.051268065973183855),\n",
              "  ('plants', 0.048976971639739866),\n",
              "  ('steam', 0.04198944632996936),\n",
              "  ('fossil', 0.026786369673877642),\n",
              "  ('tower', 0.02296532636584725),\n",
              "  ('hot', 0.019248422548567828),\n",
              "  ('cool', 0.018792752490467392)],\n",
              " 125: [('sale', 0.0371794666982042),\n",
              "  ('miles', 0.0322493306767982),\n",
              "  ('cassette', 0.031536327827863274),\n",
              "  ('excellent', 0.028192617750647464),\n",
              "  ('fm', 0.026743607219692052),\n",
              "  ('honda', 0.026120475812536702),\n",
              "  ('toyota', 0.026066179565393456),\n",
              "  ('condition', 0.0247223650140131),\n",
              "  ('obo', 0.02013047740754432),\n",
              "  ('sunroof', 0.02003857972286797)],\n",
              " 126: [('ini', 0.08929516490415322),\n",
              "  ('updating', 0.03965232258861127),\n",
              "  ('spss', 0.03676183589213989),\n",
              "  ('utility', 0.031787632842984304),\n",
              "  ('svein', 0.029292752733850357),\n",
              "  ('sysedit', 0.0271806772339787),\n",
              "  ('files', 0.026766737770053655),\n",
              "  ('bytes', 0.023864757658966274),\n",
              "  ('file', 0.023169604851086587),\n",
              "  ('windows', 0.02284654508637013)],\n",
              " 127: [('tape', 0.08174840660926229),\n",
              "  ('backup', 0.06771450876165398),\n",
              "  ('jumbo', 0.03362271573694419),\n",
              "  ('drive', 0.030958401950308887),\n",
              "  ('tapes', 0.021853916655862685),\n",
              "  ('250', 0.01810905006021781),\n",
              "  ('colorado', 0.013839012664682439),\n",
              "  ('40sc', 0.01372090835358904),\n",
              "  ('husak', 0.013007007045567641),\n",
              "  ('software', 0.012814990765902872)],\n",
              " 128: [('energy', 0.030089213943949457),\n",
              "  ('uranium', 0.023103513095887578),\n",
              "  ('nuclear', 0.020286017252853696),\n",
              "  ('plutonium', 0.01944161948717054),\n",
              "  ('oil', 0.018221314129219856),\n",
              "  ('cato', 0.016700441345213465),\n",
              "  ('waste', 0.01219152907506032),\n",
              "  ('conservation', 0.011882168088488609),\n",
              "  ('reiken', 0.009998590896945714),\n",
              "  ('tons', 0.009887357581104695)],\n",
              " 129: [('scripture', 0.02804660554496974),\n",
              "  ('prophecy', 0.028032899498221305),\n",
              "  ('prophecies', 0.019327087666923932),\n",
              "  ('god', 0.016166158323307627),\n",
              "  ('jesus', 0.013542105233550506),\n",
              "  ('canon', 0.011278943392252702),\n",
              "  ('revelation', 0.011106055966803814),\n",
              "  ('spirit', 0.011029535818979645),\n",
              "  ('apostles', 0.010548669364766216),\n",
              "  ('bible', 0.010409545505229205)],\n",
              " 130: [('stephanopoulos', 0.04009217507522948),\n",
              "  ('mr', 0.02894282524750561),\n",
              "  ('bosnia', 0.02048919043521766),\n",
              "  ('serbs', 0.015016458457379193),\n",
              "  ('president', 0.014416124826833831),\n",
              "  ('embargo', 0.014313018200326266),\n",
              "  ('somalia', 0.014118218892640784),\n",
              "  ('we', 0.011297304138600856),\n",
              "  ('said', 0.009009009561707899),\n",
              "  ('jobs', 0.008127678452497065)],\n",
              " 131: [('lyme', 0.07821324346358088),\n",
              "  ('quack', 0.03615116770606751),\n",
              "  ('gordon', 0.020965312023925715),\n",
              "  ('banks', 0.020473687548114988),\n",
              "  ('physician', 0.019993866425047482),\n",
              "  ('patients', 0.018962843939353838),\n",
              "  ('quacks', 0.016225414522613377),\n",
              "  ('disease', 0.015782992625366),\n",
              "  ('ld', 0.015653564229318545),\n",
              "  ('poo', 0.014552637266060075)],\n",
              " 132: [('homosexuality', 0.033672760808388894),\n",
              "  ('gay', 0.02464518247919706),\n",
              "  ('sex', 0.01610270083737252),\n",
              "  ('caligiuri', 0.015664156324426855),\n",
              "  ('homosexuals', 0.014868248432050403),\n",
              "  ('homosexual', 0.013140905697013136),\n",
              "  ('gays', 0.01183139540107146),\n",
              "  ('people', 0.009967295217536946),\n",
              "  ('ex', 0.009526397696310819),\n",
              "  ('who', 0.008626746079115616)],\n",
              " 133: [('god', 0.03481623706177005),\n",
              "  ('exist', 0.02673021750707431),\n",
              "  ('existence', 0.02218536990841585),\n",
              "  ('gods', 0.02161866513022281),\n",
              "  ('faith', 0.02085905285096934),\n",
              "  ('atheism', 0.019039888267891873),\n",
              "  ('believe', 0.018443320476260468),\n",
              "  ('belief', 0.016221376460138298),\n",
              "  ('atheist', 0.01421747551396913),\n",
              "  ('strong', 0.012546129096799204)],\n",
              " 134: [('books', 0.04103195580230749),\n",
              "  ('chemistry', 0.03985106729643578),\n",
              "  ('spice', 0.03409951882657524),\n",
              "  ('book', 0.031435007731999365),\n",
              "  ('cover', 0.028066586028686166),\n",
              "  ('paperback', 0.023335216390642913),\n",
              "  ('condition', 0.01961888183193127),\n",
              "  ('edition', 0.019446895977705474),\n",
              "  ('00', 0.018576416649892718),\n",
              "  ('guide', 0.01827687601627902)],\n",
              " 135: [('odometer', 0.05576742061153217),\n",
              "  ('sensor', 0.03577153724897091),\n",
              "  ('car', 0.022289983780889668),\n",
              "  ('odometers', 0.021476464236088864),\n",
              "  ('mileage', 0.01969922231115588),\n",
              "  ('sprint', 0.01923832351243283),\n",
              "  ('speedo', 0.01847094298863426),\n",
              "  ('oxygen', 0.017966457888825657),\n",
              "  ('bmw', 0.01778908581073717),\n",
              "  ('electronic', 0.017070116289951347)],\n",
              " 136: [('ax', 0.11673869700582351),\n",
              "  ('max', 0.05455491090818411),\n",
              "  ('g9v', 0.02611144883290618),\n",
              "  ('b8f', 0.025271721944463355),\n",
              "  ('a86', 0.021940244079704328),\n",
              "  ('pl', 0.01858673819962237),\n",
              "  ('1d9', 0.016583986551100192),\n",
              "  ('145', 0.016527487391336096),\n",
              "  ('1t', 0.012803894754426403),\n",
              "  ('0t', 0.011860476219328574)],\n",
              " 137: [('cramer', 0.04270373911936121),\n",
              "  ('clayton', 0.034080061561340844),\n",
              "  ('orientation', 0.02522793202075546),\n",
              "  ('sexual', 0.02310689477447726),\n",
              "  ('optilink', 0.022779876994795435),\n",
              "  ('nambla', 0.02269924686633867),\n",
              "  ('homosexuals', 0.020628465655554647),\n",
              "  ('gay', 0.01936814986304342),\n",
              "  ('pedophilia', 0.015351511311283482),\n",
              "  ('consent', 0.014554603638750456)],\n",
              " 138: [('split', 0.05277144149181746),\n",
              "  ('graphics', 0.04893203501160617),\n",
              "  ('ch', 0.03555652914558221),\n",
              "  ('newsgroup', 0.03544478414411782),\n",
              "  ('aspects', 0.035237890096266385),\n",
              "  ('comp', 0.028406940854670874),\n",
              "  ('group', 0.027599288955521405),\n",
              "  ('wate', 0.024826248454024195),\n",
              "  ('herringshaw', 0.023409160204453862),\n",
              "  ('groups', 0.022250244782691022)],\n",
              " 139: [('drink', 0.04070390717003856),\n",
              "  ('drinking', 0.0331389406744671),\n",
              "  ('dwi', 0.030458264437863113),\n",
              "  ('riding', 0.025778407593123873),\n",
              "  ('drunk', 0.023860516610157086),\n",
              "  ('hours', 0.02336624017026511),\n",
              "  ('alcohol', 0.022986994008207622),\n",
              "  ('intoxicated', 0.01947240655940875),\n",
              "  ('ride', 0.01925692779934428),\n",
              "  ('driving', 0.01887336744616254)],\n",
              " 140: [('bus', 0.06316975133264227),\n",
              "  ('vlb', 0.04833477539538662),\n",
              "  ('controller', 0.047425656226441716),\n",
              "  ('sec', 0.043902839615762315),\n",
              "  ('ide', 0.040777056947808074),\n",
              "  ('mb', 0.03220689649691199),\n",
              "  ('transfer', 0.026881060871477267),\n",
              "  ('isa', 0.025862286547099694),\n",
              "  ('card', 0.02166721844499475),\n",
              "  ('harddisk', 0.01887158973224025)],\n",
              " 141: [('3do', 0.056977872791005114),\n",
              "  ('studio', 0.03766772017935627),\n",
              "  ('3d', 0.036255213414946315),\n",
              "  ('v2', 0.023268131865240446),\n",
              "  ('imagine', 0.020945843391374876),\n",
              "  ('rumours', 0.020811897418397757),\n",
              "  ('3ds', 0.01977289826880038),\n",
              "  ('01', 0.019272600060524732),\n",
              "  ('ipas', 0.016683036441805153),\n",
              "  ('differences', 0.01667948977739648)],\n",
              " 142: [('easter', 0.0694759128169877),\n",
              "  ('resurrection', 0.031057018732687697),\n",
              "  ('pagan', 0.027842008629097175),\n",
              "  ('celebration', 0.027306631187371404),\n",
              "  ('goddess', 0.023580578855360017),\n",
              "  ('passover', 0.022610407880242166),\n",
              "  ('ishtar', 0.020852890628868175),\n",
              "  ('word', 0.01749693734175904),\n",
              "  ('feast', 0.016827229357453975),\n",
              "  ('celebrate', 0.01669689125939425)],\n",
              " 143: [('comet', 0.11386601222825808),\n",
              "  ('jupiter', 0.10380385738666102),\n",
              "  ('gehrels', 0.05849961359605074),\n",
              "  ('orbit', 0.05196155799967986),\n",
              "  ('temporary', 0.04700442723079245),\n",
              "  ('radii', 0.032209008396446144),\n",
              "  ('brader', 0.028652026253835904),\n",
              "  ('1993e', 0.027843736999851668),\n",
              "  ('1970', 0.027644926218009663),\n",
              "  ('1973', 0.027540377508334237)],\n",
              " 144: [('lane', 0.030664093512800278),\n",
              "  ('drivers', 0.015184554725493501),\n",
              "  ('stupid', 0.014765791151797991),\n",
              "  ('speed', 0.013451851270403568),\n",
              "  ('car', 0.013104676363885823),\n",
              "  ('rush', 0.012514179238165796),\n",
              "  ('differential', 0.012368066575567577),\n",
              "  ('you', 0.011320586833983202),\n",
              "  ('pass', 0.010521581219229933),\n",
              "  ('cars', 0.010312278963065795)],\n",
              " 145: [('book', 0.033284218488995265),\n",
              "  ('commentary', 0.02137930035240541),\n",
              "  ('scrolls', 0.019619282988116896),\n",
              "  ('books', 0.019232888437919923),\n",
              "  ('butera', 0.016391937513968526),\n",
              "  ('christian', 0.0157210030404403),\n",
              "  ('bible', 0.014607198077137954),\n",
              "  ('reading', 0.013996362421780532),\n",
              "  ('recommend', 0.013994480125245934),\n",
              "  ('harper', 0.01339129512454513)],\n",
              " 146: [('abortion', 0.03644372766156476),\n",
              "  ('child', 0.018243173997128433),\n",
              "  ('fetus', 0.017631846613249597),\n",
              "  ('human', 0.015604960719662447),\n",
              "  ('margolis', 0.015170123426870675),\n",
              "  ('siamese', 0.013414979409028197),\n",
              "  ('pro', 0.013287665948458987),\n",
              "  ('abortions', 0.012602764821872548),\n",
              "  ('lm', 0.012593114270141668),\n",
              "  ('life', 0.011675683049294673)],\n",
              " 147: [('nubus', 0.10389065988737169),\n",
              "  ('pds', 0.09132165014510948),\n",
              "  ('lc', 0.06583083899957906),\n",
              "  ('powerpc', 0.049972354568514614),\n",
              "  ('slot', 0.04843778073970384),\n",
              "  ('lciii', 0.03721036291193578),\n",
              "  ('iii', 0.030631118821405973),\n",
              "  ('marvin', 0.024124102401484555),\n",
              "  ('macs', 0.02164874554595094),\n",
              "  ('mirsky', 0.01944142886737459)],\n",
              " 148: [('mask', 0.09321467166108577),\n",
              "  ('goalie', 0.06202302650844063),\n",
              "  ('masks', 0.032513567291958624),\n",
              "  ('hrivnak', 0.027647640784078786),\n",
              "  ('votes', 0.026103564959678038),\n",
              "  ('keller', 0.01784222492656088),\n",
              "  ('vanbiesbrouck', 0.017123031470186605),\n",
              "  ('forehead', 0.01691577914129976),\n",
              "  ('go', 0.016850860611107656),\n",
              "  ('poll', 0.016191534802753628)],\n",
              " 149: [('xputimage', 0.05152571794919841),\n",
              "  ('server', 0.04353494060691723),\n",
              "  ('memory', 0.04345820544926384),\n",
              "  ('shared', 0.037602482975862675),\n",
              "  ('animation', 0.03202123273928187),\n",
              "  ('sunview', 0.0310755521745011),\n",
              "  ('pixmaps', 0.02924532434888468),\n",
              "  ('shm', 0.025607287078395255),\n",
              "  ('client', 0.024206500906959704),\n",
              "  ('gooch', 0.024141734977185535)],\n",
              " 150: [('ham', 0.03404778595673426),\n",
              "  ('tesla', 0.02399954459613865),\n",
              "  ('radio', 0.020067395843257847),\n",
              "  ('phonograph', 0.0152001549329405),\n",
              "  ('interference', 0.015059209614809435),\n",
              "  ('power', 0.014738100504872426),\n",
              "  ('operator', 0.014498692547387358),\n",
              "  ('hams', 0.014490463584280027),\n",
              "  ('chokes', 0.01260007565335705),\n",
              "  ('surges', 0.01260007565335705)],\n",
              " 151: [('koresh', 0.07266612013682855),\n",
              "  ('backing', 0.04117293590721293),\n",
              "  ('cites', 0.036159982826649104),\n",
              "  ('enclosed', 0.03555511650681222),\n",
              "  ('biblical', 0.03474138565053313),\n",
              "  ('tape', 0.028800317850825492),\n",
              "  ('he', 0.024321128151428697),\n",
              "  ('messenger', 0.02188007169649983),\n",
              "  ('structure', 0.02160495280555394),\n",
              "  ('solid', 0.02128251945551298)],\n",
              " 152: [('software', 0.05217741682722718),\n",
              "  ('level', 0.04308039252142768),\n",
              "  ('shuttle', 0.026846875865865382),\n",
              "  ('process', 0.024271564148772136),\n",
              "  ('flight', 0.023234006593171357),\n",
              "  ('challenger', 0.021478748674369674),\n",
              "  ('wingert', 0.018930872755093243),\n",
              "  ('rated', 0.018669368080648008),\n",
              "  ('bret', 0.016886102944537698),\n",
              "  ('standdown', 0.016859225471362396)],\n",
              " 153: [('batf', 0.048908576054543726),\n",
              "  ('blast', 0.02214393671231451),\n",
              "  ('compound', 0.01865497544719381),\n",
              "  ('waco', 0.01804515104252783),\n",
              "  ('they', 0.016460158737405653),\n",
              "  ('assault', 0.015978953104702303),\n",
              "  ('oldham', 0.015319213032069444),\n",
              "  ('then', 0.014430208731086467),\n",
              "  ('equipment', 0.01366172436765713),\n",
              "  ('proper', 0.013597278075016081)],\n",
              " 154: [('barbecued', 0.03299025028733103),\n",
              "  ('meat', 0.03278213813057429),\n",
              "  ('carcinogenic', 0.032513510982216616),\n",
              "  ('foods', 0.026217135282007663),\n",
              "  ('smoke', 0.025210999511957013),\n",
              "  ('charcoal', 0.021963058388714377),\n",
              "  ('risk', 0.021725067802457487),\n",
              "  ('food', 0.01675217431953848),\n",
              "  ('wood', 0.016599516124890596),\n",
              "  ('cooking', 0.01574850528861161)],\n",
              " 155: [('floptical', 0.05853214961789386),\n",
              "  ('bernoulli', 0.03404950192514246),\n",
              "  ('drives', 0.03313420756234312),\n",
              "  ('drive', 0.03191413954078452),\n",
              "  ('hard', 0.01997865042211404),\n",
              "  ('storage', 0.01749763125857107),\n",
              "  ('cartridge', 0.016718532084231724),\n",
              "  ('megabyte', 0.015608573231438366),\n",
              "  ('akhiani', 0.015402861345242092),\n",
              "  ('homayoon', 0.015402861345242092)],\n",
              " 156: [('ide', 0.07321956547041579),\n",
              "  ('scsi', 0.06574017635707942),\n",
              "  ('bus', 0.036691505881673414),\n",
              "  ('dma', 0.03170606100900278),\n",
              "  ('isa', 0.030970705118084148),\n",
              "  ('drive', 0.02507420628643885),\n",
              "  ('drives', 0.02474689205421472),\n",
              "  ('data', 0.01945923907614443),\n",
              "  ('controller', 0.018660673697284816),\n",
              "  ('pc', 0.017423348321823837)],\n",
              " 157: [('tempest', 0.09374316357142946),\n",
              "  ('measures', 0.026519655037197425),\n",
              "  ('tapped', 0.024564697746006364),\n",
              "  ('emmisions', 0.0231523405222398),\n",
              "  ('classified', 0.021589069665223445),\n",
              "  ('protect', 0.021377844021339007),\n",
              "  ('surveillance', 0.020957464997601066),\n",
              "  ('holland', 0.020649729857373685),\n",
              "  ('electromagnetic', 0.020176464730640352),\n",
              "  ('stampfli', 0.01856136418613541)],\n",
              " 158: [('expose', 0.09019441166610588),\n",
              "  ('window', 0.055142375691001166),\n",
              "  ('event', 0.04553927340568608),\n",
              "  ('handler', 0.04442456384783554),\n",
              "  ('mapped', 0.03949076646579537),\n",
              "  ('item', 0.039233524369067116),\n",
              "  ('xcopyarea', 0.029699179611349668),\n",
              "  ('tickle', 0.026081991380210907),\n",
              "  ('events', 0.0253900834211269),\n",
              "  ('moschetti', 0.02248939167734277)],\n",
              " 159: [('key', 0.024902368466268497),\n",
              "  ('fifth', 0.024774867971049955),\n",
              "  ('keyphrase', 0.01928116855031139),\n",
              "  ('amendment', 0.01800782024617459),\n",
              "  ('compelled', 0.016418720247348396),\n",
              "  ('defendant', 0.015408921659646819),\n",
              "  ('passwords', 0.01516581958655205),\n",
              "  ('judge', 0.014750512326375725),\n",
              "  ('evidence', 0.01358778527505899),\n",
              "  ('disclosure', 0.01357493467260879)],\n",
              " 160: [('insurance', 0.0632338584937166),\n",
              "  ('health', 0.0426423137599421),\n",
              "  ('private', 0.02349131946504283),\n",
              "  ('care', 0.01905386004026022),\n",
              "  ('malpractice', 0.015601053011906101),\n",
              "  ('canadian', 0.014873382981304287),\n",
              "  ('dare', 0.014238095601247586),\n",
              "  ('canada', 0.014189898214302641),\n",
              "  ('doctors', 0.012864694691666264),\n",
              "  ('gary', 0.012232497903585542)],\n",
              " 161: [('quadra', 0.06806142062250085),\n",
              "  ('scsi', 0.049739635874848746),\n",
              "  ('mac', 0.03153375225000753),\n",
              "  ('cartridge', 0.03143485741743374),\n",
              "  ('venerable', 0.024142814973978553),\n",
              "  ('blind', 0.023177381020005192),\n",
              "  ('problems', 0.022624523990885388),\n",
              "  ('worked', 0.022601856199672775),\n",
              "  ('jagielski', 0.022434933132390922),\n",
              "  ('quadras', 0.022418510335259084)],\n",
              " 162: [('photography', 0.07140626207205696),\n",
              "  ('krillean', 0.05254913743645247),\n",
              "  ('kirlian', 0.05161329700186194),\n",
              "  ('object', 0.03501100033356658),\n",
              "  ('energy', 0.031731070458229316),\n",
              "  ('pictures', 0.031080872830967384),\n",
              "  ('spikes', 0.024385461683076457),\n",
              "  ('patterns', 0.0213440540090695),\n",
              "  ('huey', 0.02090730832877505),\n",
              "  ('fields', 0.018929881226844962)],\n",
              " 163: [('motif', 0.09502761489172601),\n",
              "  ('linux', 0.026803639938666694),\n",
              "  ('bindings', 0.021894976377491996),\n",
              "  ('athena', 0.01795108135538062),\n",
              "  ('list', 0.015901600616349622),\n",
              "  ('software', 0.015022925644169861),\n",
              "  ('386bsd', 0.014711964606291355),\n",
              "  ('devguide', 0.014709989556765779),\n",
              "  ('interviews', 0.013267864384621427),\n",
              "  ('mailing', 0.013115885605829656)],\n",
              " 164: [('appcontext', 0.03638861044449484),\n",
              "  ('widget', 0.034596498597129904),\n",
              "  ('application', 0.02992440139977604),\n",
              "  ('value', 0.027353969868772657),\n",
              "  ('xtappaddtimeout', 0.019338451564705022),\n",
              "  ('lazarus', 0.018688143969768253),\n",
              "  ('context', 0.01793429227689479),\n",
              "  ('greene', 0.017409404369158343),\n",
              "  ('doit', 0.017409404369158343),\n",
              "  ('appcon', 0.016604314700749377)],\n",
              " 165: [('pluto', 0.05434580510456402),\n",
              "  ('kuiper', 0.049963592478787976),\n",
              "  ('object', 0.045624939605624075),\n",
              "  ('planet', 0.04080630811971836),\n",
              "  ('observations', 0.03214814628468784),\n",
              "  ('karla', 0.02776995216735103),\n",
              "  ('mercury', 0.0273112851567111),\n",
              "  ('charon', 0.026591806392386046),\n",
              "  ('smiley', 0.026029512809039954),\n",
              "  ('orbit', 0.025141603444916824)],\n",
              " 166: [('jagr', 0.0654469527052297),\n",
              "  ('francis', 0.05047318016058079),\n",
              "  ('minus', 0.04329782273632324),\n",
              "  ('player', 0.042040052481879885),\n",
              "  ('stat', 0.04164558297390574),\n",
              "  ('ice', 0.03073289652549234),\n",
              "  ('plus', 0.027363079186906402),\n",
              "  ('bowman', 0.022243229542552024),\n",
              "  ('play', 0.021558724262810868),\n",
              "  ('points', 0.02001785966398339)],\n",
              " 167: [('tv', 0.08094558112106337),\n",
              "  ('exploding', 0.033525692313127596),\n",
              "  ('subliminal', 0.031764410697292966),\n",
              "  ('emerson', 0.019815406080291136),\n",
              "  ('tube', 0.01766701607935098),\n",
              "  ('prasad', 0.01743263721736954),\n",
              "  ('researcher', 0.017199239296492533),\n",
              "  ('flashed', 0.015882205348646483),\n",
              "  ('flyback', 0.015238951051421635),\n",
              "  ('crt', 0.015103419223738363)],\n",
              " 168: [('selective', 0.03561870836516459),\n",
              "  ('service', 0.026879606029837657),\n",
              "  ('pork', 0.026793620716095118),\n",
              "  ('borden', 0.018447887268147813),\n",
              "  ('abolish', 0.01841807242170108),\n",
              "  ('naval', 0.017761003328027266),\n",
              "  ('reserve', 0.017577087629602337),\n",
              "  ('navy', 0.01499021876007419),\n",
              "  ('mueller', 0.01477389017093364),\n",
              "  ('admistration', 0.013609306020936796)],\n",
              " 169: [('car', 0.02453577455785977),\n",
              "  ('price', 0.018981879765830306),\n",
              "  ('sales', 0.01708179182073579),\n",
              "  ('dealership', 0.014802449776010355),\n",
              "  ('dealer', 0.012637333415515494),\n",
              "  ('truck', 0.012503037586866089),\n",
              "  ('deal', 0.01239252626556325),\n",
              "  ('finance', 0.010943270286293688),\n",
              "  ('buying', 0.010767680018357624),\n",
              "  ('agreed', 0.010562756195819633)],\n",
              " 170: [('crohn', 0.07067466627079382),\n",
              "  ('inflammation', 0.03125811521098931),\n",
              "  ('disease', 0.02799963595557719),\n",
              "  ('ibd', 0.024532152665591556),\n",
              "  ('diet', 0.024168055244417216),\n",
              "  ('recurrence', 0.021975567497929595),\n",
              "  ('inhibitors', 0.021251809768471245),\n",
              "  ('turmeric', 0.021251809768471245),\n",
              "  ('eyles', 0.021251809768471245),\n",
              "  ('discomfort', 0.021099003462272645)],\n",
              " 171: [('parking', 0.021508534508781336),\n",
              "  ('cages', 0.021186356753421974),\n",
              "  ('ggrrrrrr', 0.019189630870933554),\n",
              "  ('bdi', 0.01795154173575379),\n",
              "  ('cagers', 0.015387035773503248),\n",
              "  ('stickers', 0.012311063225475469),\n",
              "  ('driver', 0.01212482534703808),\n",
              "  ('motorc', 0.012009272786281736),\n",
              "  ('double', 0.011883449429785748),\n",
              "  ('geake', 0.011734692743835347)],\n",
              " 172: [('captain', 0.13873279704704533),\n",
              "  ('traded', 0.10352815135998066),\n",
              "  ('captains', 0.056611222486326496),\n",
              "  ('cyt', 0.027147903214327588),\n",
              "  ('striped', 0.02679265834294398),\n",
              "  ('team', 0.02610793370347526),\n",
              "  ('resigned', 0.025170487538852298),\n",
              "  ('captaincy', 0.024786571800001895),\n",
              "  ('collingridge', 0.024433112892894827),\n",
              "  ('trivia', 0.023886050347111974)],\n",
              " 173: [('dock', 0.0670233822627498),\n",
              "  ('portable', 0.0553886278976782),\n",
              "  ('duo', 0.05233849945175044),\n",
              "  ('powerbook', 0.0365976026934164),\n",
              "  ('coprocessor', 0.022758393313252332),\n",
              "  ('processor', 0.021670473593170484),\n",
              "  ('apple', 0.02139661851423447),\n",
              "  ('docks', 0.020278408174622953),\n",
              "  ('mini', 0.019623287956765368),\n",
              "  ('backlit', 0.018847727356640767)],\n",
              " 174: [('context', 0.046557199945878314),\n",
              "  ('jim', 0.04342248761913782),\n",
              "  ('meritt', 0.033051997000113104),\n",
              "  ('telepathy', 0.02339956707898525),\n",
              "  ('weiss', 0.021502195230078576),\n",
              "  ('articles', 0.02131116578163757),\n",
              "  ('hypocrite', 0.02077028308721344),\n",
              "  ('quotes', 0.020085008557415053),\n",
              "  ('taking', 0.019557093626965054),\n",
              "  ('joslin', 0.019480254075801694)],\n",
              " 175: [('stove', 0.061842895702544615),\n",
              "  ('electric', 0.05686955539425957),\n",
              "  ('stoves', 0.039405592462284963),\n",
              "  ('ovens', 0.037159962901554336),\n",
              "  ('brent', 0.03616525431594722),\n",
              "  ('microwaves', 0.03552716334991848),\n",
              "  ('dividian', 0.035511888188686044),\n",
              "  ('ranch', 0.03442283124331806),\n",
              "  ('burns', 0.03231952580662897),\n",
              "  ('survivors', 0.031972103651342884)],\n",
              " 176: [('rpc', 0.04565083927660989),\n",
              "  ('server', 0.034891898185902465),\n",
              "  ('x11r6', 0.02728670721138507),\n",
              "  ('hp', 0.021870167907545843),\n",
              "  ('nui', 0.02182936576910806),\n",
              "  ('tektronix', 0.021630129164771607),\n",
              "  ('motif', 0.01955185618033242),\n",
              "  ('chan', 0.019131145937878662),\n",
              "  ('notifyenablerpcsvc', 0.018188368078870573),\n",
              "  ('xrpc', 0.018188368078870573)],\n",
              " 177: [('saturn', 0.08582966677760848),\n",
              "  ('dealer', 0.04108737452675477),\n",
              "  ('profit', 0.033905204448955326),\n",
              "  ('sl2', 0.025437508866001133),\n",
              "  ('cars', 0.025149593382936333),\n",
              "  ('car', 0.024147324492161826),\n",
              "  ('price', 0.023351698613376055),\n",
              "  ('sc1', 0.022669452867073298),\n",
              "  ('sc2', 0.020780331794817187),\n",
              "  ('civic', 0.018964470399861457)],\n",
              " 178: [('pope', 0.05411472537028794),\n",
              "  ('schism', 0.03554828435480715),\n",
              "  ('church', 0.03341192080532637),\n",
              "  ('catholic', 0.025684494465617776),\n",
              "  ('sspx', 0.022539838586201122),\n",
              "  ('bishop', 0.020012754100100415),\n",
              "  ('archbishop', 0.019669542709983598),\n",
              "  ('schismatic', 0.016965213175499146),\n",
              "  ('bishops', 0.01347176624982478),\n",
              "  ('papal', 0.013284576495794013)],\n",
              " 179: [('ir', 0.042205097231349215),\n",
              "  ('cycle', 0.025140923564884277),\n",
              "  ('xxxxx', 0.0198226960165561),\n",
              "  ('detector', 0.018936428728513786),\n",
              "  ('light', 0.018485397685817497),\n",
              "  ('wazing', 0.018453526025106266),\n",
              "  ('led', 0.017546963714923067),\n",
              "  ('emitting', 0.017456751167958792),\n",
              "  ('brightness', 0.017200054058770377),\n",
              "  ('duty', 0.015971151767825705)],\n",
              " 180: [('lens', 0.09890335700858038),\n",
              "  ('camera', 0.06856387067060019),\n",
              "  ('nikon', 0.03745666116863529),\n",
              "  ('zoom', 0.033929959024099725),\n",
              "  ('canon', 0.030657187832642666),\n",
              "  ('minolta', 0.025273160468910916),\n",
              "  ('mm', 0.024885474136889178),\n",
              "  ('flash', 0.024369228493961535),\n",
              "  ('condition', 0.02389846036724158),\n",
              "  ('vivitar', 0.020784348268407908)],\n",
              " 181: [('midi', 0.10180766939367517),\n",
              "  ('driver', 0.06337625848738433),\n",
              "  ('sound', 0.05184904752231608),\n",
              "  ('speaker', 0.048657357735472896),\n",
              "  ('soundblaster', 0.0470105947669064),\n",
              "  ('drv', 0.04003996138570651),\n",
              "  ('blaster', 0.03316032055594514),\n",
              "  ('alec', 0.028762701475637954),\n",
              "  ('files', 0.02485822548890313),\n",
              "  ('windows', 0.02481084355345946)],\n",
              " 182: [('corn', 0.0761170013627524),\n",
              "  ('seizures', 0.06460677382642055),\n",
              "  ('seizure', 0.05224493462096336),\n",
              "  ('cereals', 0.036092061133070945),\n",
              "  ('kellog', 0.031141489708106658),\n",
              "  ('sharon', 0.02772759258295225),\n",
              "  ('paulson', 0.02608887553291265),\n",
              "  ('ozonoff', 0.025115089205525278),\n",
              "  ('she', 0.02355743212972007),\n",
              "  ('diet', 0.022846892804155406)],\n",
              " 183: [('orion', 0.06312633461401486),\n",
              "  ('prototype', 0.044536640681183966),\n",
              "  ('film', 0.042380263773907086),\n",
              "  ('vacuum', 0.03973900041868094),\n",
              "  ('leigh', 0.038898560371645505),\n",
              "  ('henry', 0.03036859986482828),\n",
              "  ('museum', 0.025775656378754577),\n",
              "  ('spencer', 0.024102809591149257),\n",
              "  ('pusher', 0.02366732426334472),\n",
              "  ('nasm', 0.023126194503327528)],\n",
              " 184: [('ether', 0.05535262555989225),\n",
              "  ('twist', 0.016999751300364997),\n",
              "  ('planets', 0.016186899182845948),\n",
              "  ('mcaloon', 0.015144161162997676),\n",
              "  ('orbit', 0.013196212211747218),\n",
              "  ('earth', 0.011675883270550833),\n",
              "  ('images', 0.011189049177440931),\n",
              "  ('sun', 0.010856272947308666),\n",
              "  ('moon', 0.010679476338662501),\n",
              "  ('its', 0.010383430497026287)],\n",
              " 185: [('logo', 0.09033549498818462),\n",
              "  ('rle', 0.054970967082036165),\n",
              "  ('startup', 0.05247778931047121),\n",
              "  ('galogo', 0.0378686412682686),\n",
              "  ('windows', 0.036784463708512176),\n",
              "  ('ystem', 0.03511034187911707),\n",
              "  ('vgalogo', 0.03457481737554895),\n",
              "  ('files', 0.033504283625912366),\n",
              "  ('indows', 0.032518081385325096),\n",
              "  ('screen', 0.03054465439560004)],\n",
              " 186: [('w4wg', 0.09309765814355718),\n",
              "  ('lastdrive', 0.035543299600784006),\n",
              "  ('novell', 0.029218517797327065),\n",
              "  ('workplace', 0.027727004713036598),\n",
              "  ('workgroups', 0.026196874225538905),\n",
              "  ('lan', 0.025106652852945544),\n",
              "  ('tcpview', 0.023746665095468928),\n",
              "  ('windows', 0.022367805189859438),\n",
              "  ('netx', 0.019272154865558843),\n",
              "  ('netware', 0.01815010841381756)],\n",
              " 187: [('00', 0.06875222530966461),\n",
              "  ('50', 0.05571212168303223),\n",
              "  ('wolverine', 0.043265438762531985),\n",
              "  ('1st', 0.04234204402927308),\n",
              "  ('hulk', 0.035109949645487325),\n",
              "  ('cover', 0.03494886920368797),\n",
              "  ('appears', 0.03379696371948947),\n",
              "  ('art', 0.032303642613855856),\n",
              "  ('comics', 0.03197723522900096),\n",
              "  ('40', 0.025826393491595715)],\n",
              " 188: [('color', 0.030597847197290563),\n",
              "  ('256', 0.023985423459195705),\n",
              "  ('speedstar', 0.02287329100816136),\n",
              "  ('signal', 0.022850180124310372),\n",
              "  ('video', 0.02198192520762057),\n",
              "  ('image', 0.02092768839419189),\n",
              "  ('pov', 0.020585961907345223),\n",
              "  ('view', 0.020581368654444872),\n",
              "  ('input', 0.020306032296507747),\n",
              "  ('tga', 0.01898665471856145)],\n",
              " 189: [('god', 0.03537329944897822),\n",
              "  ('muslims', 0.01812916778492365),\n",
              "  ('islam', 0.015532790623002885),\n",
              "  ('religion', 0.015301772656968022),\n",
              "  ('serbs', 0.014570825035749947),\n",
              "  ('moslems', 0.01231277901445378),\n",
              "  ('serbian', 0.010505575830467384),\n",
              "  ('man', 0.010004418074032592),\n",
              "  ('genocide', 0.0091264116096281),\n",
              "  ('christians', 0.009073987094203055)],\n",
              " 190: [('scanner', 0.05650634533473475),\n",
              "  ('scanners', 0.05169747109015324),\n",
              "  ('grayscale', 0.03867540095887837),\n",
              "  ('scanman', 0.035551461275856386),\n",
              "  ('ocr', 0.029566768524419335),\n",
              "  ('dpi', 0.028565470142188052),\n",
              "  ('logitech', 0.02601428611263664),\n",
              "  ('hand', 0.022263559544915667),\n",
              "  ('satam', 0.02219154855739964),\n",
              "  ('onescanner', 0.02219154855739964)],\n",
              " 191: [('weight', 0.08860234049967022),\n",
              "  ('wa7kgx', 0.04508232495071988),\n",
              "  ('forsberg', 0.043221036679333774),\n",
              "  ('obesity', 0.04193659159862908),\n",
              "  ('diet', 0.039262169747216503),\n",
              "  ('omen', 0.038905091575153095),\n",
              "  ('chuck', 0.036423572119773505),\n",
              "  ('rebound', 0.03350590319054198),\n",
              "  ('cycle', 0.03085218242754855),\n",
              "  ('gordon', 0.03071633196246954)],\n",
              " 192: [('funny', 0.030537407534622026),\n",
              "  ('joke', 0.022809175412292607),\n",
              "  ('tsk', 0.020224622341839363),\n",
              "  ('rahe', 0.018702850332929382),\n",
              "  ('ipser', 0.01780557103306445),\n",
              "  ('arbitron', 0.014962280266343504),\n",
              "  ('newbie', 0.01458224511161199),\n",
              "  ('ed', 0.013538652014738368),\n",
              "  ('humour', 0.012901422515145403),\n",
              "  ('ten', 0.012855053989631843)],\n",
              " 193: [('celp', 0.03596013806653793),\n",
              "  ('compression', 0.03255424347397691),\n",
              "  ('speech', 0.03088290406166226),\n",
              "  ('voice', 0.02346223084275834),\n",
              "  ('toal', 0.022389473656289016),\n",
              "  ('dsp', 0.02085542477657529),\n",
              "  ('samples', 0.01925116133222335),\n",
              "  ('graham', 0.019169164836839135),\n",
              "  ('32bis', 0.017230382976489964),\n",
              "  ('1016', 0.016797066213450028)],\n",
              " 194: [('depression', 0.027768123991054407),\n",
              "  ('islamic', 0.02732312999199678),\n",
              "  ('marriage', 0.023889166129501285),\n",
              "  ('sex', 0.021645494951693),\n",
              "  ('rice', 0.021547581109072695),\n",
              "  ('fred', 0.019728497038199577),\n",
              "  ('women', 0.017575529651237907),\n",
              "  ('men', 0.016525124998232752),\n",
              "  ('maternity', 0.015556218057013804),\n",
              "  ('sexist', 0.015198313450107468)],\n",
              " 195: [('they', 0.016512112947872488),\n",
              "  ('were', 0.015319657884104965),\n",
              "  ('she', 0.015168894030294167),\n",
              "  ('her', 0.01351893489581058),\n",
              "  ('he', 0.013396699798850678),\n",
              "  ('was', 0.013168350709000897),\n",
              "  ('sumgait', 0.011442197435166451),\n",
              "  ('apartment', 0.011257233080043325),\n",
              "  ('had', 0.01090290392209398),\n",
              "  ('we', 0.010787946377202552)],\n",
              " 196: [('sunrise', 0.11326874621709329),\n",
              "  ('sunset', 0.11114454997242633),\n",
              "  ('wetstein', 0.05209987724954349),\n",
              "  ('times', 0.05135151146236819),\n",
              "  ('compute', 0.03830175045168595),\n",
              "  ('daylight', 0.03300019347394516),\n",
              "  ('rostamian', 0.02852954478129743),\n",
              "  ('rouben', 0.02852954478129743),\n",
              "  ('degrees', 0.024816219753132647),\n",
              "  ('hello', 0.023737588413764596)],\n",
              " 197: [('s1', 0.05980570320886185),\n",
              "  ('s2', 0.05807737734605185),\n",
              "  ('key', 0.027244105944919425),\n",
              "  ('serial', 0.025619496669458078),\n",
              "  ('bit', 0.023478180835192847),\n",
              "  ('unit', 0.021654039801020614),\n",
              "  ('u1', 0.019846219366492147),\n",
              "  ('u2', 0.019612680247554304),\n",
              "  ('escrow', 0.018452716175509844),\n",
              "  ('number', 0.017585698292290033)],\n",
              " 198: [('xdm', 0.041213740399474584),\n",
              "  ('xopendisplay', 0.03839688443417848),\n",
              "  ('xwd', 0.03649963195403036),\n",
              "  ('pid', 0.029559593331979425),\n",
              "  ('error', 0.02899752862475572),\n",
              "  ('xterm', 0.027131268844496226),\n",
              "  ('server', 0.023623918621027574),\n",
              "  ('xsession', 0.022595453528015),\n",
              "  ('window', 0.02180913369564676),\n",
              "  ('display', 0.021127646636703135)],\n",
              " 199: [('exhaust', 0.05070241161501397),\n",
              "  ('carbs', 0.03765488548286409),\n",
              "  ('intake', 0.0361037123460999),\n",
              "  ('idle', 0.0267824930677595),\n",
              "  ('air', 0.026384425085787526),\n",
              "  ('pressure', 0.024055976392131268),\n",
              "  ('stroke', 0.01978463500869117),\n",
              "  ('idling', 0.018678723893135935),\n",
              "  ('rpm', 0.01743697826892217),\n",
              "  ('combustion', 0.015460845402770452)],\n",
              " 200: [('witnesses', 0.06958134817929276),\n",
              "  ('trial', 0.05723832458124487),\n",
              "  ('gm', 0.04947933911030437),\n",
              "  ('new', 0.040071690002985084),\n",
              "  ('judge', 0.03047137971694403),\n",
              "  ('justice', 0.02494840999211031),\n",
              "  ('jury', 0.02396044391408152),\n",
              "  ('occupant', 0.02023778689383157),\n",
              "  ('denied', 0.01991654777022294),\n",
              "  ('verdict', 0.019437130621790935)],\n",
              " 201: [('cview', 0.13119713860286847),\n",
              "  ('temp', 0.06045210181125447),\n",
              "  ('zenkar', 0.027105179147988063),\n",
              "  ('files', 0.02666168188870217),\n",
              "  ('directory', 0.026010202099559628),\n",
              "  ('rene', 0.02362122700172529),\n",
              "  ('floppy', 0.022978678582705574),\n",
              "  ('file', 0.022816409655185877),\n",
              "  ('walter', 0.019549558282268648),\n",
              "  ('dir', 0.019132134058056)],\n",
              " 202: [('adl', 0.06891257957476296),\n",
              "  ('bullock', 0.042782899151546136),\n",
              "  ('arens', 0.03169904323959731),\n",
              "  ('yigal', 0.031326026772700494),\n",
              "  ('francisco', 0.0299033929928779),\n",
              "  ('defamation', 0.02915160026382169),\n",
              "  ('anti', 0.024870396666195074),\n",
              "  ('police', 0.024378719959984262),\n",
              "  ('san', 0.023648951564215034),\n",
              "  ('gerard', 0.022960519988275172)],\n",
              " 203: [('iran', 0.028860494857282253),\n",
              "  ('war', 0.02125144329390294),\n",
              "  ('hussein', 0.0145939188284296),\n",
              "  ('iraqi', 0.013294174371363333),\n",
              "  ('bombing', 0.01325719417466567),\n",
              "  ('iraq', 0.011849235326254834),\n",
              "  ('gulf', 0.011805095285307645),\n",
              "  ('kuwait', 0.010255536647546337),\n",
              "  ('jury', 0.009666037390794727),\n",
              "  ('civilians', 0.009248413918375643)],\n",
              " 204: [('tire', 0.0347912077649949),\n",
              "  ('tires', 0.03354622608958313),\n",
              "  ('fluids', 0.022986521862407634),\n",
              "  ('dot', 0.021632858607658095),\n",
              "  ('abs', 0.021051276534282074),\n",
              "  ('brake', 0.020847511288191213),\n",
              "  ('braking', 0.018997318555610716),\n",
              "  ('oil', 0.015726360947789612),\n",
              "  ('car', 0.014573149685389832),\n",
              "  ('front', 0.0140203902223097)],\n",
              " 205: [('ampere', 0.045138513104791306),\n",
              "  ('amp', 0.04206253374481088),\n",
              "  ('company', 0.04178169760631156),\n",
              "  ('bell', 0.0398493150276282),\n",
              "  ('db', 0.03967648561249194),\n",
              "  ('ohmite', 0.03559951178817015),\n",
              "  ('bel', 0.03028630990886976),\n",
              "  ('dbm', 0.027781270483196105),\n",
              "  ('chiu', 0.027533009008063415),\n",
              "  ('ohms', 0.02677194857048796)],\n",
              " 206: [('drive', 0.0512647742255358),\n",
              "  ('boot', 0.046470145569045),\n",
              "  ('floppy', 0.045844412075302),\n",
              "  ('hd', 0.026847943006661983),\n",
              "  ('disk', 0.02638326212558402),\n",
              "  ('problem', 0.020389308933799793),\n",
              "  ('cmos', 0.02005389087566944),\n",
              "  ('balog', 0.019658264996984108),\n",
              "  ('light', 0.019561579872709255),\n",
              "  ('bios', 0.015466382478851954)],\n",
              " 207: [('solar', 0.08100808812798915),\n",
              "  ('sail', 0.06765001406090382),\n",
              "  ('accelerations', 0.042975255797187195),\n",
              "  ('sails', 0.039905918895044086),\n",
              "  ('45g', 0.02544940225915181),\n",
              "  ('laxman', 0.018978940028320245),\n",
              "  ('amruth', 0.018978940028320245),\n",
              "  ('project', 0.018375763640236614),\n",
              "  ('sunlight', 0.018068415176885875),\n",
              "  ('craft', 0.01663432371368607)],\n",
              " 208: [('nc', 0.03457102534564166),\n",
              "  ('civil', 0.02917508547590104),\n",
              "  ('verdict', 0.02473855273145263),\n",
              "  ('rights', 0.02468477192587019),\n",
              "  ('trial', 0.022553044895121277),\n",
              "  ('wilmington', 0.0220982540546746),\n",
              "  ('jeopardy', 0.022021486535087113),\n",
              "  ('farrell', 0.02034862611420103),\n",
              "  ('king', 0.01981323352039472),\n",
              "  ('marine', 0.01889515664220097)],\n",
              " 209: [('ati', 0.0728520174139946),\n",
              "  ('1280x1024', 0.04971325093700812),\n",
              "  ('ultra', 0.048086035867533766),\n",
              "  ('drivers', 0.045649718291770926),\n",
              "  ('2m', 0.03162473502318824),\n",
              "  ('gateway', 0.0309430946658047),\n",
              "  ('card', 0.02875980869898065),\n",
              "  ('flex', 0.02737258211703746),\n",
              "  ('ghosted', 0.024635431656470357),\n",
              "  ('pro', 0.02357537538562345)],\n",
              " 210: [('shaft', 0.1538782912782501),\n",
              "  ('wheelies', 0.09293219849684808),\n",
              "  ('wheelie', 0.0835288713795243),\n",
              "  ('stafford', 0.04340260584902418),\n",
              "  ('motorcycle', 0.04185465250167944),\n",
              "  ('winona', 0.033805851742200556),\n",
              "  ('drives', 0.03266681965185),\n",
              "  ('wheel', 0.028878839534420025),\n",
              "  ('imposible', 0.02845218508617992),\n",
              "  ('centripital', 0.02845218508617992)],\n",
              " 211: [('windows', 0.03130595669590099),\n",
              "  ('galaxy', 0.022020414231615316),\n",
              "  ('gopath', 0.02082396265519649),\n",
              "  ('visix', 0.020257064325017796),\n",
              "  ('window', 0.02004112514446218),\n",
              "  ('amancio', 0.016166154347102883),\n",
              "  ('unify', 0.014551754786046143),\n",
              "  ('gui', 0.013868835590850037),\n",
              "  ('hasty', 0.013100189666347612),\n",
              "  ('toolkit', 0.012200790482272662)],\n",
              " 212: [('octopus', 0.1265528192607531),\n",
              "  ('detroit', 0.08081543707028238),\n",
              "  ('ice', 0.06591014080283929),\n",
              "  ('zazula', 0.0402874257347623),\n",
              "  ('tradition', 0.03882568819766197),\n",
              "  ('octopi', 0.03435842591333837),\n",
              "  ('harold', 0.03197131165997173),\n",
              "  ('game', 0.02887751973060037),\n",
              "  ('cup', 0.02407432014976017),\n",
              "  ('ysebaert', 0.023700802913599012)],\n",
              " 213: [('god', 0.03502204186933541),\n",
              "  ('malcolm', 0.02119172934765409),\n",
              "  ('jews', 0.020128157550545683),\n",
              "  ('law', 0.01960943129941),\n",
              "  ('chosen', 0.019372120667871043),\n",
              "  ('jesus', 0.018389770050095143),\n",
              "  ('israelites', 0.018302986358332593),\n",
              "  ('laws', 0.01818217853785218),\n",
              "  ('salvation', 0.01728631478682117),\n",
              "  ('punishable', 0.01576315746126209)],\n",
              " 214: [('130', 0.0343743216839322),\n",
              "  ('road', 0.017164733129873815),\n",
              "  ('autobahns', 0.016403850443283855),\n",
              "  ('texas', 0.01592756118686064),\n",
              "  ('driving', 0.015833527186072366),\n",
              "  ('autobahn', 0.015724616569823744),\n",
              "  ('speed', 0.014490420323012997),\n",
              "  ('highways', 0.01437625170116987),\n",
              "  ('rail', 0.013850333091494265),\n",
              "  ('miles', 0.013580749444004643)],\n",
              " 215: [('ashok', 0.08488597417345373),\n",
              "  ('trumpet', 0.07837577458602704),\n",
              "  ('wintrumpet', 0.06595974608621455),\n",
              "  ('wim', 0.05978168263822699),\n",
              "  ('aiyar', 0.059150831729430024),\n",
              "  ('ndis', 0.0570118522149525),\n",
              "  ('leuven', 0.050739872707862926),\n",
              "  ('holder', 0.05001126218150916),\n",
              "  ('token', 0.043090691160612565),\n",
              "  ('packet', 0.0359387673943968)],\n",
              " 216: [('her', 0.04239949843684095),\n",
              "  ('denning', 0.03743683136171244),\n",
              "  ('respectability', 0.03644958415378746),\n",
              "  ('she', 0.02853029862066709),\n",
              "  ('dorothy', 0.024442970381214745),\n",
              "  ('toal', 0.023372932301679108),\n",
              "  ('dorthy', 0.02248562711394782),\n",
              "  ('tapped', 0.021430722331558832),\n",
              "  ('baffled', 0.0200174029612842),\n",
              "  ('graham', 0.020011171271340635)]}"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_model.topic_representations_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "def remove_word_from_list(word_to_remove, string_list):\n",
        "    return [string.replace(word_to_remove, '') for string in string_list]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 11314/11314 [00:00<00:00, 11671.07it/s]\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "      <th>Count</th>\n",
              "      <th>Name</th>\n",
              "      <th>Representation</th>\n",
              "      <th>Representative_Docs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1</td>\n",
              "      <td>4112</td>\n",
              "      <td>-1_the_to_and_for</td>\n",
              "      <td>[the, to, and, for, of, in, is, from, it, you]</td>\n",
              "      <td>[From Jonas Flygare Subject Re 18 Israelis mur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>321</td>\n",
              "      <td>0_clipper_encryption_key_chip</td>\n",
              "      <td>[clipper, encryption, key, chip, escrow, keys,...</td>\n",
              "      <td>[From Clipper Chip Announcement Subject text o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>199</td>\n",
              "      <td>1_scsi_drive_ide_controller</td>\n",
              "      <td>[scsi, drive, ide, controller, drives, disk, b...</td>\n",
              "      <td>[From Wayne Smith Subject Re IDE vs SCSI Organ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>196</td>\n",
              "      <td>2_fbi_batf_fire_compound</td>\n",
              "      <td>[fbi, batf, fire, compound, atf, koresh, waco,...</td>\n",
              "      <td>[From Jim De Arras Subject Re BATF FBI Murders...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>106</td>\n",
              "      <td>3_car_cars_integra_convertible</td>\n",
              "      <td>[car, cars, integra, convertible, engine, wago...</td>\n",
              "      <td>[From Chuck Kesler Subject Re Ford Probe Opini...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232</th>\n",
              "      <td>231</td>\n",
              "      <td>10</td>\n",
              "      <td>231_xman_source_9000_xrn</td>\n",
              "      <td>[xman, source, 9000, xrn, rao, hp, xarchie, ba...</td>\n",
              "      <td>[From Chun Hung Lin Subject Re xman source Nnt...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>233</th>\n",
              "      <td>232</td>\n",
              "      <td>10</td>\n",
              "      <td>232_nhl_stars_hockey_halifax</td>\n",
              "      <td>[nhl, stars, hockey, halifax, names, team, div...</td>\n",
              "      <td>[From Karim Edvard Ahmed Subject Re Truly a sa...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>234</th>\n",
              "      <td>233</td>\n",
              "      <td>10</td>\n",
              "      <td>233_wbt_wycliffe_ideological_translators</td>\n",
              "      <td>[wbt, wycliffe, ideological, translators, deno...</td>\n",
              "      <td>[From catherine c bareiss Subject Re phone num...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>235</th>\n",
              "      <td>234</td>\n",
              "      <td>10</td>\n",
              "      <td>234_ir_cycle_xxxxx_wazing</td>\n",
              "      <td>[ir, cycle, xxxxx, wazing, emitting, detector,...</td>\n",
              "      <td>[From David Prutchi Subject Re Long distance I...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>236</th>\n",
              "      <td>235</td>\n",
              "      <td>10</td>\n",
              "      <td>235_manta_opel_buick_kadett</td>\n",
              "      <td>[manta, opel, buick, kadett, blah, gt, gibbons...</td>\n",
              "      <td>[From matthew liggett Subject Re Opel owners N...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>237 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     Topic  Count                                      Name  \\\n",
              "0       -1   4112                         -1_the_to_and_for   \n",
              "1        0    321             0_clipper_encryption_key_chip   \n",
              "2        1    199               1_scsi_drive_ide_controller   \n",
              "3        2    196                  2_fbi_batf_fire_compound   \n",
              "4        3    106            3_car_cars_integra_convertible   \n",
              "..     ...    ...                                       ...   \n",
              "232    231     10                  231_xman_source_9000_xrn   \n",
              "233    232     10              232_nhl_stars_hockey_halifax   \n",
              "234    233     10  233_wbt_wycliffe_ideological_translators   \n",
              "235    234     10                 234_ir_cycle_xxxxx_wazing   \n",
              "236    235     10               235_manta_opel_buick_kadett   \n",
              "\n",
              "                                        Representation  \\\n",
              "0       [the, to, and, for, of, in, is, from, it, you]   \n",
              "1    [clipper, encryption, key, chip, escrow, keys,...   \n",
              "2    [scsi, drive, ide, controller, drives, disk, b...   \n",
              "3    [fbi, batf, fire, compound, atf, koresh, waco,...   \n",
              "4    [car, cars, integra, convertible, engine, wago...   \n",
              "..                                                 ...   \n",
              "232  [xman, source, 9000, xrn, rao, hp, xarchie, ba...   \n",
              "233  [nhl, stars, hockey, halifax, names, team, div...   \n",
              "234  [wbt, wycliffe, ideological, translators, deno...   \n",
              "235  [ir, cycle, xxxxx, wazing, emitting, detector,...   \n",
              "236  [manta, opel, buick, kadett, blah, gt, gibbons...   \n",
              "\n",
              "                                   Representative_Docs  \n",
              "0    [From Jonas Flygare Subject Re 18 Israelis mur...  \n",
              "1    [From Clipper Chip Announcement Subject text o...  \n",
              "2    [From Wayne Smith Subject Re IDE vs SCSI Organ...  \n",
              "3    [From Jim De Arras Subject Re BATF FBI Murders...  \n",
              "4    [From Chuck Kesler Subject Re Ford Probe Opini...  \n",
              "..                                                 ...  \n",
              "232  [From Chun Hung Lin Subject Re xman source Nnt...  \n",
              "233  [From Karim Edvard Ahmed Subject Re Truly a sa...  \n",
              "234  [From catherine c bareiss Subject Re phone num...  \n",
              "235  [From David Prutchi Subject Re Long distance I...  \n",
              "236  [From matthew liggett Subject Re Opel owners N...  \n",
              "\n",
              "[237 rows x 5 columns]"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "docs = []\n",
        "count = 0\n",
        "for text in tqdm(newsgroups_train[\"data\"]):\n",
        "  count+=1\n",
        "  docs.append(clean_text(text) + f\" c_{count}\")\n",
        "\n",
        "topic_model = BERTopic()\n",
        "topics, probs = topic_model.fit_transform(docs)\n",
        "topic_model.get_topic_info()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Document</th>\n",
              "      <th>Topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>From where s my thing Subject WHAT car is this...</td>\n",
              "      <td>235</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>From Guy Kuo Subject SI Clock Poll Final Call ...</td>\n",
              "      <td>205</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>From Thomas E Willis Subject PB questions Orga...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>From Joe Green Subject Re Weitek P9000 Organiz...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>From Jonathan McDowell Subject Re Shuttle Laun...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            Document  Topic\n",
              "0  From where s my thing Subject WHAT car is this...    235\n",
              "1  From Guy Kuo Subject SI Clock Poll Final Call ...    205\n",
              "2  From Thomas E Willis Subject PB questions Orga...     -1\n",
              "3  From Joe Green Subject Re Weitek P9000 Organiz...     -1\n",
              "4  From Jonathan McDowell Subject Re Shuttle Laun...     -1"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df = pd.DataFrame({\"Document\": docs, \"Topic\": topics})\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'total_changes': 2,\n",
              " 'total_same': 3,\n",
              " 'topic_to_noise': 1,\n",
              " 'all_to_noise': 2,\n",
              " 'topic_change': 1,\n",
              " 'topic_same': 0}"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "def compare_topics(df1, df2,topic_num):\n",
        "    ## 1/ Count the number of elements that changed between the two dataframes in Topic column\n",
        "    # Select only the \"Topic\" column from each dataframe\n",
        "    topics1 = df1[\"Topic\"]\n",
        "    topics2 = df2[\"Topic\"]\n",
        "    # Compare the two columns and count the number of changes\n",
        "    changes = (topics1 != topics2).sum()\n",
        "    # Count the number of elements that remained the same\n",
        "    same = (topics1 == topics2).sum()\n",
        "\n",
        "    ## 2/ Count the number of elements that changed to -1 or noise from \"topic_num\" topic. \n",
        "    # Find rows where \"Topic\" changed from a non-negative value to -1\n",
        "    changed_rows = (df1['Topic'] == topic_num) & (df2['Topic'] == -1)\n",
        "    changed_rows_2 = (df1['Topic'] >= 0) & (df2['Topic'] == -1)\n",
        "    # Extract the rows that satisfy the condition\n",
        "    changed_rows_df = df1[changed_rows]\n",
        "    changed_rows_df_2 = df1[changed_rows_2]\n",
        "    # Get the number of rows that changed\n",
        "    top2noise = len(changed_rows_df)\n",
        "    all2noise = len(changed_rows_df_2)\n",
        "\n",
        "    ## 3/ Check number of changes in topic constricted to \"topic_num\" topic. \n",
        "    # Select rows in df1 where \"Topic\" is equal to the given topic\n",
        "    rows_with_given_topic_df1 = df1[df1['Topic'] == topic_num]\n",
        "    # Find the corresponding rows in df2\n",
        "    corresponding_rows_df2 = df2.loc[rows_with_given_topic_df1.index]\n",
        "    # Count the number of rows where the \"Topic\" value changed\n",
        "    num_changed_rows = (rows_with_given_topic_df1['Topic'] != corresponding_rows_df2['Topic']).sum()\n",
        "    # Count the number of rows where the \"Topic\" value remained the same\n",
        "    num_same_rows = (rows_with_given_topic_df1['Topic'] == corresponding_rows_df2['Topic']).sum()\n",
        "\n",
        "\n",
        "    results = {\n",
        "        \"total_changes\" : changes, \n",
        "        \"total_same\" : same,\n",
        "        \"topic_to_noise\" : top2noise,\n",
        "        \"all_to_noise\" : all2noise,\n",
        "        \"topic_change\" : num_changed_rows,\n",
        "        \"topic_same\" : num_same_rows,\n",
        "    }\n",
        "\n",
        "    return results\n",
        "\n",
        "\n",
        "df1 = pd.DataFrame({\"Topic\" : [1, 2, 3, 4, 5]})\n",
        "df2 = pd.DataFrame({\"Topic\" : [-1, 2, 3, 4, -1]})\n",
        "\n",
        "compare_topics(df1, df2, 5)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "def remove_word_from_list(word_to_remove, string_list):\n",
        "    return [string.replace(word_to_remove, '') for string in string_list]\n",
        "\n",
        "def check_topic_for_comprehensiveness_fit( docs: List[str], k: int) -> Dict[str, str]:\n",
        "    \"\"\"\n",
        "    Take as input a list of topics and the initial documents, perturbs the documents\n",
        "    by removing one topic word after another and repeats the modeling to find if the\n",
        "    topic changes.\n",
        "    \"\"\"\n",
        "    ablation_mappings = {}\n",
        "    anchor_topic_model = BERTopic()\n",
        "    topics, probs = anchor_topic_model.fit_transform(docs)\n",
        "    topic_list = anchor_topic_model.get_topic_info()[\"Representation\"]\n",
        "\n",
        "    # forming doc -> topic pairing\n",
        "    df_basic_mapping = pd.DataFrame({\"Document\": docs, \"Topic\": topics})\n",
        "\n",
        "\n",
        "    for word in tqdm(topic_list[k+1]):\n",
        "        new_docs = remove_word_from_list(word, docs)\n",
        "        new_topics, probs = anchor_topic_model.transform(new_docs)\n",
        "        df_new_mapping = pd.DataFrame({\"Document\": docs, \"Topic\": new_topics})\n",
        "\n",
        "        ablation_mappings[word] =  compare_topics(df_basic_mapping, df_new_mapping,k)\n",
        "\n",
        "    return ablation_mappings\n",
        "\n",
        "def save_topic_for_comprehensiveness_fit( docs: List[str], k: int) -> pd.DataFrame:\n",
        "    \"\"\"\n",
        "    Take as input a list of topics and the initial documents, perturbs the documents\n",
        "    by removing one topic word after another and repeats the modeling to find if the\n",
        "    topic changes.\n",
        "    \"\"\"\n",
        "    ablation_mappings = {}\n",
        "    anchor_topic_model = BERTopic()\n",
        "    topics, probs = anchor_topic_model.fit_transform(docs)\n",
        "    topic_list = anchor_topic_model.get_topic_info()#[\"Representation\"]\n",
        "    c_tf_idf_mappings = anchor_topic_model.topic_representations_\n",
        "\n",
        "    # forming doc -> topic pairing\n",
        "    df_basic_mapping = pd.DataFrame({\"Document\": docs, \"Topic\": topics})\n",
        "\n",
        "\n",
        "    for word in tqdm(topic_list[k+1]):\n",
        "        new_docs = remove_word_from_list(word, docs)\n",
        "        new_topics, probs = anchor_topic_model.transform(new_docs)\n",
        "        df_new_mapping = pd.DataFrame({\"Document\": docs, \"Topic\": new_topics})\n",
        "\n",
        "        ablation_mappings[word] =  df_new_mapping\n",
        "\n",
        "    return ablation_mappings,c_tf_idf_mappings,df_basic_mapping,topic_list\n",
        "\n",
        "def save_topic_for_comprehensiveness_fit_cumulative(docs: List[str], k: int) -> pd.DataFrame:\n",
        "    \"\"\"\n",
        "    Take as input a list of topics and the initial documents, perturbs the documents\n",
        "    by removing one topic word after another and repeats the modeling to find if the\n",
        "    topic changes.\n",
        "    \"\"\"\n",
        "    ablation_mappings = {}\n",
        "    anchor_topic_model = BERTopic()\n",
        "    topics, probs = anchor_topic_model.fit_transform(docs)\n",
        "    topic_list = anchor_topic_model.get_topic_info()[\"Representation\"]\n",
        "    c_tf_idf_mappings = anchor_topic_model.topic_representations_\n",
        "\n",
        "    # forming doc -> topic pairing\n",
        "    df_basic_mapping = pd.DataFrame({\"Document\": docs, \"Topic\": topics})\n",
        "\n",
        "    new_docs = docs # Initialize new_docs with the original documents\n",
        "\n",
        "    for  word in tqdm(topic_list[k+1]): \n",
        "        new_docs = remove_word_from_list(word, new_docs)\n",
        "\n",
        "        new_topics, probs = anchor_topic_model.transform(new_docs)\n",
        "        df_new_mapping = pd.DataFrame({\"Document\": new_docs, \"Topic\": new_topics})\n",
        "\n",
        "        ablation_mappings[word] = df_new_mapping\n",
        "\n",
        "    return ablation_mappings, c_tf_idf_mappings, df_basic_mapping, topic_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 10/10 [02:03<00:00, 12.40s/it]\n"
          ]
        }
      ],
      "source": [
        "ablation = check_topic_for_comprehensiveness_fit(docs,2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'clipper': {'total_changes': 1710,\n",
              "  'total_same': 9604,\n",
              "  'topic_to_noise': 50,\n",
              "  'all_to_noise': 1269,\n",
              "  'topic_change': 51,\n",
              "  'topic_same': 293},\n",
              " 'encryption': {'total_changes': 1674,\n",
              "  'total_same': 9640,\n",
              "  'topic_to_noise': 61,\n",
              "  'all_to_noise': 1274,\n",
              "  'topic_change': 64,\n",
              "  'topic_same': 280},\n",
              " 'key': {'total_changes': 1714,\n",
              "  'total_same': 9600,\n",
              "  'topic_to_noise': 47,\n",
              "  'all_to_noise': 1276,\n",
              "  'topic_change': 47,\n",
              "  'topic_same': 297},\n",
              " 'chip': {'total_changes': 1629,\n",
              "  'total_same': 9685,\n",
              "  'topic_to_noise': 53,\n",
              "  'all_to_noise': 1223,\n",
              "  'topic_change': 55,\n",
              "  'topic_same': 289},\n",
              " 'escrow': {'total_changes': 1632,\n",
              "  'total_same': 9682,\n",
              "  'topic_to_noise': 58,\n",
              "  'all_to_noise': 1224,\n",
              "  'topic_change': 60,\n",
              "  'topic_same': 284},\n",
              " 'keys': {'total_changes': 1706,\n",
              "  'total_same': 9608,\n",
              "  'topic_to_noise': 42,\n",
              "  'all_to_noise': 1271,\n",
              "  'topic_change': 42,\n",
              "  'topic_same': 302},\n",
              " 'government': {'total_changes': 1673,\n",
              "  'total_same': 9641,\n",
              "  'topic_to_noise': 41,\n",
              "  'all_to_noise': 1246,\n",
              "  'topic_change': 43,\n",
              "  'topic_same': 301},\n",
              " 'algorithm': {'total_changes': 1687,\n",
              "  'total_same': 9627,\n",
              "  'topic_to_noise': 54,\n",
              "  'all_to_noise': 1269,\n",
              "  'topic_change': 55,\n",
              "  'topic_same': 289},\n",
              " 'enforcement': {'total_changes': 1673,\n",
              "  'total_same': 9641,\n",
              "  'topic_to_noise': 53,\n",
              "  'all_to_noise': 1245,\n",
              "  'topic_change': 54,\n",
              "  'topic_same': 290},\n",
              " 'secure': {'total_changes': 1721,\n",
              "  'total_same': 9593,\n",
              "  'topic_to_noise': 45,\n",
              "  'all_to_noise': 1278,\n",
              "  'topic_change': 46,\n",
              "  'topic_same': 298}}"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ablation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 10/10 [02:04<00:00, 12.45s/it]\n"
          ]
        }
      ],
      "source": [
        "ablation_mappings_cumulative,c_tf_idf_mappings,df_basic_mapping,topic_list = save_topic_for_comprehensiveness_fit_cumulative(docs,2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "      <th>Count</th>\n",
              "      <th>Name</th>\n",
              "      <th>Representation</th>\n",
              "      <th>Representative_Docs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1</td>\n",
              "      <td>3679</td>\n",
              "      <td>-1_to_the_and_of</td>\n",
              "      <td>[to, the, and, of, for, in, is, it, you, from]</td>\n",
              "      <td>[From Brian Ceccarelli 602 621 9615 Subject Re...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>511</td>\n",
              "      <td>0_he_year_baseball_team</td>\n",
              "      <td>[he, year, baseball, team, game, runs, braves,...</td>\n",
              "      <td>[Subject Re Eck vs Rickey was Re Rickey s whin...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>289</td>\n",
              "      <td>1_clipper_key_chip_encryption</td>\n",
              "      <td>[clipper, key, chip, encryption, escrow, keys,...</td>\n",
              "      <td>[From Pat Myrto Subject Re Once tapped your co...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>240</td>\n",
              "      <td>2_gun_guns_firearms_militia</td>\n",
              "      <td>[gun, guns, firearms, militia, weapons, amendm...</td>\n",
              "      <td>[From C D Tavares Subject Re My Gun is like my...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>195</td>\n",
              "      <td>3_tobacco_health_mail_address</td>\n",
              "      <td>[tobacco, health, mail, address, smokeless, ni...</td>\n",
              "      <td>[Subject Space FAQ 02 15 Network Resources Fro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>221</th>\n",
              "      <td>220</td>\n",
              "      <td>10</td>\n",
              "      <td>220_software_level_process_challenger</td>\n",
              "      <td>[software, level, process, challenger, shuttle...</td>\n",
              "      <td>[From fred j mccall 575 3539 Subject Re Level ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>222</th>\n",
              "      <td>221</td>\n",
              "      <td>10</td>\n",
              "      <td>221_solar_sail_sails_accelerations</td>\n",
              "      <td>[solar, sail, sails, accelerations, 45g, proje...</td>\n",
              "      <td>[From Bill Higgins Beam Jockey Subject Re Sola...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>223</th>\n",
              "      <td>222</td>\n",
              "      <td>10</td>\n",
              "      <td>222_server_x11_xwd_xdm</td>\n",
              "      <td>[server, x11, xwd, xdm, usr, pid, sigpipe, ram...</td>\n",
              "      <td>[From S Ramakrishnan Subject Re Mwm title drag...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>224</th>\n",
              "      <td>223</td>\n",
              "      <td>10</td>\n",
              "      <td>223_appcontext_application_callback_huub</td>\n",
              "      <td>[appcontext, application, callback, huub, xtap...</td>\n",
              "      <td>[From Patrick L Mahan Subject Re How do I find...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>225</th>\n",
              "      <td>224</td>\n",
              "      <td>10</td>\n",
              "      <td>224_portable_dock_duo_powerbook</td>\n",
              "      <td>[portable, dock, duo, powerbook, processor, mi...</td>\n",
              "      <td>[From Gene Wright Subject re mac portable vs p...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>226 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     Topic  Count                                      Name  \\\n",
              "0       -1   3679                          -1_to_the_and_of   \n",
              "1        0    511                   0_he_year_baseball_team   \n",
              "2        1    289             1_clipper_key_chip_encryption   \n",
              "3        2    240               2_gun_guns_firearms_militia   \n",
              "4        3    195             3_tobacco_health_mail_address   \n",
              "..     ...    ...                                       ...   \n",
              "221    220     10     220_software_level_process_challenger   \n",
              "222    221     10        221_solar_sail_sails_accelerations   \n",
              "223    222     10                    222_server_x11_xwd_xdm   \n",
              "224    223     10  223_appcontext_application_callback_huub   \n",
              "225    224     10           224_portable_dock_duo_powerbook   \n",
              "\n",
              "                                        Representation  \\\n",
              "0       [to, the, and, of, for, in, is, it, you, from]   \n",
              "1    [he, year, baseball, team, game, runs, braves,...   \n",
              "2    [clipper, key, chip, encryption, escrow, keys,...   \n",
              "3    [gun, guns, firearms, militia, weapons, amendm...   \n",
              "4    [tobacco, health, mail, address, smokeless, ni...   \n",
              "..                                                 ...   \n",
              "221  [software, level, process, challenger, shuttle...   \n",
              "222  [solar, sail, sails, accelerations, 45g, proje...   \n",
              "223  [server, x11, xwd, xdm, usr, pid, sigpipe, ram...   \n",
              "224  [appcontext, application, callback, huub, xtap...   \n",
              "225  [portable, dock, duo, powerbook, processor, mi...   \n",
              "\n",
              "                                   Representative_Docs  \n",
              "0    [From Brian Ceccarelli 602 621 9615 Subject Re...  \n",
              "1    [Subject Re Eck vs Rickey was Re Rickey s whin...  \n",
              "2    [From Pat Myrto Subject Re Once tapped your co...  \n",
              "3    [From C D Tavares Subject Re My Gun is like my...  \n",
              "4    [Subject Space FAQ 02 15 Network Resources Fro...  \n",
              "..                                                 ...  \n",
              "221  [From fred j mccall 575 3539 Subject Re Level ...  \n",
              "222  [From Bill Higgins Beam Jockey Subject Re Sola...  \n",
              "223  [From S Ramakrishnan Subject Re Mwm title drag...  \n",
              "224  [From Patrick L Mahan Subject Re How do I find...  \n",
              "225  [From Gene Wright Subject re mac portable vs p...  \n",
              "\n",
              "[226 rows x 5 columns]"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Document</th>\n",
              "      <th>Topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>From where s my thing Subject WHAT car is this...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>From Guy Kuo Subject SI Clock Poll Final Call ...</td>\n",
              "      <td>193</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>From Thomas E Willis Subject PB questions Orga...</td>\n",
              "      <td>20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>From Joe Green Subject Re Weitek P9000 Organiz...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>From Jonathan McDowell Subject Re Shuttle Laun...</td>\n",
              "      <td>220</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11309</th>\n",
              "      <td>From Jim Zisfein Subject Re Migraines and scan...</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11310</th>\n",
              "      <td>From Subject Screen Death Mac Plus 512 Lines 2...</td>\n",
              "      <td>54</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11311</th>\n",
              "      <td>From Will Estes Subject Mounting CPU Cooler in...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11312</th>\n",
              "      <td>From Steven Collins Subject Re Sphere from 4 p...</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11313</th>\n",
              "      <td>From Kevin J Gunning Subject stolen CBR900RR O...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>11314 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                Document  Topic\n",
              "0      From where s my thing Subject WHAT car is this...     -1\n",
              "1      From Guy Kuo Subject SI Clock Poll Final Call ...    193\n",
              "2      From Thomas E Willis Subject PB questions Orga...     20\n",
              "3      From Joe Green Subject Re Weitek P9000 Organiz...     -1\n",
              "4      From Jonathan McDowell Subject Re Shuttle Laun...    220\n",
              "...                                                  ...    ...\n",
              "11309  From Jim Zisfein Subject Re Migraines and scan...     17\n",
              "11310  From Subject Screen Death Mac Plus 512 Lines 2...     54\n",
              "11311  From Will Estes Subject Mounting CPU Cooler in...     -1\n",
              "11312  From Steven Collins Subject Re Sphere from 4 p...     10\n",
              "11313  From Kevin J Gunning Subject stolen CBR900RR O...     -1\n",
              "\n",
              "[11314 rows x 2 columns]"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_basic_mapping"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {},
      "outputs": [],
      "source": [
        "topic_list.to_csv(\"/home/abpal/WorkFiles/Results/Comprehensiveness_Raw/Base_Results/base.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {},
      "outputs": [],
      "source": [
        "df_basic_mapping.to_csv(\"/home/abpal/WorkFiles/Results/Comprehensiveness_Raw/Base_Results/df_basic_mapping.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {},
      "outputs": [],
      "source": [
        "import json\n",
        "\n",
        "# Specify the file path where you want to save the JSON file\n",
        "file_path = \"/home/abpal/WorkFiles/Results/Comprehensiveness_Raw/Base_Results/c_tf_idf_mappings.json\"\n",
        "\n",
        "# Open the file in write mode and use json.dump to write the dictionary to the file\n",
        "with open(file_path, 'w') as json_file:\n",
        "    json.dump(c_tf_idf_mappings, json_file)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Centroid Test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The cat sits outside \t\t The dog plays in the garden \t\t Score: 0.2838\n",
            "A man is playing guitar \t\t A woman watches TV \t\t Score: -0.0327\n",
            "The new movie is awesome \t\t The new movie is so great \t\t Score: 0.8939\n"
          ]
        }
      ],
      "source": [
        "from sentence_transformers import SentenceTransformer, util\n",
        "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Two lists of sentences\n",
        "sentences1 = ['The cat sits outside',\n",
        "             'A man is playing guitar',\n",
        "             'The new movie is awesome']\n",
        "\n",
        "sentences2 = ['The dog plays in the garden',\n",
        "              'A woman watches TV',\n",
        "              'The new movie is so great']\n",
        "\n",
        "#Compute embedding for both lists\n",
        "embeddings1 = model.encode(sentences1, convert_to_tensor=True)\n",
        "embeddings2 = model.encode(sentences2, convert_to_tensor=True)\n",
        "\n",
        "#Compute cosine-similarities\n",
        "cosine_scores = util.cos_sim(embeddings1, embeddings2)\n",
        "\n",
        "#Output the pairs with their score\n",
        "for i in range(len(sentences1)):\n",
        "    print(\"{} \\t\\t {} \\t\\t Score: {:.4f}\".format(sentences1[i], sentences2[i], cosine_scores[i][i]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([[0.5800]], device='cuda:0')\n",
            "tensor([[0.5995]], device='cuda:0')\n",
            "tensor([[0.5645]], device='cuda:0')\n"
          ]
        }
      ],
      "source": [
        "# distance of each sentence from the centroid\n",
        "centroid = embeddings1.mean(axis=0)\n",
        "for i in range(len(embeddings1)):\n",
        "    centriod_dist = util.cos_sim(centroid,embeddings1[i])\n",
        "    print(centriod_dist)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {},
      "outputs": [],
      "source": [
        "import torch\n",
        "\n",
        "tensor = torch.randn(3, 2)\n",
        "row_to_remove = 0\n",
        "\n",
        "new_tensor = torch.cat((tensor[:row_to_remove], tensor[row_to_remove+1:]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.40729418\n",
            "0.40028247\n",
            "0.41278866\n"
          ]
        }
      ],
      "source": [
        "import numpy as np \n",
        "\n",
        "# movement of centroid for the absence of each sentence\n",
        "centroid_base = embeddings1.mean(axis=0).cpu()\n",
        "for i in range(len(embeddings1)):\n",
        "    embeddings_new = torch.cat((embeddings1[:i], embeddings1[i+1:]))\n",
        "    new_centroid = embeddings_new.mean(axis=0).cpu()\n",
        "    centriod_movt =  np.linalg.norm(centroid_base - new_centroid)\n",
        "\n",
        "    print(centriod_movt)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import torch\n",
        "\n",
        "def centroid_tests(topic_list):\n",
        "    centroid_distance = {}\n",
        "    centroid_movement = {}\n",
        "\n",
        "    #find embeddings\n",
        "    embeddings = model.encode(topic_list, convert_to_tensor=True)\n",
        "    centroid = embeddings.mean(axis=0)\n",
        "\n",
        "    # distance of each sentence from the centroid\n",
        "    for i in range(len(embeddings)):\n",
        "        centriod_dist = util.cos_sim(centroid,embeddings[i])\n",
        "        centroid_distance[topic_list[i]] = float(centriod_dist[0,0])\n",
        "    \n",
        "    # movement of centroid for the absence of each sentence\n",
        "    for i in range(len(embeddings)):\n",
        "        embeddings_new = torch.cat((embeddings[:i], embeddings[i+1:]))\n",
        "        new_centroid = embeddings_new.mean(axis=0).cpu()\n",
        "        centriod_movt =  np.linalg.norm(centroid.cpu() - new_centroid)\n",
        "        centroid_movement[topic_list[i]] = float(centriod_movt)\n",
        "\n",
        "    return {\"centroid_distance\":centroid_distance , \"centroid_movement\":centroid_movement}\n",
        "        \n",
        "# centroid_tests(list(ablation_mappings.keys()))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Comprehensiveness + Centroid Test : "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "k = 200\n",
        "ablation_top_k_topics = {}\n",
        "centroid_test_results = {}\n",
        "\n",
        "for topic_i in tqdm(range(100,k)):\n",
        "    ablation_top_k_topics[f\"Topic_{topic_i}\"] = check_topic_for_comprehensiveness_fit(docs,topic_i)\n",
        "print(\"========Comprehensiveness Ablation Tests done========\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "k = 100\n",
        "ablation_top_k_topics_RAW = {}\n",
        "centroid_test_results = {}\n",
        "\n",
        "for topic_i in tqdm(range(k)):\n",
        "    ablation_top_k_topics_RAW[f\"Topic_{topic_i}\"] = save_topic_for_comprehensiveness_fit(docs,topic_i)\n",
        "print(\"========Comprehensiveness Ablation Tests done========\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 10/10 [01:57<00:00, 11.71s/it]\n",
            "100%|██████████| 10/10 [01:56<00:00, 11.66s/it]\n",
            "100%|██████████| 10/10 [01:58<00:00, 11.82s/it]\n",
            "100%|██████████| 10/10 [01:58<00:00, 11.86s/it]\n",
            "100%|██████████| 10/10 [01:58<00:00, 11.80s/it]\n",
            "100%|██████████| 10/10 [01:59<00:00, 11.94s/it]\n",
            "100%|██████████| 10/10 [01:59<00:00, 11.93s/it]\n",
            "100%|██████████| 10/10 [01:59<00:00, 11.96s/it]\n",
            "100%|██████████| 10/10 [02:00<00:00, 12.05s/it]\n",
            "100%|██████████| 10/10 [01:59<00:00, 11.98s/it]\n",
            "100%|██████████| 10/10 [01:59<00:00, 11.97s/it]]\n",
            "100%|██████████| 10/10 [02:01<00:00, 12.10s/it]]\n",
            "100%|██████████| 10/10 [01:59<00:00, 11.99s/it]]\n",
            "100%|██████████| 10/10 [02:01<00:00, 12.13s/it]]\n",
            "100%|██████████| 10/10 [02:00<00:00, 12.04s/it]]\n",
            "100%|██████████| 10/10 [02:01<00:00, 12.12s/it]]\n",
            "100%|██████████| 10/10 [02:02<00:00, 12.23s/it]]\n",
            "100%|██████████| 10/10 [02:02<00:00, 12.24s/it]]\n",
            "100%|██████████| 10/10 [02:00<00:00, 12.09s/it]]\n",
            "100%|██████████| 10/10 [02:01<00:00, 12.10s/it]]\n",
            "100%|██████████| 20/20 [45:19<00:00, 135.95s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "========Comprehensiveness Ablation Tests done========\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "k = 20\n",
        "ablation_top_k_topics_RAW = {}\n",
        "centroid_test_results = {}\n",
        "\n",
        "for topic_i in tqdm(range(k)):\n",
        "    ablation_top_k_topics_RAW[f\"Topic_{topic_i}\"],c_tf_idf_mappings,df_basic_mapping,topic_list = save_topic_for_comprehensiveness_fit_cumulative(docs,topic_i)\n",
        "print(\"========Comprehensiveness Ablation Tests done========\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0         [the, to, and, for, of, in, is, from, it, you]\n",
              "1      [gun, guns, firearms, weapons, militia, contro...\n",
              "2      [clipper, encryption, key, chip, escrow, keys,...\n",
              "3      [nist, address, mail, ncsl, japanese, computer...\n",
              "4      [moon, lunar, billion, space, mining, prize, h...\n",
              "                             ...                        \n",
              "237    [diamond, cursor, lindbergh, 1024x768x256, dri...\n",
              "238    [aids, god, punishment, sin, jesus, atonement,...\n",
              "239    [eisa, isa, motherboard, bus, 16mb, 486, schau...\n",
              "240    [analog, mf, sampling, digital, 5v, signal, sw...\n",
              "241    [book, scrolls, butera, sea, books, religions,...\n",
              "Name: Representation, Length: 242, dtype: object"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topic_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Document</th>\n",
              "      <th>Topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>From where s my thing Subject WHAT car is this...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>From Guy Kuo Subject SI Clock Poll Final Call ...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>From Thomas E Willis Subject PB questions Orga...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>From Joe Green Subject Re Weitek P9000 Organiz...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>From Jonathan McDowell Subject Re Shuttle Laun...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11309</th>\n",
              "      <td>From Jim Zisfein Subject Re Migraines and scan...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11310</th>\n",
              "      <td>From Subject Screen Death Mac Plus 512 Lines 2...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11311</th>\n",
              "      <td>From Will Estes Subject Mounting CPU Cooler in...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11312</th>\n",
              "      <td>From Steven Collins Subject Re Sphere from 4 p...</td>\n",
              "      <td>78</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11313</th>\n",
              "      <td>From Kevin J Gunning Subject stolen CBR900RR O...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>11314 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                Document  Topic\n",
              "0      From where s my thing Subject WHAT car is this...      5\n",
              "1      From Guy Kuo Subject SI Clock Poll Final Call ...     -1\n",
              "2      From Thomas E Willis Subject PB questions Orga...     -1\n",
              "3      From Joe Green Subject Re Weitek P9000 Organiz...     -1\n",
              "4      From Jonathan McDowell Subject Re Shuttle Laun...     -1\n",
              "...                                                  ...    ...\n",
              "11309  From Jim Zisfein Subject Re Migraines and scan...      6\n",
              "11310  From Subject Screen Death Mac Plus 512 Lines 2...     -1\n",
              "11311  From Will Estes Subject Mounting CPU Cooler in...     -1\n",
              "11312  From Steven Collins Subject Re Sphere from 4 p...     78\n",
              "11313  From Kevin J Gunning Subject stolen CBR900RR O...     -1\n",
              "\n",
              "[11314 rows x 2 columns]"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ablation_top_k_topics_RAW[\"Topic_0\"][\"algorithm\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {},
      "outputs": [],
      "source": [
        "import os \n",
        "def save_comprehensiveness_raw(data,path):\n",
        "    for topic in range(20):\n",
        "        top_data = data[f\"Topic_{topic}\"]\n",
        "        try: \n",
        "            os.mkdir(path+f\"/Topic_{topic}\") \n",
        "        except : \n",
        "            pass\n",
        "        for word in top_data.keys():\n",
        "            top_data[word].to_csv(path_or_buf=path+f\"/Topic_{topic}/{word}.csv\",\n",
        "                                  columns=[\"Topic\"],\n",
        "                                  index=False,\n",
        "                                  header=False\n",
        "                                  )\n",
        "\n",
        "save_comprehensiveness_raw(ablation_top_k_topics_RAW,\"/home/abpal/WorkFiles/Results/Comprehensiveness_Raw_Cumulative\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {},
      "outputs": [],
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "\n",
        "def load_comprehensiveness_raw(path):\n",
        "    loaded_data = {}\n",
        "    for topic in range(100):\n",
        "        topic_path = os.path.join(path, f\"Topic_{topic}\")\n",
        "        if os.path.exists(topic_path):\n",
        "            topic_data = {}\n",
        "            for file_name in os.listdir(topic_path):\n",
        "                word = os.path.splitext(file_name)[0]\n",
        "                file_path = os.path.join(topic_path, file_name)\n",
        "                topic_df = pd.read_csv(file_path, names=[\"Topic\"])\n",
        "                topic_data[word] = topic_df\n",
        "            loaded_data[f\"Topic_{topic}\"] = topic_data\n",
        "    return loaded_data\n",
        "\n",
        "# Example usage:\n",
        "loaded_data = load_comprehensiveness_raw(\"/home/abpal/WorkFiles/Results/Comprehensiveness_Raw_Cumulative\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11309</th>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11310</th>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11311</th>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11312</th>\n",
              "      <td>78</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11313</th>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>11314 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       Topic\n",
              "0          5\n",
              "1         -1\n",
              "2         -1\n",
              "3         -1\n",
              "4         -1\n",
              "...      ...\n",
              "11309      6\n",
              "11310     -1\n",
              "11311     -1\n",
              "11312     78\n",
              "11313     -1\n",
              "\n",
              "[11314 rows x 1 columns]"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "loaded_data[\"Topic_0\"][\"algorithm\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {},
      "outputs": [],
      "source": [
        "df_basic_mapping \n",
        "\n",
        "results = {}\n",
        "\n",
        "for topic_i in loaded_data.keys():\n",
        "    results[topic_i] = {}\n",
        "    for words in loaded_data[topic_i].keys():\n",
        "        results[topic_i][words] = compare_topics(df_basic_mapping,loaded_data[topic_i][words],int(topic_i.split(\"_\")[-1]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'chip': {'total_changes': 8075,\n",
              "  'total_same': 3239,\n",
              "  'topic_to_noise': 179,\n",
              "  'all_to_noise': 1934,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0},\n",
              " 'key': {'total_changes': 8104,\n",
              "  'total_same': 3210,\n",
              "  'topic_to_noise': 163,\n",
              "  'all_to_noise': 1862,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0},\n",
              " 'encryption': {'total_changes': 8074,\n",
              "  'total_same': 3240,\n",
              "  'topic_to_noise': 163,\n",
              "  'all_to_noise': 1798,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0},\n",
              " 'nsa': {'total_changes': 8069,\n",
              "  'total_same': 3245,\n",
              "  'topic_to_noise': 185,\n",
              "  'all_to_noise': 1872,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0},\n",
              " 'keys': {'total_changes': 8097,\n",
              "  'total_same': 3217,\n",
              "  'topic_to_noise': 176,\n",
              "  'all_to_noise': 1883,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0},\n",
              " 'escrow': {'total_changes': 8104,\n",
              "  'total_same': 3210,\n",
              "  'topic_to_noise': 170,\n",
              "  'all_to_noise': 1895,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0},\n",
              " 'government': {'total_changes': 8079,\n",
              "  'total_same': 3235,\n",
              "  'topic_to_noise': 178,\n",
              "  'all_to_noise': 1984,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0},\n",
              " 'algorithm': {'total_changes': 8098,\n",
              "  'total_same': 3216,\n",
              "  'topic_to_noise': 170,\n",
              "  'all_to_noise': 1846,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0},\n",
              " 'secure': {'total_changes': 8136,\n",
              "  'total_same': 3178,\n",
              "  'topic_to_noise': 170,\n",
              "  'all_to_noise': 1906,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0},\n",
              " 'clipper': {'total_changes': 8103,\n",
              "  'total_same': 3211,\n",
              "  'topic_to_noise': 171,\n",
              "  'all_to_noise': 1850,\n",
              "  'topic_change': 369,\n",
              "  'topic_same': 0}}"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results[\"Topic_0\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {},
      "outputs": [],
      "source": [
        "import json\n",
        "import numpy as np\n",
        "\n",
        "class NpEncoder(json.JSONEncoder):\n",
        "    def default(self, obj):\n",
        "        if isinstance(obj, np.integer):\n",
        "            return int(obj)\n",
        "        if isinstance(obj, np.floating):\n",
        "            return float(obj)\n",
        "        if isinstance(obj, np.ndarray):\n",
        "            return obj.tolist()\n",
        "        return super(NpEncoder, self).default(obj)\n",
        "    \n",
        "# Specify the file path where you want to save the JSON file\n",
        "file_path = \"/home/abpal/WorkFiles/Results/Comprehensiveness_Raw_Cumulative/Stats/results.json\"\n",
        "\n",
        "# Open the file in write mode and use json.dump to write the dictionary to the file\n",
        "with open(file_path, 'w') as json_file:\n",
        "    json.dump(results, json_file,cls=NpEncoder)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Understand Stochasticity"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Document</th>\n",
              "      <th>Topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>From where s my thing Subject WHAT car is this...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>From Guy Kuo Subject SI Clock Poll Final Call ...</td>\n",
              "      <td>180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>From Thomas E Willis Subject PB questions Orga...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>From Joe Green Subject Re Weitek P9000 Organiz...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>From Jonathan McDowell Subject Re Shuttle Laun...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11309</th>\n",
              "      <td>From Jim Zisfein Subject Re Migraines and scan...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11310</th>\n",
              "      <td>From Subject Screen Death Mac Plus 512 Lines 2...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11311</th>\n",
              "      <td>From Will Estes Subject Mounting CPU Cooler in...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11312</th>\n",
              "      <td>From Steven Collins Subject Re Sphere from 4 p...</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11313</th>\n",
              "      <td>From Kevin J Gunning Subject stolen CBR900RR O...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>11314 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                Document  Topic\n",
              "0      From where s my thing Subject WHAT car is this...     -1\n",
              "1      From Guy Kuo Subject SI Clock Poll Final Call ...    180\n",
              "2      From Thomas E Willis Subject PB questions Orga...     -1\n",
              "3      From Joe Green Subject Re Weitek P9000 Organiz...     -1\n",
              "4      From Jonathan McDowell Subject Re Shuttle Laun...     -1\n",
              "...                                                  ...    ...\n",
              "11309  From Jim Zisfein Subject Re Migraines and scan...     -1\n",
              "11310  From Subject Screen Death Mac Plus 512 Lines 2...      5\n",
              "11311  From Will Estes Subject Mounting CPU Cooler in...     -1\n",
              "11312  From Steven Collins Subject Re Sphere from 4 p...     13\n",
              "11313  From Kevin J Gunning Subject stolen CBR900RR O...     -1\n",
              "\n",
              "[11314 rows x 2 columns]"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "anchor_topic_model = BERTopic()\n",
        "topics, probs = anchor_topic_model.fit_transform(docs)\n",
        "topic_list = anchor_topic_model.get_topic_info()[\"Representation\"]\n",
        "\n",
        "# forming doc -> topic pairing\n",
        "df_basic_mapping_1 = pd.DataFrame({\"Document\": docs, \"Topic\": topics})\n",
        "df_basic_mapping_1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[-1, 180, -1, -1, -1]"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "topics[:5]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Document</th>\n",
              "      <th>Topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>From where s my thing Subject WHAT car is this...</td>\n",
              "      <td>190</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>From Guy Kuo Subject SI Clock Poll Final Call ...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>From Thomas E Willis Subject PB questions Orga...</td>\n",
              "      <td>121</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>From Joe Green Subject Re Weitek P9000 Organiz...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>From Jonathan McDowell Subject Re Shuttle Laun...</td>\n",
              "      <td>175</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11309</th>\n",
              "      <td>From Jim Zisfein Subject Re Migraines and scan...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11310</th>\n",
              "      <td>From Subject Screen Death Mac Plus 512 Lines 2...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11311</th>\n",
              "      <td>From Will Estes Subject Mounting CPU Cooler in...</td>\n",
              "      <td>210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11312</th>\n",
              "      <td>From Steven Collins Subject Re Sphere from 4 p...</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11313</th>\n",
              "      <td>From Kevin J Gunning Subject stolen CBR900RR O...</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>11314 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                Document  Topic\n",
              "0      From where s my thing Subject WHAT car is this...    190\n",
              "1      From Guy Kuo Subject SI Clock Poll Final Call ...     -1\n",
              "2      From Thomas E Willis Subject PB questions Orga...    121\n",
              "3      From Joe Green Subject Re Weitek P9000 Organiz...     -1\n",
              "4      From Jonathan McDowell Subject Re Shuttle Laun...    175\n",
              "...                                                  ...    ...\n",
              "11309  From Jim Zisfein Subject Re Migraines and scan...     -1\n",
              "11310  From Subject Screen Death Mac Plus 512 Lines 2...     -1\n",
              "11311  From Will Estes Subject Mounting CPU Cooler in...    210\n",
              "11312  From Steven Collins Subject Re Sphere from 4 p...      9\n",
              "11313  From Kevin J Gunning Subject stolen CBR900RR O...     -1\n",
              "\n",
              "[11314 rows x 2 columns]"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "anchor_topic_model = BERTopic()\n",
        "topics, probs = anchor_topic_model.fit_transform(docs)\n",
        "topic_list = anchor_topic_model.get_topic_info()[\"Representation\"]\n",
        "\n",
        "# forming doc -> topic pairing\n",
        "df_basic_mapping_2 = pd.DataFrame({\"Document\": docs, \"Topic\": topics})\n",
        "df_basic_mapping_2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "name 'compare_topics' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mcompare_topics\u001b[49m(df_basic_mapping_1, df_basic_mapping_2,\u001b[38;5;241m6\u001b[39m)\n",
            "\u001b[0;31mNameError\u001b[0m: name 'compare_topics' is not defined"
          ]
        }
      ],
      "source": [
        "compare_topics(df_basic_mapping_1, df_basic_mapping_2,6)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Common Buckets: 4\n"
          ]
        }
      ],
      "source": [
        "def count_common_buckets(list1, list2):\n",
        "    # Create dictionaries to store buckets for each list\n",
        "    buckets1 = {}\n",
        "    buckets2 = {}\n",
        "\n",
        "    # Iterate through the lists to form buckets\n",
        "    for index, (num1, num2) in enumerate(zip(list1, list2)):\n",
        "        if num1 != -1:\n",
        "            if num1 not in buckets1:\n",
        "                buckets1[num1] = []\n",
        "            buckets1[num1].append(index)\n",
        "\n",
        "        if num2 != -1:\n",
        "            if num2 not in buckets2:\n",
        "                buckets2[num2] = []\n",
        "            buckets2[num2].append(index)\n",
        "\n",
        "    # Compare the similarity of bucket structures\n",
        "    common_buckets_count = 0\n",
        "    for key1 in buckets1:\n",
        "        for key2 in buckets2:\n",
        "            common_indices = set(buckets1[key1]) & set(buckets2[key2])\n",
        "            if common_indices:\n",
        "                common_buckets_count += 1\n",
        "\n",
        "    return common_buckets_count\n",
        "\n",
        "# Example usage:\n",
        "list1 = [1,2,3,4,1,2,3,4]\n",
        "list2 = [2,3,4,1,2,3,4,1]\n",
        "\n",
        "common_buckets = count_common_buckets(list1, list2)\n",
        "\n",
        "print(\"Common Buckets:\", common_buckets)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[292, 285, 292, 284, 291]"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "def run_stochasticity_count(runs: int):\n",
        "    counts = []\n",
        "    for i in tqdm(range(runs)):\n",
        "        anchor_topic_model = BERTopic()\n",
        "        topics, probs = anchor_topic_model.fit_transform(docs)\n",
        "        topic_list1 = anchor_topic_model.get_topic_info()[\"Representation\"]\n",
        "\n",
        "        # forming doc -> topic pairing\n",
        "        df_basic_mapping_1 = pd.DataFrame({\"Document\": docs, \"Topic\": topics})\n",
        "\n",
        "        anchor_topic_model = BERTopic()\n",
        "        topics, probs = anchor_topic_model.fit_transform(docs)\n",
        "        topic_list2 = anchor_topic_model.get_topic_info()[\"Representation\"]\n",
        "\n",
        "        # forming doc -> topic pairing\n",
        "        df_basic_mapping_2 = pd.DataFrame({\"Document\": docs, \"Topic\": topics})\n",
        "\n",
        "        counts.append(count_common_buckets(df_basic_mapping_1[\"Topic\"], df_basic_mapping_2[\"Topic\"]))\n",
        "\n",
        "    return counts, f\"Total Topics in mapping 1 : {len(topic_list1)} mapping 2 : {len(topic_list2)}\"\n",
        "\n",
        "run_stochasticity_count(\n",
        "    runs = 5\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Run centroid tests, save etc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 77,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 100/100 [00:00<00:00, 181.88it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "========Centroid Tests done========\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "import numpy as np \n",
        "import torch\n",
        "for topic_i in tqdm(range(100,k)):\n",
        "    centroid_test_results[f\"Topic_{topic_i}\"] = centroid_tests(list(ablation_top_k_topics[f\"Topic_{topic_i}\"].keys()))\n",
        "print(\"========Centroid Tests done========\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {},
      "outputs": [],
      "source": [
        "import json\n",
        "import numpy as np\n",
        "\n",
        "class NpEncoder(json.JSONEncoder):\n",
        "    def default(self, obj):\n",
        "        if isinstance(obj, np.integer):\n",
        "            return int(obj)\n",
        "        if isinstance(obj, np.floating):\n",
        "            return float(obj)\n",
        "        if isinstance(obj, np.ndarray):\n",
        "            return obj.tolist()\n",
        "        return super(NpEncoder, self).default(obj)\n",
        "\n",
        "\n",
        "with open('Results/Ablation_Final/ablation_RAW_100.json', 'w') as fp:\n",
        "    json.dump(ablation_top_k_topics_RAW, fp,cls=NpEncoder)\n",
        "\n",
        "# with open('Results/Ablation_Final/centroid_result_200.json', 'w') as fp:\n",
        "    # json.dump(centroid_test_results, fp,cls=NpEncoder)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "while True:\n",
        "    pass"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# c-tf-idf numbers and rank correlation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {},
      "outputs": [],
      "source": [
        "anchor_topic_model = BERTopic()\n",
        "topics, probs = anchor_topic_model.fit_transform(docs)\n",
        "topic_list = anchor_topic_model.get_topic_info()[\"Representation\"]\n",
        "\n",
        "# forming doc -> topic pairing\n",
        "df_basic_mapping_ = pd.DataFrame({\"Document\": docs, \"Topic\": topics})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "245"
            ]
          },
          "execution_count": 45,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(set(topics))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{-1: [('the', 0.004001173393700668),\n",
              "  ('to', 0.0038867828699836234),\n",
              "  ('for', 0.0037828558747706113),\n",
              "  ('of', 0.0037471069328812957),\n",
              "  ('and', 0.0037246135155925863),\n",
              "  ('in', 0.0036972309257993475),\n",
              "  ('is', 0.003580020514716617),\n",
              "  ('from', 0.003566082473923622),\n",
              "  ('it', 0.003534473143340908),\n",
              "  ('you', 0.0035188762860559066)],\n",
              " 0: [('clipper', 0.017168714838490067),\n",
              "  ('key', 0.015362246416456277),\n",
              "  ('encryption', 0.01518368823616912),\n",
              "  ('chip', 0.015018226152477948),\n",
              "  ('escrow', 0.010645646121140998),\n",
              "  ('keys', 0.00945704480498497),\n",
              "  ('government', 0.008645721787624669),\n",
              "  ('crypto', 0.007864839285614805),\n",
              "  ('secure', 0.007650991382692927),\n",
              "  ('nsa', 0.00749946145173624)],\n",
              " 1: [('keyboard', 0.00977220058729129),\n",
              "  ('mail', 0.007593759020955795),\n",
              "  ('health', 0.007308016243972676),\n",
              "  ('tobacco', 0.0070277384452378765),\n",
              "  ('typing', 0.0062473224906844085),\n",
              "  ('address', 0.006163866360147878),\n",
              "  ('rsi', 0.0061630756713715405),\n",
              "  ('nist', 0.005780655340203438),\n",
              "  ('list', 0.0055840289281771606),\n",
              "  ('smokeless', 0.00556179999611578)],\n",
              " 2: [('car', 0.021954616978653453),\n",
              "  ('cars', 0.012377957305741366),\n",
              "  ('engine', 0.009217834497181934),\n",
              "  ('convertible', 0.007220454012807442),\n",
              "  ('integra', 0.007168813669299091),\n",
              "  ('toyota', 0.007019816768254487),\n",
              "  ('wagon', 0.006465251032708518),\n",
              "  ('ford', 0.006237961466707493),\n",
              "  ('nissan', 0.005961053256769641),\n",
              "  ('gt', 0.005871982298411333)],\n",
              " 3: [('windows', 0.020481338248660615),\n",
              "  ('swap', 0.017613315895194683),\n",
              "  ('memory', 0.016090782255729543),\n",
              "  ('dos', 0.013836927468768875),\n",
              "  ('allocation', 0.012974600939903193),\n",
              "  ('file', 0.011465629981368936),\n",
              "  ('disk', 0.011117704435589928),\n",
              "  ('ram', 0.010810377377504735),\n",
              "  ('linked', 0.009967265597442524),\n",
              "  ('duo', 0.00996189093469256)],\n",
              " 4: [('team', 0.01243123720573452),\n",
              "  ('he', 0.01061405632431241),\n",
              "  ('pitching', 0.010106193676178979),\n",
              "  ('year', 0.009920503361471062),\n",
              "  ('win', 0.008356814675114328),\n",
              "  ('game', 0.008346994279037252),\n",
              "  ('sox', 0.008199564933049287),\n",
              "  ('cubs', 0.007876886247610087),\n",
              "  ('braves', 0.007380176628847028),\n",
              "  ('last', 0.007056217974003195)],\n",
              " 5: [('moon', 0.027367357470541694),\n",
              "  ('lunar', 0.016681883756698268),\n",
              "  ('billion', 0.012418735685542045),\n",
              "  ('mining', 0.011535825911753778),\n",
              "  ('prize', 0.011111564273156748),\n",
              "  ('space', 0.010907743099877236),\n",
              "  ('cost', 0.008751235430321533),\n",
              "  ('henry', 0.007522240341828749),\n",
              "  ('base', 0.007433158863617747),\n",
              "  ('launch', 0.007203948428037355)],\n",
              " 6: [('sale', 0.030035289040566494),\n",
              "  ('shipping', 0.0176883034097093),\n",
              "  ('offer', 0.013493078686116142),\n",
              "  ('00', 0.013201802344439047),\n",
              "  ('software', 0.012508806126951207),\n",
              "  ('borland', 0.011248573018903568),\n",
              "  ('pc', 0.010776820163103516),\n",
              "  ('card', 0.010655838933561621),\n",
              "  ('items', 0.010632941507295456),\n",
              "  ('manuals', 0.010437865303408234)],\n",
              " 7: [('israeli', 0.02448212643278338),\n",
              "  ('israel', 0.014327881252127724),\n",
              "  ('anas', 0.010386357608365924),\n",
              "  ('soldiers', 0.010148836009938189),\n",
              "  ('arab', 0.009837898713995933),\n",
              "  ('omran', 0.007691846181718866),\n",
              "  ('policy', 0.006307005210000524),\n",
              "  ('israelis', 0.006062398497793137),\n",
              "  ('yassin', 0.006054516082760001),\n",
              "  ('terrorism', 0.006016508284855252)],\n",
              " 8: [('armenian', 0.024756663872342214),\n",
              "  ('turkish', 0.02006881099788295),\n",
              "  ('armenians', 0.018237020058961145),\n",
              "  ('argic', 0.013645038479712747),\n",
              "  ('serdar', 0.01353207961223496),\n",
              "  ('genocide', 0.012934094299656043),\n",
              "  ('turks', 0.011774529978080567),\n",
              "  ('armenia', 0.011053913944614933),\n",
              "  ('soviet', 0.010080645588636352),\n",
              "  ('turkey', 0.008799704363804624)],\n",
              " 9: [('gay', 0.02907542519698843),\n",
              "  ('cramer', 0.02335165327563151),\n",
              "  ('men', 0.021762298959643014),\n",
              "  ('homosexual', 0.021471939747295016),\n",
              "  ('clayton', 0.018486352002452286),\n",
              "  ('sexual', 0.0176643964357363),\n",
              "  ('homosexuals', 0.013777384809969333),\n",
              "  ('male', 0.01294411011825512),\n",
              "  ('promiscuous', 0.012838898040877154),\n",
              "  ('study', 0.012322681593874903)],\n",
              " 10: [('jews', 0.016257580646922173),\n",
              "  ('holocaust', 0.013038887886521064),\n",
              "  ('nazi', 0.012735588636058046),\n",
              "  ('zionism', 0.011501156703912992),\n",
              "  ('hitler', 0.009822194185315644),\n",
              "  ('anti', 0.009080128371507706),\n",
              "  ('germany', 0.008468343706468989),\n",
              "  ('german', 0.008366242742811088),\n",
              "  ('nazis', 0.007301139304627226),\n",
              "  ('racism', 0.007190993153885324)],\n",
              " 11: [('espn', 0.03433413379516571),\n",
              "  ('abc', 0.02660361689993632),\n",
              "  ('coverage', 0.019107033226598253),\n",
              "  ('games', 0.01871498122736845),\n",
              "  ('game', 0.018437310102874964),\n",
              "  ('hockey', 0.01604840563897196),\n",
              "  ('sports', 0.0154764387850431),\n",
              "  ('wip', 0.014719683058762425),\n",
              "  ('tv', 0.012698592095295157),\n",
              "  ('baseball', 0.012078102327039786)],\n",
              " 12: [('db', 0.05709518957499985),\n",
              "  ('0d', 0.038666127733645),\n",
              "  ('cx', 0.031114030824852125),\n",
              "  ('mov', 0.021200222828273056),\n",
              "  ('bh', 0.019683708374447485),\n",
              "  ('145', 0.017638578150035552),\n",
              "  ('s6', 0.015394059095740518),\n",
              "  ('si', 0.013350532862896558),\n",
              "  ('6um', 0.012840092238672066),\n",
              "  ('sc', 0.011811393713136959)],\n",
              " 13: [('gun', 0.024924034068677728),\n",
              "  ('guns', 0.01860268623472035),\n",
              "  ('000', 0.010613082671416271),\n",
              "  ('crime', 0.010007192325549013),\n",
              "  ('homicides', 0.009525031526806657),\n",
              "  ('handgun', 0.008584045403459247),\n",
              "  ('deaths', 0.008422083074104176),\n",
              "  ('rate', 0.007978011008296983),\n",
              "  ('self', 0.007850407725559647),\n",
              "  ('knife', 0.007721318730675423)],\n",
              " 14: [('islamic', 0.02684314589038166),\n",
              "  ('islam', 0.026427542873323284),\n",
              "  ('rushdie', 0.02298945995622116),\n",
              "  ('gregg', 0.01841953497606197),\n",
              "  ('muslim', 0.0162148875308869),\n",
              "  ('jaeger', 0.013348337690222365),\n",
              "  ('muslims', 0.013271142438597281),\n",
              "  ('law', 0.009023083545757666),\n",
              "  ('khomeini', 0.009011248152251948),\n",
              "  ('fatwa', 0.008597343256526665)],\n",
              " 15: [('traffic', 0.012871705563410617),\n",
              "  ('lane', 0.011263690127122946),\n",
              "  ('speed', 0.010083800809327371),\n",
              "  ('130', 0.009538565687549901),\n",
              "  ('nmm', 0.009385758231164124),\n",
              "  ('car', 0.009245936498952122),\n",
              "  ('cop', 0.00879117278214474),\n",
              "  ('ticket', 0.008238034858794662),\n",
              "  ('fast', 0.007657601033524206),\n",
              "  ('driving', 0.007579595282117307)],\n",
              " 16: [('drive', 0.0478636637586035),\n",
              "  ('drives', 0.027241140713382257),\n",
              "  ('controller', 0.024771674826989516),\n",
              "  ('disk', 0.024151923226173874),\n",
              "  ('bios', 0.02251527092664455),\n",
              "  ('slave', 0.021023930306557335),\n",
              "  ('jumper', 0.020568553967068137),\n",
              "  ('hard', 0.019358895713129425),\n",
              "  ('master', 0.017956879709572948),\n",
              "  ('floppy', 0.016054607106699623)],\n",
              " 17: [('printer', 0.04940263859481829),\n",
              "  ('deskjet', 0.028177847436965053),\n",
              "  ('ink', 0.026874626229513164),\n",
              "  ('printers', 0.02573800231887038),\n",
              "  ('hp', 0.02391623291876704),\n",
              "  ('laser', 0.021725666769914238),\n",
              "  ('bubblejet', 0.019119910050252542),\n",
              "  ('canon', 0.015569728577836802),\n",
              "  ('toner', 0.01556968890530761),\n",
              "  ('print', 0.013424293528641593)],\n",
              " 18: [('card', 0.039800751382915146),\n",
              "  ('winbench', 0.024462166032860083),\n",
              "  ('video', 0.023334694053404936),\n",
              "  ('diamond', 0.021958282805701692),\n",
              "  ('stealth', 0.020120766750883104),\n",
              "  ('vlb', 0.01884491091429263),\n",
              "  ('winmarks', 0.015052075524309112),\n",
              "  ('ati', 0.013751691748267594),\n",
              "  ('s3', 0.013731725677803662),\n",
              "  ('cards', 0.013465356063920238)],\n",
              " 19: [('jesus', 0.018718205301039282),\n",
              "  ('matthew', 0.013680983183882308),\n",
              "  ('judas', 0.011110997398187432),\n",
              "  ('he', 0.00918966995329814),\n",
              "  ('prophecy', 0.00917856110881479),\n",
              "  ('his', 0.00830723714765471),\n",
              "  ('luke', 0.008213636300108337),\n",
              "  ('resurrection', 0.007663179763647753),\n",
              "  ('messiah', 0.007310324431212795),\n",
              "  ('christ', 0.006875841219065571)],\n",
              " 20: [('3d', 0.04144908511914659),\n",
              "  ('graphics', 0.031004960759264847),\n",
              "  ('library', 0.021126207202158017),\n",
              "  ('package', 0.016716055251159902),\n",
              "  ('grafsys', 0.016457116158042493),\n",
              "  ('phigs', 0.015693349851770216),\n",
              "  ('cad', 0.015414430591160839),\n",
              "  ('carpenter', 0.012359640723881049),\n",
              "  ('mac', 0.012185575816176701),\n",
              "  ('please', 0.010668685298064931)],\n",
              " 21: [('32', 0.016572119038147405),\n",
              "  ('motherboard', 0.014969505582486654),\n",
              "  ('os', 0.013992820181720815),\n",
              "  ('ibm', 0.0138620036054732),\n",
              "  ('bit', 0.013585962362491887),\n",
              "  ('mac', 0.011119324478834486),\n",
              "  ('486', 0.010521886358567044),\n",
              "  ('ram', 0.010424704202110169),\n",
              "  ('cpu', 0.009514485942487235),\n",
              "  ('scsi', 0.009510640227673185)],\n",
              " 22: [('modem', 0.07677956417184394),\n",
              "  ('fax', 0.033536767609869705),\n",
              "  ('baud', 0.01801779211827186),\n",
              "  ('modems', 0.016865335638866035),\n",
              "  ('9600', 0.013037105491553473),\n",
              "  ('4k', 0.012485631072259125),\n",
              "  ('2400', 0.010783993871488876),\n",
              "  ('bps', 0.010028542685810365),\n",
              "  ('internal', 0.009787556943630966),\n",
              "  ('td', 0.009481533099422908)],\n",
              " 23: [('scsi', 0.10552916806506023),\n",
              "  ('ide', 0.041918252488553465),\n",
              "  ('drive', 0.02023228175021546),\n",
              "  ('mac', 0.01501881832056214),\n",
              "  ('controller', 0.014030990947855806),\n",
              "  ('bus', 0.013328799189197845),\n",
              "  ('drives', 0.012922396999643695),\n",
              "  ('asynchronous', 0.012755564475026118),\n",
              "  ('pc', 0.012635603751359541),\n",
              "  ('isa', 0.012257730554777342)],\n",
              " 24: [('israel', 0.03336101307479928),\n",
              "  ('lebanese', 0.028579594557412605),\n",
              "  ('lebanon', 0.021865292803168138),\n",
              "  ('israeli', 0.02010949686250773),\n",
              "  ('hezbollah', 0.015242363759460515),\n",
              "  ('civilians', 0.014741632408584823),\n",
              "  ('attacks', 0.014646196438979676),\n",
              "  ('villages', 0.014194257147762441),\n",
              "  ('soldiers', 0.010725915056666816),\n",
              "  ('peace', 0.01053939427349796)],\n",
              " 25: [('objective', 0.04271131898441718),\n",
              "  ('morality', 0.041585745934052475),\n",
              "  ('values', 0.02399683610903115),\n",
              "  ('dwyer', 0.016868704256167908),\n",
              "  ('2000', 0.013842338026921691),\n",
              "  ('moral', 0.012327543664233316),\n",
              "  ('frank', 0.012219492142797548),\n",
              "  ('christian', 0.01142810887693412),\n",
              "  ('hudson', 0.010552344898951098),\n",
              "  ('say', 0.010452045497035151)],\n",
              " 26: [('alomar', 0.039439574962793884),\n",
              "  ('runs', 0.027743030278082392),\n",
              "  ('baerga', 0.026027259584827683),\n",
              "  ('obp', 0.017405318997243773),\n",
              "  ('rbi', 0.01672300212512755),\n",
              "  ('pitching', 0.015067646545973564),\n",
              "  ('rbis', 0.014809744554102087),\n",
              "  ('team', 0.01406077475443779),\n",
              "  ('sandberg', 0.013040551429040205),\n",
              "  ('slg', 0.01167482997372116)],\n",
              " 27: [('gun', 0.01756370753804802),\n",
              "  ('firearms', 0.01536634302910455),\n",
              "  ('firearm', 0.014153492443959652),\n",
              "  ('guns', 0.012253535153074186),\n",
              "  ('weapon', 0.010993286844594581),\n",
              "  ('weapons', 0.009178640628354292),\n",
              "  ('2923', 0.00879675157320122),\n",
              "  ('section', 0.008627001434964586),\n",
              "  ('license', 0.008592327938683659),\n",
              "  ('hb', 0.008526009004887694)],\n",
              " 28: [('atheism', 0.02346749677323722),\n",
              "  ('atheists', 0.019488717466247623),\n",
              "  ('atheist', 0.010684743875035633),\n",
              "  ('argument', 0.010242174664201307),\n",
              "  ('god', 0.01015949393331548),\n",
              "  ('fallacy', 0.009587355208491737),\n",
              "  ('religion', 0.009534367908701597),\n",
              "  ('alt', 0.00874860560815523),\n",
              "  ('religious', 0.008664073570701748),\n",
              "  ('belief', 0.006118806044819313)],\n",
              " 29: [('sky', 0.02199249207425913),\n",
              "  ('rockets', 0.01754169015503924),\n",
              "  ('billboard', 0.016882152230630075),\n",
              "  ('advertising', 0.016229946714138667),\n",
              "  ('space', 0.015407161217079801),\n",
              "  ('inflatable', 0.014401599720404838),\n",
              "  ('vandalizing', 0.012587155304749081),\n",
              "  ('nasa', 0.011407646828051284),\n",
              "  ('launch', 0.011166331412802521),\n",
              "  ('rocketry', 0.011144352149863746)],\n",
              " 30: [('radar', 0.06227979115954339),\n",
              "  ('detector', 0.03983524491628768),\n",
              "  ('detectors', 0.025481464384482278),\n",
              "  ('car', 0.011964639008685003),\n",
              "  ('traffic', 0.0111567390102422),\n",
              "  ('police', 0.009543464870082152),\n",
              "  ('alarm', 0.009305220932231113),\n",
              "  ('pavement', 0.007057190122943906),\n",
              "  ('beam', 0.00685018570444929),\n",
              "  ('photo', 0.0066829961195978595)],\n",
              " 31: [('msg', 0.06315316882308174),\n",
              "  ('food', 0.0285163038068782),\n",
              "  ('sensitivity', 0.021225729713303092),\n",
              "  ('chinese', 0.020127847141925347),\n",
              "  ('glutamate', 0.019152199176700236),\n",
              "  ('superstition', 0.017716819085147165),\n",
              "  ('foods', 0.013147840734474643),\n",
              "  ('olney', 0.012772628404117055),\n",
              "  ('aspartame', 0.011601933545364792),\n",
              "  ('restaurant', 0.011577126538460998)],\n",
              " 32: [('space', 0.024103521479678577),\n",
              "  ('station', 0.02247124385622339),\n",
              "  ('redesign', 0.018123209459385673),\n",
              "  ('shuttle', 0.01644109210709083),\n",
              "  ('dc', 0.015114562684694346),\n",
              "  ('nasa', 0.013489395755602918),\n",
              "  ('ssf', 0.011388732348942),\n",
              "  ('option', 0.00978903572551998),\n",
              "  ('sdio', 0.008383095529118919),\n",
              "  ('orbit', 0.007719481987284769)],\n",
              " 33: [('space', 0.0276520890874163),\n",
              "  ('satellite', 0.01477047583759041),\n",
              "  ('nasa', 0.012719837581663528),\n",
              "  ('astronaut', 0.009066521910398015),\n",
              "  ('satellites', 0.008774326043399967),\n",
              "  ('astronomy', 0.007076375889865879),\n",
              "  ('flight', 0.006981879483306843),\n",
              "  ('propulsion', 0.006948629294550302),\n",
              "  ('shuttle', 0.006913210011818387),\n",
              "  ('spacecraft', 0.006661787150298886)],\n",
              " 34: [('truth', 0.02539269349519567),\n",
              "  ('arrogance', 0.018931639708840247),\n",
              "  ('absolute', 0.017964824506023488),\n",
              "  ('christians', 0.017786113938599215),\n",
              "  ('absolutes', 0.016741523910269566),\n",
              "  ('arrogant', 0.012654344180553946),\n",
              "  ('darren', 0.01145836350533583),\n",
              "  ('belief', 0.011420504450113065),\n",
              "  ('believe', 0.010923561507247137),\n",
              "  ('we', 0.010391933568507374)],\n",
              " 35: [('god', 0.01961475681228706),\n",
              "  ('faith', 0.01926409448985415),\n",
              "  ('sin', 0.017449051809299104),\n",
              "  ('jesus', 0.016301623834553684),\n",
              "  ('prayers', 0.01212112389401591),\n",
              "  ('our', 0.011914958442469849),\n",
              "  ('saved', 0.011471673469483634),\n",
              "  ('we', 0.01084298265543288),\n",
              "  ('romans', 0.010162429039403887),\n",
              "  ('prayer', 0.008945356006333368)],\n",
              " 36: [('hell', 0.046723881069197885),\n",
              "  ('eternal', 0.021823753200935478),\n",
              "  ('heaven', 0.017131923218789362),\n",
              "  ('god', 0.0161915882176334),\n",
              "  ('atheists', 0.010802464977770158),\n",
              "  ('believe', 0.010675843622987588),\n",
              "  ('death', 0.010435330427513382),\n",
              "  ('die', 0.010417999095487223),\n",
              "  ('jesus', 0.009552504732429004),\n",
              "  ('he', 0.008200924134278175)],\n",
              " 37: [('colormap', 0.040429036043449176),\n",
              "  ('visual', 0.0382131051151494),\n",
              "  ('color', 0.03652273708159876),\n",
              "  ('colormaps', 0.023113762113424693),\n",
              "  ('colors', 0.019471851039707683),\n",
              "  ('dpy', 0.01748728186116841),\n",
              "  ('window', 0.01676288925996453),\n",
              "  ('depth', 0.016424223829486807),\n",
              "  ('default', 0.013595835958732427),\n",
              "  ('clients', 0.013107193481755415)],\n",
              " 38: [('qur', 0.0159617417312701),\n",
              "  ('atheism', 0.014470980595320341),\n",
              "  ('religion', 0.01210848805602375),\n",
              "  ('allah', 0.011627513479778227),\n",
              "  ('motto', 0.010114534388500906),\n",
              "  ('muhammad', 0.010074781681436162),\n",
              "  ('genocide', 0.009852508359709829),\n",
              "  ('prophethood', 0.009843529062350068),\n",
              "  ('islam', 0.009149556403329564),\n",
              "  ('caused', 0.00872518979736147)],\n",
              " 39: [('games', 0.04220750257265951),\n",
              "  ('genesis', 0.03430119334452498),\n",
              "  ('sega', 0.028516574753460467),\n",
              "  ('game', 0.02847956604491959),\n",
              "  ('sale', 0.020989160141479425),\n",
              "  ('snes', 0.020905596706901595),\n",
              "  ('trade', 0.016772640732838316),\n",
              "  ('super', 0.015197068303953124),\n",
              "  ('nintendo', 0.01478212135225565),\n",
              "  ('docs', 0.012556409713710926)],\n",
              " 40: [('speed', 0.02747122670185524),\n",
              "  ('processor', 0.02153848409489723),\n",
              "  ('68070', 0.019734348298107805),\n",
              "  ('68040', 0.01869114895235827),\n",
              "  ('clock', 0.017706600100049225),\n",
              "  ('3do', 0.01670914822116479),\n",
              "  ('mhz', 0.01641721453350366),\n",
              "  ('040', 0.015744213641687918),\n",
              "  ('680x0', 0.015623025736002012),\n",
              "  ('quicktime', 0.014147298714523937)],\n",
              " 41: [('simms', 0.07108068987600678),\n",
              "  ('simm', 0.044363170161390186),\n",
              "  ('ram', 0.02313022325802376),\n",
              "  ('memory', 0.019094117257120794),\n",
              "  ('vram', 0.01841400861591663),\n",
              "  ('meg', 0.018018781288147764),\n",
              "  ('dram', 0.015023212732371711),\n",
              "  ('256k', 0.014411991958619703),\n",
              "  ('pin', 0.014215861242808298),\n",
              "  ('60ns', 0.012725526198542895)],\n",
              " 42: [('oil', 0.06506861318259606),\n",
              "  ('drain', 0.019480418988925344),\n",
              "  ('injector', 0.016428484562043953),\n",
              "  ('changing', 0.014663388451430068),\n",
              "  ('plug', 0.013605162550933747),\n",
              "  ('fuel', 0.012512203904134937),\n",
              "  ('cleaning', 0.010738277995418821),\n",
              "  ('orlin', 0.010255193146919682),\n",
              "  ('miles', 0.010068503898355872),\n",
              "  ('self', 0.00969941493198086)],\n",
              " 43: [('jewish', 0.08410483489492923),\n",
              "  ('baseball', 0.0475317666444323),\n",
              "  ('players', 0.03965174002691688),\n",
              "  ('lowenstein', 0.03015739261291469),\n",
              "  ('koufax', 0.028652223639118145),\n",
              "  ('stankowitz', 0.024351732563323587),\n",
              "  ('hank', 0.022740544951596903),\n",
              "  ('vb30', 0.02261232309451476),\n",
              "  ('berg', 0.022430503736592274),\n",
              "  ('moe', 0.02196954051257877)],\n",
              " 44: [('moral', 0.037517649136343874),\n",
              "  ('morality', 0.03530787255362465),\n",
              "  ('keith', 0.024465893245779864),\n",
              "  ('objective', 0.021830292610120438),\n",
              "  ('animals', 0.021271042045369757),\n",
              "  ('natural', 0.019360751829248707),\n",
              "  ('schneider', 0.01901128886335045),\n",
              "  ('species', 0.018626019611552257),\n",
              "  ('livesey', 0.016714354386787943),\n",
              "  ('system', 0.01632280717020415)],\n",
              " 45: [('tax', 0.03260313205416673),\n",
              "  ('taxes', 0.020277404605319044),\n",
              "  ('vat', 0.01452880508749357),\n",
              "  ('broward', 0.013114773862530502),\n",
              "  ('clinton', 0.012636257600714568),\n",
              "  ('reagan', 0.012625067994482929),\n",
              "  ('income', 0.012559701302115691),\n",
              "  ('deficit', 0.012486642531490726),\n",
              "  ('sales', 0.011386666890908415),\n",
              "  ('horne', 0.0103934938885524)],\n",
              " 46: [('sleeve', 0.04712851650704139),\n",
              "  ('cds', 0.03909887265061061),\n",
              "  ('sale', 0.03779206775127546),\n",
              "  ('picture', 0.03555028598118252),\n",
              "  ('cd', 0.029283556154663208),\n",
              "  ('45', 0.028206406873197847),\n",
              "  ('promo', 0.026371938323018672),\n",
              "  ('hiram', 0.022959700061835942),\n",
              "  ('jazz', 0.021547389156401805),\n",
              "  ('capitol', 0.017205562592324124)],\n",
              " 47: [('mormons', 0.0202358042928299),\n",
              "  ('church', 0.01232976214441354),\n",
              "  ('christian', 0.011989453860077774),\n",
              "  ('lds', 0.009933679418979387),\n",
              "  ('mormon', 0.009685726566083625),\n",
              "  ('religion', 0.007958430865556148),\n",
              "  ('brigham', 0.007501843302896359),\n",
              "  ('koresh', 0.006685711751124552),\n",
              "  ('malcolm', 0.006492014771465339),\n",
              "  ('davidians', 0.006281285523752965)],\n",
              " 48: [('windows', 0.021501244778031444),\n",
              "  ('window', 0.014309169688229205),\n",
              "  ('motif', 0.012872978985109605),\n",
              "  ('ms', 0.012646476452759521),\n",
              "  ('toolkits', 0.012037509370129809),\n",
              "  ('andrew', 0.011765300797870967),\n",
              "  ('x11', 0.010904364170712727),\n",
              "  ('nt', 0.010830201780468575),\n",
              "  ('consortium', 0.009622731266907),\n",
              "  ('toolkit', 0.009123245242379015)],\n",
              " 49: [('cancer', 0.03536037102377064),\n",
              "  ('centers', 0.015485738562522415),\n",
              "  ('medical', 0.014414363468901847),\n",
              "  ('center', 0.013412887794020913),\n",
              "  ('asthma', 0.012989792016030443),\n",
              "  ('research', 0.011873719383971961),\n",
              "  ('acne', 0.011580436605719767),\n",
              "  ('comprehensive', 0.011270774606080227),\n",
              "  ('hismanal', 0.010980882290963386),\n",
              "  ('oily', 0.01086299989915322)],\n",
              " 50: [('nt', 0.03913561852525087),\n",
              "  ('windows', 0.02746016246201913),\n",
              "  ('os', 0.027417854049785927),\n",
              "  ('xfree86', 0.027267798569430356),\n",
              "  ('chicogo', 0.014026694465991306),\n",
              "  ('server', 0.012667891627824262),\n",
              "  ('386bsd', 0.012152592824790615),\n",
              "  ('x386', 0.011856482066008431),\n",
              "  ('unix', 0.01111436481782883),\n",
              "  ('dos', 0.010845627312697562)],\n",
              " 51: [('cruel', 0.026835531240387463),\n",
              "  ('punishment', 0.023575494492424555),\n",
              "  ('penalty', 0.022478999417642007),\n",
              "  ('death', 0.022317306436593024),\n",
              "  ('innocent', 0.017407486323669962),\n",
              "  ('keith', 0.016663294676911238),\n",
              "  ('murder', 0.01615367615455575),\n",
              "  ('atheists', 0.01283976525882258),\n",
              "  ('we', 0.01275891478325875),\n",
              "  ('schneider', 0.011593208131042925)],\n",
              " 52: [('revolver', 0.02378008578745644),\n",
              "  ('semi', 0.02028345996084562),\n",
              "  ('kratz', 0.019731120870189505),\n",
              "  ('gun', 0.018550065027725566),\n",
              "  ('cops', 0.017159827812031285),\n",
              "  ('auto', 0.016895775139892398),\n",
              "  ('police', 0.01670370258836696),\n",
              "  ('gang', 0.016223520390467254),\n",
              "  ('glock', 0.01602737031766732),\n",
              "  ('safety', 0.0145362957999763)],\n",
              " 53: [('drugs', 0.05354916889039302),\n",
              "  ('drug', 0.02384346056236172),\n",
              "  ('legalization', 0.013158312338309285),\n",
              "  ('alcohol', 0.01300238400325301),\n",
              "  ('marijuana', 0.012718900887021491),\n",
              "  ('cocaine', 0.012103634193252824),\n",
              "  ('cigarettes', 0.011495807592256322),\n",
              "  ('hypocrisy', 0.011118325797649943),\n",
              "  ('lsd', 0.010500930512532708),\n",
              "  ('heroin', 0.010301211089970255)],\n",
              " 54: [('doctor', 0.02063734160044593),\n",
              "  ('banks', 0.02034463792006775),\n",
              "  ('gordon', 0.019915296076676744),\n",
              "  ('burzynski', 0.017207122788845335),\n",
              "  ('radiologist', 0.014376333176533628),\n",
              "  ('cancer', 0.014370275478320302),\n",
              "  ('clinic', 0.013195844436433684),\n",
              "  ('homeopathy', 0.012807432820554921),\n",
              "  ('treatment', 0.011311796776782825),\n",
              "  ('patients', 0.010977490672596702)],\n",
              " 55: [('sho', 0.03225191048317965),\n",
              "  ('automatic', 0.018957746858057702),\n",
              "  ('shift', 0.017835117548875977),\n",
              "  ('manual', 0.01644957051167857),\n",
              "  ('brakes', 0.014672100000066515),\n",
              "  ('speed', 0.013834126022309776),\n",
              "  ('auto', 0.013573392542809535),\n",
              "  ('transmission', 0.012742332547645719),\n",
              "  ('shifting', 0.011621916104247969),\n",
              "  ('clutch', 0.011240714734856402)],\n",
              " 56: [('government', 0.023154789683809223),\n",
              "  ('hendricks', 0.017841940434327873),\n",
              "  ('libertarians', 0.016732988622981313),\n",
              "  ('libertarian', 0.013584182402569348),\n",
              "  ('regulation', 0.01186725372991817),\n",
              "  ('libertarianism', 0.010379003734652538),\n",
              "  ('steve', 0.01034258389493566),\n",
              "  ('limiting', 0.010168909715837369),\n",
              "  ('govt', 0.009598749854170653),\n",
              "  ('gritz', 0.00952032589891401)],\n",
              " 57: [('window', 0.08702995266586122),\n",
              "  ('manager', 0.02933343842959558),\n",
              "  ('mainwin', 0.026726465396029013),\n",
              "  ('xsizehints', 0.02511683488762998),\n",
              "  ('detailwin', 0.02182392201587831),\n",
              "  ('myhint', 0.020136268595907815),\n",
              "  ('mydisplay', 0.01875633763165542),\n",
              "  ('win', 0.013980469951374697),\n",
              "  ('pposition', 0.013959502183653858),\n",
              "  ('focus', 0.013264770651220524)],\n",
              " 58: [('phone', 0.04791245474427989),\n",
              "  ('hook', 0.0413125784875974),\n",
              "  ('number', 0.03366015440236276),\n",
              "  ('line', 0.031492783185154555),\n",
              "  ('dial', 0.019004322630049014),\n",
              "  ('led', 0.018966905257677313),\n",
              "  ('telephone', 0.01591118386997197),\n",
              "  ('find', 0.014975206878613596),\n",
              "  ('operator', 0.014768750443578098),\n",
              "  ('al', 0.013870486028672201)],\n",
              " 59: [('lib', 0.07559841074507961),\n",
              "  ('libxmu', 0.05295029600863454),\n",
              "  ('xmu', 0.047047927504498196),\n",
              "  ('symbol', 0.03298255507698344),\n",
              "  ('undefined', 0.028553714142774772),\n",
              "  ('doug', 0.026535620909299555),\n",
              "  ('x11r5', 0.024923657823546722),\n",
              "  ('gcc', 0.021464728390886745),\n",
              "  ('libx11', 0.02104624221732942),\n",
              "  ('ld', 0.018354401168317595)],\n",
              " 60: [('rocks', 0.0250164004275314),\n",
              "  ('kids', 0.01909737282566485),\n",
              "  ('teenagers', 0.018094115800357576),\n",
              "  ('warning', 0.01632352326437673),\n",
              "  ('overpass', 0.012338560561661788),\n",
              "  ('cars', 0.011288817808292584),\n",
              "  ('read', 0.010732426301551502),\n",
              "  ('erik', 0.01052615307193841),\n",
              "  ('please', 0.010068738531647362),\n",
              "  ('triumf', 0.009369186352561395)],\n",
              " 61: [('mouse', 0.0685152629058114),\n",
              "  ('irq', 0.050125936753598145),\n",
              "  ('com3', 0.035925049534628516),\n",
              "  ('com1', 0.03090267093622359),\n",
              "  ('port', 0.02863765490206105),\n",
              "  ('interrupt', 0.025247602302674352),\n",
              "  ('windows', 0.02169850617298767),\n",
              "  ('com2', 0.018437512661380617),\n",
              "  ('modem', 0.018282676355570782),\n",
              "  ('irq5', 0.017050569772884897)],\n",
              " 62: [('existence', 0.01447415846014552),\n",
              "  ('why', 0.013176142093032597),\n",
              "  ('god', 0.012876253564656178),\n",
              "  ('exist', 0.01269268157624341),\n",
              "  ('science', 0.01157949698482748),\n",
              "  ('exists', 0.010011053252296882),\n",
              "  ('universe', 0.009997239071103062),\n",
              "  ('question', 0.00826649732824019),\n",
              "  ('taoism', 0.00786677506181998),\n",
              "  ('evolution', 0.007686693861954638)],\n",
              " 63: [('polio', 0.028488583555551495),\n",
              "  ('syndrome', 0.01921613785919361),\n",
              "  ('disease', 0.018618569462995945),\n",
              "  ('she', 0.017380193299063543),\n",
              "  ('patients', 0.014228542613207584),\n",
              "  ('chronic', 0.012179481702417278),\n",
              "  ('cirrhosis', 0.011185945807812915),\n",
              "  ('her', 0.01027936472497489),\n",
              "  ('zisfein', 0.010261268429817358),\n",
              "  ('banks', 0.010020534871513177)],\n",
              " 64: [('militia', 0.04200536431018847),\n",
              "  ('amendment', 0.023077895123992647),\n",
              "  ('arms', 0.020632064515626485),\n",
              "  ('regulated', 0.018074948369199588),\n",
              "  ('right', 0.017994659643518866),\n",
              "  ('bear', 0.01790073280720619),\n",
              "  ('constitution', 0.014947253305970175),\n",
              "  ('people', 0.011312325215817348),\n",
              "  ('well', 0.01125877517631094),\n",
              "  ('second', 0.01123220287506126)],\n",
              " 65: [('keenan', 0.03922312145605799),\n",
              "  ('coach', 0.031614110368505116),\n",
              "  ('rangers', 0.02517290710126631),\n",
              "  ('team', 0.016410469890932925),\n",
              "  ('jets', 0.015592029621919617),\n",
              "  ('jokerit', 0.014907974880235916),\n",
              "  ('messier', 0.01449084154274392),\n",
              "  ('gerald', 0.014165299028456878),\n",
              "  ('tps', 0.014042471496824246),\n",
              "  ('dare', 0.013110697516533706)],\n",
              " 66: [('precision', 0.033468560553465185),\n",
              "  ('ftp', 0.025098475188351035),\n",
              "  ('package', 0.0221770871996896),\n",
              "  ('bignum', 0.01662360616941463),\n",
              "  ('multiple', 0.014398883980765623),\n",
              "  ('mpeg', 0.014235730938001654),\n",
              "  ('includes', 0.014213955575060716),\n",
              "  ('tar', 0.0136925704275169),\n",
              "  ('filename', 0.013237679076907255),\n",
              "  ('integer', 0.01313694498348357)],\n",
              " 67: [('insurance', 0.056752359303369714),\n",
              "  ('car', 0.01638588067802728),\n",
              "  ('fault', 0.015776601502167798),\n",
              "  ('deductible', 0.014717989609389737),\n",
              "  ('rates', 0.011262482262226908),\n",
              "  ('lawyers', 0.010692528797590998),\n",
              "  ('alamo', 0.010413832300978752),\n",
              "  ('pay', 0.010396430302724111),\n",
              "  ('farm', 0.010228245848206688),\n",
              "  ('company', 0.009676456168223371)],\n",
              " 68: [('bike', 0.024793085901223165),\n",
              "  ('honda', 0.021776747906145446),\n",
              "  ('ducati', 0.017661872772759983),\n",
              "  ('400', 0.015161773245438662),\n",
              "  ('750', 0.015093540979598401),\n",
              "  ('sprocket', 0.01385621853668861),\n",
              "  ('yamaha', 0.01175764582408708),\n",
              "  ('dod', 0.011435798437153345),\n",
              "  ('miles', 0.010445002855140002),\n",
              "  ('bikes', 0.010110632956424672)],\n",
              " 69: [('taste', 0.02432895696865902),\n",
              "  ('carcinogenic', 0.024081310513053355),\n",
              "  ('food', 0.024046842630199707),\n",
              "  ('foods', 0.022512288639041828),\n",
              "  ('barbecued', 0.02221909225479725),\n",
              "  ('meat', 0.020911608957634668),\n",
              "  ('smoke', 0.016040402581295857),\n",
              "  ('risk', 0.014439907611179461),\n",
              "  ('charcoal', 0.014065905024058191),\n",
              "  ('substances', 0.013326097465567297)],\n",
              " 70: [('dod', 0.04334141105251368),\n",
              "  ('denizens', 0.014885563056900539),\n",
              "  ('motorcycle', 0.01400120440362022),\n",
              "  ('motorcycles', 0.012384439333392809),\n",
              "  ('bike', 0.012203275452803669),\n",
              "  ('doom', 0.011250098906784953),\n",
              "  ('muck', 0.010698002403920572),\n",
              "  ('ride', 0.010627564254569703),\n",
              "  ('kotl', 0.010457727018862578),\n",
              "  ('rec', 0.01039112950189863)],\n",
              " 71: [('entry', 0.0162694899207769),\n",
              "  ('output', 0.0127077843373312),\n",
              "  ('file', 0.011088640145822061),\n",
              "  ('anonymous', 0.010950292599681121),\n",
              "  ('ripem', 0.00955569478287112),\n",
              "  ('privacy', 0.009542653818874553),\n",
              "  ('key', 0.008992302805663096),\n",
              "  ('oname', 0.008926727668690914),\n",
              "  ('printf', 0.008175058782747272),\n",
              "  ('entries', 0.008112327826841674)],\n",
              " 72: [('gif', 0.04741568288359113),\n",
              "  ('otis', 0.03833504153488037),\n",
              "  ('images', 0.037803767970804245),\n",
              "  ('ics', 0.023909900066122473),\n",
              "  ('uci', 0.023771305928962923),\n",
              "  ('incoming', 0.022553820062653742),\n",
              "  ('image', 0.01837975255456331),\n",
              "  ('earth', 0.014067635744454219),\n",
              "  ('jpg', 0.011918786945562558),\n",
              "  ('files', 0.01130456361741232)],\n",
              " 73: [('p3', 0.04859923162967457),\n",
              "  ('p2', 0.04788997664903466),\n",
              "  ('den', 0.047646241552285876),\n",
              "  ('sphere', 0.04739607562629915),\n",
              "  ('p1', 0.04463410992841348),\n",
              "  ('points', 0.04300257318675185),\n",
              "  ('radius', 0.04077894106794491),\n",
              "  ('ellipse', 0.02266305304500768),\n",
              "  ('circle', 0.0221785295816982),\n",
              "  ('plane', 0.020993185892284227)],\n",
              " 74: [('monitor', 0.06239771554073717),\n",
              "  ('vga', 0.04054721208795104),\n",
              "  ('monitors', 0.02667631627002081),\n",
              "  ('lc', 0.024042437561658365),\n",
              "  ('video', 0.023349825136065844),\n",
              "  ('nec', 0.01677645035740069),\n",
              "  ('lciii', 0.01557664792686063),\n",
              "  ('macintosh', 0.015474888259902298),\n",
              "  ('adapter', 0.015252027306332542),\n",
              "  ('multisync', 0.014598936103954834)],\n",
              " 75: [('christianity', 0.021401441080697765),\n",
              "  ('convenient', 0.01765121068389941),\n",
              "  ('christian', 0.013851541916841592),\n",
              "  ('christians', 0.009704776246931949),\n",
              "  ('religion', 0.008938612923061962),\n",
              "  ('god', 0.008578870766769357),\n",
              "  ('bible', 0.008330374690510232),\n",
              "  ('religious', 0.0072262137614984324),\n",
              "  ('christ', 0.007084331783638401),\n",
              "  ('life', 0.006549468758134153)],\n",
              " 76: [('greek', 0.03904849865652183),\n",
              "  ('greece', 0.030971151054989675),\n",
              "  ('turkish', 0.027715773543752504),\n",
              "  ('greeks', 0.02723106239520626),\n",
              "  ('turks', 0.021268280992759108),\n",
              "  ('cyprus', 0.02036671980135412),\n",
              "  ('turkey', 0.018907619272972846),\n",
              "  ('minority', 0.011037223429698078),\n",
              "  ('cypriot', 0.009647767708038171),\n",
              "  ('napoleon', 0.008264374802589229)],\n",
              " 77: [('pts', 0.04857320060540776),\n",
              "  ('period', 0.03354167260245096),\n",
              "  ('11', 0.02985812181713577),\n",
              "  ('pp', 0.029791737951951187),\n",
              "  ('10', 0.029148606915899232),\n",
              "  ('la', 0.02885184949196517),\n",
              "  ('16', 0.028058935099429794),\n",
              "  ('12', 0.027804735614220442),\n",
              "  ('pt', 0.0274608094771128),\n",
              "  ('14', 0.02675133453653088)],\n",
              " 78: [('monitor', 0.04654715147750736),\n",
              "  ('screen', 0.027190486153493),\n",
              "  ('problem', 0.020655004961834558),\n",
              "  ('video', 0.019314748809493822),\n",
              "  ('colors', 0.019228475548314132),\n",
              "  ('610', 0.017103873201169278),\n",
              "  ('display', 0.015066792003260629),\n",
              "  ('apple', 0.014647486228388807),\n",
              "  ('centris', 0.013222261070078546),\n",
              "  ('scrolling', 0.009689562978888988)],\n",
              " 79: [('dog', 0.07205557424182206),\n",
              "  ('dogs', 0.022847622010732274),\n",
              "  ('springer', 0.015185866443551025),\n",
              "  ('spagthorpe', 0.013856468481473865),\n",
              "  ('chase', 0.011797402397772458),\n",
              "  ('bike', 0.011576388739304081),\n",
              "  ('riding', 0.010507669437547457),\n",
              "  ('viking', 0.010069814043658946),\n",
              "  ('dod', 0.009867587267913638),\n",
              "  ('ryan', 0.00948296908948219)],\n",
              " 80: [('spacecraft', 0.03088940917181221),\n",
              "  ('probe', 0.021651269006853293),\n",
              "  ('venus', 0.020815130894302415),\n",
              "  ('galileo', 0.01784922123390341),\n",
              "  ('mars', 0.017681311925874233),\n",
              "  ('magellan', 0.01752564622324812),\n",
              "  ('probes', 0.01690061706800228),\n",
              "  ('mission', 0.015048253070028389),\n",
              "  ('mariner', 0.013908240314075246),\n",
              "  ('command', 0.013783931025310416)],\n",
              " 81: [('des', 0.03545013636593695),\n",
              "  ('key', 0.02439620947192941),\n",
              "  ('keys', 0.02199126534984323),\n",
              "  ('keyseach', 0.02084971765766406),\n",
              "  ('machine', 0.019811249841584472),\n",
              "  ('skipjack', 0.01813637274937839),\n",
              "  ('shelf', 0.01782600636490224),\n",
              "  ('bits', 0.017461833787310057),\n",
              "  ('80', 0.014862023281697162),\n",
              "  ('bellovin', 0.014756751670439988)],\n",
              " 82: [('jews', 0.03062011265437633),\n",
              "  ('israel', 0.026870541255134776),\n",
              "  ('arabs', 0.02301618771982755),\n",
              "  ('land', 0.022129876052122653),\n",
              "  ('arab', 0.017520513739343824),\n",
              "  ('palestine', 0.016745283910964307),\n",
              "  ('palestinians', 0.01390693033850927),\n",
              "  ('jewish', 0.013813689451170265),\n",
              "  ('isreal', 0.011232897123555623),\n",
              "  ('jerusalem', 0.010949546065626058)],\n",
              " 83: [('polygon', 0.08084435629484721),\n",
              "  ('algorithm', 0.022983727544510717),\n",
              "  ('routine', 0.020997369858373104),\n",
              "  ('graeme', 0.01852771382422146),\n",
              "  ('line', 0.017268924962870275),\n",
              "  ('polygons', 0.01578426874071104),\n",
              "  ('spline', 0.013198412931574957),\n",
              "  ('concave', 0.01317923788228294),\n",
              "  ('problem', 0.012575452184829757),\n",
              "  ('method', 0.011969480097800566)],\n",
              " 84: [('clinton', 0.03366114077058511),\n",
              "  ('administration', 0.025442836239222003),\n",
              "  ('bush', 0.013750146046696195),\n",
              "  ('qualcomm', 0.010322946682570134),\n",
              "  ('chip', 0.009553895468307656),\n",
              "  ('privacy', 0.009402093574283376),\n",
              "  ('freedoms', 0.008483036571913861),\n",
              "  ('na', 0.008294863779336947),\n",
              "  ('been', 0.008226553574207745),\n",
              "  ('myrto', 0.008205719003796408)],\n",
              " 85: [('countersteering', 0.04003740505523566),\n",
              "  ('bike', 0.02302526846974484),\n",
              "  ('countersteeringfaq', 0.020652492753674957),\n",
              "  ('motorcycle', 0.019500895758975303),\n",
              "  ('technique', 0.01802927564944381),\n",
              "  ('mjs', 0.017693977649118463),\n",
              "  ('rider', 0.015228439421385201),\n",
              "  ('neck', 0.014448722985195341),\n",
              "  ('horse', 0.01442565406915067),\n",
              "  ('left', 0.014373025988925333)],\n",
              " 86: [('candida', 0.029238016236675133),\n",
              "  ('yeast', 0.023318906155837565),\n",
              "  ('albicans', 0.010523838071330999),\n",
              "  ('bloom', 0.00944004894509755),\n",
              "  ('sinus', 0.008975245120776996),\n",
              "  ('vitamin', 0.008963851854101874),\n",
              "  ('gonorrhea', 0.008685425029400073),\n",
              "  ('symptoms', 0.008526604374878214),\n",
              "  ('patients', 0.008383131640786511),\n",
              "  ('nystatin', 0.00792933115649957)],\n",
              " 87: [('biggest', 0.030517093464595572),\n",
              "  ('selanne', 0.01614026383933123),\n",
              "  ('disappointment', 0.015978094214397248),\n",
              "  ('team', 0.015397596511910621),\n",
              "  ('mvp', 0.014181792625977484),\n",
              "  ('year', 0.013910657916748207),\n",
              "  ('he', 0.013103625944860699),\n",
              "  ('player', 0.012552486980830078),\n",
              "  ('game', 0.012414321794531363),\n",
              "  ('stauber', 0.01211099198528869)],\n",
              " 88: [('monitors', 0.042618329361199546),\n",
              "  ('nanao', 0.03487289713817454),\n",
              "  ('17', 0.028759524832461674),\n",
              "  ('monitor', 0.027114909396951173),\n",
              "  ('t560i', 0.024090288500773684),\n",
              "  ('mag', 0.022691984246712003),\n",
              "  ('f550iw', 0.021722297800684534),\n",
              "  ('trinitron', 0.021272022185576698),\n",
              "  ('nec', 0.021106742032520897),\n",
              "  ('viewsonic', 0.01963571278668618)],\n",
              " 89: [('16550', 0.03293273637616779),\n",
              "  ('uart', 0.028162868162545384),\n",
              "  ('modem', 0.022117232407810786),\n",
              "  ('serial', 0.02146364390192231),\n",
              "  ('ports', 0.01725201009452551),\n",
              "  ('procomm', 0.015668044485747414),\n",
              "  ('windows', 0.013988214301001993),\n",
              "  ('card', 0.013283888939633828),\n",
              "  ('uarts', 0.012366333995216522),\n",
              "  ('turbocom', 0.011855915967528314)],\n",
              " 90: [('nhl', 0.04316593745185126),\n",
              "  ('europeans', 0.037648510218526964),\n",
              "  ('players', 0.02904894193995088),\n",
              "  ('european', 0.025163019017819473),\n",
              "  ('teams', 0.02295821694366641),\n",
              "  ('canadian', 0.02035111707191514),\n",
              "  ('hockey', 0.01987745447051388),\n",
              "  ('league', 0.019459731849393088),\n",
              "  ('canadians', 0.018541710780402555),\n",
              "  ('euros', 0.016038388045801707)],\n",
              " 91: [('drivers', 0.09068473589077786),\n",
              "  ('driver', 0.03954044072296701),\n",
              "  ('speedstar', 0.03897214631900345),\n",
              "  ('trident', 0.038913163733425155),\n",
              "  ('card', 0.03847233879537627),\n",
              "  ('diamond', 0.030007821645871905),\n",
              "  ('video', 0.02982450293877253),\n",
              "  ('ftp', 0.02773650581581006),\n",
              "  ('stealth', 0.021225525099615804),\n",
              "  ('ati', 0.02041690358136787)],\n",
              " 92: [('key', 0.035706296242626584),\n",
              "  ('accelerators', 0.031013955621417517),\n",
              "  ('ctrl', 0.02832410525365688),\n",
              "  ('keyboard', 0.027433682068401705),\n",
              "  ('keycode', 0.02571684456300537),\n",
              "  ('numlock', 0.025408956450460155),\n",
              "  ('keys', 0.02375849591784943),\n",
              "  ('emacs', 0.019890184633924137),\n",
              "  ('modifiers', 0.01679105047703149),\n",
              "  ('xterm', 0.0167493847337265)],\n",
              " 93: [('card', 0.03283126976506163),\n",
              "  ('6551', 0.020849690511313402),\n",
              "  ('dip', 0.016906904637274296),\n",
              "  ('board', 0.016631830288628),\n",
              "  ('pin', 0.016302317709349288),\n",
              "  ('switches', 0.015232735504018104),\n",
              "  ('dram', 0.014667941371433147),\n",
              "  ('chip', 0.013826470006505517),\n",
              "  ('pc', 0.012144286011177749),\n",
              "  ('3com', 0.011773684240236138)],\n",
              " 94: [('henrik', 0.041737050110069965),\n",
              "  ('armenia', 0.03942512664754329),\n",
              "  ('bm', 0.03391797893164011),\n",
              "  ('planes', 0.029563300770913285),\n",
              "  ('armenians', 0.02903819917111678),\n",
              "  ('turkey', 0.026888940080034184),\n",
              "  ('turkish', 0.024786719121613475),\n",
              "  ('karabakh', 0.024114334023008964),\n",
              "  ('azeris', 0.022234031675441173),\n",
              "  ('azerbaijan', 0.016320678125206926)],\n",
              " 95: [('bible', 0.026924634758419573),\n",
              "  ('texts', 0.015181898572452363),\n",
              "  ('hebrew', 0.012854912947802931),\n",
              "  ('translations', 0.012389898832323559),\n",
              "  ('ancient', 0.010668346067598321),\n",
              "  ('manuscripts', 0.010592069791694048),\n",
              "  ('text', 0.010312592862444517),\n",
              "  ('testament', 0.009665741729798807),\n",
              "  ('translation', 0.009239101359774978),\n",
              "  ('copies', 0.008293577254021205)],\n",
              " 96: [('god', 0.032052463253045754),\n",
              "  ('exist', 0.02470053810683531),\n",
              "  ('existence', 0.02165275486594156),\n",
              "  ('gods', 0.017185550854750865),\n",
              "  ('faith', 0.015959726432014633),\n",
              "  ('believe', 0.015487777575039042),\n",
              "  ('atheism', 0.01500100967821577),\n",
              "  ('belief', 0.014670000729775234),\n",
              "  ('atheist', 0.013394782871193098),\n",
              "  ('atheists', 0.01051306201006616)],\n",
              " 97: [('witnesses', 0.03279915232540732),\n",
              "  ('trial', 0.0268820032887169),\n",
              "  ('gm', 0.022587600063829347),\n",
              "  ('new', 0.01798179199812318),\n",
              "  ('judge', 0.016714092098564955),\n",
              "  ('jury', 0.012864982908355983),\n",
              "  ('justice', 0.012837565518609033),\n",
              "  ('denied', 0.010082065706488902),\n",
              "  ('1960', 0.010031809221839048),\n",
              "  ('schiff', 0.009364778930940138)],\n",
              " 98: [('white', 0.019838630165599173),\n",
              "  ('ted', 0.018749660665557936),\n",
              "  ('black', 0.015207595466035928),\n",
              "  ('kyle', 0.011963729152603695),\n",
              "  ('males', 0.011035451952744245),\n",
              "  ('adjective', 0.010920250671910747),\n",
              "  ('frank', 0.00984324340959684),\n",
              "  ('blacks', 0.009177284937772612),\n",
              "  ('cramm', 0.008681871295334678),\n",
              "  ('koppenhoefer', 0.008681871295334678)],\n",
              " 99: [('fbi', 0.03458563312432561),\n",
              "  ('roby', 0.022291745208956066),\n",
              "  ('batf', 0.020208390136292952),\n",
              "  ('koresh', 0.018028964275953855),\n",
              "  ('compound', 0.012337195715710498),\n",
              "  ('children', 0.012330094630353198),\n",
              "  ('scott', 0.01206834732512178),\n",
              "  ('they', 0.011705005455204781),\n",
              "  ('fire', 0.011664745394322376),\n",
              "  ('never', 0.011336965893157658)],\n",
              " 100: [('aftermarket', 0.021671911085511638),\n",
              "  ('air', 0.01867112211780044),\n",
              "  ('substitutes', 0.017496306914227027),\n",
              "  ('r12', 0.016913412153595663),\n",
              "  ('heat', 0.01579010625747617),\n",
              "  ('conditioning', 0.012598834671326352),\n",
              "  ('les', 0.011537336569387996),\n",
              "  ('honda', 0.011407158586487063),\n",
              "  ('installed', 0.011193063966051681),\n",
              "  ('unit', 0.010916046121024356)],\n",
              " 101: [('bike', 0.0406741963274705),\n",
              "  ('sale', 0.02186010486704185),\n",
              "  ('honda', 0.019814284444705607),\n",
              "  ('hawk', 0.01935345456522896),\n",
              "  ('bikes', 0.01816309407431347),\n",
              "  ('miles', 0.017740289589934115),\n",
              "  ('tank', 0.01686613894832831),\n",
              "  ('sold', 0.013568479552352727),\n",
              "  ('rear', 0.013532833936384605),\n",
              "  ('seat', 0.01254374394915578)],\n",
              " 102: [('gaza', 0.030684382813376286),\n",
              "  ('israel', 0.027770838894210455),\n",
              "  ('israeli', 0.02447734748242509),\n",
              "  ('palestinian', 0.023326658499998827),\n",
              "  ('gazans', 0.020259800925099394),\n",
              "  ('jews', 0.017899937717424232),\n",
              "  ('ghetto', 0.016610410307454908),\n",
              "  ('solution', 0.012577282574266973),\n",
              "  ('palestinians', 0.01197148699983827),\n",
              "  ('rights', 0.010998773228240678)],\n",
              " 103: [('condition', 0.026364024747167127),\n",
              "  ('excellent', 0.021276004750393474),\n",
              "  ('miles', 0.020135853879583974),\n",
              "  ('checker', 0.020007496171674057),\n",
              "  ('cassette', 0.01875523984441575),\n",
              "  ('honda', 0.017991928904406915),\n",
              "  ('car', 0.01719432966454848),\n",
              "  ('sale', 0.016375913009088004),\n",
              "  ('fm', 0.01584973791229308),\n",
              "  ('interior', 0.01421539611064199)],\n",
              " 104: [('pens', 0.035220101090139046),\n",
              "  ('vs', 0.03057145638420565),\n",
              "  ('pitt', 0.022553670301203237),\n",
              "  ('win', 0.019740198372139854),\n",
              "  ('hawks', 0.018780866421794953),\n",
              "  ('playoffs', 0.018396713471147137),\n",
              "  ('caps', 0.018334030726802755),\n",
              "  ('cup', 0.015053069895817192),\n",
              "  ('chi', 0.015047569350203193),\n",
              "  ('bos', 0.014823333655757942)],\n",
              " 105: [('hall', 0.03286834718539205),\n",
              "  ('fame', 0.02944857205627659),\n",
              "  ('kingman', 0.026550338282902663),\n",
              "  ('dave', 0.01985680554299128),\n",
              "  ('winfield', 0.01923382670839111),\n",
              "  ('players', 0.017037975829697946),\n",
              "  ('nolan', 0.016179967787183672),\n",
              "  ('yount', 0.014831583265864265),\n",
              "  ('garvey', 0.014523235006693034),\n",
              "  ('career', 0.014425841072234519)],\n",
              " 106: [('grass', 0.03724133054151133),\n",
              "  ('valley', 0.036837741094050636),\n",
              "  ('lord', 0.0343781058398787),\n",
              "  ('verse', 0.030500579376077357),\n",
              "  ('god', 0.02786648209193886),\n",
              "  ('buffalo', 0.02635802745594181),\n",
              "  ('chuck', 0.026255145621783403),\n",
              "  ('daily', 0.025032449025877913),\n",
              "  ('petch', 0.024762531770641614),\n",
              "  ('promise', 0.024607752963568374)],\n",
              " 107: [('phillies', 0.07455383100584051),\n",
              "  ('1964', 0.030593134860417703),\n",
              "  ('phils', 0.029023296006820354),\n",
              "  ('bunning', 0.02524968143306489),\n",
              "  ('cardinals', 0.020448315839251486),\n",
              "  ('reds', 0.016147469836034194),\n",
              "  ('season', 0.01535626468932144),\n",
              "  ('game', 0.014679282498461642),\n",
              "  ('wins', 0.01393251676177449),\n",
              "  ('deja', 0.013835061862615591)],\n",
              " 108: [('rosicrucian', 0.04226738516571111),\n",
              "  ('order', 0.03930920986931057),\n",
              "  ('alicea', 0.02925236899617049),\n",
              "  ('tony', 0.028004755626648514),\n",
              "  ('reuss', 0.027273668631835026),\n",
              "  ('amorc', 0.024818657135474022),\n",
              "  ('oto', 0.023157570989300328),\n",
              "  ('crucis', 0.019746238551415215),\n",
              "  ('ordo', 0.018438658018265046),\n",
              "  ('templars', 0.018227297124383274)],\n",
              " 109: [('800', 0.06959623610294237),\n",
              "  ('books', 0.03025300066201543),\n",
              "  ('chemistry', 0.027282867935061718),\n",
              "  ('spice', 0.023546463212335323),\n",
              "  ('book', 0.02221245442523955),\n",
              "  ('cover', 0.01906570582579131),\n",
              "  ('paperback', 0.01607683400126369),\n",
              "  ('korea', 0.01378014342965459),\n",
              "  ('software', 0.013605661614077807),\n",
              "  ('analytic', 0.013488875462597258)],\n",
              " 110: [('antenna', 0.02913472528266158),\n",
              "  ('receiver', 0.027212599012542027),\n",
              "  ('td', 0.021933791642986897),\n",
              "  ('channel', 0.01724938772286387),\n",
              "  ('fsk', 0.01622907280565112),\n",
              "  ('frequencies', 0.014532406154149427),\n",
              "  ('cb', 0.013792650050024904),\n",
              "  ('wave', 0.013487221542915349),\n",
              "  ('radios', 0.011551002580739878),\n",
              "  ('mhz', 0.011531282487595128)],\n",
              " 111: [('marriage', 0.05909359352891828),\n",
              "  ('married', 0.05377694924228292),\n",
              "  ('ceremony', 0.02220588724592303),\n",
              "  ('eyes', 0.01995761633834525),\n",
              "  ('church', 0.01648407932179939),\n",
              "  ('marry', 0.016126874913748257),\n",
              "  ('god', 0.015435502067413856),\n",
              "  ('commitment', 0.015271525831551816),\n",
              "  ('wedding', 0.014983212991223543),\n",
              "  ('priest', 0.014192483477011448)],\n",
              " 112: [('helmet', 0.08641036539058812),\n",
              "  ('helmets', 0.03140348602933039),\n",
              "  ('shoei', 0.02552027866055567),\n",
              "  ('passenger', 0.019363875946647388),\n",
              "  ('liner', 0.015319031569280606),\n",
              "  ('paint', 0.014063210048927183),\n",
              "  ('cb', 0.013896055012558257),\n",
              "  ('hamer', 0.013286611566620034),\n",
              "  ('size', 0.012814237350973208),\n",
              "  ('cbd', 0.01280522376118681)],\n",
              " 113: [('leafs', 0.0765773862546988),\n",
              "  ('wings', 0.04480953248318168),\n",
              "  ('toronto', 0.033487306175841815),\n",
              "  ('detroit', 0.03033891884524283),\n",
              "  ('game', 0.02195358569502869),\n",
              "  ('gilmour', 0.021424958000056785),\n",
              "  ('red', 0.02000139013994232),\n",
              "  ('potvin', 0.018954509529121432),\n",
              "  ('puck', 0.017809197412905077),\n",
              "  ('bay', 0.01635597550762526)],\n",
              " 114: [('bus', 0.07159441215185194),\n",
              "  ('vlb', 0.048503060161170466),\n",
              "  ('ide', 0.042887622190274935),\n",
              "  ('controller', 0.04270749647068672),\n",
              "  ('isa', 0.03192303807910392),\n",
              "  ('eisa', 0.021171295827119595),\n",
              "  ('speed', 0.020906123847873662),\n",
              "  ('mhz', 0.020545378443467616),\n",
              "  ('sec', 0.018972567369942208),\n",
              "  ('local', 0.01854817343120498)],\n",
              " 115: [('moa', 0.0675930440241924),\n",
              "  ('bmw', 0.05098550469607669),\n",
              "  ('bmwmoa', 0.029918242323032612),\n",
              "  ('vechorik', 0.019377663750676694),\n",
              "  ('members', 0.017630954559528896),\n",
              "  ('senner', 0.01453324781300752),\n",
              "  ('ra', 0.014088019577878773),\n",
              "  ('rathole', 0.013313609351741699),\n",
              "  ('karr', 0.013179640101609337),\n",
              "  ('ranck', 0.012846240918105165)],\n",
              " 116: [('disks', 0.04689293586622232),\n",
              "  ('floptical', 0.04254807707139014),\n",
              "  ('drives', 0.03041732830294023),\n",
              "  ('drive', 0.023974650420174028),\n",
              "  ('bernoulli', 0.023868659525995976),\n",
              "  ('megabyte', 0.022393724774415865),\n",
              "  ('floppy', 0.020154051031630205),\n",
              "  ('per', 0.017875077874757624),\n",
              "  ('disk', 0.016856582569456963),\n",
              "  ('syquest', 0.014637572981842588)],\n",
              " 117: [('wave', 0.04631557366346979),\n",
              "  ('riders', 0.043935041270822615),\n",
              "  ('motorcycling', 0.03671917287946693),\n",
              "  ('bike', 0.030557187715744054),\n",
              "  ('harley', 0.030092596465540432),\n",
              "  ('cruiser', 0.02412338300958551),\n",
              "  ('bikers', 0.019044656691122024),\n",
              "  ('declerck', 0.018881337441075452),\n",
              "  ('sport', 0.01777287079922923),\n",
              "  ('riding', 0.01730421350274823)],\n",
              " 118: [('mp', 0.009172697508477188),\n",
              "  ('mv', 0.009078805129334191),\n",
              "  ('m8', 0.007456885140460097),\n",
              "  ('m5', 0.007283550346770965),\n",
              "  ('m0', 0.007097797845088756),\n",
              "  ('mh', 0.006950269369384769),\n",
              "  ('mz', 0.006915313285747368),\n",
              "  ('m9', 0.006900561373589218),\n",
              "  ('printer', 0.006776147826231404),\n",
              "  ('mw', 0.006696614167926344)],\n",
              " 119: [('batf', 0.04067448740458759),\n",
              "  ('blast', 0.01854859357678869),\n",
              "  ('waco', 0.017637069050931917),\n",
              "  ('assault', 0.017536800378950817),\n",
              "  ('compound', 0.01689732514364044),\n",
              "  ('they', 0.013680180815483431),\n",
              "  ('fire', 0.011942299744625273),\n",
              "  ('proper', 0.011937602800038818),\n",
              "  ('oldham', 0.011817575323362275),\n",
              "  ('then', 0.011608856287538154)],\n",
              " 120: [('speakers', 0.0379338346218287),\n",
              "  ('sale', 0.033796595324227886),\n",
              "  ('condition', 0.03017700353291444),\n",
              "  ('shipping', 0.027259007102893952),\n",
              "  ('asking', 0.024566112713196614),\n",
              "  ('sony', 0.024056220463003538),\n",
              "  ('paid', 0.023799922600623708),\n",
              "  ('offer', 0.023358170933106683),\n",
              "  ('receiver', 0.022089933450034113),\n",
              "  ('stereo', 0.021972944509113187)],\n",
              " 121: [('02', 0.06546112983875858),\n",
              "  ('03', 0.06487413887677125),\n",
              "  ('00', 0.05649764335675319),\n",
              "  ('01', 0.055204829343684245),\n",
              "  ('04', 0.04034704753251232),\n",
              "  ('333', 0.039039973062101614),\n",
              "  ('lost', 0.02859789915062869),\n",
              "  ('05', 0.026437473419899275),\n",
              "  ('won', 0.023553954169353054),\n",
              "  ('500', 0.019568654545894838)],\n",
              " 122: [('stephanopoulos', 0.025540160424070302),\n",
              "  ('president', 0.025108456532267336),\n",
              "  ('mr', 0.01897043150328689),\n",
              "  ('myers', 0.01669045616905928),\n",
              "  ('jobs', 0.015843509610292083),\n",
              "  ('ms', 0.011641625494033789),\n",
              "  ('we', 0.011180946443977463),\n",
              "  ('secretary', 0.011039136901753122),\n",
              "  ('going', 0.008674911425118895),\n",
              "  ('he', 0.007948341362900919)],\n",
              " 123: [('battery', 0.0637956185098358),\n",
              "  ('batteries', 0.03831393192538842),\n",
              "  ('ups', 0.020917295562435254),\n",
              "  ('copper', 0.020804084526254932),\n",
              "  ('zinc', 0.020479978878664262),\n",
              "  ('switchover', 0.01817705641582006),\n",
              "  ('electrolyte', 0.016599966302909767),\n",
              "  ('inverter', 0.016599966302909767),\n",
              "  ('dc', 0.015250562898967734),\n",
              "  ('anneser', 0.01499464226131685)],\n",
              " 124: [('ticket', 0.06612429357697691),\n",
              "  ('airline', 0.04061752375374644),\n",
              "  ('hotel', 0.030819337587600342),\n",
              "  ('tickets', 0.028119979714739944),\n",
              "  ('coupons', 0.024672242851698785),\n",
              "  ('hotelco', 0.01799208541832021),\n",
              "  ('northwest', 0.017389975412042923),\n",
              "  ('transferable', 0.017088889106636702),\n",
              "  ('vacation', 0.016535420815814772),\n",
              "  ('sale', 0.016081486616673406)],\n",
              " 125: [('mary', 0.052377889489235996),\n",
              "  ('maria', 0.026909672074112982),\n",
              "  ('her', 0.021209083269578183),\n",
              "  ('immaculate', 0.02106337850195042),\n",
              "  ('conception', 0.02021440628085869),\n",
              "  ('virgin', 0.01884485108077708),\n",
              "  ('she', 0.01775923558438014),\n",
              "  ('church', 0.015068697281458477),\n",
              "  ('sin', 0.01402318762657377),\n",
              "  ('revelation', 0.013109191449241532)],\n",
              " 126: [('amp', 0.06242696128302525),\n",
              "  ('adcom', 0.049851235245085966),\n",
              "  ('components', 0.024531344722513667),\n",
              "  ('amps', 0.022411161581496423),\n",
              "  ('amplifier', 0.020053504438075874),\n",
              "  ('audio', 0.015492268883806993),\n",
              "  ('sound', 0.014073011438390657),\n",
              "  ('products', 0.013126748637959812),\n",
              "  ('shore', 0.012437562770822202),\n",
              "  ('555', 0.012254522849706384)],\n",
              " 127: [('ini', 0.07922755986783657),\n",
              "  ('updating', 0.03536817300519227),\n",
              "  ('spss', 0.032778400841694234),\n",
              "  ('utility', 0.028150560638597614),\n",
              "  ('svein', 0.026176206192283714),\n",
              "  ('sysedit', 0.0242938765901229),\n",
              "  ('windows', 0.024204441636784868),\n",
              "  ('files', 0.023318965895514767),\n",
              "  ('file', 0.021888227957409124),\n",
              "  ('bytes', 0.021099607422988283)],\n",
              " 128: [('cd', 0.06056050118833495),\n",
              "  ('apple', 0.04360513788033976),\n",
              "  ('cd300', 0.03024656116402429),\n",
              "  ('cd300i', 0.02499597115644172),\n",
              "  ('toshiba', 0.02436070311305444),\n",
              "  ('drive', 0.022180575548384356),\n",
              "  ('cdrom', 0.021550716491358063),\n",
              "  ('rom', 0.020863137123258206),\n",
              "  ('nec', 0.01857857875873688),\n",
              "  ('bauer', 0.01544522148974858)],\n",
              " 129: [('ra', 0.042917646126227636),\n",
              "  ('satan', 0.030507961528553795),\n",
              "  ('god', 0.02634721680548772),\n",
              "  ('lucifer', 0.02009902053858603),\n",
              "  ('evil', 0.017615187419253844),\n",
              "  ('bible', 0.013042190785286008),\n",
              "  ('created', 0.011057191497348625),\n",
              "  ('angels', 0.010541010630550523),\n",
              "  ('ye', 0.009903325315737227),\n",
              "  ('free', 0.0097359908993062)],\n",
              " 130: [('ati', 0.046649016706701234),\n",
              "  ('drivers', 0.04010295409186434),\n",
              "  ('ultra', 0.03163341452847409),\n",
              "  ('card', 0.029731726419552587),\n",
              "  ('gateway', 0.028588581525328876),\n",
              "  ('1280x1024', 0.02400899489155094),\n",
              "  ('24x', 0.019599199300287903),\n",
              "  ('windows', 0.01746903069056006),\n",
              "  ('experienced', 0.01744689707281036),\n",
              "  ('pro', 0.01688496400319031)],\n",
              " 131: [('25', 0.18812401805374943),\n",
              "  ('35', 0.03579135342838595),\n",
              "  ('playoff', 0.03287764790868615),\n",
              "  ('pool', 0.031733674052533925),\n",
              "  ('34', 0.03105941944301545),\n",
              "  ('36', 0.0301536537256073),\n",
              "  ('37', 0.02979893243354612),\n",
              "  ('stats', 0.02803438254569876),\n",
              "  ('32', 0.023651178811816782),\n",
              "  ('pick', 0.021588960283663066)],\n",
              " 132: [('summer', 0.03392496272540333),\n",
              "  ('room', 0.03078901898745808),\n",
              "  ('sublet', 0.024340025237042772),\n",
              "  ('bedroom', 0.02170940889705357),\n",
              "  ('washer', 0.017586251084099717),\n",
              "  ('dryer', 0.01746799460433479),\n",
              "  ('kitchen', 0.017302884127245198),\n",
              "  ('furnished', 0.016180714478173642),\n",
              "  ('condo', 0.01548756426071451),\n",
              "  ('rent', 0.015401199233650081)],\n",
              " 133: [('lopez', 0.03663245685931817),\n",
              "  ('catchers', 0.021746079042447804),\n",
              "  ('olson', 0.015344087609125083),\n",
              "  ('aaa', 0.014687767251750233),\n",
              "  ('catcher', 0.013538097612875033),\n",
              "  ('braves', 0.013485855472080586),\n",
              "  ('berryhill', 0.013293043501417176),\n",
              "  ('he', 0.013039161670979894),\n",
              "  ('year', 0.012493493930000401),\n",
              "  ('singer', 0.012455049803845031)],\n",
              " 134: [('koresh', 0.0762360879969579),\n",
              "  ('backing', 0.03587488222693838),\n",
              "  ('cites', 0.03159658783168161),\n",
              "  ('enclosed', 0.031057898486282206),\n",
              "  ('biblical', 0.030106095445392184),\n",
              "  ('tape', 0.02482648806305004),\n",
              "  ('he', 0.022242062245969585),\n",
              "  ('messenger', 0.019112552914635202),\n",
              "  ('structure', 0.01873090682309339),\n",
              "  ('solid', 0.018444125738086948)],\n",
              " 135: [('comet', 0.1011366110706751),\n",
              "  ('jupiter', 0.09219925451513428),\n",
              "  ('gehrels', 0.05947464871449133),\n",
              "  ('orbit', 0.04761794359471752),\n",
              "  ('temporary', 0.04264039374488301),\n",
              "  ('brader', 0.02973722517745406),\n",
              "  ('radii', 0.028300933291621092),\n",
              "  ('1993e', 0.02448059102787257),\n",
              "  ('1970', 0.024181302586252878),\n",
              "  ('1973', 0.02412809737015021)],\n",
              " 136: [('font', 0.06444570589340895),\n",
              "  ('fonts', 0.06394442812118062),\n",
              "  ('atm', 0.05297010113162711),\n",
              "  ('tt', 0.04861861543168362),\n",
              "  ('truetype', 0.04352979243929944),\n",
              "  ('type', 0.026991077154341618),\n",
              "  ('characters', 0.025493827414729803),\n",
              "  ('character', 0.021386562105268028),\n",
              "  ('atmanager', 0.021227272495959848),\n",
              "  ('coreldraw', 0.01716870328940572)],\n",
              " 137: [('split', 0.050003339538730936),\n",
              "  ('graphics', 0.0456716318472991),\n",
              "  ('ch', 0.033732718583733494),\n",
              "  ('aspects', 0.03342315064585499),\n",
              "  ('newsgroup', 0.03337425168033822),\n",
              "  ('comp', 0.026672854499573337),\n",
              "  ('group', 0.026332625937924185),\n",
              "  ('wate', 0.023734478991472446),\n",
              "  ('herringshaw', 0.022356543583376128),\n",
              "  ('groups', 0.020883845746868888)],\n",
              " 138: [('stadium', 0.053561632578919306),\n",
              "  ('uniforms', 0.04933851942640295),\n",
              "  ('baseball', 0.024421499547073507),\n",
              "  ('mets', 0.024070310884217197),\n",
              "  ('new', 0.02024561378827641),\n",
              "  ('seats', 0.01838559700329299),\n",
              "  ('phillies', 0.015950592951175697),\n",
              "  ('reds', 0.015826696610097397),\n",
              "  ('marlins', 0.013082021218917841),\n",
              "  ('football', 0.012724602686879713)],\n",
              " 139: [('tiff', 0.08880776774666271),\n",
              "  ('42', 0.06005935064161611),\n",
              "  ('philosophical', 0.04495132624550429),\n",
              "  ('significance', 0.04208699931876561),\n",
              "  ('spec', 0.021714197085182046),\n",
              "  ('alice', 0.019313174178310216),\n",
              "  ('question', 0.015121250652600499),\n",
              "  ('gripe', 0.014716365645912782),\n",
              "  ('joachim', 0.01377808247029461),\n",
              "  ('cartwright', 0.01331757880064757)],\n",
              " 140: [('nubus', 0.08993239301626701),\n",
              "  ('pds', 0.07922217879682761),\n",
              "  ('lc', 0.056954706158977624),\n",
              "  ('powerpc', 0.04338924538371345),\n",
              "  ('slot', 0.041902037454161024),\n",
              "  ('lciii', 0.032287407544404845),\n",
              "  ('iii', 0.026482628037956635),\n",
              "  ('marvin', 0.02099370724666754),\n",
              "  ('macs', 0.018765031338324824),\n",
              "  ('mirsky', 0.016966633873298265)],\n",
              " 141: [('cooling', 0.07046843422007623),\n",
              "  ('towers', 0.05999999626696333),\n",
              "  ('water', 0.05041709234700873),\n",
              "  ('nuclear', 0.049922327824383404),\n",
              "  ('plants', 0.04806394146422188),\n",
              "  ('steam', 0.04123607820501755),\n",
              "  ('fossil', 0.02637175429533253),\n",
              "  ('tower', 0.022515486829449982),\n",
              "  ('hot', 0.018746430862346068),\n",
              "  ('cool', 0.018376121222462467)],\n",
              " 142: [('stephanopoulos', 0.03890117192642885),\n",
              "  ('mr', 0.02774796249493472),\n",
              "  ('bosnia', 0.020005864100807146),\n",
              "  ('serbs', 0.014705092145523225),\n",
              "  ('embargo', 0.014050677429053793),\n",
              "  ('president', 0.013872185512617046),\n",
              "  ('somalia', 0.01385562640079264),\n",
              "  ('we', 0.010455492795226074),\n",
              "  ('said', 0.008547212437218538),\n",
              "  ('jobs', 0.007900305099538325)],\n",
              " 143: [('odometer', 0.05484216337323458),\n",
              "  ('sensor', 0.03513090933292812),\n",
              "  ('car', 0.021398381892768655),\n",
              "  ('odometers', 0.021198373852784363),\n",
              "  ('mileage', 0.019330455457676275),\n",
              "  ('sprint', 0.01897920506805588),\n",
              "  ('speedo', 0.018208683290689785),\n",
              "  ('oxygen', 0.017674184969091296),\n",
              "  ('bmw', 0.01736087178111857),\n",
              "  ('electronic', 0.01661484993163507)],\n",
              " 144: [('mask', 0.08013398886882718),\n",
              "  ('goalie', 0.05816652780385909),\n",
              "  ('hrivnak', 0.03857851653075215),\n",
              "  ('masks', 0.02865834754732421),\n",
              "  ('votes', 0.0282276798719063),\n",
              "  ('go', 0.019585285174622474),\n",
              "  ('vanbiesbrouck', 0.017676169966083507),\n",
              "  ('mike', 0.017124989263212646),\n",
              "  ('keller', 0.015713292846062748),\n",
              "  ('forehead', 0.014964516875746034)],\n",
              " 145: [('requests', 0.04491453878396939),\n",
              "  ('send', 0.029106268049200534),\n",
              "  ('mailing', 0.026159609317853473),\n",
              "  ('junk', 0.023611127565507265),\n",
              "  ('list', 0.02193149532521716),\n",
              "  ('lists', 0.020991929914997137),\n",
              "  ('catalog', 0.02071546861048067),\n",
              "  ('mail', 0.019765541341190376),\n",
              "  ('etiquette', 0.016609519187769393),\n",
              "  ('improper', 0.015356907729795026)],\n",
              " 146: [('energy', 0.03266910662323388),\n",
              "  ('uranium', 0.023178715678637652),\n",
              "  ('oil', 0.021762203116352234),\n",
              "  ('nuclear', 0.019555319397550123),\n",
              "  ('cato', 0.017795813648580634),\n",
              "  ('plutonium', 0.01625153376460697),\n",
              "  ('waste', 0.012857102930608036),\n",
              "  ('conservation', 0.012656704891777778),\n",
              "  ('reiken', 0.010670647940542512),\n",
              "  ('japan', 0.010656049402907677)],\n",
              " 147: [('xputimage', 0.04829480366948267),\n",
              "  ('memory', 0.04128122159501442),\n",
              "  ('server', 0.03923330181060641),\n",
              "  ('shared', 0.0330068305149844),\n",
              "  ('animation', 0.028127638462296208),\n",
              "  ('sunview', 0.027422495655361208),\n",
              "  ('pixmaps', 0.025833641594973176),\n",
              "  ('shm', 0.02264648565054574),\n",
              "  ('gooch', 0.021356346737570527),\n",
              "  ('client', 0.02121035927471441)],\n",
              " 148: [('ax', 0.10473946937759479),\n",
              "  ('max', 0.051176871466178),\n",
              "  ('g9v', 0.02500838633007454),\n",
              "  ('b8f', 0.024218697286548133),\n",
              "  ('a86', 0.021043202689430482),\n",
              "  ('pl', 0.017898767613908087),\n",
              "  ('1d9', 0.01598853169353712),\n",
              "  ('145', 0.015874822243010053),\n",
              "  ('1t', 0.012381191380824209),\n",
              "  ('0t', 0.011469134953097468)],\n",
              " 149: [('sabbath', 0.04006302681622799),\n",
              "  ('law', 0.03411405674194579),\n",
              "  ('worship', 0.029661480283537434),\n",
              "  ('paul', 0.02591957776591888),\n",
              "  ('jesus', 0.020279985761526106),\n",
              "  ('ceremonial', 0.019992224371996006),\n",
              "  ('christians', 0.017461312734461172),\n",
              "  ('day', 0.017086208635373152),\n",
              "  ('sunday', 0.01410773004671823),\n",
              "  ('gentiles', 0.013630467038339915)],\n",
              " 150: [('lyme', 0.08296027754027242),\n",
              "  ('quack', 0.03379376520590879),\n",
              "  ('physician', 0.022615422155294156),\n",
              "  ('gordon', 0.022015367506402712),\n",
              "  ('banks', 0.021512158151227395),\n",
              "  ('patients', 0.01888294485088094),\n",
              "  ('ld', 0.016591428625242193),\n",
              "  ('disease', 0.01557291514757285),\n",
              "  ('quacks', 0.01554683478902324),\n",
              "  ('poo', 0.015524247882176193)],\n",
              " 151: [('dos6', 0.04406079739838046),\n",
              "  ('stacker', 0.041247279754915864),\n",
              "  ('dos', 0.03843289653133305),\n",
              "  ('compression', 0.02963360263535352),\n",
              "  ('norton', 0.028918434844547956),\n",
              "  ('doubledisk', 0.026023354817220807),\n",
              "  ('speedisk', 0.020940271996453326),\n",
              "  ('defrag', 0.02056838717035531),\n",
              "  ('ms', 0.019841274393909078),\n",
              "  ('doublespace', 0.019681627147745052)],\n",
              " 152: [('plastic', 0.053058639997187455),\n",
              "  ('paint', 0.029160311657097187),\n",
              "  ('detailing', 0.02328814424107774),\n",
              "  ('scratches', 0.022678604368481497),\n",
              "  ('wax', 0.022408266318899923),\n",
              "  ('tip', 0.022164394438264668),\n",
              "  ('polish', 0.019485573583705163),\n",
              "  ('compound', 0.01917592463498432),\n",
              "  ('scuffed', 0.018313321314294105),\n",
              "  ('coat', 0.016083419808779923)],\n",
              " 153: [('scope', 0.07121766028051675),\n",
              "  ('scopes', 0.04120319081200079),\n",
              "  ('phosphor', 0.01736573494736058),\n",
              "  ('oscilloscope', 0.016955377081515646),\n",
              "  ('medin', 0.016939532834522795),\n",
              "  ('digital', 0.016205189837068302),\n",
              "  ('tektronix', 0.015565648093734086),\n",
              "  ('huntsville', 0.015333681519355847),\n",
              "  ('mhz', 0.014689187854470787),\n",
              "  ('205', 0.01425155736482115)],\n",
              " 154: [('expose', 0.08001975060985331),\n",
              "  ('window', 0.05156953805883611),\n",
              "  ('event', 0.04010981264671817),\n",
              "  ('handler', 0.03948165181311573),\n",
              "  ('mapped', 0.03513389776350757),\n",
              "  ('item', 0.034735879351743874),\n",
              "  ('xcopyarea', 0.026474501638312685),\n",
              "  ('selepntr', 0.0238155070847577),\n",
              "  ('tickle', 0.02328977730868241),\n",
              "  ('events', 0.02236861389175819)],\n",
              " 155: [('cd', 0.06127566368529157),\n",
              "  ('rom', 0.05745941301463136),\n",
              "  ('toshiba', 0.05447649220460693),\n",
              "  ('adaptec', 0.04277174497481772),\n",
              "  ('3401', 0.04113489403649952),\n",
              "  ('buslogic', 0.033538264183863506),\n",
              "  ('cdrom', 0.02891560477947041),\n",
              "  ('drivers', 0.027048607056752248),\n",
              "  ('os', 0.02444538958429231),\n",
              "  ('scsi', 0.023338472478619046)],\n",
              " 156: [('saturn', 0.07966242694526005),\n",
              "  ('dealer', 0.03844372833556226),\n",
              "  ('profit', 0.0302219223444575),\n",
              "  ('cars', 0.02385622838890819),\n",
              "  ('sl2', 0.02283281970140587),\n",
              "  ('car', 0.022267164622174863),\n",
              "  ('price', 0.020527750106317105),\n",
              "  ('sc1', 0.020362187240329743),\n",
              "  ('sc2', 0.018665338303635596),\n",
              "  ('civic', 0.016977012772778716)],\n",
              " 157: [('tape', 0.0789719809471022),\n",
              "  ('backup', 0.07161088638293793),\n",
              "  ('drive', 0.026829441615030463),\n",
              "  ('tapes', 0.024109893239131916),\n",
              "  ('jumbo', 0.021566667318323383),\n",
              "  ('250', 0.015906021245350597),\n",
              "  ('40sc', 0.015271103192704718),\n",
              "  ('husak', 0.01446413702862234),\n",
              "  ('dh', 0.014005474189313175),\n",
              "  ('colorado', 0.01397505593941426)],\n",
              " 158: [('lens', 0.08755639484624914),\n",
              "  ('camera', 0.05831890225838403),\n",
              "  ('nikon', 0.03212120498783056),\n",
              "  ('lenses', 0.03049704349205727),\n",
              "  ('zoom', 0.028981598079269894),\n",
              "  ('canon', 0.026065992775661816),\n",
              "  ('minolta', 0.021699509852984664),\n",
              "  ('mm', 0.021166076963264648),\n",
              "  ('flash', 0.020780404915784086),\n",
              "  ('condition', 0.020184728731247357)],\n",
              " 159: [('pantheism', 0.021993575521669827),\n",
              "  ('disasters', 0.02030281629306241),\n",
              "  ('environmentalism', 0.017869780111356735),\n",
              "  ('environmental', 0.015050131172763439),\n",
              "  ('watt', 0.012975551927804254),\n",
              "  ('disaster', 0.01184019967390166),\n",
              "  ('environment', 0.011443027867228363),\n",
              "  ('paganism', 0.011184208185917796),\n",
              "  ('shk', 0.011095119859753377),\n",
              "  ('religious', 0.01002808731746437)],\n",
              " 160: [('dl', 0.0621611001339204),\n",
              "  ('harkey', 0.0554822661823988),\n",
              "  ('dolven', 0.050607323919049055),\n",
              "  ('mel', 0.049474322690412076),\n",
              "  ('wetteland', 0.04058742228745683),\n",
              "  ('cubs', 0.03794259061260786),\n",
              "  ('melido', 0.035418222369043274),\n",
              "  ('expos', 0.035411577312848344),\n",
              "  ('roster', 0.03217850937328827),\n",
              "  ('hall', 0.030907327617443334)],\n",
              " 161: [('mouse', 0.11058904807245899),\n",
              "  ('motion', 0.022614274904093533),\n",
              "  ('smoothly', 0.020930420866941458),\n",
              "  ('ball', 0.02037914528472053),\n",
              "  ('trackball', 0.01881834084230809),\n",
              "  ('jumpy', 0.016936506758077282),\n",
              "  ('apple', 0.016763409852694772),\n",
              "  ('jump', 0.015772845746550683),\n",
              "  ('eckton', 0.015242282136540313),\n",
              "  ('problem', 0.014441129299848405)],\n",
              " 162: [('xdm', 0.029625922198382875),\n",
              "  ('server', 0.027011865171912064),\n",
              "  ('font', 0.026266874938180893),\n",
              "  ('xterm', 0.0260814992240025),\n",
              "  ('error', 0.023679190719650262),\n",
              "  ('pid', 0.02304071977883958),\n",
              "  ('xopendisplay', 0.020917103616591135),\n",
              "  ('display', 0.01881507723592463),\n",
              "  ('xon', 0.018812417389093138),\n",
              "  ('console', 0.018483790509154885)],\n",
              " 163: [('dock', 0.05516929435759761),\n",
              "  ('portable', 0.04531891564708966),\n",
              "  ('duo', 0.044558772549217064),\n",
              "  ('powerbook', 0.02439329838823259),\n",
              "  ('apple', 0.021946045467562285),\n",
              "  ('68000', 0.017742725135739587),\n",
              "  ('processor', 0.01771246380341666),\n",
              "  ('docks', 0.01676111415456955),\n",
              "  ('mac', 0.016278389106108168),\n",
              "  ('mini', 0.016122857918515324)],\n",
              " 164: [('gl', 0.09040761528746162),\n",
              "  ('widget', 0.04335485323256632),\n",
              "  ('widgets', 0.028734841272434586),\n",
              "  ('xmdrawingarea', 0.02834968012971305),\n",
              "  ('motif', 0.02495649307086795),\n",
              "  ('xt', 0.018641418565334827),\n",
              "  ('glxmdraw', 0.01738263465040397),\n",
              "  ('mix', 0.017121548237647482),\n",
              "  ('circular', 0.017096391416359727),\n",
              "  ('application', 0.016745968857988614)],\n",
              " 165: [('easter', 0.06968461506672016),\n",
              "  ('resurrection', 0.031091701612797028),\n",
              "  ('celebration', 0.027514729256251553),\n",
              "  ('pagan', 0.02667526408160831),\n",
              "  ('passover', 0.022775268550694935),\n",
              "  ('goddess', 0.022285910928038336),\n",
              "  ('ishtar', 0.02105399789183304),\n",
              "  ('feast', 0.016975239185146464),\n",
              "  ('celebrate', 0.016813503034924668),\n",
              "  ('word', 0.016615310321623354)],\n",
              " 166: [('pain', 0.04365770909858193),\n",
              "  ('hernia', 0.026218642581905118),\n",
              "  ('massager', 0.014847599000519428),\n",
              "  ('neck', 0.014719291160414052),\n",
              "  ('pressure', 0.014551795320397682),\n",
              "  ('bone', 0.014456998555982878),\n",
              "  ('back', 0.012007233901667365),\n",
              "  ('compartment', 0.011941093432994727),\n",
              "  ('banks', 0.011043501217460965),\n",
              "  ('gordon', 0.010810445353364282)],\n",
              " 167: [('3do', 0.06347932673835144),\n",
              "  ('studio', 0.04201438584463984),\n",
              "  ('3d', 0.038209807293933855),\n",
              "  ('v2', 0.025912019922381874),\n",
              "  ('rumours', 0.023229698566678506),\n",
              "  ('3ds', 0.022103996966773096),\n",
              "  ('imagine', 0.021333612570444655),\n",
              "  ('01', 0.021263211459226977),\n",
              "  ('ipas', 0.01870148324156706),\n",
              "  ('differences', 0.018474601418888677)],\n",
              " 168: [('insurance', 0.05637825623365766),\n",
              "  ('health', 0.037356376000410405),\n",
              "  ('private', 0.020568671489731332),\n",
              "  ('care', 0.016665057814171103),\n",
              "  ('malpractice', 0.013910066309285124),\n",
              "  ('canadian', 0.013140587607962455),\n",
              "  ('dare', 0.012591080315184553),\n",
              "  ('canada', 0.012399563052925982),\n",
              "  ('doctors', 0.011385687681747038),\n",
              "  ('gary', 0.010760908978655146)],\n",
              " 169: [('migraine', 0.0606658088445779),\n",
              "  ('pain', 0.05067527421141551),\n",
              "  ('migraines', 0.03456796638429118),\n",
              "  ('headache', 0.023317665710914044),\n",
              "  ('banks', 0.022479064198783958),\n",
              "  ('gordon', 0.022004678618724512),\n",
              "  ('exercise', 0.021994118802834393),\n",
              "  ('zisfein', 0.021763514381632727),\n",
              "  ('jl', 0.02143913487110828),\n",
              "  ('headaches', 0.01649164220854483)],\n",
              " 170: [('ground', 0.0861567012298981),\n",
              "  ('outlet', 0.04658629543908393),\n",
              "  ('neutral', 0.04605984759173753),\n",
              "  ('wire', 0.04272490378641189),\n",
              "  ('grounding', 0.04135023803481256),\n",
              "  ('wiring', 0.0351003229273733),\n",
              "  ('outlets', 0.03350155680139242),\n",
              "  ('conductor', 0.03310433187770768),\n",
              "  ('connected', 0.02272426297774252),\n",
              "  ('breaker', 0.022498359214474076)],\n",
              " 171: [('fbi', 0.02625113472614781),\n",
              "  ('president', 0.01634064396087426),\n",
              "  ('federal', 0.013083696677610765),\n",
              "  ('batf', 0.012631740132284764),\n",
              "  ('waco', 0.011302390601818313),\n",
              "  ('were', 0.010210974792892073),\n",
              "  ('attorney', 0.009671475604239123),\n",
              "  ('koresh', 0.009472317980673113),\n",
              "  ('compound', 0.008770954877012271),\n",
              "  ('yesterday', 0.008737270335762359)],\n",
              " 172: [('copy', 0.0515450237069537),\n",
              "  ('protection', 0.029702311684596448),\n",
              "  ('protected', 0.026020545376609173),\n",
              "  ('babak', 0.02153946526578013),\n",
              "  ('sehari', 0.02153946526578013),\n",
              "  ('disks', 0.017020131174027613),\n",
              "  ('crack', 0.013996339914898695),\n",
              "  ('ketil', 0.011888071872819279),\n",
              "  ('albertsen', 0.011888071872819279),\n",
              "  ('software', 0.01127349793340911)],\n",
              " 173: [('fifth', 0.02425228997194751),\n",
              "  ('key', 0.023812756541817658),\n",
              "  ('keyphrase', 0.019028120826650268),\n",
              "  ('amendment', 0.0175263132991615),\n",
              "  ('compelled', 0.016155654659377395),\n",
              "  ('defendant', 0.015200574751047038),\n",
              "  ('passwords', 0.014927485088004589),\n",
              "  ('judge', 0.014371028550948958),\n",
              "  ('disclosure', 0.013360180036121284),\n",
              "  ('evidence', 0.013109458818941598)],\n",
              " 174: [('abortion', 0.05429759748127098),\n",
              "  ('abortions', 0.021380778632837206),\n",
              "  ('lm', 0.021329272761479258),\n",
              "  ('coverage', 0.01727244770379896),\n",
              "  ('pro', 0.01693668073881429),\n",
              "  ('margolis', 0.015842929945386875),\n",
              "  ('viability', 0.01491022062906519),\n",
              "  ('health', 0.01490398897977664),\n",
              "  ('protesters', 0.014694097863885505),\n",
              "  ('clinic', 0.013923357267533255)],\n",
              " 175: [('bike', 0.053248786765127024),\n",
              "  ('buying', 0.04473702184489832),\n",
              "  ('motorcycle', 0.04356079817457519),\n",
              "  ('motorcycles', 0.038816102216658245),\n",
              "  ('advice', 0.027951278888977214),\n",
              "  ('guide', 0.019627192587592945),\n",
              "  ('belliveau', 0.019582837044441314),\n",
              "  ('humphreys', 0.018848602280728346),\n",
              "  ('bikes', 0.015590171796445037),\n",
              "  ('cbr', 0.013795809783904303)],\n",
              " 176: [('quadra', 0.06650211008316685),\n",
              "  ('scsi', 0.047944021550163),\n",
              "  ('cartridge', 0.030899600490330217),\n",
              "  ('mac', 0.03042825473197435),\n",
              "  ('venerable', 0.0238378663606577),\n",
              "  ('blind', 0.02271911305460557),\n",
              "  ('jagielski', 0.022159971252362163),\n",
              "  ('quadras', 0.02210562253383015),\n",
              "  ('worked', 0.022035352241783407),\n",
              "  ('jasmine', 0.021905953236102307)],\n",
              " 177: [('book', 0.03978523674821417),\n",
              "  ('scrolls', 0.026334147601003582),\n",
              "  ('books', 0.01907376894354855),\n",
              "  ('butera', 0.01840767879645516),\n",
              "  ('sea', 0.016613532313971342),\n",
              "  ('religions', 0.014694829082134781),\n",
              "  ('mystery', 0.012627088328615508),\n",
              "  ('dead', 0.012402616885716169),\n",
              "  ('eisenman', 0.012042562429001846),\n",
              "  ('christian', 0.011617063182067117)],\n",
              " 178: [('photography', 0.07387695443019326),\n",
              "  ('krillean', 0.05389888181558225),\n",
              "  ('kirlian', 0.05124728114860422),\n",
              "  ('object', 0.03438831589355227),\n",
              "  ('energy', 0.031233255007794215),\n",
              "  ('pictures', 0.029364103262413923),\n",
              "  ('spikes', 0.024271240186408617),\n",
              "  ('huey', 0.02267429868401783),\n",
              "  ('patterns', 0.021158220542231983),\n",
              "  ('fields', 0.018682122199257918)],\n",
              " 179: [('motif', 0.09217651696547087),\n",
              "  ('linux', 0.026229140759593234),\n",
              "  ('bindings', 0.021557835419890002),\n",
              "  ('athena', 0.01763146491740633),\n",
              "  ('list', 0.015273041340935545),\n",
              "  ('devguide', 0.01450734090545452),\n",
              "  ('386bsd', 0.014476858422090127),\n",
              "  ('software', 0.01437840450996709),\n",
              "  ('interviews', 0.01302745769742466),\n",
              "  ('mailing', 0.01278228676626508)],\n",
              " 180: [('science', 0.03170370480443783),\n",
              "  ('methodology', 0.030846171982339892),\n",
              "  ('scientific', 0.016533496511557083),\n",
              "  ('turpin', 0.015551308384334134),\n",
              "  ('lady', 0.015445223288600212),\n",
              "  ('homeopathy', 0.013585353967954059),\n",
              "  ('empirical', 0.012551562834263105),\n",
              "  ('psychology', 0.012392472126719801),\n",
              "  ('lee', 0.012194777777016694),\n",
              "  ('russell', 0.012135838785382899)],\n",
              " 181: [('stove', 0.06563101586057929),\n",
              "  ('electric', 0.0629850233206217),\n",
              "  ('stoves', 0.043942815753676465),\n",
              "  ('ovens', 0.04145944933886765),\n",
              "  ('brent', 0.040134820683683896),\n",
              "  ('microwaves', 0.039605903212382707),\n",
              "  ('dividian', 0.03642271830699241),\n",
              "  ('ranch', 0.035281941302842804),\n",
              "  ('metro', 0.03348656543123585),\n",
              "  ('burns', 0.033079074921858465)],\n",
              " 182: [('midi', 0.10047033254372907),\n",
              "  ('driver', 0.061607376701458806),\n",
              "  ('sound', 0.05044934299679646),\n",
              "  ('speaker', 0.047870038858708615),\n",
              "  ('soundblaster', 0.04641464805700043),\n",
              "  ('drv', 0.03959611911710174),\n",
              "  ('blaster', 0.03280035642516677),\n",
              "  ('alec', 0.028506545622711772),\n",
              "  ('files', 0.02401885876646536),\n",
              "  ('windows', 0.023782777016142197)],\n",
              " 183: [('w4wg', 0.08761607116688602),\n",
              "  ('lastdrive', 0.03357935706842728),\n",
              "  ('novell', 0.027461744366229814),\n",
              "  ('workplace', 0.026175577378732672),\n",
              "  ('workgroups', 0.024741615694561535),\n",
              "  ('lan', 0.023656179568333192),\n",
              "  ('tcpview', 0.022460385367273104),\n",
              "  ('netx', 0.02051166215903943),\n",
              "  ('windows', 0.020453038985203176),\n",
              "  ('netware', 0.019018099601674582)],\n",
              " 184: [('jagr', 0.0642388336642331),\n",
              "  ('francis', 0.04945818029794929),\n",
              "  ('minus', 0.04253827330259609),\n",
              "  ('player', 0.04083980748507975),\n",
              "  ('stat', 0.040809963700007966),\n",
              "  ('ice', 0.030002199214193347),\n",
              "  ('plus', 0.0265625834246639),\n",
              "  ('bowman', 0.021901385095316254),\n",
              "  ('play', 0.02079463813320567),\n",
              "  ('points', 0.019402104303629535)],\n",
              " 185: [('tempest', 0.09526352002744469),\n",
              "  ('measures', 0.02688323714473594),\n",
              "  ('tapped', 0.024907570211347978),\n",
              "  ('emmisions', 0.023657412349785963),\n",
              "  ('classified', 0.021872892660572938),\n",
              "  ('surveillance', 0.021270391188059918),\n",
              "  ('holland', 0.02095103600627497),\n",
              "  ('electromagnetic', 0.02056744701824976),\n",
              "  ('protect', 0.01984812817031258),\n",
              "  ('stampfli', 0.018945038677815954)],\n",
              " 186: [('crohn', 0.06945283275395442),\n",
              "  ('inflammation', 0.030760534594308858),\n",
              "  ('disease', 0.02727726703260429),\n",
              "  ('ibd', 0.02420300297608668),\n",
              "  ('diet', 0.023648019448729896),\n",
              "  ('recurrence', 0.021669292179483886),\n",
              "  ('eyles', 0.020975112671493802),\n",
              "  ('inhibitors', 0.020975112671493802),\n",
              "  ('turmeric', 0.020975112671493802),\n",
              "  ('discomfort', 0.02078955767719414)],\n",
              " 187: [('pope', 0.0485780063596984),\n",
              "  ('schism', 0.03242004320374955),\n",
              "  ('church', 0.02890547352908096),\n",
              "  ('catholic', 0.022770213762419448),\n",
              "  ('sspx', 0.020599779257476997),\n",
              "  ('archbishop', 0.018732000638872447),\n",
              "  ('bishop', 0.01758359916437612),\n",
              "  ('schismatic', 0.015520297278314929),\n",
              "  ('saint', 0.014480469344016082),\n",
              "  ('bishops', 0.01232269509302722)],\n",
              " 188: [('joke', 0.02683370237367842),\n",
              "  ('funny', 0.025632905838276973),\n",
              "  ('arbitron', 0.01708942678452297),\n",
              "  ('rahe', 0.01708942678452297),\n",
              "  ('pat', 0.01621866496917144),\n",
              "  ('humour', 0.014699714604312486),\n",
              "  ('readership', 0.012299333887713308),\n",
              "  ('ross', 0.01191803540888225),\n",
              "  ('borden', 0.011259479052931648),\n",
              "  ('ipser', 0.0108034100430903)],\n",
              " 189: [('weight', 0.08145895322741968),\n",
              "  ('wa7kgx', 0.041879858708235),\n",
              "  ('forsberg', 0.040119250664984424),\n",
              "  ('obesity', 0.038946711526168055),\n",
              "  ('diet', 0.0378553162005336),\n",
              "  ('omen', 0.03616413185944925),\n",
              "  ('chuck', 0.033608222495963694),\n",
              "  ('rebound', 0.031057140698766056),\n",
              "  ('cycle', 0.028465054949208375),\n",
              "  ('gordon', 0.028140220925480616)],\n",
              " 190: [('homosexuality', 0.038226717054194774),\n",
              "  ('homosexual', 0.015866221201434407),\n",
              "  ('gay', 0.013516248248570941),\n",
              "  ('gays', 0.011918754641129807),\n",
              "  ('sin', 0.011370781435832799),\n",
              "  ('love', 0.01125816417404745),\n",
              "  ('homosexuals', 0.010510230503668883),\n",
              "  ('hudson', 0.009994095614475512),\n",
              "  ('people', 0.00938860731677264),\n",
              "  ('christianity', 0.00856047318475842)],\n",
              " 191: [('gas', 0.041645111117899064),\n",
              "  ('tear', 0.026711397906964827),\n",
              "  ('cs', 0.02661450281938328),\n",
              "  ('toxic', 0.019369996838145175),\n",
              "  ('projectile', 0.019316510024439792),\n",
              "  ('gun', 0.015675148419223375),\n",
              "  ('fuller', 0.014549549198229175),\n",
              "  ('aluminum', 0.014534618935067597),\n",
              "  ('mix', 0.014105427384081198),\n",
              "  ('eder', 0.014051655853082688)],\n",
              " 192: [('atf', 0.0350146510288204),\n",
              "  ('fbi', 0.026398221863267603),\n",
              "  ('survivors', 0.01915472282439506),\n",
              "  ('ranch', 0.019068472307597933),\n",
              "  ('fires', 0.018170433582178822),\n",
              "  ('dividian', 0.01804459842470934),\n",
              "  ('burns', 0.016388085540315567),\n",
              "  ('reported', 0.011517663822742094),\n",
              "  ('koresh', 0.011323892718826351),\n",
              "  ('actions', 0.010346973643182978)],\n",
              " 193: [('context', 0.03951607721391389),\n",
              "  ('jim', 0.03581906582644907),\n",
              "  ('meritt', 0.02777303572383468),\n",
              "  ('telepathy', 0.019684487347293864),\n",
              "  ('quote', 0.01853193645728788),\n",
              "  ('weiss', 0.018008629540809857),\n",
              "  ('articles', 0.017723338382510126),\n",
              "  ('hypocrite', 0.017483621719666425),\n",
              "  ('quotes', 0.016770682922878557),\n",
              "  ('joslin', 0.016378581873844614)],\n",
              " 194: [('corn', 0.0667321302307395),\n",
              "  ('seizures', 0.06000348922980645),\n",
              "  ('seizure', 0.04224411190475946),\n",
              "  ('cereals', 0.0377239560408987),\n",
              "  ('kellog', 0.029550586359330173),\n",
              "  ('sharon', 0.028129958275311055),\n",
              "  ('paulson', 0.02716263476481282),\n",
              "  ('diet', 0.02567516858855067),\n",
              "  ('food', 0.022179161934489013),\n",
              "  ('ozonoff', 0.022147663417543127)],\n",
              " 195: [('captain', 0.13597398229650148),\n",
              "  ('traded', 0.10152113573285435),\n",
              "  ('captains', 0.05582774548701557),\n",
              "  ('cyt', 0.026839613110423467),\n",
              "  ('striped', 0.02648253650175991),\n",
              "  ('team', 0.0251233402942179),\n",
              "  ('resigned', 0.024852048654591428),\n",
              "  ('captaincy', 0.024510936370775654),\n",
              "  ('collingridge', 0.02415565179938112),\n",
              "  ('trivia', 0.02356110012719345)],\n",
              " 196: [('drink', 0.05770503812278353),\n",
              "  ('drinking', 0.04711026292961599),\n",
              "  ('riding', 0.036465923334801406),\n",
              "  ('hours', 0.03296492143950977),\n",
              "  ('alcohol', 0.032656534216063474),\n",
              "  ('drinks', 0.026821695707931713),\n",
              "  ('ride', 0.02549079475178892),\n",
              "  ('sobriety', 0.025122031916552306),\n",
              "  ('sober', 0.023851254919930846),\n",
              "  ('cycling', 0.02254032458889681)],\n",
              " 197: [('exhaust', 0.037766852222559155),\n",
              "  ('carbs', 0.03182552588743473),\n",
              "  ('idle', 0.025382455240043903),\n",
              "  ('intake', 0.020961409235518823),\n",
              "  ('idling', 0.017796113751548402),\n",
              "  ('splitfires', 0.013203801167744946),\n",
              "  ('rpm', 0.012875654563494182),\n",
              "  ('plugs', 0.01244810138127379),\n",
              "  ('carburetor', 0.012120268031126903),\n",
              "  ('carb', 0.012083228076625954)],\n",
              " 198: [('mydisplay', 0.05173365795439847),\n",
              "  ('gc', 0.0406228347179026),\n",
              "  ('pixmap', 0.04003950852807332),\n",
              "  ('gxxor', 0.03985025038596839),\n",
              "  ('drawindex', 0.03760137602887997),\n",
              "  ('palettecolors', 0.03760137602887997),\n",
              "  ('draw', 0.03631254194771143),\n",
              "  ('xdrawline', 0.03159962922551508),\n",
              "  ('drawing', 0.027698558310814258),\n",
              "  ('y3', 0.026333024354595905)],\n",
              " 199: [('lds', 0.031664962815492984),\n",
              "  ('bible', 0.02819350578863311),\n",
              "  ('mormon', 0.026050426164952876),\n",
              "  ('mcconkie', 0.024224808955399163),\n",
              "  ('ra', 0.020118257783211038),\n",
              "  ('jesus', 0.019423469349122122),\n",
              "  ('weiss', 0.01843075911713718),\n",
              "  ('church', 0.017572888535038154),\n",
              "  ('doctrine', 0.015429926529973183),\n",
              "  ('teaches', 0.014713849101985994)],\n",
              " 200: [('car', 0.024657998075080165),\n",
              "  ('price', 0.018533838012742877),\n",
              "  ('sales', 0.017466891994030394),\n",
              "  ('dealership', 0.015258149729025262),\n",
              "  ('dealer', 0.01290484835675688),\n",
              "  ('truck', 0.012815395948285221),\n",
              "  ('deal', 0.012575619558566294),\n",
              "  ('finance', 0.011283765746716082),\n",
              "  ('buying', 0.010993819450149886),\n",
              "  ('scam', 0.010121080954316027)],\n",
              " 201: [('breaker', 0.09313773413900334),\n",
              "  ('tie', 0.09282306952531347),\n",
              "  ('devils', 0.0652266682737626),\n",
              "  ('islanders', 0.05317858562803604),\n",
              "  ('record', 0.04746660803732393),\n",
              "  ('win', 0.03278787803600261),\n",
              "  ('tiebreaker', 0.031866999739788265),\n",
              "  ('ahve', 0.028293740870958428),\n",
              "  ('points', 0.026746723921071934),\n",
              "  ('retard', 0.026574150003715945)],\n",
              " 202: [('sunrise', 0.10530602477882102),\n",
              "  ('sunset', 0.103292047642042),\n",
              "  ('wetstein', 0.0485955520471822),\n",
              "  ('times', 0.04672067982693727),\n",
              "  ('compute', 0.03559247177555956),\n",
              "  ('daylight', 0.030726606193497617),\n",
              "  ('rouben', 0.026649296214561614),\n",
              "  ('rostamian', 0.026649296214561614),\n",
              "  ('degrees', 0.02296834016199597),\n",
              "  ('shirlene', 0.021925182162854565)],\n",
              " 203: [('celp', 0.03543267245689758),\n",
              "  ('compression', 0.03181343052635521),\n",
              "  ('speech', 0.030170869930254845),\n",
              "  ('voice', 0.02277182555697476),\n",
              "  ('toal', 0.021986618760854944),\n",
              "  ('dsp', 0.02048155846449001),\n",
              "  ('samples', 0.018906053967221545),\n",
              "  ('graham', 0.018756660342515757),\n",
              "  ('32bis', 0.016912684837255273),\n",
              "  ('1016', 0.016546365951453746)],\n",
              " 204: [('kuiper', 0.04485400202516082),\n",
              "  ('mercury', 0.043691472493707355),\n",
              "  ('pluto', 0.04101712048917124),\n",
              "  ('planet', 0.04001855651307103),\n",
              "  ('vulcan', 0.036745173433957924),\n",
              "  ('object', 0.03385820768394569),\n",
              "  ('smiley', 0.028718975365503177),\n",
              "  ('orbit', 0.02742501989382439),\n",
              "  ('sightings', 0.026896298059308977),\n",
              "  ('karla', 0.026896298059308977)],\n",
              " 205: [('adl', 0.06663470026587329),\n",
              "  ('bullock', 0.04103943694696794),\n",
              "  ('yigal', 0.030880218844382263),\n",
              "  ('arens', 0.030433063079248007),\n",
              "  ('francisco', 0.028509434408821536),\n",
              "  ('defamation', 0.028004755910117085),\n",
              "  ('anti', 0.023543903268873095),\n",
              "  ('police', 0.023049734936787674),\n",
              "  ('san', 0.022352240275193086),\n",
              "  ('gerard', 0.022023283015591744)],\n",
              " 206: [('liar', 0.03794999127560718),\n",
              "  ('lunatic', 0.027143508430351557),\n",
              "  ('he', 0.02311841197080744),\n",
              "  ('gathered', 0.014563205985917982),\n",
              "  ('drawn', 0.014261708923721444),\n",
              "  ('bissell', 0.013547904563880594),\n",
              "  ('bible', 0.013132056871761817),\n",
              "  ('fool', 0.013095187263916678),\n",
              "  ('crazy', 0.012849830503269455),\n",
              "  ('christian', 0.012341895243919996)],\n",
              " 207: [('rational', 0.025790609878505302),\n",
              "  ('science', 0.022753999111760487),\n",
              "  ('fulk', 0.0217918547259654),\n",
              "  ('theory', 0.020787567730675017),\n",
              "  ('fantasy', 0.020037480700247596),\n",
              "  ('kekule', 0.01978505460876989),\n",
              "  ('merrill', 0.017406872758068508),\n",
              "  ('scientific', 0.016151245130220933),\n",
              "  ('sas', 0.01540892771149629),\n",
              "  ('methodology', 0.01540892771149629)],\n",
              " 208: [('shaft', 0.15441951129539133),\n",
              "  ('wheelies', 0.08207289015723924),\n",
              "  ('wheelie', 0.07380381299695253),\n",
              "  ('stafford', 0.03828200307199695),\n",
              "  ('motorcycle', 0.03662031503060112),\n",
              "  ('winona', 0.02983888166616362),\n",
              "  ('wheel', 0.028936257426834644),\n",
              "  ('drives', 0.02837982576754542),\n",
              "  ('drive', 0.02558298233625128),\n",
              "  ('imposible', 0.02521139213574143)],\n",
              " 209: [('selective', 0.039469041715978266),\n",
              "  ('pork', 0.029717275614137104),\n",
              "  ('service', 0.025785262767323177),\n",
              "  ('borden', 0.020508452799194346),\n",
              "  ('abolish', 0.020440506017649684),\n",
              "  ('naval', 0.019629557877984004),\n",
              "  ('reserve', 0.01932447644496658),\n",
              "  ('navy', 0.016561173540893488),\n",
              "  ('mueller', 0.016417544763250386),\n",
              "  ('admistration', 0.015167809473786263)],\n",
              " 210: [('cview', 0.12892317032544945),\n",
              "  ('temp', 0.05947546496676579),\n",
              "  ('zenkar', 0.026816031437359566),\n",
              "  ('files', 0.025691834453707242),\n",
              "  ('directory', 0.025303074950352014),\n",
              "  ('rene', 0.023313509426680926),\n",
              "  ('floppy', 0.02240864438802965),\n",
              "  ('file', 0.021797914521695407),\n",
              "  ('walter', 0.019221054751876934),\n",
              "  ('dir', 0.01880160985294627)],\n",
              " 211: [('00', 0.06653188772185115),\n",
              "  ('50', 0.05414947392433737),\n",
              "  ('wolverine', 0.04291822733371714),\n",
              "  ('1st', 0.041687811558939934),\n",
              "  ('hulk', 0.034845306101713824),\n",
              "  ('cover', 0.03428537607407259),\n",
              "  ('appears', 0.03320767784970729),\n",
              "  ('art', 0.03181568548728501),\n",
              "  ('comics', 0.031718490571612815),\n",
              "  ('liefeld', 0.02558935282377924)],\n",
              " 212: [('hiv', 0.05338670783339996),\n",
              "  ('aids', 0.030507375256933783),\n",
              "  ('health', 0.02237191654806115),\n",
              "  ('care', 0.01895960987028219),\n",
              "  ('infected', 0.01600835557843384),\n",
              "  ('immunization', 0.014422188686085623),\n",
              "  ('children', 0.014386283413025896),\n",
              "  ('immunizations', 0.013337776634243811),\n",
              "  ('medical', 0.012623965307520793),\n",
              "  ('cancer', 0.011380007489528837)],\n",
              " 213: [('clock', 0.06728487321275872),\n",
              "  ('iisi', 0.03952177931896481),\n",
              "  ('kuo', 0.03400295131992691),\n",
              "  ('mhz', 0.029418914052046444),\n",
              "  ('si', 0.02762263695249504),\n",
              "  ('oscillator', 0.023086712732698005),\n",
              "  ('speed', 0.023027280894127075),\n",
              "  ('quadra', 0.02039110154628012),\n",
              "  ('upgrade', 0.02006360662302144),\n",
              "  ('washington', 0.019767217558955833)],\n",
              " 214: [('god', 0.03440262439173176),\n",
              "  ('muslims', 0.01764536048246128),\n",
              "  ('religion', 0.015517296028490838),\n",
              "  ('serbs', 0.014987735475073542),\n",
              "  ('islam', 0.014181660084894368),\n",
              "  ('moslems', 0.01269046872839121),\n",
              "  ('serbian', 0.010841662664950293),\n",
              "  ('man', 0.01008475121074311),\n",
              "  ('cleansing', 0.009324720527774907),\n",
              "  ('genocide', 0.009316729883131572)],\n",
              " 215: [('simms', 0.07767667408722384),\n",
              "  ('256k', 0.0730142892269465),\n",
              "  ('jh', 0.04463327526046824),\n",
              "  ('old', 0.036298325365950374),\n",
              "  ('inovative', 0.030984399945871592),\n",
              "  ('bunch', 0.027773594066531817),\n",
              "  ('them', 0.027110862408441568),\n",
              "  ('haines', 0.02375439619876068),\n",
              "  ('gotten', 0.023003392794018397),\n",
              "  ('wondering', 0.02278731691328035)],\n",
              " 216: [('meter', 0.0354955413334277),\n",
              "  ('generator', 0.03525353693359414),\n",
              "  ('fluke', 0.029401763610091645),\n",
              "  ('dmm', 0.024186088485242827),\n",
              "  ('voltage', 0.02261079494240743),\n",
              "  ('function', 0.01972587420497354),\n",
              "  ('platinum', 0.015995935277803255),\n",
              "  ('circuit', 0.01409564509586804),\n",
              "  ('little', 0.013516356063462184),\n",
              "  ('output', 0.012311433447041014)],\n",
              " 217: [('ham', 0.03970065849099131),\n",
              "  ('tesla', 0.02804418017884389),\n",
              "  ('radio', 0.023100505305362406),\n",
              "  ('phonograph', 0.017837220500200326),\n",
              "  ('interference', 0.017548806754911703),\n",
              "  ('hams', 0.016991140629314876),\n",
              "  ('operator', 0.016880993412045606),\n",
              "  ('chokes', 0.014778041664034965),\n",
              "  ('unlikely', 0.014000084511584476),\n",
              "  ('power', 0.013716101352696103)],\n",
              " 218: [('tire', 0.03402516000047428),\n",
              "  ('tires', 0.030475152808554364),\n",
              "  ('fluids', 0.022551654762250937),\n",
              "  ('dot', 0.021158704569902636),\n",
              "  ('abs', 0.020633491305005722),\n",
              "  ('brake', 0.020391626840800297),\n",
              "  ('braking', 0.01863475487385001),\n",
              "  ('oil', 0.015270727394257258),\n",
              "  ('front', 0.013567894448297315),\n",
              "  ('car', 0.013563793824068145)],\n",
              " 219: [('orion', 0.06972592771597975),\n",
              "  ('prototype', 0.04460816792167504),\n",
              "  ('film', 0.04226492989749635),\n",
              "  ('vacuum', 0.03971042263699766),\n",
              "  ('leigh', 0.039067994753759515),\n",
              "  ('henry', 0.030150719743028104),\n",
              "  ('museum', 0.025741768269765746),\n",
              "  ('spencer', 0.023965392835803973),\n",
              "  ('pusher', 0.023804980790812166),\n",
              "  ('nasm', 0.023252392318384534)],\n",
              " 220: [('encryption', 0.02905883141679396),\n",
              "  ('clipper', 0.02805639299165418),\n",
              "  ('chip', 0.02465196296362587),\n",
              "  ('technology', 0.01595199689038993),\n",
              "  ('protect', 0.014221887114160982),\n",
              "  ('devices', 0.01217042063560664),\n",
              "  ('will', 0.011910474588627635),\n",
              "  ('industry', 0.011694860530663907),\n",
              "  ('key', 0.011656838545507064),\n",
              "  ('privacy', 0.011396418022869111)],\n",
              " 221: [('ampere', 0.04317942264612074),\n",
              "  ('amp', 0.039900505096733374),\n",
              "  ('company', 0.039238336661578506),\n",
              "  ('bell', 0.03765868016131062),\n",
              "  ('db', 0.037238754810379526),\n",
              "  ('ohmite', 0.034084489984903626),\n",
              "  ('bel', 0.02895900419690798),\n",
              "  ('dbm', 0.026616721213704668),\n",
              "  ('chiu', 0.026326367451734525),\n",
              "  ('ohms', 0.025585282589260967)],\n",
              " 222: [('stark', 0.03711758149870899),\n",
              "  ('writer', 0.024539876371525122),\n",
              "  ('jayson', 0.023833933906381593),\n",
              "  ('hobokenite', 0.023516845122151055),\n",
              "  ('gajarsky', 0.023516845122151055),\n",
              "  ('homers', 0.023175327019005026),\n",
              "  ('pace', 0.022528204131731636),\n",
              "  ('324', 0.021366404449591314),\n",
              "  ('baseball', 0.02069029842169406),\n",
              "  ('bob', 0.020296103630547766)],\n",
              " 223: [('weapons', 0.06327917756374275),\n",
              "  ('destruction', 0.030835188291638223),\n",
              "  ('weapon', 0.02655371273414773),\n",
              "  ('foxvog', 0.024635523073255828),\n",
              "  ('nuclear', 0.023687373349121923),\n",
              "  ('mass', 0.02300454063550931),\n",
              "  ('militia', 0.02111278833609538),\n",
              "  ('nerve', 0.01651983126956085),\n",
              "  ('biological', 0.01611284716551061),\n",
              "  ('rewording', 0.015480271705441659)],\n",
              " 224: [('ether', 0.06100743143574631),\n",
              "  ('twist', 0.018741514374221446),\n",
              "  ('planets', 0.017827423911009353),\n",
              "  ('orbit', 0.014395044368879272),\n",
              "  ('mcaloon', 0.014372775605926707),\n",
              "  ('earth', 0.012658507655929978),\n",
              "  ('images', 0.012177696694305257),\n",
              "  ('sun', 0.011741993514149555),\n",
              "  ('moon', 0.011639838787002381),\n",
              "  ('reality', 0.011107285920211575)],\n",
              " 225: [('ir', 0.0495053139038665),\n",
              "  ('cycle', 0.03195615988486279),\n",
              "  ('xxxxx', 0.02535562023556105),\n",
              "  ('wazing', 0.023666894278583473),\n",
              "  ('emitting', 0.022367804249278107),\n",
              "  ('detector', 0.022209975923112267),\n",
              "  ('led', 0.02220787677732585),\n",
              "  ('duty', 0.020291469628332268),\n",
              "  ('burton', 0.01859570726783837),\n",
              "  ('distance', 0.018181277357283802)],\n",
              " 226: [('ball', 0.04203609031332282),\n",
              "  ('fan', 0.0393421655116529),\n",
              "  ('strawberry', 0.032313683404868274),\n",
              "  ('dodger', 0.032129109970649834),\n",
              "  ('darryl', 0.02908231506438145),\n",
              "  ('singer', 0.020272177163485935),\n",
              "  ('lasorda', 0.019454698611315817),\n",
              "  ('catch', 0.01794288527170494),\n",
              "  ('daryl', 0.01775468212566192),\n",
              "  ('fans', 0.016982112047043797)],\n",
              " 227: [('mjm', 0.06809285770169915),\n",
              "  ('circuits', 0.05856740013897743),\n",
              "  ('mixer', 0.05354572536626913),\n",
              "  ('theory', 0.04681619812358256),\n",
              "  ('rf', 0.03800963434410353),\n",
              "  ('fm', 0.037913449064811595),\n",
              "  ('books', 0.03430972427410331),\n",
              "  ('klinkner', 0.029055820373093455),\n",
              "  ('lot', 0.02874416559771862),\n",
              "  ('diagrams', 0.028292825621284674)],\n",
              " 228: [('s1', 0.06115868974500402),\n",
              "  ('s2', 0.05953393382361812),\n",
              "  ('serial', 0.026032245538874527),\n",
              "  ('key', 0.02547470891320345),\n",
              "  ('bit', 0.023498728382847715),\n",
              "  ('unit', 0.022026647492354876),\n",
              "  ('u1', 0.020409067632266357),\n",
              "  ('u2', 0.020163967816985005),\n",
              "  ('escrow', 0.017775378644028197),\n",
              "  ('number', 0.01758429162636993)],\n",
              " 229: [('solar', 0.07921885542400507),\n",
              "  ('sail', 0.06662789935643368),\n",
              "  ('accelerations', 0.04245358122388075),\n",
              "  ('sails', 0.03936857550696135),\n",
              "  ('45g', 0.02518045153427035),\n",
              "  ('laxman', 0.018791006856892896),\n",
              "  ('amruth', 0.018791006856892896),\n",
              "  ('project', 0.01785910512894112),\n",
              "  ('sunlight', 0.017818644951396664),\n",
              "  ('craft', 0.016377431093550693)],\n",
              " 230: [('they', 0.015179299625521956),\n",
              "  ('she', 0.01453744928619842),\n",
              "  ('were', 0.014309874392911705),\n",
              "  ('her', 0.012971193723992504),\n",
              "  ('he', 0.01242554325592083),\n",
              "  ('was', 0.012070238073494086),\n",
              "  ('sumgait', 0.01118348317040909),\n",
              "  ('apartment', 0.01098422931981216),\n",
              "  ('went', 0.010380981677798658),\n",
              "  ('had', 0.010232717827842996)],\n",
              " 231: [('god', 0.030360973870980412),\n",
              "  ('law', 0.02688438522227083),\n",
              "  ('jesus', 0.021664287702813395),\n",
              "  ('malcolm', 0.017575678317827397),\n",
              "  ('jews', 0.0164041519101418),\n",
              "  ('chosen', 0.01603313573969145),\n",
              "  ('israelites', 0.015218981591024548),\n",
              "  ('laws', 0.014896970250846332),\n",
              "  ('salvation', 0.014310696248208587),\n",
              "  ('christ', 0.01316266318289985)],\n",
              " 232: [('diamond', 0.06815458996441003),\n",
              "  ('cursor', 0.0670303955320893),\n",
              "  ('lindbergh', 0.045352729450962935),\n",
              "  ('1024x768x256', 0.04115687954907916),\n",
              "  ('driver', 0.039815173876039466),\n",
              "  ('gateway', 0.034842563639001685),\n",
              "  ('ss24x', 0.03317153752459917),\n",
              "  ('mouse', 0.032064783502074444),\n",
              "  ('seen', 0.02849917604604809),\n",
              "  ('latest', 0.027692746361383618)],\n",
              " 233: [('scanner', 0.059447349386307045),\n",
              "  ('scanners', 0.058255065377201334),\n",
              "  ('scanman', 0.04017712952716372),\n",
              "  ('ocr', 0.033397160885925065),\n",
              "  ('dpi', 0.03215485511174597),\n",
              "  ('grayscale', 0.030441088776193965),\n",
              "  ('logitech', 0.029317630189374),\n",
              "  ('onescanner', 0.02511336961577354),\n",
              "  ('satam', 0.02511336961577354),\n",
              "  ('hand', 0.02459590863301058)],\n",
              " 234: [('color', 0.03995340990442192),\n",
              "  ('speedstar', 0.03277835877901897),\n",
              "  ('mpeg', 0.02919002734239768),\n",
              "  ('vga', 0.028578757389363094),\n",
              "  ('pov', 0.02549427905034809),\n",
              "  ('nobles', 0.02434736332159843),\n",
              "  ('mode', 0.02215628223945964),\n",
              "  ('24x', 0.021987673150876925),\n",
              "  ('tga', 0.02166871168614263),\n",
              "  ('view', 0.021550211597214618)],\n",
              " 235: [('nc', 0.033871971087495525),\n",
              "  ('civil', 0.028417478795509845),\n",
              "  ('verdict', 0.024324670498109914),\n",
              "  ('rights', 0.023846518332843795),\n",
              "  ('trial', 0.02199690926950656),\n",
              "  ('wilmington', 0.02181413706200081),\n",
              "  ('jeopardy', 0.0216900076919298),\n",
              "  ('farrell', 0.020104330701902845),\n",
              "  ('king', 0.019294615387746),\n",
              "  ('marine', 0.018597747913620462)],\n",
              " 236: [('gainey', 0.037532680373772755),\n",
              "  ('bob', 0.019347308503931644),\n",
              "  ('hockey', 0.018888551110605816),\n",
              "  ('gilmour', 0.01867295908453225),\n",
              "  ('player', 0.0163995659300995),\n",
              "  ('cherry', 0.012052721726042314),\n",
              "  ('he', 0.01196186657391708),\n",
              "  ('don', 0.011895488036207323),\n",
              "  ('roger', 0.01154389753362114),\n",
              "  ('doug', 0.011287336124228385)],\n",
              " 237: [('ottawa', 0.09130479780309524),\n",
              "  ('lindros', 0.0815886368207987),\n",
              "  ('montreal', 0.05707635682839024),\n",
              "  ('phillie', 0.04889829483286842),\n",
              "  ('trade', 0.03315406582456257),\n",
              "  ('pick', 0.03290927373639471),\n",
              "  ('damphousse', 0.02880833035498697),\n",
              "  ('senators', 0.02661420063038846),\n",
              "  ('rumor', 0.025238774519498748),\n",
              "  ('reported', 0.02215661710527394)],\n",
              " 238: [('tint', 0.03600666406715261),\n",
              "  ('parking', 0.02551193650505899),\n",
              "  ('cages', 0.024667628740054376),\n",
              "  ('sticker', 0.02060525273249294),\n",
              "  ('stickers', 0.019367334415575227),\n",
              "  ('ggrrrrrr', 0.018003332033576305),\n",
              "  ('car', 0.017968420461820657),\n",
              "  ('bdi', 0.016822404894535615),\n",
              "  ('motorc', 0.01579654459720292),\n",
              "  ('nick', 0.013308564376828279)],\n",
              " 239: [('negev', 0.05289887897056686),\n",
              "  ('bedouin', 0.04279530569469448),\n",
              "  ('israel', 0.025865874078626155),\n",
              "  ('war', 0.020577906885448555),\n",
              "  ('jordan', 0.019442616124003814),\n",
              "  ('river', 0.017759144629581478),\n",
              "  ('arabs', 0.017471298073151228),\n",
              "  ('1948', 0.013953719221067345),\n",
              "  ('israeli', 0.01360919082041119),\n",
              "  ('cultivated', 0.013188076832746018)],\n",
              " 240: [('1999', 0.044679153835989034),\n",
              "  ('space', 0.0378974700593646),\n",
              "  ('infinity', 0.03713163196695032),\n",
              "  ('schiewer', 0.03270046205090809),\n",
              "  ('keesler', 0.028107740057340968),\n",
              "  ('mars', 0.023398884985977426),\n",
              "  ('debris', 0.022023419490939885),\n",
              "  ('doppelganger', 0.019271834603480992),\n",
              "  ('inland', 0.018964510386091867),\n",
              "  ('theories', 0.017656302792709234)],\n",
              " 241: [('her', 0.0415508849689805),\n",
              "  ('denning', 0.04134548971967436),\n",
              "  ('respectability', 0.03858383786386818),\n",
              "  ('she', 0.027298543032241905),\n",
              "  ('dorothy', 0.02629308072325111),\n",
              "  ('toal', 0.02511817900268986),\n",
              "  ('baffled', 0.023160554283677325),\n",
              "  ('tapped', 0.02298605175607195),\n",
              "  ('graham', 0.021428176706041406),\n",
              "  ('dorthy', 0.020852277994182874)],\n",
              " 242: [('jacket', 0.05759664067509583),\n",
              "  ('leather', 0.05066273920842778),\n",
              "  ('gloves', 0.048778545440001056),\n",
              "  ('pants', 0.04269259252745679),\n",
              "  ('piece', 0.04228437848105805),\n",
              "  ('protective', 0.041068161510473294),\n",
              "  ('gear', 0.03647694345942864),\n",
              "  ('aerostitch', 0.026022619165000024),\n",
              "  ('fit', 0.019834934313324188),\n",
              "  ('helmet', 0.019816932404265603)],\n",
              " 243: [('x11r6', 0.04917909803205942),\n",
              "  ('mahan', 0.039666677637785926),\n",
              "  ('tektronix', 0.038669633615061906),\n",
              "  ('sunview', 0.03747073583108911),\n",
              "  ('lazarus', 0.03156207459416649),\n",
              "  ('server', 0.03074948301544515),\n",
              "  ('klaus', 0.03011245675367756),\n",
              "  ('patrick', 0.027500066175099112),\n",
              "  ('mentioning', 0.02330974897726768),\n",
              "  ('terminal', 0.02321940063871024)]}"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "anchor_topic_model.topic_representations_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import scipy.stats\n",
        "\n",
        "# Example data\n",
        "x = [25, 15, 10, 5, 30]\n",
        "y = [5, 10, 15, 25, 30]\n",
        "\n",
        "# Calculate Spearman's rank correlation coefficient\n",
        "rho, p_value = scipy.stats.spearmanr(x, y)\n",
        "\n",
        "# Print the result\n",
        "print(f\"Spearman's rank correlation coefficient: {rho}\")\n",
        "print(f\"P-value: {p_value}\")\n",
        "\n",
        "# Interpret the result\n",
        "if p_value < 0.05:\n",
        "    print(\"The correlation is statistically significant.\")\n",
        "else:\n",
        "    print(\"The correlation is not statistically significant.\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Sufficiency"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def filter_documents(documents, allowed_words):\n",
        "    filtered_documents = []\n",
        "\n",
        "    for document in documents:\n",
        "        filtered_document = []\n",
        "        for word in document.split():\n",
        "            if word in allowed_words:\n",
        "                filtered_document.append(word)\n",
        "        filtered_documents.append(\" \".join(filtered_document))\n",
        "\n",
        "    return filtered_documents"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "def check_topic_for_comprehensiveness_fit( docs: List[str], k: int) -> Dict[str, str]:\n",
        "    \"\"\"\n",
        "    Take as input a list of topics and the initial documents, perturbs the documents\n",
        "    by keeping only the topic words present in the top-k topics and see how many of the \n",
        "    documents fall into the same bin as before. \n",
        "    \"\"\"\n",
        "    ablation_mappings = {}\n",
        "    anchor_topic_model = BERTopic()\n",
        "    topics, probs = anchor_topic_model.fit_transform(docs)\n",
        "    topic_list = anchor_topic_model.get_topic_info()[\"Representation\"]\n",
        "\n",
        "    # forming doc -> topic pairing\n",
        "    df_basic_mapping = pd.DataFrame({\"Document\": docs, \"Topic\": topics})\n",
        "\n",
        "\n",
        "    for topic in tqdm(topic_list[:k+1]):\n",
        "        new_docs = remove_word_from_list(word, docs)\n",
        "        new_topics, probs = anchor_topic_model.transform(new_docs)\n",
        "        df_new_mapping = pd.DataFrame({\"Document\": docs, \"Topic\": new_topics})\n",
        "\n",
        "        ablation_mappings[word] =  compare_topics(df_basic_mapping, df_new_mapping)\n",
        "\n",
        "    return ablation_mappings\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Stats regarding datasets "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import re\n",
        "\n",
        "def analyze_csv(csv_path):\n",
        "    # Read the CSV file into a DataFrame\n",
        "    df = pd.read_csv(csv_path)\n",
        "\n",
        "    # Check if the 'text' column exists in the DataFrame\n",
        "    if 'text' not in df.columns:\n",
        "        raise ValueError(\"The 'text' column is not present in the CSV file.\")\n",
        "\n",
        "    # Calculate the total number of data points\n",
        "    total_datapoints = len(df)\n",
        "\n",
        "    # Calculate the average number of words per data point in the 'text' column\n",
        "    df['word_count'] = df['text'].apply(lambda x: len(re.findall(r'\\w+', str(x))))\n",
        "    average_words_per_datapoint = df['word_count'].mean()\n",
        "\n",
        "    # Drop the temporary 'word_count' column\n",
        "    df = df.drop(columns=['word_count'])\n",
        "\n",
        "    return total_datapoints, average_words_per_datapoint"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total Datapoints: 18846\n",
            "Average Words per Datapoint: 202.69309137217448\n"
          ]
        }
      ],
      "source": [
        "# Example usage:\n",
        "csv_path = \"/home/abpal/WorkFiles/Faithful-Topic-Modeling/data/20newsgroups.csv\"\n",
        "total_datapoints, avg_words_per_datapoint = analyze_csv(csv_path)\n",
        "print(f\"Total Datapoints: {total_datapoints}\")\n",
        "print(f\"Average Words per Datapoint: {avg_words_per_datapoint}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total Datapoints: 10000\n",
            "Average Words per Datapoint: 1483.8913\n"
          ]
        }
      ],
      "source": [
        "# Example usage:\n",
        "csv_path = \"/home/abpal/WorkFiles/Faithful-Topic-Modeling/data/wiki_en_10000.csv\"\n",
        "total_datapoints, avg_words_per_datapoint = analyze_csv(csv_path)\n",
        "print(f\"Total Datapoints: {total_datapoints}\")\n",
        "print(f\"Average Words per Datapoint: {avg_words_per_datapoint}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total Datapoints: 16784\n",
            "Average Words per Datapoint: 31.040216873212582\n"
          ]
        }
      ],
      "source": [
        "# Example usage:\n",
        "csv_path = \"/home/abpal/WorkFiles/Faithful-Topic-Modeling/data/nyt2020.csv\"\n",
        "total_datapoints, avg_words_per_datapoint = analyze_csv(csv_path)\n",
        "print(f\"Total Datapoints: {total_datapoints}\")\n",
        "print(f\"Average Words per Datapoint: {avg_words_per_datapoint}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.6"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "00c5b5d42c7d44bdb31f86aa88670287": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_676ee070a88b434b8f574a9eb521ed6f",
            "placeholder": "​",
            "style": "IPY_MODEL_b2c4e9ce8e0c44679e49b59ba41e9ee1",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "017fad538ebc4f6db14d9292d52ff9a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_316529a1093a457a97b13889cd7e2145",
              "IPY_MODEL_c921cb15118545e28c9f1b6fd71fbdca",
              "IPY_MODEL_333d86a0056a47bdb78fd2351581c595"
            ],
            "layout": "IPY_MODEL_5361a9c5f80d471bad6c6e2c7244df36"
          }
        },
        "02c98ebf1b0e459a921c3c6feae0c4c3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "043eff82f0844aaba4bdb88eaf1d06e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0567a3e903a94a018b1cc00236d9c408": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "05b3053fcc1f4b57a93e16c63727f9e5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "05dc10b6f01442b9890644cce976d926": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d3b409b1e4fd4f7ea4ca824291775c0d",
            "placeholder": "​",
            "style": "IPY_MODEL_caac5be227b14379bf49f906304341c6",
            "value": " 466k/466k [00:00&lt;00:00, 3.71MB/s]"
          }
        },
        "0831c763dccc41e29c786bdf63162129": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb846c087d944ce59fef5c1f510d390a",
            "max": 39265,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_56efca88567a425dbbbf1941050c2067",
            "value": 39265
          }
        },
        "098a56a5943c465fb24fbbffe7f5127f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0ab7b9547b8b4e74b7e4e04371cbc1e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0fbc6195c29040a28b253c8ec0b0ff27",
            "placeholder": "​",
            "style": "IPY_MODEL_4d3b6828adf5497dac6663a4eb88d824",
            "value": "Downloading (…)5de9125/modules.json: 100%"
          }
        },
        "0fbc6195c29040a28b253c8ec0b0ff27": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10a5a30e1da1408780520d1f4a6efcc4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "11c159c5366e4882accc5ae477538993": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14468f6a20a24ee98bc28942a28a5120": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1ab5d69341b64ff2bffecbf1b61d0221": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c10b9c687cc4d138807b539eaf6ab4c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1e94b637c0704abd870c695a97db50aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1eae17cb6afa49b2b6c7b3b88855dd3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_21ce0d225502429d867eadbd450ba193",
            "max": 350,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_47a7901de55a4c45b04db4b7bef3f4e5",
            "value": 350
          }
        },
        "1fb84f38cb254e3a86e1576f43535930": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_895ab4c6b74645378821e609a8741005",
              "IPY_MODEL_e33e636cff49493bb5c639c171656dce",
              "IPY_MODEL_9533d7f423f54a4593655046d1eb8d34"
            ],
            "layout": "IPY_MODEL_02c98ebf1b0e459a921c3c6feae0c4c3"
          }
        },
        "203ce0f1d11748108b3ee7bea3eba465": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_62dac3ee8a6047ed999541146d2e78c0",
            "max": 13156,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a28e80d1336d46ec8391a1809640520c",
            "value": 13156
          }
        },
        "21ce0d225502429d867eadbd450ba193": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "237117009cc942f69385dc3a84aec21a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "238dcafe014441828fe16323d8695ea5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb1a2812381143ad9343088d0bc384d5",
            "max": 112,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ef446c74fbc741b398705cd11de8c3c9",
            "value": 112
          }
        },
        "23bd8095a91d4b1985972cd2a51abf64": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2a230a21139f4021991f7b5fb3bfa1b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b312dd33dff7496eba56caacc3d2e451",
              "IPY_MODEL_34b600d4d57345e39920aa86219254a2",
              "IPY_MODEL_884658cc10f94971b9617a0b1ab5e79e"
            ],
            "layout": "IPY_MODEL_30764587368f40db9899401f445aa822"
          }
        },
        "2a5b683e04684a55aeb85b9eba3de702": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2c847bffa35c4131b2b3af40b1c06616": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2df5e2c81fe748739477749d2681faad": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2e84f36123504dce8b1bcb084725fd3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_acdf5712c6e8499194b9ee06e76ba5cc",
            "max": 190,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_af5b64672d2046c9a35b28ebf0f94e30",
            "value": 190
          }
        },
        "30764587368f40db9899401f445aa822": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "316529a1093a457a97b13889cd7e2145": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_49f02432bbbf4cab9d669ef9d771fb79",
            "placeholder": "​",
            "style": "IPY_MODEL_b5b9fcec2fdf4bf38c4f5237bd9f9e45",
            "value": "Downloading (…)7e55de9125/vocab.txt: 100%"
          }
        },
        "333d86a0056a47bdb78fd2351581c595": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fda9a61f29094cc2a88e269256ac9b85",
            "placeholder": "​",
            "style": "IPY_MODEL_6ee1aece7f8249d3949c54208eac1ac7",
            "value": " 232k/232k [00:00&lt;00:00, 3.57MB/s]"
          }
        },
        "33cea769ae4b4605ad912a94df435121": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3738a599a2f645b78adf0acf12429d21",
              "IPY_MODEL_0831c763dccc41e29c786bdf63162129",
              "IPY_MODEL_b116e473a5fb4f58b773f8a5d0be4d5c"
            ],
            "layout": "IPY_MODEL_bce7ad625c294564bcc293287f1553e5"
          }
        },
        "34b600d4d57345e39920aa86219254a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d14b01ce3be84b04a5e9d2326e0f1843",
            "max": 10610,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8f23c56e124e4583b4217170c9de0f3a",
            "value": 10610
          }
        },
        "370ba460468a4ecc9a71d63b95fc3d67": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3738a599a2f645b78adf0acf12429d21": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd36886e06034bccaaf9938719f0b727",
            "placeholder": "​",
            "style": "IPY_MODEL_fb11412116b8439eba35ebb45dc9d7a6",
            "value": "Downloading (…)125/data_config.json: 100%"
          }
        },
        "382b48023d80425fabce141d29cd606f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd71abcf6ee44b1897edd29c0f5f16e2",
            "max": 90888945,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_23bd8095a91d4b1985972cd2a51abf64",
            "value": 90888945
          }
        },
        "3ae65f8b465d4b10a6218016916e65d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f387b6b18f614a7e85675679fd905dda",
            "placeholder": "​",
            "style": "IPY_MODEL_11c159c5366e4882accc5ae477538993",
            "value": " 116/116 [00:00&lt;00:00, 5.49kB/s]"
          }
        },
        "3d7541ebcfa8424db40c7faba7e1f913": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_60ee7b919b5441ef827c82b491f5a934",
            "placeholder": "​",
            "style": "IPY_MODEL_474783f23f034d22861fefc5fff2c588",
            "value": "Downloading (…)nce_bert_config.json: 100%"
          }
        },
        "3ea5137f09014e7fb65d20a3cfc4e5c6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "417a06cd26c24b1883881c54eefdb7d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3d7541ebcfa8424db40c7faba7e1f913",
              "IPY_MODEL_e62a1e40613849b5a69460f932f1bf84",
              "IPY_MODEL_44b1b6ae6c574dcf8e2b4c343b3241d2"
            ],
            "layout": "IPY_MODEL_d55c9fb955184963b1f0cf393a7c7443"
          }
        },
        "44ac6a5d8ced41c1b8ab38d91522ca6f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44b1b6ae6c574dcf8e2b4c343b3241d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f474319c1a734f4b9276cd2209bd6abf",
            "placeholder": "​",
            "style": "IPY_MODEL_ba435803e0b140638efef8704a97d3b6",
            "value": " 53.0/53.0 [00:00&lt;00:00, 2.23kB/s]"
          }
        },
        "44e678162db746909ab1fa73f16f94b9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46494435d80842bab62a2666b05d90e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "46a2e1afb5dd49ae937ecc1c37e2ddf7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "474783f23f034d22861fefc5fff2c588": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "47a7901de55a4c45b04db4b7bef3f4e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "47ed733a76834917947791a2b64cad1d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "49f02432bbbf4cab9d669ef9d771fb79": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d3b6828adf5497dac6663a4eb88d824": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "50047e0fc0764fa68c22ac4ead81b5a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d1fb66d2fb954acebc164c5b5fec8935",
            "placeholder": "​",
            "style": "IPY_MODEL_80fa7ab50b074b0ea62a443eb4d5cc8f",
            "value": " 13.2k/13.2k [00:00&lt;00:00, 488kB/s]"
          }
        },
        "5253fae4ecd24ca0a9647e324382ee12": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5361a9c5f80d471bad6c6e2c7244df36": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55429736de1b4f72b90a0099021d4d91": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "56efca88567a425dbbbf1941050c2067": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5838420dc36547c0883de21658d56fdf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5aae157a0c4c46a98cf971122b96fbbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5ab6a0c3226b4344ae20222365082979": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60ee7b919b5441ef827c82b491f5a934": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "62369b513d344ab694c291091d28ccdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_992cfbb44b0e4ea6b08d79ab9f45522a",
              "IPY_MODEL_2e84f36123504dce8b1bcb084725fd3a",
              "IPY_MODEL_a58aff58197346bdb71e867cb3df8e8b"
            ],
            "layout": "IPY_MODEL_2a5b683e04684a55aeb85b9eba3de702"
          }
        },
        "62dac3ee8a6047ed999541146d2e78c0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6471df3a56c942559895108361e0cf64": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "652b9fb3b8f84d91a4237872d48fcfc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7c1a23d3d4374b2292ab4bbce19aae40",
            "max": 466247,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7fdd3ce3521f418caa07a2049d56c091",
            "value": 466247
          }
        },
        "676ee070a88b434b8f574a9eb521ed6f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ec3bf17fc1347b3885b0ef391442d59": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ee1aece7f8249d3949c54208eac1ac7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6fada5bf924847088dafe0b5f921847d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "730665e715b84a669b7da360308195d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "75edbc68c2944fcba13b993657dbf601": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1ab5d69341b64ff2bffecbf1b61d0221",
            "max": 116,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_14468f6a20a24ee98bc28942a28a5120",
            "value": 116
          }
        },
        "78bbe83a642e471b91ef79ceb401652c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7947457e0a8f4d098b35e038cdc1ff61": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47ed733a76834917947791a2b64cad1d",
            "placeholder": "​",
            "style": "IPY_MODEL_55429736de1b4f72b90a0099021d4d91",
            "value": "Downloading (…)9125/train_script.py: 100%"
          }
        },
        "794e3ae8144e40e1a4af18bd6c117d65": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7bd2107f445b459187744351088a7d3e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c1a23d3d4374b2292ab4bbce19aae40": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c91ff7c22fe41bc9974937980421f6e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7dfa55fbc0f443a597903d19bdd43a69": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5838420dc36547c0883de21658d56fdf",
            "placeholder": "​",
            "style": "IPY_MODEL_cf399b71c608407ea269e7bb30f26f95",
            "value": " 112/112 [00:00&lt;00:00, 4.51kB/s]"
          }
        },
        "7fdd3ce3521f418caa07a2049d56c091": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "80fa7ab50b074b0ea62a443eb4d5cc8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "820987ff57ed4ebdb5d7b5616a1e537b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "85a535cf8a654ca383296c4c67ceda80": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7c91ff7c22fe41bc9974937980421f6e",
            "max": 1175,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c01fc41773d4455fbdcd393895d6132c",
            "value": 1175
          }
        },
        "884658cc10f94971b9617a0b1ab5e79e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ddcd62668f514870a6e0582f966bbd37",
            "placeholder": "​",
            "style": "IPY_MODEL_46a2e1afb5dd49ae937ecc1c37e2ddf7",
            "value": " 10.6k/10.6k [00:00&lt;00:00, 473kB/s]"
          }
        },
        "895ab4c6b74645378821e609a8741005": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3ea5137f09014e7fb65d20a3cfc4e5c6",
            "placeholder": "​",
            "style": "IPY_MODEL_730665e715b84a669b7da360308195d0",
            "value": "Downloading (…)55de9125/config.json: 100%"
          }
        },
        "8b6fd07501ac413683b5ecd639c72e34": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2c847bffa35c4131b2b3af40b1c06616",
            "placeholder": "​",
            "style": "IPY_MODEL_370ba460468a4ecc9a71d63b95fc3d67",
            "value": "Downloading (…)cial_tokens_map.json: 100%"
          }
        },
        "8f23c56e124e4583b4217170c9de0f3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9197371044d94125aae81fa3f0024017": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "91d117e8cda24e5e8b94fe36d3bd5053": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_00c5b5d42c7d44bdb31f86aa88670287",
              "IPY_MODEL_1eae17cb6afa49b2b6c7b3b88855dd3a",
              "IPY_MODEL_fdb920fc3baf47a5b0265d4e5832bb8c"
            ],
            "layout": "IPY_MODEL_ad373a2eb5b54f9587ea2b571cab1ad5"
          }
        },
        "93e8e933a3614c9bb39114f024ad2e7e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9533d7f423f54a4593655046d1eb8d34": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44e678162db746909ab1fa73f16f94b9",
            "placeholder": "​",
            "style": "IPY_MODEL_f3b0416618a246a7a6e749ed3a980883",
            "value": " 612/612 [00:00&lt;00:00, 23.6kB/s]"
          }
        },
        "96b4bea95dd1422080dba1dd09fd7200": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7947457e0a8f4d098b35e038cdc1ff61",
              "IPY_MODEL_203ce0f1d11748108b3ee7bea3eba465",
              "IPY_MODEL_50047e0fc0764fa68c22ac4ead81b5a2"
            ],
            "layout": "IPY_MODEL_f3150b32f34547988187af0740043129"
          }
        },
        "992cfbb44b0e4ea6b08d79ab9f45522a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc66147a51b141e79d648a796392ffa5",
            "placeholder": "​",
            "style": "IPY_MODEL_0567a3e903a94a018b1cc00236d9c408",
            "value": "Downloading (…)_Pooling/config.json: 100%"
          }
        },
        "a28e80d1336d46ec8391a1809640520c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a46ac10c196d4a86b7638bb0a59e3c2f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a526cdedd76845a0b78db3c67539714c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a58aff58197346bdb71e867cb3df8e8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_237117009cc942f69385dc3a84aec21a",
            "placeholder": "​",
            "style": "IPY_MODEL_e99b2ca556e5466087f6eb5b8df2647b",
            "value": " 190/190 [00:00&lt;00:00, 8.47kB/s]"
          }
        },
        "a642860afc8e43519c249c23786d202b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a46ac10c196d4a86b7638bb0a59e3c2f",
            "placeholder": "​",
            "style": "IPY_MODEL_9197371044d94125aae81fa3f0024017",
            "value": "Downloading (…)e9125/.gitattributes: 100%"
          }
        },
        "a66293dd44d24775ab010bf4e848f560": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cc80198b67bf4d05a332905ce26d23d9",
              "IPY_MODEL_652b9fb3b8f84d91a4237872d48fcfc0",
              "IPY_MODEL_05dc10b6f01442b9890644cce976d926"
            ],
            "layout": "IPY_MODEL_93e8e933a3614c9bb39114f024ad2e7e"
          }
        },
        "a9624546276f4df7b7b92bcd5d31730a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "acdf5712c6e8499194b9ee06e76ba5cc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ad373a2eb5b54f9587ea2b571cab1ad5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ad7f02de9427422a93f0a5e5d1a2500f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae3f51248d6b4b21a4fda3fec5410efd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "af5b64672d2046c9a35b28ebf0f94e30": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b0424b94a2014249b55e278a2063a502": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_10a5a30e1da1408780520d1f4a6efcc4",
            "max": 349,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_46494435d80842bab62a2666b05d90e4",
            "value": 349
          }
        },
        "b050d76b86f64dea870c5178c393949e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b116e473a5fb4f58b773f8a5d0be4d5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c10b9c687cc4d138807b539eaf6ab4c",
            "placeholder": "​",
            "style": "IPY_MODEL_e1b867546edf451dac52dc39ab1b7c20",
            "value": " 39.3k/39.3k [00:00&lt;00:00, 1.38MB/s]"
          }
        },
        "b2c4e9ce8e0c44679e49b59ba41e9ee1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b312dd33dff7496eba56caacc3d2e451": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b518c97575c04d04abc4db1f87b2648d",
            "placeholder": "​",
            "style": "IPY_MODEL_78bbe83a642e471b91ef79ceb401652c",
            "value": "Downloading (…)7e55de9125/README.md: 100%"
          }
        },
        "b4e54988f386430498e3030b2fc6813a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d9d53da4e554488ab4588f7eaa9b9449",
            "placeholder": "​",
            "style": "IPY_MODEL_2df5e2c81fe748739477749d2681faad",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "b518c97575c04d04abc4db1f87b2648d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b5b9fcec2fdf4bf38c4f5237bd9f9e45": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba435803e0b140638efef8704a97d3b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb846c087d944ce59fef5c1f510d390a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bce7ad625c294564bcc293287f1553e5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bff127a806144effa8538c9aef8912f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8b6fd07501ac413683b5ecd639c72e34",
              "IPY_MODEL_238dcafe014441828fe16323d8695ea5",
              "IPY_MODEL_7dfa55fbc0f443a597903d19bdd43a69"
            ],
            "layout": "IPY_MODEL_6471df3a56c942559895108361e0cf64"
          }
        },
        "c01fc41773d4455fbdcd393895d6132c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c39c93007e114d2e96c2c9533d92c38a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5fa5cc4f98947f2b028a6ca7ed8ad09": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c921cb15118545e28c9f1b6fd71fbdca": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_794e3ae8144e40e1a4af18bd6c117d65",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_098a56a5943c465fb24fbbffe7f5127f",
            "value": 231508
          }
        },
        "caac5be227b14379bf49f906304341c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cbbe326fb04c4199baa9fe538196bf42": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c39c93007e114d2e96c2c9533d92c38a",
            "placeholder": "​",
            "style": "IPY_MODEL_a9624546276f4df7b7b92bcd5d31730a",
            "value": " 349/349 [00:00&lt;00:00, 11.5kB/s]"
          }
        },
        "cc66147a51b141e79d648a796392ffa5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cc80198b67bf4d05a332905ce26d23d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_05b3053fcc1f4b57a93e16c63727f9e5",
            "placeholder": "​",
            "style": "IPY_MODEL_d265c731e78c4ff188eae9c234d53d6c",
            "value": "Downloading (…)e9125/tokenizer.json: 100%"
          }
        },
        "cd36886e06034bccaaf9938719f0b727": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd71abcf6ee44b1897edd29c0f5f16e2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce63080393604afeb104bb087018f79d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5253fae4ecd24ca0a9647e324382ee12",
            "placeholder": "​",
            "style": "IPY_MODEL_043eff82f0844aaba4bdb88eaf1d06e4",
            "value": "Downloading (…)ce_transformers.json: 100%"
          }
        },
        "cf399b71c608407ea269e7bb30f26f95": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d14b01ce3be84b04a5e9d2326e0f1843": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d1fb66d2fb954acebc164c5b5fec8935": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d265c731e78c4ff188eae9c234d53d6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d3b409b1e4fd4f7ea4ca824291775c0d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d55c9fb955184963b1f0cf393a7c7443": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9d53da4e554488ab4588f7eaa9b9449": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbc459f0024a4c20ad1295f3967bf430": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ddcd62668f514870a6e0582f966bbd37": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e1b867546edf451dac52dc39ab1b7c20": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e33e636cff49493bb5c639c171656dce": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a526cdedd76845a0b78db3c67539714c",
            "max": 612,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c5fa5cc4f98947f2b028a6ca7ed8ad09",
            "value": 612
          }
        },
        "e62a1e40613849b5a69460f932f1bf84": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44ac6a5d8ced41c1b8ab38d91522ca6f",
            "max": 53,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1e94b637c0704abd870c695a97db50aa",
            "value": 53
          }
        },
        "e99b2ca556e5466087f6eb5b8df2647b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "eac58eaad7614f6d8e21c87d04fefb4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ad7f02de9427422a93f0a5e5d1a2500f",
            "placeholder": "​",
            "style": "IPY_MODEL_5aae157a0c4c46a98cf971122b96fbbd",
            "value": " 90.9M/90.9M [00:00&lt;00:00, 137MB/s]"
          }
        },
        "ed12e0d17fc94bc98f03d0a3d90766fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a642860afc8e43519c249c23786d202b",
              "IPY_MODEL_85a535cf8a654ca383296c4c67ceda80",
              "IPY_MODEL_f4b16ec425d54ccaaf59d916527e5f1f"
            ],
            "layout": "IPY_MODEL_7bd2107f445b459187744351088a7d3e"
          }
        },
        "ef446c74fbc741b398705cd11de8c3c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f3150b32f34547988187af0740043129": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f387b6b18f614a7e85675679fd905dda": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3b0416618a246a7a6e749ed3a980883": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f474319c1a734f4b9276cd2209bd6abf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4b16ec425d54ccaaf59d916527e5f1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ab6a0c3226b4344ae20222365082979",
            "placeholder": "​",
            "style": "IPY_MODEL_dbc459f0024a4c20ad1295f3967bf430",
            "value": " 1.18k/1.18k [00:00&lt;00:00, 57.7kB/s]"
          }
        },
        "f591569039cc4d7ea889d211fb96d687": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ce63080393604afeb104bb087018f79d",
              "IPY_MODEL_75edbc68c2944fcba13b993657dbf601",
              "IPY_MODEL_3ae65f8b465d4b10a6218016916e65d7"
            ],
            "layout": "IPY_MODEL_b050d76b86f64dea870c5178c393949e"
          }
        },
        "f606a5e5649340b9aff8c756e15582df": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b4e54988f386430498e3030b2fc6813a",
              "IPY_MODEL_382b48023d80425fabce141d29cd606f",
              "IPY_MODEL_eac58eaad7614f6d8e21c87d04fefb4a"
            ],
            "layout": "IPY_MODEL_6ec3bf17fc1347b3885b0ef391442d59"
          }
        },
        "fb11412116b8439eba35ebb45dc9d7a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fb1a2812381143ad9343088d0bc384d5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc380d312db34dbbb2c3c013cff13492": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0ab7b9547b8b4e74b7e4e04371cbc1e7",
              "IPY_MODEL_b0424b94a2014249b55e278a2063a502",
              "IPY_MODEL_cbbe326fb04c4199baa9fe538196bf42"
            ],
            "layout": "IPY_MODEL_820987ff57ed4ebdb5d7b5616a1e537b"
          }
        },
        "fda9a61f29094cc2a88e269256ac9b85": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fdb920fc3baf47a5b0265d4e5832bb8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6fada5bf924847088dafe0b5f921847d",
            "placeholder": "​",
            "style": "IPY_MODEL_ae3f51248d6b4b21a4fda3fec5410efd",
            "value": " 350/350 [00:00&lt;00:00, 12.3kB/s]"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}