tfidf_model.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ryderwishart/biblical-machine-learning/blob/main/tfidf_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "32a5c902",
      "metadata": {
        "id": "32a5c902"
      },
      "source": [
        "## Import dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "id": "9fefcaed",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9fefcaed",
        "outputId": "84201a19-1b2d-477e-c8db-baeea703ad54"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting gensim\n",
            "  Using cached gensim-4.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (26.5 MB)\n",
            "Collecting unidecode\n",
            "  Using cached Unidecode-1.3.6-py3-none-any.whl (235 kB)\n",
            "Collecting smart-open>=1.8.1\n",
            "  Using cached smart_open-6.3.0-py3-none-any.whl (56 kB)\n",
            "Collecting numpy>=1.18.5\n",
            "  Using cached numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
            "Collecting scipy>=1.7.0\n",
            "  Using cached scipy-1.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.5 MB)\n",
            "Installing collected packages: unidecode, smart-open, numpy, scipy, gensim\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "numba 0.56.4 requires numpy<1.24,>=1.18, but you have numpy 1.24.2 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed gensim-4.3.1 numpy-1.24.2 scipy-1.10.1 smart-open-6.3.0 unidecode-1.3.6\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import codecs\n",
        "import time\n",
        "try:\n",
        "    from unidecode import unidecode\n",
        "    from collections.abc import Mapping\n",
        "    from gensim.models.tfidfmodel import TfidfModel\n",
        "    from gensim.corpora import Dictionary\n",
        "except:\n",
        "#     print(\"Depencies not found. Make sure you have installed GenSim.\")\n",
        "    !pip install -I gensim unidecode\n",
        "#     !pip install -Iv gensim==3.2.0\n",
        "    from collections.abc import Mapping\n",
        "    from unidecode import unidecode\n",
        "    from gensim.models.tfidfmodel import TfidfModel"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "if 'biblical-machine-learning' not in [path for path in os.listdir()]:\n",
        "    !git clone https://github.com/ryderwishart/biblical-machine-learning.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yRWtDZtlNo9d",
        "outputId": "6c57cfea-bb51-41dc-9246-04c37c02e426"
      },
      "id": "yRWtDZtlNo9d",
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'biblical-machine-learning'...\n",
            "remote: Enumerating objects: 1321, done.\u001b[K\n",
            "remote: Counting objects: 100% (37/37), done.\u001b[K\n",
            "remote: Compressing objects: 100% (24/24), done.\u001b[K\n",
            "remote: Total 1321 (delta 11), reused 35 (delta 11), pack-reused 1284\u001b[K\n",
            "Receiving objects: 100% (1321/1321), 63.78 MiB | 15.52 MiB/s, done.\n",
            "Resolving deltas: 100% (40/40), done.\n",
            "Updating files: 100% (1302/1302), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "id": "2372ef89",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2372ef89",
        "outputId": "2fcb6cf6-0499-4e60-f01a-19c466d99ed7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Directories found in data folder: ['texts', 'lemmas']\n"
          ]
        }
      ],
      "source": [
        "corpus_directories = [path for path in os.listdir('biblical-machine-learning/data') if not(path.startswith('.'))]\n",
        "print('Directories found in data folder:', corpus_directories)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2e00a44f",
      "metadata": {
        "id": "2e00a44f"
      },
      "source": [
        "## Locate corpus data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 72,
      "id": "0423832f",
      "metadata": {
        "id": "0423832f"
      },
      "outputs": [],
      "source": [
        "force_lowercase = True\n",
        "use_lemma_disambiguation = False # Some lemmas are indicated with a numeric suffix (e.g., 'ὅτι2')\n",
        "\n",
        "# Change to the directory entered (this is necessary to use the codecs.open() method). \n",
        "# TODO: rewrite this corpus iterator without the codecs module.\n",
        "\n",
        "# if not(os.getcwd().split('/')[-1].endswith('corpus')):\n",
        "#     os.chdir(corpus_directory)\n",
        "\n",
        "# This class streams through the corpus when called.\n",
        "\n",
        "perseus_stopwords = \"μή, ἑαυτοῦ, ἄν, ἀλλ', ἀλλά, ἄλλος, ἀπό, ἄρα, αὐτός, δ', δέ, δή, διά, δαί, δαίς, ἔτι, ἐγώ, ἐκ, ἐμός, ἐν, ἐπί, εἰ, εἰμί, εἴμι, εἰς, γάρ, γε, γα, ἡ, ἤ, καί, κατά, μέν, μετά, μή, ὁ, ὅδε, ὅς, ὅστις, ὅτι, οὕτως, οὗτος, οὔτε, οὖν, οὐδείς, οἱ, οὐ, οὐδέ, οὐκ, περί, πρός, σύ, σύν, τά, τε, τήν, τῆς, τῇ, τι, τί, τις, τίς, τό, τοί, τοιοῦτος, τόν, τούς, τοῦ, τῶν, τῷ, ὑμός, ὑπέρ, ὑπό, ὡς, ὦ, ὥστε, ἐάν, παρά, σός\".split(', ')\n",
        "perseus_stopwords += \"συ δ μοι\".split(' ')\n",
        "perseus_stopwords = [unidecode(w) for w in perseus_stopwords]\n",
        "\n",
        "def tokenize(string):\n",
        "    output = string\n",
        "    if use_lemma_disambiguation:\n",
        "        pass\n",
        "    else:\n",
        "        # Filter numeric digits from token\n",
        "        output = ''.join(filter(lambda x: x.isalpha() or x == ' ', string))\n",
        "    if force_lowercase:\n",
        "        return [token.lower() for token in output.split() if unidecode(token.lower()) not in perseus_stopwords] # use unidecode to strip accents temporarily\n",
        "    else:\n",
        "        return output.split()\n",
        "    \n",
        "class Texts:\n",
        "    def __init__(self, selected_corpus):\n",
        "        self.selected_corpus = selected_corpus\n",
        "\n",
        "    def __iter__(self):\n",
        "        for file in os.listdir(f'biblical-machine-learning/data/{self.selected_corpus}'): \n",
        "            if file.endswith(\".txt\"):\n",
        "                text = []\n",
        "                for line in codecs.open(f'biblical-machine-learning/data/{self.selected_corpus}/{file}', 'r+'):\n",
        "                    tokens = tokenize(line)\n",
        "                    if len(tokens) > 1: # skip one-word lines, since these are often enumerations\n",
        "                        text += tokens \n",
        "                if len(text) > 1: # skip one-word texts, if they exist\n",
        "                  yield text"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "85c16c57",
      "metadata": {
        "id": "85c16c57"
      },
      "source": [
        "## Create corpus and dictionary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 73,
      "id": "d93a0de1",
      "metadata": {
        "id": "d93a0de1"
      },
      "outputs": [],
      "source": [
        "texts = Texts('texts')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 74,
      "id": "f3d12cab",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f3d12cab",
        "outputId": "45fa757e-c047-4d02-8346-6ead3ac7d4c3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['μαξίμου', 'καταρχῶν', 'μεταφρασθὲν', 'πεζῇ', 'λέξει', 'ἡρωικῶν', 'μέτρων', 'ἄγε', 'κούρη', 'πιμπληιὰς']\n"
          ]
        }
      ],
      "source": [
        "# Output should resemble: ['ἐν', 'ὁ', 'πρότερος', 'ὅτι2', 'εὔχομαι', 'νύξ', 'καί', 'ἡμέρα', 'ὁράω', ...etc.\n",
        "count = 0\n",
        "for i in texts:\n",
        "    print(i[0:10])\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Generating dictionary . . . \")\n",
        "start = time.time()\n",
        "dictionary = Dictionary(texts)\n",
        "print(\"\\nDictionary initialized in {0:.2f} seconds.\".format(time.time() - start))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lJxxqHWvcWvV",
        "outputId": "c85805af-e182-4a1e-b52a-c71fdcf38d67"
      },
      "id": "lJxxqHWvcWvV",
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Generating dictionary . . . \n",
            "\n",
            "Dictionary initialized in 86.41 seconds.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Generating corpus . . . \")\n",
        "start = time.time()\n",
        "corpus = [dictionary.doc2bow(text) for text in texts]\n",
        "print(\"\\nCorpus initialized in {0:.2f} seconds.\".format(time.time() - start))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zh9-Z4-ygJa7",
        "outputId": "24f073ac-9c8b-4669-a9da-c5e4be7d8ebc"
      },
      "id": "zh9-Z4-ygJa7",
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Generating corpus . . . \n",
            "\n",
            "Corpus initialized in 80.50 seconds.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2a086553",
      "metadata": {
        "id": "2a086553"
      },
      "source": [
        "## Train model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 77,
      "id": "682c49fd",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "682c49fd",
        "outputId": "f7cdff52-da93-4074-b39b-88ebd64b728c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Generating model . . . \n",
            "\n",
            "Model initialized in 2.33 seconds.\n"
          ]
        }
      ],
      "source": [
        "print(\"Generating model . . . \")\n",
        "start = time.time()\n",
        "model = TfidfModel(corpus)\n",
        "# Create a list of all the unique words in the corpus, in case user wants to query all words.\n",
        "#     words_seen = set() # holds lines already seen\n",
        "#     allWords = []\n",
        "#     for line in sentences:\n",
        "#         for word in line:\n",
        "#             if word not in words_seen: # not a duplicate\n",
        "#                 allWords.append(word)\n",
        "#                 words_seen.add(word)   \n",
        "\n",
        "print(\"\\nModel initialized in {0:.2f} seconds.\".format(time.time() - start))"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "7478c819",
      "metadata": {
        "id": "7478c819"
      },
      "source": [
        "## Query model"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input_text = \"Ἐν ἀρχῇ ἦν ὁ λόγος, καὶ ὁ λόγος ἦν πρὸς τὸν θεόν, καὶ θεὸς ἦν ὁ λόγος.\""
      ],
      "metadata": {
        "id": "6reFdUI9zHeV"
      },
      "id": "6reFdUI9zHeV",
      "execution_count": 94,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 96,
      "id": "a3056306",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a3056306",
        "outputId": "4b147fb8-f606-411b-85a4-025d7e3c17db"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Most significant words in input text: \n",
            "0.58: θεόν\n",
            "0.54: λόγος\n",
            "0.47: ἀρχῇ\n",
            "0.40: θεὸς\n"
          ]
        }
      ],
      "source": [
        "input_tokens = [w for w in tokenize(input_text)]\n",
        "input_bow = dictionary.doc2bow(input_tokens)\n",
        "input_tfidf = model[input_bow]\n",
        "summary = sorted(input_tfidf, key=lambda x: x[1], reverse=True)[:10]\n",
        "print('Most significant words in input text: ')\n",
        "for result in summary:\n",
        "    id, score = result\n",
        "    token = dictionary[id]\n",
        "    print(f'{score:.2f}: {token}')"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.13"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}