FrequencyToolkit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readFile(filepath): \n",
    "    \n",
    "    with open(filepath, 'r') as f:\n",
    "        stories = json.load(f)\n",
    "        \n",
    "    return stories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "stories = readFile('Cleaned/cleaned_scale.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def frequency(stories):\n",
    "    \n",
    "    frequent = {}\n",
    "    \n",
    "    for story in stories:\n",
    "        \n",
    "        text = story['cleaned_text']\n",
    "        \n",
    "        for word in text:\n",
    "            \n",
    "            if word.isnumeric():\n",
    "                continue\n",
    "            \n",
    "            elif word in frequent:\n",
    "                frequent[word] += 1\n",
    "                \n",
    "            else:\n",
    "                frequent[word] = 1\n",
    "                  \n",
    "    frequent = [(k, v) for k, v in frequent.items()]  \n",
    "    frequent = sorted(frequent, key=lambda x: x[-1], reverse = True)\n",
    "                \n",
    "    return frequent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "def formatFrequencyList(frequency_list, remove_count):\n",
    "    \n",
    "    formatted = []\n",
    "    \n",
    "    if (remove_count):\n",
    "        \n",
    "        for tuple in frequency_list:\n",
    "        \n",
    "            obj = tuple[0]\n",
    "          \n",
    "            formatted.append(obj)\n",
    "        \n",
    "    else:\n",
    "    \n",
    "        for tuple in frequency_list:\n",
    "\n",
    "            obj = {\n",
    "                'word': tuple[0],\n",
    "                'count': tuple[1]\n",
    "            } \n",
    "\n",
    "            formatted.append(obj)\n",
    "        \n",
    "    return formatted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def removeDuplicates(collection) :\n",
    "    \n",
    "    dictionary = {}\n",
    "    no_duplicates = []\n",
    "    \n",
    "    for story in collection:\n",
    "        \n",
    "        title = story['title']\n",
    "        \n",
    "        if (title in dictionary):\n",
    "            continue\n",
    "        else:\n",
    "            dictionary[title] = True\n",
    "            no_duplicates.append(story)\n",
    "            \n",
    "            \n",
    "    return no_duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createCollection(filepaths):\n",
    "    \n",
    "    if filepaths == [] or None: \n",
    "        \n",
    "        raise ValueError('var input filepaths is an Empty Array') \n",
    "        \n",
    "    elif isinstance(filepaths, str):\n",
    "        \n",
    "        return readFile(filepaths)\n",
    "    \n",
    "    else:\n",
    "    \n",
    "        stories = []\n",
    "\n",
    "        for filepath in filepaths:\n",
    "\n",
    "            collection = readFile(filepath)\n",
    "            stories += collection\n",
    "            \n",
    "    stories = removeDuplicates(stories)\n",
    "        \n",
    "    return stories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract(filepaths, keyword, export_file_extension):\n",
    "    \n",
    "    stories = createCollection(filepaths)\n",
    "    \n",
    "    frequency_list = frequency(stories)\n",
    "    formatted = formatFrequencyList(frequency_list)\n",
    "\n",
    "    with open('Frequency/frequency_list_' + keyword + export_file_extension, 'w') as data:\n",
    "        json.dump(formatted, data)\n",
    "    \n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract conversion lists and export to JSON file\n",
    "\n",
    "extract(['Cleaned/cleaned_scale.json', 'Cleaned/cleaned_not_scale.json'], 'scale_all', '.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display(frequency_list, list_size) :\n",
    "    \n",
    "    for i in range(list_size):\n",
    "        print(frequency_list[i])\n",
    "        \n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "inclusive = createCollection(['Cleaned/cleaned_scale.json', 'Cleaned/cleaned_not_scale.json'])\n",
    "contains = createCollection('Cleaned/cleaned_scale.json')\n",
    "not_contains = createCollection('Cleaned/cleaned_not_scale.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label(stories, frequency_list):\n",
    "    \n",
    "    all_lists = []\n",
    "    \n",
    "    # Format frequency_list for labels\n",
    "    \n",
    "    frequency_list = formatFrequencyList(frequency_list, True)\n",
    "    \n",
    "    for story in stories:\n",
    "        \n",
    "        dictionary = {}\n",
    "        \n",
    "        text = story['cleaned_text']\n",
    "          \n",
    "        for word in frequency_list:\n",
    "            \n",
    "            # Why is 'word' in the formate of: (word, index) ? \n",
    "            # TODO: Fix this formatting\n",
    "            \n",
    "            if word in text: \n",
    "                dictionary[word] = True\n",
    "            else:\n",
    "                dictionary[word] = False\n",
    "        \n",
    "        all_lists.append(dictionary)\n",
    "    \n",
    "    \n",
    "    return all_lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Examples "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'one': True,\n",
       "  'also': True,\n",
       "  'scale': True,\n",
       "  'people': True,\n",
       "  'year': True,\n",
       "  'like': True,\n",
       "  'years': True,\n",
       "  'time': True,\n",
       "  'many': True,\n",
       "  'could': True}]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stories = readFile('Cleaned/cleaned_scale.json')[0:1]\n",
    "\n",
    "frequency_list = frequency(contains)[0:10]\n",
    "\n",
    "label(stories, frequency_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('one', 1489)\n",
      "('also', 1422)\n",
      "('scale', 1412)\n",
      "('people', 1373)\n",
      "('year', 1368)\n",
      "('like', 1358)\n",
      "('years', 1331)\n",
      "('time', 1283)\n",
      "('many', 1258)\n",
      "('could', 1248)\n"
     ]
    }
   ],
   "source": [
    "# Tuple[1] is occurence count\n",
    "\n",
    "display(frequency(contains), 10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}