text_preprocessing_modules.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running Tweets Sentiment Analysis on All Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is an edited version of the prior plan -- I can't get rid of similar lines using fuzz \n",
    "# Assuming that private tweets (removing those with url) do not overlap, the data does not incude retweets anyway"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tweets to Dict -- Retaining Relevant Tweets\n",
    "\n",
    "def tweets_to_dict(tweet):\n",
    "    \"\"\"\n",
    "    For each dictionary in the output, create a new dictionary that includes\n",
    "    information on name, username, timestamp, text, the url shared by the user, the\n",
    "    title of the url shared, and a boolean indicator showing whether or not the\n",
    "    tweet included a url\n",
    "\n",
    "    Parameters\n",
    "    tweet: a dictionary\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    dictionary\n",
    "    \"\"\"\n",
    "    # keys: username, url, time, title\n",
    "    tweets_dict = {}\n",
    "    # creates {'username':'', ..}\n",
    "    tweets_dict['fullname'] = tweet['fullname']\n",
    "    tweets_dict['user'] = tweet['user']\n",
    "    tweets_dict['timestamp'] = tweet['timestamp']\n",
    "    tweets_dict['text'] = tweet['text']\n",
    "    tweets_dict['url_dum'] = False\n",
    "    tweets_dict['url'] = None\n",
    "    tweets_dict['title'] = None\n",
    "    tweets_dict['url_source'] = None\n",
    "    # for the url\n",
    "    html_text = tweet['html']\n",
    "    f_str = \"data-expanded-url\"\n",
    "    # for the title\n",
    "    text = tweet['text']\n",
    "    # get the titles\n",
    "    if f_str in html_text:\n",
    "        tweets_dict['url_dum'] = True\n",
    "    return tweets_dict\n",
    "\n",
    "def CSVtoDict(filename):\n",
    "    '''\n",
    "    Read CSV file yield a row as dictionary in a list\n",
    "\n",
    "    Parameters\n",
    "    ---------\n",
    "    filename: a string\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    a list\n",
    "    '''\n",
    "    import csv\n",
    "    dict_list = []\n",
    "    with open(filename) as f:\n",
    "        reader = csv.DictReader(f)\n",
    "        for line in reader:\n",
    "            dict_list.append(tweets_to_dict(dict(line)))\n",
    "    return(dict_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code that Cleans Corpus According to Time Frame\n",
    "\n",
    "def ConvertDates(string_date):\n",
    "    '''\n",
    "    Convert string dates (\"Y-M-D\") to datetime object\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    string_date: a string\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    datetime object\n",
    "    '''\n",
    "    import datetime\n",
    "    string_date = string_date[0:10]\n",
    "    string_date = string_date.replace(\"-\",\"\")\n",
    "    string_date = string_date.strip()\n",
    "    date = datetime.datetime.strptime(string_date,\"%Y%m%d\").date()\n",
    "    return date\n",
    "\n",
    "def ChangeDicts(data):\n",
    "    '''\n",
    "    Convert string dates (\"Y-M-D\") to datetime object\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    string_date: a string\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    datetime object\n",
    "    '''\n",
    "    import datetime\n",
    "    for d in data:\n",
    "        d['date_conv'] = ConvertDates(d['timestamp'])\n",
    "    # remove those with url\n",
    "    changed = [i for i in data if not (i['url_dum'] == True)]\n",
    "    return changed\n",
    "\n",
    "\n",
    "# Create String Before Conducting Word Count\n",
    "\n",
    "def PreCorpusList(data,key,s_date,e_date):\n",
    "    \"\"\"\n",
    "    Remove tweets including crisis key words\n",
    "    Remove tweets including urls\n",
    "\n",
    "    String join all text under time condition\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    data : a list of dictionaries\n",
    "    key: the key in the dictionary within the list ('text' or 'title')\n",
    "    s_date: start date, in string \"Y-M-D\"\n",
    "    e_date: end date, in string \"Y-M-D\"\n",
    "\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    a joined string of all tweets\n",
    "    \"\"\"\n",
    "    import re\n",
    "    import random\n",
    "    # converte the dates to datetime format\n",
    "    data = ChangeDicts(data)\n",
    "    # parse according to date & remove tweets with url\n",
    "    text_list = []\n",
    "    for d in data:\n",
    "        if (d['date_conv'] >= ConvertDates(s_date)) & (d['date_conv'] <= ConvertDates(e_date)) & (d['url_dum'] is False):\n",
    "            text_list.append(d[key])\n",
    "    # strings in text_list\n",
    "    # remove if the text is the same\n",
    "    text_list2 = [i for n, i in enumerate(text_list) if i not in text_list[n + 1:]]\n",
    "    # Remove words related to THAAD\n",
    "    fp_words = ['THAAD','thaad','사드','싸드','#사드','#싸드','#THAAD','#Thaad']\n",
    "    pattern = re.compile(r\"|\".join(fp_words))\n",
    "    removed = [s for s in text_list2 if not pattern.search(s)]\n",
    "    all_text = ' '.join(removed)\n",
    "    return(all_text)\n",
    "\n",
    "\n",
    "def CorpusListbyHandle(data,s_date,e_date):\n",
    "    \"\"\"\n",
    "    Remove tweets including crisis key words\n",
    "    Remove tweets including urls\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    data : a list of dictionaries\n",
    "    s_date: start date, in string \"Y-M-D\"\n",
    "    e_date: end date, in string \"Y-M-D\"\n",
    "    k: the Token_Set_Ratio\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    a list of dictionaries by handle ex: [{handle:joined_string},{handle:joined_string}]\n",
    "    \"\"\"\n",
    "    import re\n",
    "    import random\n",
    "    # convert the dates to datetime format\n",
    "    data = ChangeDicts(data)\n",
    "    # parse according to date & remove tweets with url\n",
    "    alist = []\n",
    "    for d in data:\n",
    "        if (d['date_conv'] >= ConvertDates(s_date)) & (d['date_conv'] <= ConvertDates(e_date)) & (d['url_dum'] is False) & (d['text'] != 'url_source'):\n",
    "            alist.append(d)\n",
    "    import re\n",
    "    fp_words = ['THAAD','thaad','사드','싸드','#사드','#싸드','#THAAD','#Thaad']\n",
    "    pattern = re.compile(r\"|\".join(fp_words))\n",
    "    output = [x for x in alist if not pattern.search(x['text'])]\n",
    "    # remove if 'text' is null:\n",
    "    output = [x for x in output if not x['text'] == '']\n",
    "    # remove duplicates\n",
    "    check_val  = set()\n",
    "    final_output = []\n",
    "    for d in output:\n",
    "        if d['text'] not in check_val:\n",
    "            final_output.append(d)\n",
    "            check_val.add(d['text'])\n",
    "    return(final_output)\n",
    "\n",
    "\n",
    "class TweetsCorpus:\n",
    "    '''\n",
    "    Functions\n",
    "    ---------\n",
    "    __init__: returns a string to be cleaned\n",
    "    text_cleaning: returns a list of characters\n",
    "    getPreWordCorpus: join the results of text_cleaning\n",
    "    getWordCorpus: returns a string\n",
    "    getNouns: returns a list of nouns using Konlpy module\n",
    "    '''\n",
    "    def __init__(self, text):\n",
    "        self.text = text\n",
    "    def text_cleaning(self):\n",
    "        import re\n",
    "        result_list = []\n",
    "        for item in self.text:\n",
    "            cleaned_text = re.sub('[a-zA-Z]', '', item)\n",
    "            cleaned_text = re.sub('\\n', ' ', cleaned_text)\n",
    "            cleaned_text = re.sub('[\\{\\}\\[\\]\\/?.,;:|\\)*~`!^\\-_+<>@\\#$%&\\\\\\=\\(\\'\\\"...]',\n",
    "                              '', cleaned_text)\n",
    "            result_list.append(cleaned_text)\n",
    "        return result_list\n",
    "    def getPreWordCorpus(self):\n",
    "        result=self.text_cleaning()\n",
    "        doc = (''.join(result))\n",
    "        return doc\n",
    "    def getWordCorpus(self):\n",
    "        doc = self.getPreWordCorpus()\n",
    "        text = ''.join(c for c in doc if c.isalnum() or c in '+, ')\n",
    "        text = ''.join([i for i in text if not i.isdigit()])\n",
    "        return text\n",
    "    def getPhrases(self):\n",
    "        from konlpy.tag import Okt\n",
    "        phrases_tagger = Okt()\n",
    "        word_corpus = self.getWordCorpus()\n",
    "        phrases = phrases_tagger.phrases(word_corpus)\n",
    "        return phrases\n",
    "    def getNouns(self):\n",
    "        from konlpy.tag import Okt\n",
    "        from collections import Counter\n",
    "        nouns_tagger = Okt()\n",
    "        word_corpus = self.getWordCorpus()\n",
    "        nouns = nouns_tagger.nouns(word_corpus)\n",
    "        return nouns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the CSV File, Split to Pre and Post\n",
    "\n",
    "# fn = '20150101_china_tweets_20170731.csv'\n",
    "# dt = CSVtoDict(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuesday August 20 -- just create pandas using user column, tweet column, and the cleaned tweet using the class above\n",
    "\n",
    "# then follow the instructions to run LDA on the individual tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# packages to store and manipulate data\n",
    "# import pandas as pd\n",
    "# import numpy as np\n",
    "\n",
    "# # plotting packages\n",
    "# import matplotlib.pyplot as plt\n",
    "# import seaborn as sns\n",
    "\n",
    "# # model building package\n",
    "# import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the Corpus List by Handle as pd.data frame:\n",
    "# CorpusListbyHandle(data,s_date,e_date)\n",
    "# Step 1: clean the texts in corpus list by handle:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sentiment analyses\n",
    "\n",
    "# print(getSentiScore(TweetsCorpus(pre_joined_raw_string)),getSentiScore(TweetsCorpus(post_joined_raw_string)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run LDAs on the saved json files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for stm -- create files based on the writer\n",
    "# list of dictionaries by author: [{author; joined_strings},]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}