02_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Cleaning up csvs**\n",
    "\n",
    "In our intial scraping, we had an issue with certain rural areas returning **the entire planet's tweets** so we had to remove those from our final dataset, as well as the following steps:\n",
    "* Remove redundant headers\n",
    "* Remove rows with no text or date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_and_clean_shutdown_df(state_abbrev:str, cities_to_remove:list):\n",
    "    '''\n",
    "    This function combines and cleans dataframes after they were\n",
    "    scraped from dirty, dirty Twitter. We need to go through a few\n",
    "    similar steps based on our collection methods: remove broken\n",
    "    cities, remove redundant headers, remove rows with no text or date\n",
    "    \n",
    "    -----------------------\n",
    "    Parameters:\n",
    "    \n",
    "    'state_abbrev' : str, two-letter state abbreviation at the beginning\n",
    "    of your auto-generated csv title\n",
    "    \n",
    "    'cities_to_remove' : list, A list of cities to remove from your dataset\n",
    "    '''\n",
    "    \n",
    "    # Getting all of a state's filenames\n",
    "    filenames = [filename for filename in os.listdir('./data/shutdown_data/') if filename.startswith(state_abbrev.upper())]\n",
    "    # Create new df so the final df doesn't keep appending if you need to run again\n",
    "    temp_df = pd.read_csv(f'./data/shutdown_data/{filenames[0]}')\n",
    "\n",
    "    # Write a clean df\n",
    "    df = pd.DataFrame(columns=temp_df.columns).to_csv(f'./data/shutdown_data/{state_abbrev.upper()}_full_shutdown.csv', index=False)\n",
    "    \n",
    "    # Main cleaning loop\n",
    "    for filename in filenames:\n",
    "        df = pd.read_csv(f'./data/shutdown_data/{filename}')\n",
    "        df.drop_duplicates(subset=['username', 'date'], keep='first', inplace=True) # need to remove any potential duplicates from overlapping city areas\n",
    "        \n",
    "        # Clean up final df\n",
    "        df = df[(df['username'] != 'username') & # Removes headers leftover from scraping\n",
    "               (~df['city'].isin(cities_to_remove))] # Removes the cities that we need to remove\n",
    "        df.dropna(subset=['text', 'date'], inplace=True) # There were some nulls in the text and date column that are likely the result of deleted/private tweets\n",
    "        df.to_csv(f'./data/shutdown_data/{state_abbrev.upper()}_full_shutdown.csv', index=False, mode='a', header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_and_clean_reopening_df(state_abbrev:str, cities_to_remove:list):\n",
    "    '''\n",
    "    This function combines and cleans dataframes after they were\n",
    "    scraped from dirty, dirty Twitter. We need to go through a few\n",
    "    similar steps based on our collection methods: remove broken\n",
    "    cities, remove redundant headers, remove rows with no text or date\n",
    "    \n",
    "    -----------------------\n",
    "    Parameters:\n",
    "    \n",
    "    'state_abbrev' : str, two-letter state abbreviation at the beginning\n",
    "    of your auto-generated csv title\n",
    "    \n",
    "    'cities_to_remove' : list, A list of cities to remove from your dataset\n",
    "    '''\n",
    "    \n",
    "    # Getting all of a state's filenames\n",
    "    filenames = [filename for filename in os.listdir('./data/reopening_data/') if filename.startswith(state_abbrev.upper())]\n",
    "    # Create new df so the final df doesn't keep appending if you need to run again\n",
    "    temp_df = pd.read_csv(f'./data/reopening_data/{filenames[0]}')\n",
    "\n",
    "    # Write a clean df\n",
    "    df = pd.DataFrame(columns=temp_df.columns).to_csv(f'./data/reopening_data/{state_abbrev.upper()}_full_reopen.csv', index=False)\n",
    "    \n",
    "    # Main cleaning loop\n",
    "    for filename in filenames:\n",
    "        df = pd.read_csv(f'./data/reopening_data/{filename}')\n",
    "        df.drop_duplicates(subset=['username', 'date'], keep='first', inplace=True) # need to remove any potential duplicates from overlapping city areas\n",
    "        \n",
    "        # Clean up final df\n",
    "        df = df[(df['username'] != 'username') & # Removes headers leftover from scraping\n",
    "               (~df['city'].isin(cities_to_remove))] # Removes the cities that we need to remove\n",
    "        df.dropna(subset=['text', 'date'], inplace=True) # There were some nulls in the text and date column that are likely the result of deleted/private tweets\n",
    "        df.to_csv(f'./data/reopening_data/{state_abbrev.upper()}_full_reopen.csv', index=False, mode='a', header=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Cleaning Georgia Dataset - shutdown**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_cities_to_remove = ['herndon', 'sunnyside', 'windsor', 'rowena', 'dublin']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_and_clean_shutdown_df('GA', ga_cities_to_remove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Cleaning Georgia Dataset - reopen**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_cities_to_remove = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_and_clean_reopening_df('GA', ga_cities_to_remove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Cleaning Illinois Dataset - shutdown**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "il_cities_to_remove = ['ripley', 'island grove', 'pleasant grove', 'pickneyville']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_and_clean_shutdown_df('IL', il_cities_to_remove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Cleaning Illinois Dataset - reopening**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "il_cities_to_remove = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_and_clean_reopening_df('IL', il_cities_to_remove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Cleaning Texas Dataset - shutdown**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "tx_cities_to_remove = ['fort stolkhom']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_and_clean_shutdown_df('TX', tx_cities_to_remove) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Cleaning Texas Dataset - reopening**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "tx_cities_to_remove = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_and_clean_reopening_df('TX', tx_cities_to_remove) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Once the reopening dates are scraped:\n",
    "They should be added to this notebook following similar processes, but using the reopening function rather than the shutdown function"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}