Untitled1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f773b390-2d83-4125-8dcf-1581e5be4cca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "people = pd.read_csv('data/People.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5973dfaf-8ba1-4bdf-8acd-f43efa32d50f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import random\n",
    "\n",
    "def get_sample(file_path, n_samples=1000, seed=42):\n",
    "    \"\"\"Get a random sample of the data.\"\"\"\n",
    "    random.seed(seed)\n",
    "    sampled_rows = []\n",
    "\n",
    "    try:\n",
    "        with open(file_path, 'r', newline='', encoding='utf-8') as file:\n",
    "            reader = csv.DictReader(file)\n",
    "\n",
    "            # Fill the reservoir with initial rows\n",
    "            for _ in range(n_samples):\n",
    "                try:\n",
    "                    row = next(reader)\n",
    "                    sampled_rows.append(row)\n",
    "                except StopIteration:\n",
    "                    print(f\"File has fewer rows than requested samples ({len(sampled_rows)} rows).\")\n",
    "                    return sampled_rows\n",
    "\n",
    "            # Reservoir sampling for remaining rows\n",
    "            for row_num, row in enumerate(reader, start=n_samples):\n",
    "                random_pick = random.randint(1, row_num)\n",
    "                if random_pick < n_samples:\n",
    "                    sampled_rows[random_pick] = row\n",
    "                    \n",
    "    except FileNotFoundError:\n",
    "        print(f\"Error: File {file_path} not found.\")\n",
    "    except Exception as e:\n",
    "        print(f\"Error while sampling data: {e}\")\n",
    "    return sampled_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c504d24d-6f7b-40cb-998a-358f48fc1611",
   "metadata": {},
   "outputs": [],
   "source": [
    "people_sampled = get_sample('data/People.csv', 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c3675874-6824-40ed-8435-dcd09176dc8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting fuzzywuzzy\n",
      "  Using cached fuzzywuzzy-0.18.0-py2.py3-none-any.whl.metadata (4.9 kB)\n",
      "Using cached fuzzywuzzy-0.18.0-py2.py3-none-any.whl (18 kB)\n",
      "Installing collected packages: fuzzywuzzy\n",
      "Successfully installed fuzzywuzzy-0.18.0\n"
     ]
    }
   ],
   "source": [
    "!pip install fuzzywuzzy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "27dfa1f6-0283-4d3f-9e00-733ae74daae9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n",
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'PERSON_ID': 'O561555', 'PERSON_TYPE': 'DRIVER', 'RD_NO': 'JC113649', 'VEHICLE_ID': '535742.0', 'CRASH_DATE': '01/12/2019 12:01:00 AM', 'CITY': '', 'STATE': '', 'SEX': 'X', 'AGE': '', 'SAFETY_EQUIPMENT': 'USAGE UNKNOWN', 'AIRBAG_DEPLOYED': 'DEPLOYMENT UNKNOWN', 'EJECTION': 'UNKNOWN', 'INJURY_CLASSIFICATION': 'NO INDICATION OF INJURY', 'DRIVER_ACTION': 'UNKNOWN', 'DRIVER_VISION': 'UNKNOWN', 'PHYSICAL_CONDITION': 'UNKNOWN', 'BAC_RESULT': 'TEST NOT OFFERED', 'DAMAGE_CATEGORY': 'OVER $1,500', 'DAMAGE': '6069.0895692240765', 'BEST_MATCH_STATE': None}\n",
      "{'PERSON_ID': 'O155095', 'PERSON_TYPE': 'DRIVER', 'RD_NO': 'JA255517', 'VEHICLE_ID': '154240.0', 'CRASH_DATE': '05/06/2017 11:13:00 AM', 'CITY': 'CICERO', 'STATE': 'IL', 'SEX': 'M', 'AGE': '', 'SAFETY_EQUIPMENT': 'SAFETY BELT USED', 'AIRBAG_DEPLOYED': 'DID NOT DEPLOY', 'EJECTION': 'NONE', 'INJURY_CLASSIFICATION': 'NO INDICATION OF INJURY', 'DRIVER_ACTION': 'FAILED TO YIELD', 'DRIVER_VISION': 'UNKNOWN', 'PHYSICAL_CONDITION': 'UNKNOWN', 'BAC_RESULT': 'TEST NOT OFFERED', 'DAMAGE_CATEGORY': 'OVER $1,500', 'DAMAGE': '4408.8043417345125', 'BEST_MATCH_STATE': 'IL'}\n",
      "{'PERSON_ID': 'O107529', 'PERSON_TYPE': 'DRIVER', 'RD_NO': 'HZ552010', 'VEHICLE_ID': '105588.0', 'CRASH_DATE': '12/14/2016 09:19:00 PM', 'CITY': 'CHICAGO', 'STATE': 'IL', 'SEX': 'F', 'AGE': '55.0', 'SAFETY_EQUIPMENT': 'SAFETY BELT USED', 'AIRBAG_DEPLOYED': 'DID NOT DEPLOY', 'EJECTION': 'NONE', 'INJURY_CLASSIFICATION': 'NO INDICATION OF INJURY', 'DRIVER_ACTION': 'NONE', 'DRIVER_VISION': 'NOT OBSCURED', 'PHYSICAL_CONDITION': 'NORMAL', 'BAC_RESULT': 'TEST NOT OFFERED', 'DAMAGE_CATEGORY': 'OVER $1,500', 'DAMAGE': '3113.1873664148707', 'BEST_MATCH_STATE': 'IL'}\n",
      "{'PERSON_ID': 'O530714', 'PERSON_TYPE': 'DRIVER', 'RD_NO': 'JB531249', 'VEHICLE_ID': '506775.0', 'CRASH_DATE': '11/27/2018 07:10:00 PM', 'CITY': 'PHILA', 'STATE': '', 'SEX': 'M', 'AGE': '30.0', 'SAFETY_EQUIPMENT': 'USAGE UNKNOWN', 'AIRBAG_DEPLOYED': 'DID NOT DEPLOY', 'EJECTION': 'NONE', 'INJURY_CLASSIFICATION': 'NO INDICATION OF INJURY', 'DRIVER_ACTION': 'NONE', 'DRIVER_VISION': 'NOT OBSCURED', 'PHYSICAL_CONDITION': 'NORMAL', 'BAC_RESULT': 'TEST NOT OFFERED', 'DAMAGE_CATEGORY': 'OVER $1,500', 'DAMAGE': '5126.000146148784', 'BEST_MATCH_STATE': None}\n",
      "{'PERSON_ID': 'O325593', 'PERSON_TYPE': 'DRIVER', 'RD_NO': 'JB154616', 'VEHICLE_ID': '314761.0', 'CRASH_DATE': '02/16/2018 04:00:00 PM', 'CITY': 'CHICAGO', 'STATE': 'IL', 'SEX': 'M', 'AGE': '63.0', 'SAFETY_EQUIPMENT': 'SAFETY BELT USED', 'AIRBAG_DEPLOYED': 'DID NOT DEPLOY', 'EJECTION': 'NONE', 'INJURY_CLASSIFICATION': 'NO INDICATION OF INJURY', 'DRIVER_ACTION': 'NONE', 'DRIVER_VISION': 'NOT OBSCURED', 'PHYSICAL_CONDITION': 'NORMAL', 'BAC_RESULT': 'TEST NOT OFFERED', 'DAMAGE_CATEGORY': '$500 OR LESS', 'DAMAGE': '', 'BEST_MATCH_STATE': 'IL'}\n",
      "{'PERSON_ID': 'O178046', 'PERSON_TYPE': 'DRIVER', 'RD_NO': 'JA330758', 'VEHICLE_ID': '176335.0', 'CRASH_DATE': '07/01/2017 07:50:00 PM', 'CITY': '', 'STATE': '', 'SEX': 'M', 'AGE': '', 'SAFETY_EQUIPMENT': 'USAGE UNKNOWN', 'AIRBAG_DEPLOYED': 'DEPLOYMENT UNKNOWN', 'EJECTION': 'NONE', 'INJURY_CLASSIFICATION': 'NO INDICATION OF INJURY', 'DRIVER_ACTION': 'NONE', 'DRIVER_VISION': 'UNKNOWN', 'PHYSICAL_CONDITION': 'UNKNOWN', 'BAC_RESULT': 'TEST NOT OFFERED', 'DAMAGE_CATEGORY': '$501 - $1,500', 'DAMAGE': '1022.5167293578611', 'BEST_MATCH_STATE': None}\n",
      "{'PERSON_ID': 'O11941', 'PERSON_TYPE': 'DRIVER', 'RD_NO': 'HY497985', 'VEHICLE_ID': '11452.0', 'CRASH_DATE': '11/11/2015 04:15:00 PM', 'CITY': 'CHICAGO', 'STATE': 'IL', 'SEX': 'F', 'AGE': '20.0', 'SAFETY_EQUIPMENT': 'USAGE UNKNOWN', 'AIRBAG_DEPLOYED': 'DEPLOYMENT UNKNOWN', 'EJECTION': 'NONE', 'INJURY_CLASSIFICATION': 'NO INDICATION OF INJURY', 'DRIVER_ACTION': 'FOLLOWED TOO CLOSELY', 'DRIVER_VISION': 'UNKNOWN', 'PHYSICAL_CONDITION': 'UNKNOWN', 'BAC_RESULT': 'TEST NOT OFFERED', 'DAMAGE_CATEGORY': '$501 - $1,500', 'DAMAGE': '584.8737701535666', 'BEST_MATCH_STATE': 'IL'}\n",
      "{'PERSON_ID': 'O182933', 'PERSON_TYPE': 'DRIVER', 'RD_NO': 'JA345985', 'VEHICLE_ID': '180991.0', 'CRASH_DATE': '07/13/2017 09:05:00 AM', 'CITY': 'CHICAGO', 'STATE': 'IL', 'SEX': 'F', 'AGE': '23.0', 'SAFETY_EQUIPMENT': 'SAFETY BELT USED', 'AIRBAG_DEPLOYED': 'DID NOT DEPLOY', 'EJECTION': 'NONE', 'INJURY_CLASSIFICATION': 'NONINCAPACITATING INJURY', 'DRIVER_ACTION': 'FAILED TO YIELD', 'DRIVER_VISION': 'NOT OBSCURED', 'PHYSICAL_CONDITION': 'NORMAL', 'BAC_RESULT': 'TEST NOT OFFERED', 'DAMAGE_CATEGORY': 'OVER $1,500', 'DAMAGE': '2056.352490421956', 'BEST_MATCH_STATE': 'IL'}\n",
      "{'PERSON_ID': 'P72511', 'PERSON_TYPE': 'PASSENGER', 'RD_NO': 'JB165990', 'VEHICLE_ID': '321291.0', 'CRASH_DATE': '02/26/2018 12:30:00 PM', 'CITY': 'CHICAGO', 'STATE': 'IL', 'SEX': 'M', 'AGE': '4.0', 'SAFETY_EQUIPMENT': 'SAFETY BELT USED', 'AIRBAG_DEPLOYED': 'NOT APPLICABLE', 'EJECTION': 'NONE', 'INJURY_CLASSIFICATION': 'NO INDICATION OF INJURY', 'DRIVER_ACTION': '', 'DRIVER_VISION': '', 'PHYSICAL_CONDITION': '', 'BAC_RESULT': '', 'DAMAGE_CATEGORY': 'OVER $1,500', 'DAMAGE': '3909.779808182218', 'BEST_MATCH_STATE': 'IL'}\n",
      "{'PERSON_ID': 'P56384', 'PERSON_TYPE': 'PASSENGER', 'RD_NO': 'JA509064', 'VEHICLE_ID': '253067.0', 'CRASH_DATE': '11/12/2017 01:55:00 PM', 'CITY': 'CHICAGO', 'STATE': 'IL', 'SEX': 'F', 'AGE': '17.0', 'SAFETY_EQUIPMENT': 'SAFETY BELT USED', 'AIRBAG_DEPLOYED': 'DID NOT DEPLOY', 'EJECTION': 'NONE', 'INJURY_CLASSIFICATION': 'NO INDICATION OF INJURY', 'DRIVER_ACTION': '', 'DRIVER_VISION': '', 'PHYSICAL_CONDITION': '', 'BAC_RESULT': '', 'DAMAGE_CATEGORY': 'OVER $1,500', 'DAMAGE': '2713.4896944409243', 'BEST_MATCH_STATE': 'IL'}\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from fuzzywuzzy import process\n",
    "\n",
    "# Example list of valid states\n",
    "valid_states = [\"IL\", \"IN\", \"MI\", \"TX\", \"CO\", \"GA\", \"FL\", \"NJ\", \"IA\", \"MO\", \"NY\", \n",
    "                \"MA\", \"CA\", \"AR\", \"MN\", \"WI\", \"WA\", \"LA\", \"MD\", \"ME\", \"OH\", \"UT\", \n",
    "                \"NC\", \"NV\", \"TN\", \"DE\", \"VA\", \"AZ\", \"MS\", \"KS\", \"PA\", \"SC\", \"ID\", \n",
    "                \"WY\", \"KY\", \"DC\", \"OK\", \"NM\", \"AL\", \"MT\", \"HI\", \"OR\", \"RI\", \"ND\", \n",
    "                \"WV\", \"NH\", \"CT\", \"SD\", \"AK\", \"VT\"]\n",
    "\n",
    "\n",
    "\n",
    "def get_best_match(state, valid_states):\n",
    "    \"\"\"Find the best matching city using fuzzy matching.\"\"\"\n",
    "    best_match = process.extractOne(state, valid_states, score_cutoff=80)\n",
    "    return best_match[0] if best_match else None\n",
    "\n",
    "def add_best_match_column(data, city_key, valid_cities):\n",
    "    \"\"\"\n",
    "    Add a new key 'BEST_MATCH_CITY' to each dictionary in the list.\n",
    "    \n",
    "    Args:\n",
    "        data (list of dict): The sampled rows as a list of dictionaries.\n",
    "        city_key (str): The key to look up city names in the dictionaries.\n",
    "        valid_cities (list): List of valid city names for matching.\n",
    "    \n",
    "    Returns:\n",
    "        list of dict: The updated data with the new 'BEST_MATCH_CITY' key.\n",
    "    \"\"\"\n",
    "    for row in data:\n",
    "        state = row.get(\"STATE\", \"\")\n",
    "        row[\"BEST_MATCH_STATE\"] = get_best_match(state, valid_states)\n",
    "    return data\n",
    "\n",
    "# Example usage\n",
    "file_path = \"sample_file.csv\"  # Replace with your file path\n",
    "sampled_data = get_sample('data/People.csv', 100, seed=42)\n",
    "\n",
    "# Add the best match for the 'CITY' column\n",
    "sampled_data_with_matches = add_best_match_column(sampled_data, \"STATE\", valid_states)\n",
    "\n",
    "# Print the updated rows (for demonstration purposes)\n",
    "for row in sampled_data_with_matches[:10]:  # Display first 10 rows\n",
    "    print(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ea2b28b0-42d9-4e16-ad85-d4b64df21485",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'correggi_nome_citta' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[26], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m index, row \u001b[38;5;129;01min\u001b[39;00m people\u001b[38;5;241m.\u001b[39miterrows():\n\u001b[1;32m      2\u001b[0m     original_city \u001b[38;5;241m=\u001b[39m row[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCITY\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m----> 3\u001b[0m     corrected_city \u001b[38;5;241m=\u001b[39m \u001b[43mcorreggi_nome_citta\u001b[49m(original_city)\n\u001b[1;32m      4\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOriginal: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00moriginal_city\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Corrected: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcorrected_city\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'correggi_nome_citta' is not defined"
     ]
    }
   ],
   "source": [
    "for index, row in people.iterrows():\n",
    "    original_city = row[\"CITY\"]\n",
    "    corrected_city = correggi_nome_citta(original_city)\n",
    "    print(f\"Original: {original_city}, Corrected: {corrected_city}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04292e25-8773-439f-b14c-2774b4bd416f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}