scripts/scrape_ANC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = []\n",
    "base_url = \"https://anc.dc.gov/page/advisory-neighborhood-commission-\"\n",
    "for i in range(1, 9):\n",
    "    for j in ['a', 'b', 'c', 'd', 'e', 'f', 'g']:\n",
    "        url = base_url + str(i) + j\n",
    "        r = requests.get(url)\n",
    "        if r.status_code == 200:\n",
    "            df = pd.read_html(r.content, header=None, converters={'SMD': str})\n",
    "            dfs.append(df[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#missing one ANC, 1B due to a 403 error\n",
    "r = requests.get(\"https://anc.dc.gov/page/advisory-neighborhood-commission-1b-00\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_html(r.content, header=None)\n",
    "dfs.append(df[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dfs) == 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_members = pd.concat(dfs, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_members = current_members.set_index('SMD', verify_integrity= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(297, 5)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current_members.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_members.to_csv(\"current_anc_membership.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Address</th>\n",
       "      <th>Phone</th>\n",
       "      <th>Email</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SMD</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1A01</th>\n",
       "      <td>Layla Bonnot</td>\n",
       "      <td>1434 Parkwood Place NW  Washington, DC 20010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1A01@anc.dc.gov</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1A02</th>\n",
       "      <td>Teresa A. Edmondson</td>\n",
       "      <td>1477 Newton St NW  Washington, DC 20010</td>\n",
       "      <td>(240) 813-0264</td>\n",
       "      <td>1A02@anc.dc.gov</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1A03</th>\n",
       "      <td>Zach Rybarczyk</td>\n",
       "      <td>1400 Irving Street NW  Washington, DC 20010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1A03@anc.dc.gov</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1A04</th>\n",
       "      <td>Matthew Goldschimdt</td>\n",
       "      <td>1349 Parkwood Place NW  Washington, DC 20010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1A04@anc.dc.gov</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1A05</th>\n",
       "      <td>Christine Miller</td>\n",
       "      <td>1530 Monroe St NW  Washington, DC 20010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1A05@anc.dc.gov</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Name                                       Address  \\\n",
       "SMD                                                                       \n",
       "1A01         Layla Bonnot  1434 Parkwood Place NW  Washington, DC 20010   \n",
       "1A02  Teresa A. Edmondson       1477 Newton St NW  Washington, DC 20010   \n",
       "1A03       Zach Rybarczyk   1400 Irving Street NW  Washington, DC 20010   \n",
       "1A04  Matthew Goldschimdt  1349 Parkwood Place NW  Washington, DC 20010   \n",
       "1A05     Christine Miller       1530 Monroe St NW  Washington, DC 20010   \n",
       "\n",
       "               Phone            Email  \n",
       "SMD                                    \n",
       "1A01             NaN  1A01@anc.dc.gov  \n",
       "1A02  (240) 813-0264  1A02@anc.dc.gov  \n",
       "1A03             NaN  1A03@anc.dc.gov  \n",
       "1A04             NaN  1A04@anc.dc.gov  \n",
       "1A05             NaN  1A05@anc.dc.gov  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current_members.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}