scripts/01_download_fl_voter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download FL Voter Data from Dataverse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataverse API Token\n",
    "# https://dataverse.harvard.edu/dataverseuser.xhtml?selectTab=apiTokenTab\n",
    "token = '<TOKEN>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyDataverse.api import Api\n",
    "from pyDataverse.models import Dataverse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'OK'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "api = Api('https://dataverse.harvard.edu/', token)\n",
    "api.status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'OK',\n",
       " 'data': {'id': 3015860,\n",
       "  'identifier': 'DVN/UBIG3F',\n",
       "  'persistentUrl': 'https://doi.org/10.7910/DVN/UBIG3F',\n",
       "  'protocol': 'doi',\n",
       "  'authority': '10.7910',\n",
       "  'publisher': 'Harvard Dataverse',\n",
       "  'publicationDate': '2017-05-02',\n",
       "  'storageIdentifier': 's3://dvn-cloud:10.7910/DVN/UBIG3F',\n",
       "  'latestVersion': {'id': 126212,\n",
       "   'datasetId': 3015860,\n",
       "   'datasetPersistentId': 'doi:10.7910/DVN/UBIG3F',\n",
       "   'storageIdentifier': 's3://dvn-cloud:10.7910/DVN/UBIG3F',\n",
       "   'versionNumber': 1,\n",
       "   'versionMinorNumber': 0,\n",
       "   'versionState': 'RELEASED',\n",
       "   'lastUpdateTime': '2017-05-02T16:32:54Z',\n",
       "   'releaseTime': '2017-05-02T16:32:54Z',\n",
       "   'createTime': '2017-05-02T16:32:07Z',\n",
       "   'license': 'CC0',\n",
       "   'termsOfUse': 'CC0 Waiver',\n",
       "   'termsOfAccess': 'The data are available for research purposes only. ',\n",
       "   'fileAccessRequest': True,\n",
       "   'metadataBlocks': {'citation': {'displayName': 'Citation Metadata',\n",
       "     'fields': [{'typeName': 'title',\n",
       "       'multiple': False,\n",
       "       'typeClass': 'primitive',\n",
       "       'value': 'Florida Voter Registration Data'},\n",
       "      {'typeName': 'author',\n",
       "       'multiple': True,\n",
       "       'typeClass': 'compound',\n",
       "       'value': [{'authorName': {'typeName': 'authorName',\n",
       "          'multiple': False,\n",
       "          'typeClass': 'primitive',\n",
       "          'value': 'Sood, Gaurav'}}]},\n",
       "      {'typeName': 'datasetContact',\n",
       "       'multiple': True,\n",
       "       'typeClass': 'compound',\n",
       "       'value': [{'datasetContactName': {'typeName': 'datasetContactName',\n",
       "          'multiple': False,\n",
       "          'typeClass': 'primitive',\n",
       "          'value': 'Sood, Gaurav'},\n",
       "         'datasetContactEmail': {'typeName': 'datasetContactEmail',\n",
       "          'multiple': False,\n",
       "          'typeClass': 'primitive',\n",
       "          'value': 'gsood07@gmail.com'}}]},\n",
       "      {'typeName': 'dsDescription',\n",
       "       'multiple': True,\n",
       "       'typeClass': 'compound',\n",
       "       'value': [{'dsDescriptionValue': {'typeName': 'dsDescriptionValue',\n",
       "          'multiple': False,\n",
       "          'typeClass': 'primitive',\n",
       "          'value': 'Florida Voter Registration Data from February 2017. (2017-02-07). '}}]},\n",
       "      {'typeName': 'subject',\n",
       "       'multiple': True,\n",
       "       'typeClass': 'controlledVocabulary',\n",
       "       'value': ['Social Sciences']},\n",
       "      {'typeName': 'depositor',\n",
       "       'multiple': False,\n",
       "       'typeClass': 'primitive',\n",
       "       'value': 'Sood, Gaurav'},\n",
       "      {'typeName': 'dateOfDeposit',\n",
       "       'multiple': False,\n",
       "       'typeClass': 'primitive',\n",
       "       'value': '2017-05-02'}]}},\n",
       "   'files': [{'label': '02-2017 Voter Extract Disk File Layout.docx',\n",
       "     'restricted': False,\n",
       "     'version': 1,\n",
       "     'datasetVersionId': 126212,\n",
       "     'dataFile': {'id': 3015862,\n",
       "      'persistentId': 'doi:10.7910/DVN/UBIG3F/PAMH5I',\n",
       "      'pidURL': 'https://doi.org/10.7910/DVN/UBIG3F/PAMH5I',\n",
       "      'filename': '02-2017 Voter Extract Disk File Layout.docx',\n",
       "      'contentType': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',\n",
       "      'filesize': 27240,\n",
       "      'storageIdentifier': 's3://dvn-cloud:15bc9e60d49-ad8383c6acc0',\n",
       "      'rootDataFileId': -1,\n",
       "      'md5': '4e3fb0391935de5db94fe768fc3a093c',\n",
       "      'checksum': {'type': 'MD5', 'value': '4e3fb0391935de5db94fe768fc3a093c'},\n",
       "      'creationDate': '2017-05-02'}},\n",
       "    {'label': '20170207_VoterDetail.7z',\n",
       "     'restricted': True,\n",
       "     'version': 1,\n",
       "     'datasetVersionId': 126212,\n",
       "     'dataFile': {'id': 3015861,\n",
       "      'persistentId': 'doi:10.7910/DVN/UBIG3F/JRTDGY',\n",
       "      'pidURL': 'https://doi.org/10.7910/DVN/UBIG3F/JRTDGY',\n",
       "      'filename': '20170207_VoterDetail.7z',\n",
       "      'contentType': 'application/x-7z-compressed',\n",
       "      'filesize': 414108218,\n",
       "      'storageIdentifier': 's3://dvn-cloud:15bc9fcc31d-eaa1ef6b12b4',\n",
       "      'rootDataFileId': -1,\n",
       "      'md5': 'c7558bb4496f77b4dc5d6aad992e0dbb',\n",
       "      'checksum': {'type': 'MD5', 'value': 'c7558bb4496f77b4dc5d6aad992e0dbb'},\n",
       "      'creationDate': '2017-05-02'}},\n",
       "    {'label': '20170207_VoterHistory.7z',\n",
       "     'restricted': True,\n",
       "     'version': 1,\n",
       "     'datasetVersionId': 126212,\n",
       "     'dataFile': {'id': 3015863,\n",
       "      'persistentId': 'doi:10.7910/DVN/UBIG3F/LH707C',\n",
       "      'pidURL': 'https://doi.org/10.7910/DVN/UBIG3F/LH707C',\n",
       "      'filename': '20170207_VoterHistory.7z',\n",
       "      'contentType': 'application/x-7z-compressed',\n",
       "      'filesize': 223473639,\n",
       "      'storageIdentifier': 's3://dvn-cloud:15bc9e88896-90d5ad979455',\n",
       "      'rootDataFileId': -1,\n",
       "      'md5': '68f5d9a37d1eebc2cfc8c99ba50866cc',\n",
       "      'checksum': {'type': 'MD5', 'value': '68f5d9a37d1eebc2cfc8c99ba50866cc'},\n",
       "      'creationDate': '2017-05-02'}}]}}}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = api.get_dataset('doi:10.7910/DVN/UBIG3F')\n",
    "j = resp.json()\n",
    "j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3015862 02-2017 Voter Extract Disk File Layout.docx\n",
      "3015861 20170207_VoterDetail.7z\n",
      "3015863 20170207_VoterHistory.7z\n"
     ]
    }
   ],
   "source": [
    "files = []\n",
    "for file in j['data']['latestVersion']['files']:\n",
    "    fid = file['dataFile']['id']\n",
    "    fn = file['dataFile']['filename']\n",
    "    print(fid, fn)\n",
    "    files.append((fid, fn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "def download_file(url, fn):\n",
    "    local_filename = fn # url.split('/')[-1]\n",
    "    # NOTE the stream=True parameter below\n",
    "    with requests.get(url, stream=True) as r:\n",
    "        r.raise_for_status()\n",
    "        with open(local_filename, 'wb') as f:\n",
    "            for chunk in r.iter_content(chunk_size=8192): \n",
    "                if chunk: # filter out keep-alive new chunks\n",
    "                    f.write(chunk)\n",
    "                    # f.flush()\n",
    "    return local_filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "02-2017 Voter Extract Disk File Layout.docx\n",
      "20170207_VoterDetail.7z\n",
      "20170207_VoterHistory.7z\n"
     ]
    }
   ],
   "source": [
    "for fid, fn in files:\n",
    "    print(fn)\n",
    "    if not fn.startswith('20170207_VoterDetail.7z'):\n",
    "        continue\n",
    "    url = 'https://dataverse.harvard.edu//api/v1/access/datafile/%s?key=%s' % (fid, token)\n",
    "    download_file(url, fn)\n",
    "    #break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}