SI Table 4 Extract.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4b24793c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Page number 74 was parsed successfully!\n",
      "Page number 75 was parsed successfully!\n",
      "Page number 76 was parsed successfully!\n",
      "Page number 77 was parsed successfully!\n",
      "Page number 78 was parsed successfully!\n",
      "Page number 79 was parsed successfully!\n",
      "Page number 80 was parsed successfully!\n",
      "Page number 81 was parsed successfully!\n",
      "Page number 82 was parsed successfully!\n",
      "Page number 83 was parsed successfully!\n",
      "Page number 84 was parsed successfully!\n",
      "Page number 85 was parsed successfully!\n",
      "Page number 86 was parsed successfully!\n",
      "Page number 87 was parsed successfully!\n",
      "Page number 88 was parsed successfully!\n",
      "Page number 89 was parsed successfully!\n",
      "Page number 90 was parsed successfully!\n",
      "Page number 91 was parsed successfully!\n",
      "Page number 92 was parsed successfully!\n"
     ]
    }
   ],
   "source": [
    "file_name = '41587_2014_BFnbt2798_MOESM2_ESM.pdf'\n",
    "\n",
    "import tabula\n",
    "import pandas as pd\n",
    "import numpy as np \n",
    "\n",
    "def read_table(page_number):\n",
    "    #import tabula\n",
    "    table_list = tabula.read_pdf(file_name, \n",
    "                          pages=page_number,\n",
    "                               pandas_options={'header': None, \n",
    "                                               },\n",
    "                               stream=True\n",
    "                               )\n",
    "    table_df = table_list[0]\n",
    "    if table_df.shape[1] == 6:\n",
    "        table_df.columns = ['Motif', 'TF Names', 'Chromatin-Dependence', \n",
    "                                         'Motif-Dependence', 'Information Content', 'Classification']\n",
    "    else:\n",
    "        table_df.columns = ['Motif', 'Chromatin-Dependence', \n",
    "                                     'Motif-Dependence', 'Information Content', 'Classification']\n",
    "        import numpy as np\n",
    "        table_df['TF Names'] = np.nan   \n",
    "    #import pandas as pd\n",
    "    table_df = table_df[pd.to_numeric(table_df['Information Content'], errors='coerce').notna()]\n",
    "    print(f'Page number {page_number} was parsed successfully!')\n",
    "    return table_df\n",
    "    \n",
    "#parsed_table_df = read_table(75)\n",
    "table_start_page = 74\n",
    "table_end_page = 92\n",
    "page_nos_to_parse = list(range(table_start_page, table_end_page+1,1))\n",
    "parsed_table_df = pd.concat([read_table(page_no) for page_no in page_nos_to_parse], ignore_index=True)\n",
    "parsed_table_df[['Chromatin-Dependence','Motif-Dependence','Information Content']] = parsed_table_df[['Chromatin-Dependence','Motif-Dependence','Information Content']].apply(pd.to_numeric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1579d2d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Motif</th>\n",
       "      <th>TF Names</th>\n",
       "      <th>Chromatin-Dependence</th>\n",
       "      <th>Motif-Dependence</th>\n",
       "      <th>Information Content</th>\n",
       "      <th>Classification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CN0001.1 LM1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.745</td>\n",
       "      <td>0.113</td>\n",
       "      <td>12.974</td>\n",
       "      <td>Migrant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CN0002.1 LM2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.555</td>\n",
       "      <td>0.328</td>\n",
       "      <td>13.144</td>\n",
       "      <td>Migrant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CN0003.1 LM3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.774</td>\n",
       "      <td>0.089</td>\n",
       "      <td>14.587</td>\n",
       "      <td>Settler</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CN0004.1 LM4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.808</td>\n",
       "      <td>0.220</td>\n",
       "      <td>15.843</td>\n",
       "      <td>Pioneer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CN0005.1 LM5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.498</td>\n",
       "      <td>0.024</td>\n",
       "      <td>15.947</td>\n",
       "      <td>Migrant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728</th>\n",
       "      <td>SA0003.1 at_AC_acceptor</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.141</td>\n",
       "      <td>0.027</td>\n",
       "      <td>5.264</td>\n",
       "      <td>Migrant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>729</th>\n",
       "      <td>SD0003.1 at_AC_acceptor</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.446</td>\n",
       "      <td>0.015</td>\n",
       "      <td>7.133</td>\n",
       "      <td>Migrant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>730</th>\n",
       "      <td>YL_ES_Brg_3_0_c74</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.007</td>\n",
       "      <td>0.202</td>\n",
       "      <td>22.302</td>\n",
       "      <td>Migrant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>731</th>\n",
       "      <td>YL_ES_Med12_3_0_c1651</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.784</td>\n",
       "      <td>0.070</td>\n",
       "      <td>7.653</td>\n",
       "      <td>Settler</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>732</th>\n",
       "      <td>YL_ES_Med15_3_0_c356</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.873</td>\n",
       "      <td>0.002</td>\n",
       "      <td>8.236</td>\n",
       "      <td>Pioneer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>733 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Motif TF Names  Chromatin-Dependence  Motif-Dependence  \\\n",
       "0               CN0001.1 LM1      NaN                 0.745             0.113   \n",
       "1               CN0002.1 LM2      NaN                 0.555             0.328   \n",
       "2               CN0003.1 LM3      NaN                 0.774             0.089   \n",
       "3               CN0004.1 LM4      NaN                 0.808             0.220   \n",
       "4               CN0005.1 LM5      NaN                -0.498             0.024   \n",
       "..                       ...      ...                   ...               ...   \n",
       "728  SA0003.1 at_AC_acceptor      NaN                -0.141             0.027   \n",
       "729  SD0003.1 at_AC_acceptor      NaN                -0.446             0.015   \n",
       "730        YL_ES_Brg_3_0_c74      NaN                -0.007             0.202   \n",
       "731    YL_ES_Med12_3_0_c1651      NaN                 0.784             0.070   \n",
       "732     YL_ES_Med15_3_0_c356      NaN                 0.873             0.002   \n",
       "\n",
       "     Information Content Classification  \n",
       "0                 12.974        Migrant  \n",
       "1                 13.144        Migrant  \n",
       "2                 14.587        Settler  \n",
       "3                 15.843        Pioneer  \n",
       "4                 15.947        Migrant  \n",
       "..                   ...            ...  \n",
       "728                5.264        Migrant  \n",
       "729                7.133        Migrant  \n",
       "730               22.302        Migrant  \n",
       "731                7.653        Settler  \n",
       "732                8.236        Pioneer  \n",
       "\n",
       "[733 rows x 6 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parsed_table_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02982e19",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}