04-training-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "73ef2c3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1c8fcfa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19999"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs = pd.read_csv(\"data/pol/pairs.csv\")\n",
    "len(pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "30880fdb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accession</th>\n",
       "      <th>pol</th>\n",
       "      <th>query acc.ver</th>\n",
       "      <th>% identity</th>\n",
       "      <th>alignment length</th>\n",
       "      <th>mismatches</th>\n",
       "      <th>gap opens</th>\n",
       "      <th>q. start</th>\n",
       "      <th>q. end</th>\n",
       "      <th>s. start</th>\n",
       "      <th>s. end</th>\n",
       "      <th>evalue</th>\n",
       "      <th>bit score</th>\n",
       "      <th>pol_cut</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AB098330</td>\n",
       "      <td>FFRENLAFQQGEARKFSTEQTGANSPTSGALWDGGRDILPSEAGAE...</td>\n",
       "      <td>Query_1</td>\n",
       "      <td>92.735</td>\n",
       "      <td>234</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>234</td>\n",
       "      <td>156</td>\n",
       "      <td>389</td>\n",
       "      <td>5.920000e-155</td>\n",
       "      <td>458.0</td>\n",
       "      <td>PISPIDTVPVKLKPGMDGPKVKQWPLTEEKIKALTEICIDMEKEGK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AB098331</td>\n",
       "      <td>FFRENLAFQQGEARKFSTEQTGANSPTSGALWDGGRDILPSEAGTK...</td>\n",
       "      <td>Query_1</td>\n",
       "      <td>91.880</td>\n",
       "      <td>234</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>234</td>\n",
       "      <td>156</td>\n",
       "      <td>389</td>\n",
       "      <td>2.630000e-154</td>\n",
       "      <td>456.0</td>\n",
       "      <td>PISPIDTVPVKLKPGMDGPKVKQWPLTEEKVKALTEICIDMEKEGK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AB098332</td>\n",
       "      <td>FFRENLAFQQGEARKFPSEQTGANSPTSRDLWNGGRDSLPSEAGAE...</td>\n",
       "      <td>Query_1</td>\n",
       "      <td>94.872</td>\n",
       "      <td>234</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>234</td>\n",
       "      <td>156</td>\n",
       "      <td>389</td>\n",
       "      <td>1.760000e-157</td>\n",
       "      <td>464.0</td>\n",
       "      <td>PISPIETVPVTLKPGMDGPKVKQWPLTEEKIKALTEICTEMEKEGK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AB098333</td>\n",
       "      <td>FFRENLAFQQGEARKFPSEQTGANSPTSRDLWNGGRDSLPSEAGAE...</td>\n",
       "      <td>Query_1</td>\n",
       "      <td>94.872</td>\n",
       "      <td>234</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>234</td>\n",
       "      <td>156</td>\n",
       "      <td>389</td>\n",
       "      <td>1.760000e-157</td>\n",
       "      <td>464.0</td>\n",
       "      <td>PISPIETVPVTLKPGMDGPKVKQWPLTEEKIKALTEICTEMEKEGK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AB220944</td>\n",
       "      <td>FFRENLAFQQRKAGEFSSEQTRANSPTSRKLGDGGRDNLLTEAGAE...</td>\n",
       "      <td>Query_1</td>\n",
       "      <td>93.590</td>\n",
       "      <td>234</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>234</td>\n",
       "      <td>154</td>\n",
       "      <td>387</td>\n",
       "      <td>3.240000e-156</td>\n",
       "      <td>461.0</td>\n",
       "      <td>PISPIDTIPVTLKPGMDGPKVKQWPLTEEKIKALTEICKEMEEEGK...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  accession                                                pol query acc.ver  \\\n",
       "0  AB098330  FFRENLAFQQGEARKFSTEQTGANSPTSGALWDGGRDILPSEAGAE...       Query_1   \n",
       "1  AB098331  FFRENLAFQQGEARKFSTEQTGANSPTSGALWDGGRDILPSEAGTK...       Query_1   \n",
       "2  AB098332  FFRENLAFQQGEARKFPSEQTGANSPTSRDLWNGGRDSLPSEAGAE...       Query_1   \n",
       "3  AB098333  FFRENLAFQQGEARKFPSEQTGANSPTSRDLWNGGRDSLPSEAGAE...       Query_1   \n",
       "4  AB220944  FFRENLAFQQRKAGEFSSEQTRANSPTSRKLGDGGRDNLLTEAGAE...       Query_1   \n",
       "\n",
       "   % identity  alignment length  mismatches  gap opens  q. start  q. end  \\\n",
       "0      92.735               234          17          0         1     234   \n",
       "1      91.880               234          19          0         1     234   \n",
       "2      94.872               234          12          0         1     234   \n",
       "3      94.872               234          12          0         1     234   \n",
       "4      93.590               234          15          0         1     234   \n",
       "\n",
       "   s. start  s. end         evalue  bit score  \\\n",
       "0       156     389  5.920000e-155      458.0   \n",
       "1       156     389  2.630000e-154      456.0   \n",
       "2       156     389  1.760000e-157      464.0   \n",
       "3       156     389  1.760000e-157      464.0   \n",
       "4       154     387  3.240000e-156      461.0   \n",
       "\n",
       "                                             pol_cut  \n",
       "0  PISPIDTVPVKLKPGMDGPKVKQWPLTEEKIKALTEICIDMEKEGK...  \n",
       "1  PISPIDTVPVKLKPGMDGPKVKQWPLTEEKVKALTEICIDMEKEGK...  \n",
       "2  PISPIETVPVTLKPGMDGPKVKQWPLTEEKIKALTEICTEMEKEGK...  \n",
       "3  PISPIETVPVTLKPGMDGPKVKQWPLTEEKIKALTEICTEMEKEGK...  \n",
       "4  PISPIDTIPVTLKPGMDGPKVKQWPLTEEKIKALTEICKEMEEEGK...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/pol/2-pol.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6a3a1313",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "120dd52ae6a743449c0443bc4cd082ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/240393 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1434 -ABCDEFGHIKLMNPQRSTVWXYZ\n"
     ]
    }
   ],
   "source": [
    "vocab = set('-')\n",
    "max_seq_len = 0\n",
    "for s in tqdm(data.pol):\n",
    "    vocab = vocab.union(set(s))\n",
    "    if len(s) > max_seq_len:\n",
    "        max_seq_len = len(s)\n",
    "        \n",
    "vocab = sorted(vocab)\n",
    "vocab_index = dict([(char, i) for i, char in enumerate(vocab)])\n",
    "\n",
    "print(max_seq_len, ''.join(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9620f693",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(240393, 1435, 24)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_data = np.zeros(\n",
    "    (len(data), max_seq_len + 1, len(vocab)), dtype = \"short\"\n",
    ")\n",
    "input_data[:,:,0] = 1\n",
    "input_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f2883b99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf852aade6ee4b69a1bdd522b0ed68b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/240393 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i, input_seq in tqdm(list(enumerate(data.pol))):\n",
    "    for t, char in enumerate(input_seq):\n",
    "        input_data[i, t, 0] = 0\n",
    "        input_data[i, t, vocab_index[char]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "851d3838",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da62927d08a04cd68b6c7007ca8c299b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/19999 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pids, cids = [], []\n",
    "for i, pair in tqdm(list(pairs.iterrows())):\n",
    "    pid = data.index[data.accession == pair['parent']].values[0]\n",
    "    cid = data.index[data.accession == pair['child']].values[0]\n",
    "    pids.append(pid)\n",
    "    cids.append(cid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5f67dcc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/pol/train.npz', 'wb') as f:\n",
    "    np.savez(f, \n",
    "             parents = input_data[pids], \n",
    "             children = input_data[cids],\n",
    "             vocab_index = vocab_index\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32639457",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "hiv",
   "language": "python",
   "name": "hiv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}