train_tokenizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tokenizers import Tokenizer\n",
    "from tokenizers.models import BPE\n",
    "from tokenizers.trainers import BpeTrainer\n",
    "from tokenizers.pre_tokenizers import Whitespace\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(BPE())\n",
    "trainer = BpeTrainer(\n",
    "    special_tokens=[\"\\n\"], show_progress=True, #continuing_subword_prefix=\"##\", end_of_word_suffix=\"_\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.pre_tokenizer = Whitespace()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer.train(files=[\"data/tinyshakespeare.txt\"], trainer=trainer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'onato': 12122,\n",
       " 'bri': 965,\n",
       " 'JO': 1877,\n",
       " 'nes': 4459,\n",
       " 'measures': 12161,\n",
       " 'expressly': 8310,\n",
       " 'greybeard': 15930,\n",
       " 'repined': 14205,\n",
       " 'reprehended': 17215,\n",
       " 'fren': 11632,\n",
       " 'perverse': 13118,\n",
       " 'mur': 3388,\n",
       " 'atonement': 16919,\n",
       " 'Som': 2947,\n",
       " 'skin': 6965,\n",
       " 'taverns': 12959,\n",
       " 'etch': 4865,\n",
       " 'sweeter': 4590,\n",
       " 'BURY': 2740,\n",
       " 'cursed': 3161,\n",
       " 'Luke': 7302,\n",
       " 'Feather': 15961,\n",
       " 'accents': 10786,\n",
       " 'drums': 3894,\n",
       " 'DY': 742,\n",
       " 'relation': 8034,\n",
       " 'enacts': 12083,\n",
       " 'lilies': 12426,\n",
       " 'Skip': 17113,\n",
       " 'wrapt': 10375,\n",
       " 'denote': 14100,\n",
       " 'inst': 3762,\n",
       " 'stratagems': 8615,\n",
       " 'hams': 12212,\n",
       " 'harming': 14835,\n",
       " 'pless': 6714,\n",
       " 'dismember': 13316,\n",
       " 'whoever': 9461,\n",
       " 'pox': 6621,\n",
       " 'wis': 1385,\n",
       " 'curdied': 14281,\n",
       " 'Breaking': 15952,\n",
       " 'uncover': 12613,\n",
       " 'fly': 1460,\n",
       " 'Hecu': 13257,\n",
       " 'Jesu': 5110,\n",
       " 'Span': 11437,\n",
       " 'lebr': 7550,\n",
       " 'companions': 8326,\n",
       " 'dubb': 14886,\n",
       " 'sepulch': 4725,\n",
       " 'iz': 560,\n",
       " 'Read': 6098,\n",
       " 'Age': 11060,\n",
       " 'allegiance': 5641,\n",
       " 'notwithstanding': 7245,\n",
       " 'dissolved': 10651,\n",
       " 'Turks': 8468,\n",
       " 'Slu': 11445,\n",
       " 'aducement': 12511,\n",
       " 'organs': 12051,\n",
       " 'cleaving': 14178,\n",
       " 'infi': 8974,\n",
       " 'reput': 4214,\n",
       " 'unbarbed': 12703,\n",
       " 'Du': 6300,\n",
       " 'weal': 4511,\n",
       " 'confident': 9798,\n",
       " 'sers': 11868,\n",
       " 'spider': 5951,\n",
       " 'fruitful': 7061,\n",
       " 'pause': 4589,\n",
       " 'treachery': 7077,\n",
       " 'quar': 1547,\n",
       " 'oar': 11794,\n",
       " 'phe': 5738,\n",
       " 'dowdy': 14377,\n",
       " 'ulet': 2519,\n",
       " 'planteth': 16394,\n",
       " 'fights': 6869,\n",
       " 'island': 3977,\n",
       " 'metaphysics': 17790,\n",
       " 'Meantime': 7164,\n",
       " 'madest': 6751,\n",
       " 'unbid': 12582,\n",
       " 'entrance': 5293,\n",
       " 'enfranchisement': 16918,\n",
       " 'FL': 1555,\n",
       " 'render': 6220,\n",
       " 'misdeeds': 13593,\n",
       " 'vehement': 10856,\n",
       " 'Sad': 7322,\n",
       " 'Talk': 5692,\n",
       " 'SIC': 771,\n",
       " 'Teaching': 16586,\n",
       " 'thanks': 1832,\n",
       " 'descant': 10788,\n",
       " 'polled': 12942,\n",
       " 'trenches': 7698,\n",
       " 'Messenger': 1587,\n",
       " 'uming': 13207,\n",
       " 'pleading': 15629,\n",
       " 'Feed': 11217,\n",
       " 'ldom': 3248,\n",
       " 'ershade': 11990,\n",
       " 'seedness': 16718,\n",
       " 'longing': 7878,\n",
       " 'adds': 9170,\n",
       " 'hollowly': 15617,\n",
       " 'chronicle': 17419,\n",
       " 'dearer': 5436,\n",
       " 'Hew': 13250,\n",
       " 'kings': 1935,\n",
       " 'baggage': 7152,\n",
       " 'kedst': 15183,\n",
       " 'trades': 6630,\n",
       " 'freed': 4774,\n",
       " 'believe': 1444,\n",
       " 'Resides': 15652,\n",
       " 'malign': 15191,\n",
       " 'cambric': 10835,\n",
       " 'atage': 7506,\n",
       " 'remonstr': 12246,\n",
       " 'border': 11555,\n",
       " 'fraught': 10166,\n",
       " 'Interpretation': 17275,\n",
       " 'childhoods': 17317,\n",
       " 'befell': 9092,\n",
       " 'con': 180,\n",
       " 'resist': 5986,\n",
       " 'applied': 10331,\n",
       " 'Bred': 11102,\n",
       " 'vor': 11925,\n",
       " 'painted': 3671,\n",
       " 'Sta': 5168,\n",
       " 'occ': 2174,\n",
       " 'embark': 13687,\n",
       " 'stifle': 17199,\n",
       " 'ibat': 7410,\n",
       " 'kneels': 15251,\n",
       " 'granted': 3878,\n",
       " 'knocking': 15227,\n",
       " 'conspir': 5879,\n",
       " 'Rough': 11377,\n",
       " 'rouse': 4801,\n",
       " 'Answer': 5982,\n",
       " 'Bergam': 15950,\n",
       " 'nthi': 11787,\n",
       " 'visiting': 8179,\n",
       " 'Provoke': 14029,\n",
       " 'a': 37,\n",
       " 'Achieve': 17437,\n",
       " 'rather': 1161,\n",
       " 'Cer': 8643,\n",
       " 'filling': 10175,\n",
       " 'ON': 345,\n",
       " 'screen': 13850,\n",
       " 'camel': 14151,\n",
       " 'interpreters': 15044,\n",
       " 'atory': 7507,\n",
       " 'mocked': 8099,\n",
       " 'absence': 3191,\n",
       " 'peror': 7720,\n",
       " 'Please': 2837,\n",
       " 'forest': 6674,\n",
       " 'sheep': 2998,\n",
       " 'Cice': 11142,\n",
       " 'pl': 386,\n",
       " 'Mount': 8707,\n",
       " 'abject': 13617,\n",
       " 'snat': 11876,\n",
       " 'squ': 6436,\n",
       " 'corrupted': 15775,\n",
       " 'marr': 6710,\n",
       " 'dealing': 8270,\n",
       " 'enjoying': 10299,\n",
       " 'strains': 6874,\n",
       " 'matches': 16349,\n",
       " 'chains': 6818,\n",
       " 'untrodden': 14386,\n",
       " 'Phy': 11350,\n",
       " 'betwitched': 18040,\n",
       " 'share': 4896,\n",
       " 'unthrifty': 10864,\n",
       " 'widest': 12321,\n",
       " 'stud': 12134,\n",
       " 'Varlet': 17644,\n",
       " 'effects': 8248,\n",
       " 'kerns': 15275,\n",
       " 'Prick': 10250,\n",
       " 'correspondent': 18076,\n",
       " 'Wooing': 17649,\n",
       " 'Alike': 11063,\n",
       " 'Officious': 15631,\n",
       " 'bedash': 14188,\n",
       " 'lullaby': 17714,\n",
       " 'prison': 992,\n",
       " 'mere': 3561,\n",
       " 'peaces': 17889,\n",
       " 'necklace': 15247,\n",
       " 'throat': 3342,\n",
       " 'remit': 12235,\n",
       " 'concludes': 15523,\n",
       " 'dish': 1676,\n",
       " 'windows': 4633,\n",
       " 'volume': 7001,\n",
       " 'poking': 12935,\n",
       " 'transport': 4900,\n",
       " 'Accords': 16838,\n",
       " 'high': 825,\n",
       " 'indictment': 11019,\n",
       " 'livion': 7583,\n",
       " 'mixture': 13976,\n",
       " 'rivals': 10761,\n",
       " 'dignifies': 15821,\n",
       " 'dif': 11597,\n",
       " 'heralds': 10647,\n",
       " 'devoted': 10464,\n",
       " 'defy': 5487,\n",
       " 'dues': 14884,\n",
       " 'Nicanor': 10738,\n",
       " 'Titus': 4125,\n",
       " 'Forerunning': 13082,\n",
       " 'farewell': 1402,\n",
       " 'buzzard': 7163,\n",
       " 'disdains': 10444,\n",
       " 'unresolved': 12628,\n",
       " 'violate': 8191,\n",
       " 'tas': 11899,\n",
       " 'Wherefore': 4944,\n",
       " 'effusion': 17293,\n",
       " 'Spa': 8734,\n",
       " 'unconsidered': 12663,\n",
       " 'behalf': 3786,\n",
       " 'scale': 10284,\n",
       " 'Tw': 6350,\n",
       " 'Butchers': 13012,\n",
       " 'disliken': 13317,\n",
       " 'mingling': 16489,\n",
       " 'proach': 13220,\n",
       " 'Stephen': 10917,\n",
       " 'prest': 13662,\n",
       " 'Griefs': 17407,\n",
       " 'wills': 4535,\n",
       " 'shrieks': 10355,\n",
       " 'Containing': 14551,\n",
       " 'urous': 7600,\n",
       " 'therewithal': 9391,\n",
       " 'prevail': 3464,\n",
       " 'dome': 11593,\n",
       " 'desc': 6732,\n",
       " 'vio': 6709,\n",
       " 'howbeit': 13340,\n",
       " 'Believe': 3458,\n",
       " 'toys': 8420,\n",
       " 'strifes': 9894,\n",
       " 'shivers': 14232,\n",
       " 'Moun': 11311,\n",
       " 'Ven': 4436,\n",
       " 'adorns': 16945,\n",
       " 'arl': 2041,\n",
       " 'haunt': 6512,\n",
       " 'mong': 7422,\n",
       " 'shamed': 7959,\n",
       " 'tackled': 16546,\n",
       " 'infidels': 10986,\n",
       " 'death': 350,\n",
       " 'ended': 1902,\n",
       " 'delighted': 15616,\n",
       " 'wan': 4471,\n",
       " 'hears': 6721,\n",
       " 'rite': 8908,\n",
       " 'Hortensio': 2193,\n",
       " 'usual': 5851,\n",
       " 'tains': 4607,\n",
       " 'sepulchres': 16197,\n",
       " 'Flor': 4382,\n",
       " 'written': 5042,\n",
       " 'rode': 6535,\n",
       " 'gnat': 8846,\n",
       " 'necessities': 8292,\n",
       " 'Procure': 9758,\n",
       " 'personally': 9982,\n",
       " 'straying': 16982,\n",
       " 'grateful': 7210,\n",
       " 'Offi': 3133,\n",
       " 'profits': 6931,\n",
       " 'qual': 2706,\n",
       " 'Boot': 8642,\n",
       " 'onward': 12119,\n",
       " 'thir': 2767,\n",
       " 'prizes': 13861,\n",
       " 'ag': 281,\n",
       " 'Hoo': 7292,\n",
       " 'saff': 13086,\n",
       " 'pleader': 15628,\n",
       " 'staying': 13956,\n",
       " 'opes': 13899,\n",
       " 'causes': 5908,\n",
       " 'Cotus': 10937,\n",
       " 'lour': 4168,\n",
       " 'perni': 4916,\n",
       " 'sweating': 13427,\n",
       " 'Conduct': 16622,\n",
       " 'liarly': 15909,\n",
       " 'last': 895,\n",
       " 'lout': 11731,\n",
       " 'Boy': 3484,\n",
       " 'elvish': 13410,\n",
       " 'utions': 12402,\n",
       " 'Infer': 6708,\n",
       " 'ok': 2852,\n",
       " 'hing': 11682,\n",
       " 'Boils': 14311,\n",
       " 'winking': 14572,\n",
       " 'arent': 12094,\n",
       " 'reverently': 15143,\n",
       " 'erthrow': 11983,\n",
       " 'alack': 4242,\n",
       " 'boring': 15181,\n",
       " 'execute': 5026,\n",
       " 'princesses': 15845,\n",
       " 'prating': 5882,\n",
       " 'enches': 4480,\n",
       " 'hapless': 9062,\n",
       " 'precipitation': 11038,\n",
       " 'real': 12218,\n",
       " 'Cushions': 17473,\n",
       " 'record': 4216,\n",
       " 'hangeth': 17703,\n",
       " 'afflicted': 10624,\n",
       " 'bleeding': 4545,\n",
       " 'threescore': 16827,\n",
       " 'Fairly': 16025,\n",
       " 'Wid': 3547,\n",
       " 'barricado': 14932,\n",
       " 'quoth': 2655,\n",
       " 'consulships': 15082,\n",
       " 'soldiers': 1977,\n",
       " 'proud': 1287,\n",
       " 'crave': 2808,\n",
       " 'feasts': 6142,\n",
       " 'Montgomery': 8621,\n",
       " 'treacherous': 4127,\n",
       " 'budding': 16418,\n",
       " 'tush': 8927,\n",
       " 'chastely': 16121,\n",
       " 'Cancel': 17478,\n",
       " 'Thomas': 2929,\n",
       " 'cosm': 13767,\n",
       " 'slaughter': 2232,\n",
       " 'uneasy': 9745,\n",
       " 'prop': 1834,\n",
       " 'shot': 4807,\n",
       " 'Beside': 14965,\n",
       " 'titles': 8070,\n",
       " 'bears': 2529,\n",
       " 'works': 13103,\n",
       " 'resident': 13792,\n",
       " 'blench': 9369,\n",
       " 'shooting': 17244,\n",
       " 'Gentlemen': 3865,\n",
       " 'Marquis': 9603,\n",
       " 'Pronoun': 9902,\n",
       " 'bigger': 4670,\n",
       " 'suborn': 7921,\n",
       " 'Bermoo': 15951,\n",
       " 'Prepare': 5630,\n",
       " 'stewed': 14182,\n",
       " 'shrub': 16543,\n",
       " 'Shot': 11420,\n",
       " 'chidest': 13575,\n",
       " 'Marseilles': 18026,\n",
       " 'fend': 7389,\n",
       " 'unlike': 7625,\n",
       " 'bad': 1921,\n",
       " 'betray': 4409,\n",
       " 'LA': 330,\n",
       " 'POMP': 1276,\n",
       " 'umpire': 13211,\n",
       " 'fiery': 3006,\n",
       " 'comes': 604,\n",
       " 'shirt': 14233,\n",
       " 'meditation': 6871,\n",
       " 'cell': 2004,\n",
       " 'Never': 2749,\n",
       " 'tempter': 14243,\n",
       " 'harts': 11689,\n",
       " 'cheat': 9608,\n",
       " 'unfledged': 17874,\n",
       " 'Falls': 16454,\n",
       " 'pry': 6613,\n",
       " 'swe': 388,\n",
       " 'Thou': 377,\n",
       " 'hide': 2132,\n",
       " 'lifted': 10546,\n",
       " 'lectable': 12320,\n",
       " 'darkened': 16808,\n",
       " 'convenient': 9298,\n",
       " 'pated': 14226,\n",
       " 'chimney': 17665,\n",
       " 'jaws': 17710,\n",
       " 'sense': 2413,\n",
       " 'unskilfully': 17229,\n",
       " 'Obe': 5676,\n",
       " 'opposed': 7883,\n",
       " 'probation': 9440,\n",
       " 'fountain': 10838,\n",
       " 'precedent': 7252,\n",
       " 'no': 83,\n",
       " 'prouds': 16382,\n",
       " 'Rowland': 17587,\n",
       " 'North': 2143,\n",
       " 'stopping': 14901,\n",
       " 'altering': 15522,\n",
       " 'seventy': 16923,\n",
       " 'Maliciously': 17565,\n",
       " 'lucky': 10458,\n",
       " 'capar': 6646,\n",
       " 'wall': 1505,\n",
       " 'dimming': 16062,\n",
       " 'tent': 780,\n",
       " 'stronger': 4645,\n",
       " 'dst': 5193,\n",
       " 'erest': 5275,\n",
       " 'dre': 849,\n",
       " 'market': 3630,\n",
       " 'acknowledged': 16602,\n",
       " 'mine': 379,\n",
       " 'quinces': 13245,\n",
       " 'scornfully': 15151,\n",
       " 'oftener': 14976,\n",
       " 'bation': 4444,\n",
       " 'braves': 13982,\n",
       " 'want': 1190,\n",
       " 'tyranny': 4381,\n",
       " 'novices': 12193,\n",
       " 'amiable': 16531,\n",
       " 'infallible': 8984,\n",
       " 'mbers': 5046,\n",
       " 'steterat': 8601,\n",
       " 'protect': 3272,\n",
       " 'deliberate': 9962,\n",
       " 'fa': 161,\n",
       " 'c': 39,\n",
       " 'darling': 15273,\n",
       " 'realm': 3784,\n",
       " 'miser': 2796,\n",
       " 'Half': 8568,\n",
       " 'llenger': 12149,\n",
       " 'ostentation': 9916,\n",
       " 'shining': 5334,\n",
       " 'hung': 3177,\n",
       " 'folks': 5489,\n",
       " 'aly': 4516,\n",
       " 'armies': 13558,\n",
       " 'herald': 5325,\n",
       " 'Seduced': 17601,\n",
       " 'depopul': 12450,\n",
       " 'lyra': 12537,\n",
       " 'doublet': 8249,\n",
       " 'ste': 883,\n",
       " 'leap': 5818,\n",
       " 'Hilloa': 17532,\n",
       " 'invo': 12004,\n",
       " 'ormity': 12047,\n",
       " 'alderman': 15874,\n",
       " 'invincible': 16705,\n",
       " 'Untouch': 14057,\n",
       " 'disfigure': 13324,\n",
       " 'y': 61,\n",
       " 'chain': 7935,\n",
       " 'deaths': 4561,\n",
       " 'umpt': 13209,\n",
       " 'pantry': 16490,\n",
       " 'mean': 798,\n",
       " 'prevails': 7830,\n",
       " 'week': 5290,\n",
       " 'howled': 13337,\n",
       " 'soming': 10055,\n",
       " 'crafted': 16083,\n",
       " 'approaches': 8200,\n",
       " 'Somewhat': 14591,\n",
       " 'Breathe': 8357,\n",
       " 'dew': 3105,\n",
       " 'ainted': 5323,\n",
       " 'al': 129,\n",
       " 'tribes': 9689,\n",
       " 'ic': 763,\n",
       " 'forbidden': 15294,\n",
       " 'regarded': 16091,\n",
       " 'gallery': 15135,\n",
       " 'pher': 1086,\n",
       " 'design': 3134,\n",
       " 'cowardly': 5517,\n",
       " 'DW': 549,\n",
       " 'gay': 6394,\n",
       " 'meats': 9034,\n",
       " 'FID': 1556,\n",
       " 'unfill': 12615,\n",
       " 'ker': 2372,\n",
       " 'Cali': 5021,\n",
       " 'prit': 12880,\n",
       " 'fawn': 4878,\n",
       " 'Valiant': 17642,\n",
       " 'Tongue': 8582,\n",
       " 'eror': 11971,\n",
       " 'prate': 5880,\n",
       " 'covered': 13772,\n",
       " 'flatteries': 14924,\n",
       " 'party': 2798,\n",
       " 'Exposed': 15202,\n",
       " 'payment': 10154,\n",
       " 'mutter': 15277,\n",
       " 'gainsays': 14301,\n",
       " 'peremptory': 7242,\n",
       " 'task': 4184,\n",
       " 'general': 1585,\n",
       " 'monster': 4990,\n",
       " 'thirsty': 8419,\n",
       " 'fection': 1688,\n",
       " 'marchpan': 15306,\n",
       " 'interior': 10159,\n",
       " 'chine': 12374,\n",
       " 'meat': 2865,\n",
       " 'Pad': 2360,\n",
       " 'Allow': 8629,\n",
       " 'confound': 5439,\n",
       " 'forma': 12344,\n",
       " 'uto': 7466,\n",
       " 'grows': 3871,\n",
       " 'sworth': 11881,\n",
       " 'promotions': 10892,\n",
       " 'compassion': 5922,\n",
       " 'stomach': 4354,\n",
       " 'STANLE': 2910,\n",
       " 'os': 5736,\n",
       " 'warmth': 15185,\n",
       " 'gnarling': 15233,\n",
       " 'incen': 4191,\n",
       " 'impartial': 9565,\n",
       " 'form': 1055,\n",
       " 'seducing': 14363,\n",
       " 'scars': 7874,\n",
       " 'christen': 17830,\n",
       " 'logue': 12763,\n",
       " 'Ga': 4739,\n",
       " 'exactly': 10790,\n",
       " 'avenged': 15163,\n",
       " 'distance': 9487,\n",
       " 'quainted': 5362,\n",
       " 'conduits': 13679,\n",
       " 'bigamy': 15272,\n",
       " 'wont': 3501,\n",
       " 'incorporate': 10987,\n",
       " 'askance': 14508,\n",
       " 'constantly': 15571,\n",
       " 'Wash': 9956,\n",
       " 'inducement': 13095,\n",
       " 'advocate': 5845,\n",
       " 'provinces': 15625,\n",
       " 'UR': 7340,\n",
       " 'Except': 8207,\n",
       " 'model': 5899,\n",
       " 'ute': 1406,\n",
       " 'attern': 6008,\n",
       " 'thankfulness': 8401,\n",
       " 'prattle': 10869,\n",
       " 'prophecy': 6181,\n",
       " 'vices': 2898,\n",
       " 'lineally': 15728,\n",
       " 'Nor': 647,\n",
       " 'INGS': 1345,\n",
       " 'pains': 3039,\n",
       " 'clasp': 13931,\n",
       " 'gibing': 14198,\n",
       " 'Northumberland': 2242,\n",
       " 'acting': 8092,\n",
       " 'forthwith': 4973,\n",
       " 'anders': 12171,\n",
       " 'topsail': 15517,\n",
       " 'atter': 685,\n",
       " 'weakly': 14998,\n",
       " 'concerneth': 16164,\n",
       " 'plagues': 9712,\n",
       " 'text': 9343,\n",
       " 'RO': 331,\n",
       " 'ield': 1203,\n",
       " 'saw': 1288,\n",
       " 'Atlas': 14457,\n",
       " 'pure': 3390,\n",
       " 'scribed': 16409,\n",
       " 'ost': 1106,\n",
       " 'oughly': 7856,\n",
       " 'buildeth': 10472,\n",
       " 'augmenting': 17360,\n",
       " 'quake': 5936,\n",
       " 'gusts': 10682,\n",
       " 'Fright': 11210,\n",
       " 'intention': 15831,\n",
       " 'courtier': 5421,\n",
       " 'rod': 7575,\n",
       " 'icing': 9750,\n",
       " 'bedaub': 17216,\n",
       " 'kiss': 1254,\n",
       " 'politicly': 17432,\n",
       " 'penny': 6915,\n",
       " 'AUM': 1754,\n",
       " 'Contract': 14539,\n",
       " 'wilful': 6527,\n",
       " 'forbids': 10262,\n",
       " 'ambassador': 10910,\n",
       " 'ra': 141,\n",
       " 'ark': 832,\n",
       " 'grinning': 17310,\n",
       " 'Lef': 6332,\n",
       " 'sisting': 8920,\n",
       " 'blind': 2701,\n",
       " 'protesting': 15755,\n",
       " 'HERBERT': 14913,\n",
       " 'ton': 3229,\n",
       " 'disgrace': 3275,\n",
       " 'Darest': 6306,\n",
       " 'axe': 4691,\n",
       " 'adverse': 6222,\n",
       " 'coverture': 10621,\n",
       " 'cament': 11582,\n",
       " 'ison': 702,\n",
       " 'skull': 10145,\n",
       " 'France': 1537,\n",
       " 'grandfather': 6167,\n",
       " 'you': 81,\n",
       " 'amain': 5185,\n",
       " 'hostess': 8245,\n",
       " 'commends': 5989,\n",
       " 'spoons': 14461,\n",
       " 'lavish': 12912,\n",
       " 'burns': 5028,\n",
       " 'Ser': 762,\n",
       " 'corn': 2299,\n",
       " 'oler': 2551,\n",
       " 'minion': 14436,\n",
       " 'trouble': 2349,\n",
       " 'stops': 5544,\n",
       " 'distant': 13331,\n",
       " 'gess': 8842,\n",
       " 'Bear': 3537,\n",
       " 'gipsy': 14199,\n",
       " 'purses': 7913,\n",
       " 'consistory': 14700,\n",
       " 'sinner': 14608,\n",
       " 'objects': 16184,\n",
       " 'longed': 13865,\n",
       " 'surly': 8113,\n",
       " 'Lascivious': 8698,\n",
       " 'logs': 9248,\n",
       " 'Adieu': 5624,\n",
       " 'team': 12990,\n",
       " 'serp': 9616,\n",
       " 'steeped': 16582,\n",
       " 'sighted': 14629,\n",
       " 'prognostication': 17964,\n",
       " 'flour': 13738,\n",
       " 'trueborn': 13737,\n",
       " 'pamper': 17728,\n",
       " 'cage': 13045,\n",
       " 'duty': 1615,\n",
       " 'mon': 1033,\n",
       " 'Sh': 1210,\n",
       " 'extirp': 17409,\n",
       " 'bereaves': 16221,\n",
       " 'ction': 783,\n",
       " 'ade': 1920,\n",
       " 'nit': 11780,\n",
       " 'occupations': 16094,\n",
       " 'Give': 1277,\n",
       " 'broider': 13205,\n",
       " 'spare': 3627,\n",
       " 'mand': 4171,\n",
       " 'Snakes': 17595,\n",
       " 'efest': 5717,\n",
       " 'plays': 7998,\n",
       " 'despite': 3135,\n",
       " 'Compact': 14661,\n",
       " 'Hallowmas': 8569,\n",
       " 'Measured': 16797,\n",
       " 'Wise': 8767,\n",
       " 'Wales': 6355,\n",
       " 'changing': 4664,\n",
       " 'tered': 5756,\n",
       " 'Bareheaded': 17075,\n",
       " 'woods': 15666,\n",
       " 'ponder': 8903,\n",
       " 'frames': 15066,\n",
       " 'sme': 3224,\n",
       " 'old': 374,\n",
       " 'Intolerable': 17989,\n",
       " 'correspond': 14425,\n",
       " 'spites': 13354,\n",
       " 'verity': 7653,\n",
       " 'strokes': 4678,\n",
       " 'stooping': 6183,\n",
       " 'harlots': 15114,\n",
       " 'betroth': 13784,\n",
       " 'mocker': 14802,\n",
       " 'governor': 6633,\n",
       " 'madmen': 9734,\n",
       " 'caused': 6783,\n",
       " 'plain': 1327,\n",
       " 'Volumnia': 18090,\n",
       " 'figured': 8338,\n",
       " 'lica': 12423,\n",
       " 'Citizens': 2561,\n",
       " 'messages': 15071,\n",
       " 'probri': 13222,\n",
       " 'unequal': 14009,\n",
       " 'RIVERS': 2117,\n",
       " 'Flow': 11214,\n",
       " 'Commonly': 14664,\n",
       " 'MARGAR': 986,\n",
       " 'cavilling': 17935,\n",
       " 'acc': 1113,\n",
       " 'taris': 11909,\n",
       " 'boast': 4903,\n",
       " 'Correction': 14658,\n",
       " 'fliers': 8589,\n",
       " 'argosies': 16710,\n",
       " 'handkerchief': 10877,\n",
       " 'poper': 12936,\n",
       " 'occupy': 16092,\n",
       " 'Tro': 11450,\n",
       " 'iol': 1951,\n",
       " 'themes': 12038,\n",
       " 'Pewter': 15061,\n",
       " 'hot': 1709,\n",
       " 'beside': 6027,\n",
       " 'assure': 4227,\n",
       " 'forsooth': 5826,\n",
       " 'orch': 6482,\n",
       " 'unstring': 12676,\n",
       " 'cle': 881,\n",
       " 'thousands': 6053,\n",
       " 'stir': 1800,\n",
       " 'od': 1995,\n",
       " 'ual': 2449,\n",
       " 'strangers': 9534,\n",
       " 'thriving': 10118,\n",
       " 'Hostili': 16637,\n",
       " 'gun': 8841,\n",
       " 'groweth': 14665,\n",
       " 'savoury': 8534,\n",
       " \"';\": 11055,\n",
       " 'athwart': 13016,\n",
       " 'distress': 5944,\n",
       " 'givest': 13462,\n",
       " 'denying': 10183,\n",
       " 'replenished': 17431,\n",
       " 'roused': 10604,\n",
       " 'benediction': 18109,\n",
       " 'appare': 3131,\n",
       " 'frail': 4661,\n",
       " 'jealousy': 15635,\n",
       " 'copatain': 18035,\n",
       " 'Sneak': 17603,\n",
       " 'begg': 4045,\n",
       " 'lou': 6406,\n",
       " 'circumstance': 5590,\n",
       " 'consequence': 6264,\n",
       " 'berate': 9613,\n",
       " 'breach': 9557,\n",
       " 'filth': 16225,\n",
       " 'balanced': 17344,\n",
       " 'security': 8603,\n",
       " 'mum': 6412,\n",
       " 'We': 298,\n",
       " 'sleek': 14043,\n",
       " 'plucked': 8060,\n",
       " 'napes': 16035,\n",
       " 'straws': 14431,\n",
       " 'alter': 2878,\n",
       " 'Re': 1499,\n",
       " 'tials': 12805,\n",
       " 'bonfires': 15875,\n",
       " 'Cor': 1436,\n",
       " 'issued': 6485,\n",
       " 'straining': 14429,\n",
       " 'purchase': 4606,\n",
       " 'unscann': 16154,\n",
       " 'EDWARD': 552,\n",
       " 'boun': 6364,\n",
       " 'woful': 3353,\n",
       " 'natures': 6985,\n",
       " 'nigh': 5734,\n",
       " 'hatches': 7590,\n",
       " 'Cousins': 11159,\n",
       " 'confounds': 9797,\n",
       " 'industriously': 17254,\n",
       " 'declare': 16146,\n",
       " 'straited': 14430,\n",
       " 'lord': 225,\n",
       " 'Lucentio': 1786,\n",
       " 'mighty': 2054,\n",
       " 'Deep': 8102,\n",
       " 'Derby': 4403,\n",
       " 'million': 10460,\n",
       " 'contrary': 3517,\n",
       " 'riddling': 15216,\n",
       " 'discreet': 8447,\n",
       " 'uction': 2520,\n",
       " 'sobri': 12542,\n",
       " 'courtiers': 13891,\n",
       " 'CAMIL': 1112,\n",
       " 'keys': 5559,\n",
       " 'FR': 1137,\n",
       " 'upward': 12915,\n",
       " 'John': 2644,\n",
       " 'parle': 7902,\n",
       " 'stirring': 6951,\n",
       " 'trade': 3847,\n",
       " 'sureties': 14421,\n",
       " 'avoiding': 16014,\n",
       " 'romise': 12944,\n",
       " 'sat': 1704,\n",
       " 'prompter': 14480,\n",
       " 'lath': 12901,\n",
       " 'Hag': 11250,\n",
       " 'supplant': 14749,\n",
       " 'cedar': 10126,\n",
       " 'vanquish': 10439,\n",
       " 'lastly': 9821,\n",
       " 'unwillingness': 10694,\n",
       " 'Likely': 14755,\n",
       " 'wealth': 2988,\n",
       " 'Seeing': 10051,\n",
       " 'Curtis': 5602,\n",
       " 'hitherto': 14209,\n",
       " 'findings': 14129,\n",
       " 'quests': 14521,\n",
       " 'odd': 4092,\n",
       " 'wealthy': 8394,\n",
       " 'troublest': 15253,\n",
       " 'ABRAHAM': 6284,\n",
       " 'ELLA': 704,\n",
       " 'armour': 3837,\n",
       " 'simpleness': 15427,\n",
       " 'Somerset': 3192,\n",
       " 'observer': 17016,\n",
       " 'undercrest': 14139,\n",
       " 'climate': 6163,\n",
       " 'hent': 8856,\n",
       " 'lays': 14170,\n",
       " 'Martino': 17558,\n",
       " 'fowling': 17681,\n",
       " 'Dispark': 15321,\n",
       " 'ebus': 6389,\n",
       " 'Ding': 11163,\n",
       " 'tumbled': 10751,\n",
       " 'embas': 9611,\n",
       " 'Montagues': 6102,\n",
       " 'They': 897,\n",
       " 'troubles': 8167,\n",
       " 'infirmities': 17764,\n",
       " 'IET': 723,\n",
       " 'Suspicion': 16329,\n",
       " 'bred': 3836,\n",
       " 'method': 10692,\n",
       " 'vainly': 15333,\n",
       " 'eno': 8998,\n",
       " 'possessed': 6954,\n",
       " 'Back': 6292,\n",
       " 'cushions': 10507,\n",
       " 'Sprawl': 17618,\n",
       " 'lawfully': 9780,\n",
       " 'Necessity': 15499,\n",
       " 'ounds': 2626,\n",
       " 'ongest': 12120,\n",
       " 'Adoptedly': 17067,\n",
       " 'levy': 6525,\n",
       " 'proved': 4918,\n",
       " 'Quick': 8389,\n",
       " 'squash': 16692,\n",
       " 'discoveries': 15826,\n",
       " 'yers': 8946,\n",
       " 'foo': 670,\n",
       " 'sim': 2628,\n",
       " 'extir': 10789,\n",
       " 'Sycorax': 6275,\n",
       " 'dignity': 4392,\n",
       " 'aments': 4534,\n",
       " 'Fridays': 15366,\n",
       " 'agation': 13179,\n",
       " 'sleepest': 14044,\n",
       " 'estate': 4023,\n",
       " 'secure': 4209,\n",
       " 'pays': 6969,\n",
       " 'body': 1049,\n",
       " 'ually': 5757,\n",
       " 'surmises': 14862,\n",
       " 'Burgundy': 5646,\n",
       " 'adulterous': 16436,\n",
       " 'seen': 1297,\n",
       " 'Clifford': 1491,\n",
       " 'IFF': 1401,\n",
       " 'oce': 7429,\n",
       " 'grumblings': 13827,\n",
       " 'swooned': 16358,\n",
       " 'sullens': 12846,\n",
       " 'Did': 1358,\n",
       " 'rack': 5305,\n",
       " 'suddenly': 5515,\n",
       " 'Belike': 6724,\n",
       " 'laughing': 10334,\n",
       " 'ousin': 923,\n",
       " 'GLOUCES': 454,\n",
       " 'stares': 15563,\n",
       " 'Well': 720,\n",
       " 'murderers': 10073,\n",
       " 'Di': 3066,\n",
       " 'iction': 9752,\n",
       " 'more': 242,\n",
       " 'definit': 14747,\n",
       " 'league': 4502,\n",
       " 'sting': 4198,\n",
       " 'sooth': 3228,\n",
       " 'Get': 4419,\n",
       " 'kna': 2959,\n",
       " 'tied': 3814,\n",
       " 'southern': 10842,\n",
       " 'oily': 11804,\n",
       " 'breaks': 6079,\n",
       " 'sk': 1926,\n",
       " 'relents': 14555,\n",
       " 'cadoes': 17152,\n",
       " 'sport': 2402,\n",
       " 'lament': 2152,\n",
       " 'chamele': 14148,\n",
       " 'its': 807,\n",
       " 'troubler': 16601,\n",
       " 'several': 3514,\n",
       " 'lify': 8051,\n",
       " 'beseems': 16082,\n",
       " 'Hyrcan': 17090,\n",
       " 'bethought': 12287,\n",
       " 'honey': 4778,\n",
       " 'sanctuary': 4128,\n",
       " 'conjure': 5081,\n",
       " 'prolix': 9439,\n",
       " 'revived': 10693,\n",
       " 'accomplished': 16764,\n",
       " 'merits': 7855,\n",
       " 'festival': 7138,\n",
       " 'swell': 3970,\n",
       " 'Clar': 1020,\n",
       " 'dressings': 15928,\n",
       " 'shrugs': 16544,\n",
       " 'working': 9392,\n",
       " 'gentlemen': 1779,\n",
       " 'Bold': 5650,\n",
       " 'Conceives': 16446,\n",
       " 'inconstant': 6472,\n",
       " 'rian': 7594,\n",
       " 'renew': 9790,\n",
       " 'ob': 1563,\n",
       " 'perj': 4012,\n",
       " 'eys': 9556,\n",
       " 'tokens': 10637,\n",
       " 'eggs': 10836,\n",
       " 'unst': 7623,\n",
       " 'ingham': 1685,\n",
       " 'poultice': 17732,\n",
       " 'matter': 1120,\n",
       " 'canor': 9395,\n",
       " 'st': 75,\n",
       " 'tailors': 10420,\n",
       " 'Promotion': 17577,\n",
       " 'utes': 3249,\n",
       " 'ducats': 8122,\n",
       " 'unfinish': 12692,\n",
       " 'som': 1622,\n",
       " 'plainest': 14582,\n",
       " 'cur': 972,\n",
       " 'Cretan': 17481,\n",
       " 'unprofit': 12638,\n",
       " 'osier': 17725,\n",
       " 'earls': 9426,\n",
       " 'Ind': 2944,\n",
       " 'litter': 9723,\n",
       " 'ered': 1898,\n",
       " 'wit': 976,\n",
       " 'marjor': 13406,\n",
       " 'RIV': 2075,\n",
       " 'xy': 8944,\n",
       " 'assailing': 15296,\n",
       " 'something': 1741,\n",
       " 'effected': 15385,\n",
       " 'goods': 4901,\n",
       " 'Moved': 16853,\n",
       " 'tino': 10138,\n",
       " 'Mi': 11303,\n",
       " 'Senators': 8075,\n",
       " ...}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.get_vocab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = tokenizer.encode(\"Your belly's answer? What!\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 1, 1, 1, 1, 1, 1, 1, 1]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[511, 3166, 3, 55, 1019, 10, 229, 1, 0]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Your', 'belly', \"'\", 's', 'answer', '?', 'What', '!', '\\n']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.save(\"./data/tokenizer.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrained = Tokenizer.from_file(\"./data/tokenizer.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrained_outs = pretrained.encode(\"Your belly's answer? What!\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Your belly ' s answer ? What ! \\n\""
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrained.decode(pretrained_outs.ids,  skip_special_tokens=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.6 64-bit ('tfgpt')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "bfdc0eac67cf196f2171871a77a37aff362885343dc84e4d9326d459b303568d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}