play_text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-04 14:18:37.380037: I tensorflow/core/util/util.cc:169] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n"
     ]
    }
   ],
   "source": [
    "from typing import List\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from tokenizers import Tokenizer\n",
    "\n",
    "from gpt.trainer import (Trainer, TrainerConfig,)\n",
    "from gpt.modeling import (GPT, GPT1Config,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Token level GPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_up_tokenization(out_string: str) -> str:\n",
    "    # From: https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3494\n",
    "    out_string = (\n",
    "        out_string.replace(\" .\", \".\")\n",
    "        .replace(\" ?\", \"?\")\n",
    "        .replace(\" !\", \"!\")\n",
    "        .replace(\" ,\", \",\")\n",
    "        .replace(\" ' \", \"'\")\n",
    "        .replace(\" n't\", \"n't\")\n",
    "        .replace(\" 'm\", \"'m\")\n",
    "        .replace(\" 's\", \"'s\")\n",
    "        .replace(\" 've\", \"'ve\")\n",
    "        .replace(\" 're\", \"'re\")\n",
    "    )\n",
    "    return out_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TokenizedDataset(tf.data.Dataset):\n",
    "    def _gen_examples_from(data: List[int], block_size: int):\n",
    "        def _gen():\n",
    "            nb_examples = len(data)-block_size\n",
    "            for idx in range(nb_examples):\n",
    "                # grab a chunk of (block_size + 1) characters from the data\n",
    "                chunk = data[idx:idx + block_size + 1]\n",
    "                x = tf.convert_to_tensor(chunk[:-1])\n",
    "                y = tf.convert_to_tensor(chunk[1:])\n",
    "                yield x, y\n",
    "        return _gen\n",
    "\n",
    "    def __new__(\n",
    "        cls, input_ids: List[int], block_size: int, batch_size: int\n",
    "    ):\n",
    "        # nb_examples = len(data)-block_size\n",
    "        dataset =  (\n",
    "            tf.data.Dataset.from_generator(\n",
    "                cls._gen_examples_from(input_ids, block_size),\n",
    "                output_signature=(\n",
    "                    tf.TensorSpec(shape=(block_size,), dtype=tf.int32),\n",
    "                    tf.TensorSpec(shape=(block_size,), dtype=tf.int32))\n",
    "                )\n",
    "                # .shuffle(nb_examples, reshuffle_each_iteration=True)\n",
    "                .batch(batch_size, drop_remainder=True)\n",
    "                .repeat()\n",
    "                .prefetch(tf.data.experimental.AUTOTUNE)\n",
    "        )\n",
    "        return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "BLOCK_SIZE=128\n",
    "BATCH_SIZE=512\n",
    "\n",
    "def encode_text_to_ids(tokenizer, text: str):\n",
    "    output = tokenizer.encode(text)\n",
    "    return output.ids\n",
    "\n",
    "pretrained_tokenizer = Tokenizer.from_file(\"./data/tokenizer.json\")\n",
    "vocab_size = pretrained_tokenizer.get_vocab_size()\n",
    "\n",
    "text = open(\"./data/tinyshakespeare.txt\").read()\n",
    "input_ids = encode_text_to_ids(pretrained_tokenizer, text)\n",
    "train_dataset = TokenizedDataset(\n",
    "    input_ids, block_size=BLOCK_SIZE, batch_size=BATCH_SIZE\n",
    ")\n",
    "nb_examples = len(input_ids)-BLOCK_SIZE\n",
    "nb_optimization_steps = nb_examples//BATCH_SIZE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18146"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "301842"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "589"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_optimization_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number optimization steps = 2356\n"
     ]
    }
   ],
   "source": [
    "EPOCHS=4\n",
    "LEARNING_RATE=0.003#6.25e-5\n",
    "\n",
    "total_number_optimization_steps = nb_optimization_steps * EPOCHS\n",
    "\n",
    "print(f\"total number optimization steps = {total_number_optimization_steps}\")\n",
    "\n",
    "config = GPT1Config(\n",
    "    vocab_size=vocab_size, block_size=BLOCK_SIZE,\n",
    "    n_layer=8, n_head=8, n_embd=512\n",
    ")\n",
    "tconf = TrainerConfig(\n",
    "    max_epochs=EPOCHS, batch_size=BATCH_SIZE, learning_rate=LEARNING_RATE,\n",
    "    do_lr_decay=True, warmup_ratio=0.05, cosine_decay_alpha=0.0,\n",
    "    total_number_optimization_steps=total_number_optimization_steps, log_every_steps=10,\n",
    "    ckpt_path='./logs', trial_id='shakespeare_token_level'\n",
    ")\n",
    "\n",
    "model = GPT(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 2356: loss 4.67317 - acc 22.33% - lr 0.000000: 100%|██████████| 2356/2356 [30:59<00:00,  1.27it/s]\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model, train_dataset, total_number_optimization_steps, config=tconf\n",
    ")\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "O God, O God! \n",
      " \n",
      " \n",
      " \n",
      " KING HENRY VI shall be gone ; \n",
      " And thou hast not be not not I must have so to the day, \n",
      " I know \n",
      " For I am gone : \n",
      " I's head \n",
      " That thou shalt you. \n",
      " \n",
      " And he had not not not, and be, \n",
      " To have, \n",
      " To make you have I will the queen, \n",
      " The rest of your own : \n",
      " That you, that you, \n",
      " And thou art, \n",
      " But he was in a Duke of the prince. \n",
      " \n",
      " \n",
      " My \n",
      " KING RICHARD II : \n",
      " And thou, for this man \n",
      " And we do I know you be a day, \n",
      " And thou wilt thou art in this : \n",
      " That I would not, if you are a world : \n",
      " The day is I am so, \n",
      " But we shall be done, \n",
      " And so, \n",
      " To be the queen, \n",
      " For the queen! \n",
      " \n",
      " \n",
      " \n",
      " And, \n",
      " But in thy heart? \n",
      " What, \n",
      " And not the world is the rest! \n",
      " \n",
      " \n",
      " But we shall not. \n",
      " ROMEO : \n",
      " \n",
      " \n",
      " \n",
      " The crown! \n",
      " \n",
      " \n",
      " KING HENRY VI : \n",
      " \n",
      " \n",
      " I do be gone? Come, and I shall be so, if I must, \n",
      " And I do not not not so to be a Duke of his heart? \n",
      " \n",
      " O, \n",
      " I's blood of that is the world'tis a Duke of thy heart. \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " ROMEO : \n",
      " KING HENRY VI \n",
      " I am to make this. \n",
      " \n",
      " \n",
      " And thou, my Lord : \n",
      " And so, \n",
      " And thou art, \n",
      " But I am thou hast I'st, I will the crown. \n",
      " \n",
      " \n",
      " \n",
      " O, \n",
      " And he should do not, \n",
      " I'd with this, I'd \n",
      " As thou I shall be not not. \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " The crown, as thou shalt do, let'd the king, \n",
      " Which thou wert he hath I will have, \n",
      " But I'd, \n",
      " And, \n",
      " The man, \n",
      " And I will not, \n",
      " And, \n",
      " That is, \n",
      " And, \n",
      " The man, \n",
      " And thou, \n",
      " And so that he's a king \n",
      " With all, \n",
      " And so a day, \n",
      " To make it shall say'll take my son! what, \n",
      " \n",
      " To make my brother? \n",
      " \n",
      " \n",
      " \n",
      " KING HENRY VI \n",
      " \n",
      " \n",
      " And thou hast thou hast not be a king, and not, \n",
      " I would have not be gone. \n",
      " \n",
      " \n",
      " I have not the queen, \n",
      " The king, \n",
      " That I am the crown'Tis not not I's my lord, as it, \n",
      " That we, \n",
      " And, \n",
      " And to the rest ; \n",
      " And he was I have I shall be I do you. \n",
      " \n",
      " \n",
      " \n",
      " I will make his hands, \n",
      " For I'll give me, I will I'd \n",
      " For that thou canst I'd of a man's son, and be done to be, \n",
      " But we have a crown'll make me, \n",
      " And he shall, \n",
      " That is a Duke of the duke \n",
      " The world'd and that's a king's death and his father! \n",
      " \n",
      " \n",
      " And he is to the king, and a day, \n",
      " The king'd and to him with him. \n",
      " \n",
      " \n",
      " \n",
      " ROMEO : \n",
      " KING RICHARD Richard, I am I know \n",
      " Which thou art thou dost we be gone, \n",
      " And not, I shall be done, \n",
      " To make a thousand years \n",
      " And thou art the crown? \n",
      " To make your lord. \n",
      " I'er be, that thou art not so? \n",
      " \n",
      " My lord, I have not a world, \n",
      " For the king. \n",
      " What I will the man, \n",
      " But we are not be done. \n",
      " \n",
      " \n",
      " \n",
      " And, \n",
      " And so, \n",
      " For you are the king, \n",
      " That you, \n",
      " And I shall not to me, let'tis the crown, let'll be I will be gone, and my son, \n",
      " And to - hearted men! \n",
      " The king ; and the king, \n",
      " And I will not not make his son \n",
      " I'll make the king ; \n",
      " To be a man, \n",
      " And I shall do'd \n",
      " I have been thou art he hath been a crown'd \n",
      " And I am not be been, \n",
      " And I's death, \n",
      " To be, I do, \n",
      " That now ; \n",
      " The man's son \n",
      " And in the world, and my master, \n",
      " And so? \n",
      " What, and that'd. \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " O, \n",
      " And, \n",
      " And, and my brother'll make the duke'd of a man, \n",
      " And we are not, \n",
      " And he hath not, \n",
      " The ground? the crown : \n",
      " \n",
      " And so? \n",
      " \n",
      " What, \n",
      " But to - morrow,\n"
     ]
    }
   ],
   "source": [
    "context = \"O God, O God!\"\n",
    "x = tf.convert_to_tensor(pretrained_tokenizer.encode(context).ids, dtype=tf.int32)[None, ...]\n",
    "y = model.sample(x, 1000, temperature=1.0, sample=True, top_k=10)[0]\n",
    "completion = clean_up_tokenization(\n",
    "    pretrained_tokenizer.decode(y, skip_special_tokens=False)\n",
    ")\n",
    "print(completion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Char level GPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CharDataset(tf.data.Dataset):\n",
    "    @staticmethod\n",
    "    def compute_vocab_from_text(text: str):\n",
    "        chars = sorted(list(set(text)))\n",
    "        data_size, vocab_size = len(text), len(chars)\n",
    "        print('data has %d characters, %d unique.' % (data_size, vocab_size))\n",
    "        stoi = { ch:i for i,ch in enumerate(chars) }\n",
    "        itos = { i:ch for i,ch in enumerate(chars) }\n",
    "        return stoi, itos\n",
    "\n",
    "    def _gen_examples_from(text: str, block_size: int):\n",
    "        def _gen():\n",
    "            stoi, _ = CharDataset.compute_vocab_from_text(text)\n",
    "            nb_examples = len(text)-block_size\n",
    "            for idx in range(nb_examples):\n",
    "                # grab a chunk of (block_size + 1) characters from the data\n",
    "                chunk = text[idx:idx + block_size + 1]\n",
    "                # encode every character to an integer\n",
    "                dix = [stoi[s] for s in chunk]\n",
    "                x = tf.convert_to_tensor(dix[:-1])\n",
    "                y = tf.convert_to_tensor(dix[1:])\n",
    "                yield x, y\n",
    "        return _gen\n",
    "\n",
    "    def __new__(\n",
    "        cls, text: str, block_size: int, batch_size: int\n",
    "    ):\n",
    "        # nb_examples = len(text)-block_size\n",
    "        dataset =  (\n",
    "            tf.data.Dataset.from_generator(\n",
    "                cls._gen_examples_from(text, block_size),\n",
    "                output_signature=(\n",
    "                    tf.TensorSpec(shape=(block_size,), dtype=tf.int32),\n",
    "                    tf.TensorSpec(shape=(block_size,), dtype=tf.int32))\n",
    "                )\n",
    "                # .shuffle(nb_examples, reshuffle_each_iteration=True)\n",
    "                .batch(batch_size, drop_remainder=True)\n",
    "                .repeat()\n",
    "                .prefetch(tf.data.experimental.AUTOTUNE)\n",
    "        )\n",
    "        return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "BLOCK_SIZE=128\n",
    "BATCH_SIZE=512\n",
    "EPOCHS=2\n",
    "\n",
    "LEARNING_RATE=0.003#6.25e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 1115393 characters, 64 unique.\n"
     ]
    }
   ],
   "source": [
    "text = open(\"./data/tinyshakespeare.txt\").read()\n",
    "train_dataset_char = CharDataset(\n",
    "    text, block_size=BLOCK_SIZE, batch_size=BATCH_SIZE\n",
    ")\n",
    "nb_examples = len(text)-BLOCK_SIZE\n",
    "nb_optimization_steps = nb_examples//BATCH_SIZE\n",
    "total_number_optimization_steps = nb_optimization_steps*EPOCHS\n",
    "\n",
    "stoi, itos = CharDataset.compute_vocab_from_text(text)\n",
    "vocab_size = len(stoi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab size = 64\n",
      "no. examples = 1115265\n",
      "no. optimization steps = 2178\n",
      "no. total optimization steps = 4356\n"
     ]
    }
   ],
   "source": [
    "print(f\"vocab size = {len(stoi)}\")\n",
    "print(f\"no. examples = {nb_examples}\")\n",
    "print(f\"no. optimization steps = {nb_optimization_steps}\")\n",
    "print(f\"no. total optimization steps = {total_number_optimization_steps}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 1115393 characters, 64 unique.\n",
      "tf.Tensor(\n",
      "[[17 46 55 ... 57 52  1]\n",
      " [46 55 56 ... 52  1 41]\n",
      " [55 56 57 ...  1 41 46]\n",
      " ...\n",
      " [ 5  1 57 ...  0 60 45]\n",
      " [ 1 57 45 ... 60 45 52]\n",
      " [57 45 42 ... 45 52 49]], shape=(512, 128), dtype=int32) tf.Tensor(\n",
      "[[46 55 56 ... 52  1 41]\n",
      " [55 56 57 ...  1 41 46]\n",
      " [56 57  1 ... 41 46 42]\n",
      " ...\n",
      " [ 1 57 45 ... 60 45 52]\n",
      " [57 45 42 ... 45 52 49]\n",
      " [45 42  1 ... 52 49 42]], shape=(512, 128), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "for x, y in train_dataset_char:\n",
    "    print(x, y)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4356 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 1115393 characters, 64 unique.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-04 12:16:03.626045: I tensorflow/stream_executor/cuda/cuda_blas.cc:1786] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "step 2176: loss 1.69441 - acc 40.61% - lr 0.000567:  50%|████▉     | 2177/4356 [17:26<17:05,  2.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 1115393 characters, 64 unique.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 4354: loss 1.82917 - acc 44.29% - lr 0.000000: 100%|█████████▉| 4355/4356 [34:32<00:00,  2.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 1115393 characters, 64 unique.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "step 4356: loss 1.65761 - acc 44.30% - lr 0.000000: 100%|██████████| 4356/4356 [34:33<00:00,  2.10it/s]\n"
     ]
    }
   ],
   "source": [
    "config_char = GPT1Config(\n",
    "    vocab_size=vocab_size, block_size=BLOCK_SIZE,\n",
    "    n_layer=8, n_head=8, n_embd=512\n",
    ")\n",
    "tconf_char = TrainerConfig(\n",
    "    max_epochs=EPOCHS, batch_size=BATCH_SIZE, learning_rate=LEARNING_RATE,\n",
    "    do_lr_decay=True, warmup_ratio=0.3, cosine_decay_alpha=0.0,\n",
    "    total_number_optimization_steps=total_number_optimization_steps, log_every_steps=10,\n",
    "    ckpt_path='./logs', trial_id='shakespeare_token_level'\n",
    ")\n",
    "\n",
    "model_char = GPT(config_char)\n",
    "\n",
    "trainer_char = Trainer(\n",
    "    model_char, train_dataset_char, total_number_optimization_steps, config=tconf_char\n",
    ")\n",
    "\n",
    "trainer_char.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "O God, O God! thou clep hurle of hysure of,\n",
      "For that heave, fault the when boy of the btoth?\n",
      "\n",
      "BRNOLAN:\n",
      "No brings? and with thereful not of mastand\n",
      "The pasit boy, that belive father: slack.\n",
      "\n",
      "KING RALY CORNOMHERDND II:\n",
      "As to his dest in extech'd tear the citys hold\n",
      "Why, would a fair beging his with brood:\n",
      "Ime have mind, and to from in my mortor:\n",
      "Here to servent may, and ascountage is strive is mercy\n",
      "I do the cousing so be hence\n",
      "Who bown with fractice one of my sands:\n",
      "And of weet this hatharge.\n",
      "\n",
      "DY ANRY:\n",
      "And to sempoke backs:\n",
      "O, that this soons bats haition sovight,\n",
      "The sucherence shallest as incred the pride and my crreasond:\n",
      "With they strause and with waye sin abistion\n",
      "Freep that so desin me the doth our husbmit\n",
      "Age be have is which othan one, seem the servest:\n",
      "My loves feeky say the to he, split,\n",
      "What than a the come me of mighter more be was their to here think of in\n",
      "Angen, with be had angeranly of the deserved\n",
      "And the head the for wreturn be her will had of whichier:\n",
      "Their foals be streagy of here havest\n",
      "Thy brhy says of sir, his well that him his we here asts,\n",
      "And thee a folds as to mostion by beggr a price,\n",
      "My doung humself\n",
      "Beith, to the will attent will bose minto then,\n",
      "If then it to betwent for as thoughts to sworrower.\n",
      "\n",
      "DUKE:\n",
      "Which hither, forth:\n",
      "I shall swort see\n",
      "Angy and mareform'd tain blust makes to by spast;\n",
      "And be wI art they woo thing from thuse of the come our strue-suison;\n",
      "At but will hand wife is not in the stand\n",
      "With of the be arms the did father, by the deat,\n",
      "The son,-gin my ment wife, stire me it beanst and from bring him at to steet alorts,\n",
      "The there obeing them here the fanger their the we all,\n",
      "And with a soume is show thy to britter.\n",
      "In an allent impeter we sain inlow it the drenge in arms\n",
      "The morringlyss and all thice touch and letters,\n",
      "Shus? what, and fielloss of that both in\n",
      "To streeks for shory lord say of me so trous my\n",
      "And hand, and we day you death these in me hated's call:\n",
      "When sempess is breased well mistaling;\n",
      "Thou had how to would my clifes on t\n"
     ]
    }
   ],
   "source": [
    "context = \"O God, O God!\"\n",
    "x = tf.convert_to_tensor([stoi[c] for c in context], dtype=tf.int32)[None, ...]\n",
    "y = model_char.sample(x, 2000, temperature=1.0, sample=True, top_k=10)[0]\n",
    "completion = \"\".join([itos[int(i)] for i in y])\n",
    "print(completion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.6 64-bit ('tfgpt')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "bfdc0eac67cf196f2171871a77a37aff362885343dc84e4d9326d459b303568d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}