exploring-pseudolabelling-schemes-pydata.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "654456b6-e648-0379-0d66-1cc97af6d00d",
    "_uuid": "6b48ce0e361bdb67689dd2f254ecedd9ade1f5ff",
    "papermill": {
     "duration": 0.010415,
     "end_time": "2020-12-17T02:04:44.749531",
     "exception": false,
     "start_time": "2020-12-17T02:04:44.739116",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Pydata Dec. 2020 - Stanley Zheng\n",
    "\n",
    "This notebook is intended to explore various pseudolabelling schemes. Validation, model, data isn't hugely important here, so the cells are collapsed. Keep in mind this is a minimal example without much of the techniques discussed, and this dataset is very basic - adding augmentations, stochastic depth, etc. during training would result in better results. This is intended to be a minimal code example.\n",
    "\n",
    "### Reproducibility\n",
    "To make this as fair a comparison as possible, I have seeded random weights and all pseudolabels are produced from the same set of weights. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "e5b02688-c589-5a89-e11c-837c6a99eb6e",
    "_uuid": "f043e48097bfd98e41710142dd8aac41fa88a801",
    "execution": {
     "iopub.execute_input": "2020-12-17T02:04:44.777243Z",
     "iopub.status.busy": "2020-12-17T02:04:44.776563Z",
     "iopub.status.idle": "2020-12-17T02:04:49.629835Z",
     "shell.execute_reply": "2020-12-17T02:04:49.628440Z"
    },
    "papermill": {
     "duration": 4.87082,
     "end_time": "2020-12-17T02:04:49.629959",
     "exception": false,
     "start_time": "2020-12-17T02:04:44.759139",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras import  backend as K\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "import os\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "22a7fd70-ab61-432d-24cb-93e558414495",
    "_uuid": "62fbd0fe9c338b7ac0b04e688c8ee7947e6170f7",
    "papermill": {
     "duration": 0.009225,
     "end_time": "2020-12-17T02:04:49.649050",
     "exception": false,
     "start_time": "2020-12-17T02:04:49.639825",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Load Train and Test data and cross validation**\n",
    "============================"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "05226b08-226a-1a00-044d-a0e6b2101388",
    "_uuid": "4eff577bcd43479a3b7e91180393cbad9fcfca33",
    "execution": {
     "iopub.execute_input": "2020-12-17T02:04:49.689566Z",
     "iopub.status.busy": "2020-12-17T02:04:49.681222Z",
     "iopub.status.idle": "2020-12-17T02:05:12.567097Z",
     "shell.execute_reply": "2020-12-17T02:05:12.565757Z"
    },
    "papermill": {
     "duration": 22.908624,
     "end_time": "2020-12-17T02:05:12.567208",
     "exception": false,
     "start_time": "2020-12-17T02:04:49.658584",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def standardize(x): \n",
    "    return (x-mean_px)/std_px\n",
    "\n",
    "def seed_everything(seed=1234):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    tf.random.set_seed(seed)\n",
    "\n",
    "train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n",
    "test= pd.read_csv(\"../input/digit-recognizer/test.csv\")\n",
    "X_train = (train.iloc[:,1:].values).astype('float32') # all pixel values\n",
    "y_train = train.iloc[:,0].values.astype('int32') # only labels i.e targets digits\n",
    "X_test = test.values.astype('float32')\n",
    "mean_px = X_train.mean().astype(np.float32)\n",
    "std_px = X_train.std().astype(np.float32) \n",
    "\n",
    "y_train = tf.keras.utils.to_categorical(y_train)\n",
    "num_classes = y_train.shape[1]\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed_everything(seed=42)\n",
    "\n",
    "X_test = X_test.reshape(X_test.shape[0], 28, 28,1)\n",
    "X_train = X_train.reshape(X_train.shape[0], 28, 28,1)\n",
    "\n",
    "# cross validation\n",
    "X = X_train\n",
    "y = y_train\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)\n",
    "\n",
    "mnist_test = pd.read_csv(\"../input/mnist-in-csv/mnist_test.csv\")\n",
    "mnist_train = pd.read_csv(\"../input/mnist-in-csv/mnist_train.csv\")\n",
    "ground_truth = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\n",
    "\n",
    "cols = test.columns\n",
    "\n",
    "test['dataset'] = 'test'\n",
    "\n",
    "train['dataset'] = 'train'\n",
    "\n",
    "dataset = pd.concat([train.drop('label', axis=1), test]).reset_index()\n",
    "\n",
    "mnist = pd.concat([mnist_train, mnist_test]).reset_index(drop=True)\n",
    "labels = mnist['label'].values\n",
    "mnist.drop('label', axis=1, inplace=True)\n",
    "mnist.columns = cols\n",
    "\n",
    "idx_mnist = mnist.sort_values(by=list(mnist.columns)).index\n",
    "dataset_from = dataset.sort_values(by=list(mnist.columns))['dataset'].values\n",
    "original_idx = dataset.sort_values(by=list(mnist.columns))['index'].values\n",
    "\n",
    "for i in range(len(idx_mnist)):\n",
    "    if dataset_from[i] == 'test':\n",
    "        ground_truth.loc[original_idx[i], 'Label'] = labels[idx_mnist[i]]\n",
    "        \n",
    "def get_test_acc(model):\n",
    "    predictions = model.predict(X_test, verbose=0)\n",
    "    predictions = np.argmax(predictions,axis=1)\n",
    "\n",
    "    submissions = pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n",
    "                                \"Label\": predictions})\n",
    "    return accuracy_score(ground_truth['Label'].values, submissions['Label'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "8b72580fbb06f5f4f769c514cb0d7d2f15aa2c2f",
    "execution": {
     "iopub.execute_input": "2020-12-17T02:05:12.601060Z",
     "iopub.status.busy": "2020-12-17T02:05:12.599313Z",
     "iopub.status.idle": "2020-12-17T02:05:12.601720Z",
     "shell.execute_reply": "2020-12-17T02:05:12.602144Z"
    },
    "papermill": {
     "duration": 0.025195,
     "end_time": "2020-12-17T02:05:12.602246",
     "exception": false,
     "start_time": "2020-12-17T02:05:12.577051",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "verbosity = 0\n",
    "\n",
    "def get_model():\n",
    "    input_1 = tf.keras.layers.Input((28,28,1))\n",
    "    x = tf.keras.layers.Lambda(standardize)(input_1)\n",
    "    x = tf.keras.layers.Convolution2D(32,(3,3), activation='relu')(x)\n",
    "    x = tf.keras.layers.BatchNormalization(axis=1)(x)\n",
    "    x = tf.keras.layers.Convolution2D(32,(3,3), activation='relu')(x)\n",
    "    x = tf.keras.layers.MaxPooling2D()(x)\n",
    "    x = tf.keras.layers.BatchNormalization(axis=1)(x)\n",
    "    x = tf.keras.layers.Convolution2D(64,(3,3), activation='relu')(x)\n",
    "    x = tf.keras.layers.BatchNormalization(axis=1)(x)\n",
    "    x = tf.keras.layers.Convolution2D(64,(3,3), activation='relu')(x)\n",
    "    x = tf.keras.layers.MaxPooling2D()(x)\n",
    "    x = tf.keras.layers.Flatten()(x)\n",
    "    x = tf.keras.layers.BatchNormalization()(x)\n",
    "    x = tf.keras.layers.Dense(512, activation='relu')(x)\n",
    "    x = tf.keras.layers.BatchNormalization()(x)\n",
    "    out = tf.keras.layers.Dense(10, activation='softmax')(x)\n",
    "    model = tf.keras.Model(inputs=input_1, outputs=out)\n",
    "    model.compile(tf.keras.optimizers.Adam(lr=1e-3), loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.009636,
     "end_time": "2020-12-17T02:05:12.621498",
     "exception": false,
     "start_time": "2020-12-17T02:05:12.611862",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# No pseudolabelling baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T02:05:12.646346Z",
     "iopub.status.busy": "2020-12-17T02:05:12.645832Z",
     "iopub.status.idle": "2020-12-17T02:06:15.926327Z",
     "shell.execute_reply": "2020-12-17T02:06:15.926810Z"
    },
    "papermill": {
     "duration": 63.295917,
     "end_time": "2020-12-17T02:06:15.926932",
     "exception": false,
     "start_time": "2020-12-17T02:05:12.631015",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No pseudolabelling accuracy: 0.99018\n"
     ]
    }
   ],
   "source": [
    "ckp = tf.keras.callbacks.ModelCheckpoint(f'baseline.hdf5', monitor='val_accuracy', verbose=0, save_best_only=True, save_weights_only=True, mode='max')\n",
    "model = get_model()\n",
    "model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=10, batch_size=32, verbose=verbosity, callbacks=[ckp])\n",
    "\n",
    "model.load_weights('baseline.hdf5') # load best weights\n",
    "no_pseudo_acc = get_test_acc(model)\n",
    "print(f\"No pseudolabelling accuracy: {format(no_pseudo_acc, '.5g')}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.009684,
     "end_time": "2020-12-17T02:06:15.946659",
     "exception": false,
     "start_time": "2020-12-17T02:06:15.936975",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Self training\n",
    "\n",
    "First, we train on the labelled data, then produce pseudolabels and finetune on the pseudolabels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T02:06:15.975922Z",
     "iopub.status.busy": "2020-12-17T02:06:15.975133Z",
     "iopub.status.idle": "2020-12-17T02:06:51.674315Z",
     "shell.execute_reply": "2020-12-17T02:06:51.674749Z"
    },
    "papermill": {
     "duration": 35.718224,
     "end_time": "2020-12-17T02:06:51.674883",
     "exception": false,
     "start_time": "2020-12-17T02:06:15.956659",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Self training accuracy: 0.99079\n"
     ]
    }
   ],
   "source": [
    "model = get_model()\n",
    "model.load_weights('baseline.hdf5')\n",
    "\n",
    "pseudolabels = model.predict(X_test, verbose=0) # create our pseudolabels\n",
    "pseudolabels = np.argmax(pseudolabels,axis=1) # convert probabilities into classes\n",
    "pseudolabels = tf.keras.utils.to_categorical(pseudolabels) \n",
    "\n",
    "model.optimizer.lr = 1e-4 # reduce learning rate since we are finetuning\n",
    "\n",
    "ckp = tf.keras.callbacks.ModelCheckpoint(f'selftrain.hdf5', monitor='val_accuracy', verbose=0, save_best_only=True, save_weights_only=True, mode='max')\n",
    "model.fit(X_test, pseudolabels, validation_data=(X_val, y_val), epochs=10, batch_size=32, verbose=verbosity, callbacks=[ckp])\n",
    "\n",
    "model.load_weights('selftrain.hdf5') # load best weights\n",
    "self_train_acc = get_test_acc(model)\n",
    "print(f\"Self training accuracy: {format(self_train_acc, '.5g')}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.010284,
     "end_time": "2020-12-17T02:06:51.695892",
     "exception": false,
     "start_time": "2020-12-17T02:06:51.685608",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Simultaneous training\n",
    "First, we train on the labelled data, then initialize a new model and train with labelled data and pseudolabels simultaneously."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T02:06:51.725976Z",
     "iopub.status.busy": "2020-12-17T02:06:51.725027Z",
     "iopub.status.idle": "2020-12-17T02:06:53.890960Z",
     "shell.execute_reply": "2020-12-17T02:06:53.890488Z"
    },
    "papermill": {
     "duration": 2.184731,
     "end_time": "2020-12-17T02:06:53.891125",
     "exception": false,
     "start_time": "2020-12-17T02:06:51.706394",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = get_model()\n",
    "model.load_weights('baseline.hdf5')\n",
    "\n",
    "pseudolabels = model.predict(X_test, verbose=0) # create our pseudolabels\n",
    "pseudolabels = np.argmax(pseudolabels,axis=1) # convert probabilities into classes\n",
    "pseudolabels = tf.keras.utils.to_categorical(pseudolabels) \n",
    "y_combined = np.concatenate([pseudolabels, y_train]) # combine our pseudolabels with labelled data\n",
    "X_combined = np.concatenate([X_test, X_train]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T02:06:53.922521Z",
     "iopub.status.busy": "2020-12-17T02:06:53.921682Z",
     "iopub.status.idle": "2020-12-17T02:10:03.198612Z",
     "shell.execute_reply": "2020-12-17T02:10:03.199231Z"
    },
    "papermill": {
     "duration": 189.297254,
     "end_time": "2020-12-17T02:10:03.199375",
     "exception": false,
     "start_time": "2020-12-17T02:06:53.902121",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simultaneous training accuracy: 0.99057\n"
     ]
    }
   ],
   "source": [
    "ckp = tf.keras.callbacks.ModelCheckpoint('simultaneous_train.hdf5', monitor='val_accuracy', verbose=0, save_best_only=True, save_weights_only=True, mode='max')\n",
    "\n",
    "model = get_model() # reinitialize model\n",
    "model.fit(X_combined, y_combined, validation_data=(X_val, y_val), epochs=20, batch_size=32, callbacks=[ckp], verbose=verbosity) # train a new model on all data together\n",
    "\n",
    "model.load_weights('simultaneous_train.hdf5') # load best weights\n",
    "\n",
    "simultaneous_acc = get_test_acc(model) # get test accuracy\n",
    "print(f\"Simultaneous training accuracy: {format(simultaneous_acc, '.5g')}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.013285,
     "end_time": "2020-12-17T02:10:03.224377",
     "exception": false,
     "start_time": "2020-12-17T02:10:03.211092",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Pretraining\n",
    "First, we train on labelled data, then we create pseudolabels. \n",
    "\n",
    "Next, we initialize a new model and train it on "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "78e382d0b3de14312e762edc480b5d215be82269",
    "execution": {
     "iopub.execute_input": "2020-12-17T02:10:03.255753Z",
     "iopub.status.busy": "2020-12-17T02:10:03.253525Z",
     "iopub.status.idle": "2020-12-17T02:10:04.700694Z",
     "shell.execute_reply": "2020-12-17T02:10:04.699603Z"
    },
    "papermill": {
     "duration": 1.464776,
     "end_time": "2020-12-17T02:10:04.700802",
     "exception": false,
     "start_time": "2020-12-17T02:10:03.236026",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = get_model()\n",
    "model.load_weights('baseline.hdf5')\n",
    "\n",
    "pseudolabels = model.predict(X_test, verbose=0) # create our pseudolabels\n",
    "pseudolabels = np.argmax(pseudolabels,axis=1) # convert probabilities into classes\n",
    "pseudolabels = tf.keras.utils.to_categorical(pseudolabels) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T02:10:04.738774Z",
     "iopub.status.busy": "2020-12-17T02:10:04.735015Z",
     "iopub.status.idle": "2020-12-17T02:11:54.873712Z",
     "shell.execute_reply": "2020-12-17T02:11:54.874222Z"
    },
    "papermill": {
     "duration": 110.161989,
     "end_time": "2020-12-17T02:11:54.874352",
     "exception": false,
     "start_time": "2020-12-17T02:10:04.712363",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pretrain on pseudolabels\n",
      "Finetune on labelled data\n",
      "Pretraining accuracy: 0.9910357142857142\n"
     ]
    }
   ],
   "source": [
    "ckp = tf.keras.callbacks.ModelCheckpoint('pretrain.hdf5', monitor='val_accuracy', verbose=0, save_best_only=True, save_weights_only=True, mode='max')\n",
    "\n",
    "print(\"Pretrain on pseudolabels\")\n",
    "model = get_model() # reinitialize model\n",
    "model.fit(X_test, pseudolabels, validation_data=(X_val, y_val), epochs=15, batch_size=32, callbacks=[ckp], verbose=verbosity) # first train on pseudolabels only\n",
    "\n",
    "print(\"Finetune on labelled data\")\n",
    "model.optimizer.lr = 1e-4 # reduce learning rate since we are finetuning\n",
    "model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=10, batch_size=32, callbacks=[ckp], verbose=verbosity) # finetune on labelled data\n",
    "\n",
    "model.load_weights('pretrain.hdf5') # load best weights\n",
    "pretrain_acc = get_test_acc(model) # get test accuracy\n",
    "print(f\"Pretraining accuracy: {pretrain_acc}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.012177,
     "end_time": "2020-12-17T02:11:54.899520",
     "exception": false,
     "start_time": "2020-12-17T02:11:54.887343",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Conclusion\n",
    "\n",
    "In my talk, I explained the use cases for various pseudolabelling methods. Even though MNIST is not a particularly complex dataset and it's not very fit for pseudolabelling, we still see an improvement over baseline. MNIST's test set is only about half the size of the train set. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T02:11:54.931385Z",
     "iopub.status.busy": "2020-12-17T02:11:54.930529Z",
     "iopub.status.idle": "2020-12-17T02:11:54.940063Z",
     "shell.execute_reply": "2020-12-17T02:11:54.940664Z"
    },
    "papermill": {
     "duration": 0.029198,
     "end_time": "2020-12-17T02:11:54.940808",
     "exception": false,
     "start_time": "2020-12-17T02:11:54.911610",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No pseudolabelling accuracy: 0.9901785714285715\n",
      "Self training accuracy: 0.9907857142857143\n",
      "Simultaneous training accuracy: 0.9905714285714285\n",
      "Pretraining accuracy: 0.9910357142857142\n",
      "------------------------------\n",
      "Percent difference from no pseudolabelling to self training: 0.06131650135256473%\n",
      "Percent difference from self training to simultaneous training: -0.02162785667940389%\n",
      "Percent difference from simultaneous training to pretraining: 0.04687049322180364%\n",
      "------------------------------\n",
      "Percent difference from no pseudolabelling to pretraining: 0.08656447249773778%\n"
     ]
    }
   ],
   "source": [
    "print(f\"No pseudolabelling accuracy: {no_pseudo_acc}\")\n",
    "print(f\"Self training accuracy: {self_train_acc}\")\n",
    "print(f\"Simultaneous training accuracy: {simultaneous_acc}\")\n",
    "print(f\"Pretraining accuracy: {pretrain_acc}\")\n",
    "\n",
    "print(\"-\"*30)\n",
    "\n",
    "print(f\"Percent difference from no pseudolabelling to self training: {100*(self_train_acc-no_pseudo_acc)/no_pseudo_acc}%\")\n",
    "print(f\"Percent difference from self training to simultaneous training: {100*(simultaneous_acc-self_train_acc)/self_train_acc}%\")\n",
    "print(f\"Percent difference from simultaneous training to pretraining: {100*(pretrain_acc-simultaneous_acc)/simultaneous_acc}%\")\n",
    "\n",
    "print(\"-\"*30)\n",
    "\n",
    "print(f\"Percent difference from no pseudolabelling to pretraining: {100*(pretrain_acc-no_pseudo_acc)/no_pseudo_acc}%\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.019636,
     "end_time": "2020-12-17T02:11:54.980134",
     "exception": false,
     "start_time": "2020-12-17T02:11:54.960498",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 435.891324,
   "end_time": "2020-12-17T02:11:56.711865",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-12-17T02:04:40.820541",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}