Base/frameworks/keras/demo/Word_Language_Modelling_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Thx to [deepanwayx](https://github.com/deepanwayx/char-and-word-rnn-keras/blob/master/Word%20Language%20Modelling%20LSTM%20.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import sys\n",
    "import string\n",
    "import numpy as np\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Dense, Dropout, LSTM\n",
    "from keras.layers.embeddings import Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rawtext = open(r'F:/data/wonderland.txt','r').read().split('\\n')\n",
    "rawtext = ' '.join(rawtext)\n",
    "rawtext = [word.strip(string.punctuation) for word in rawtext.split()] #string.punctuation包含所有标点的字符串\n",
    "rawtext = ' '.join(rawtext)\n",
    "rawtext = rawtext.replace('-', ' ')\n",
    "rawtext = ' '.join(rawtext.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Vocab: 3063\n"
     ]
    }
   ],
   "source": [
    "all_words = rawtext.split()\n",
    "unique_words = sorted(list(set(all_words)))\n",
    "n_vocab = len(unique_words)\n",
    "print(\"Total Vocab:\", n_vocab)\n",
    "word_to_int = dict((w, i) for i, w in enumerate(unique_words))\n",
    "int_to_word = dict((i, w) for i, w in enumerate(unique_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26694\n"
     ]
    }
   ],
   "source": [
    "raw_text = rawtext.split()\n",
    "n_words = len(raw_text)\n",
    "print(n_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total patterns: 26594\n"
     ]
    }
   ],
   "source": [
    "#定义输入 输出 并数值化\n",
    "seq_length = 100\n",
    "dataX = []\n",
    "dataY = []\n",
    "for i in range(0, n_words - seq_length):\n",
    "    seq_in  = raw_text[i: i+seq_length]\n",
    "    seq_out = raw_text[i+seq_length]\n",
    "    dataX.append([word_to_int[word] for word in seq_in])\n",
    "    dataY.append(word_to_int[seq_out])\n",
    "n_patterns = len(dataX)\n",
    "print('Total patterns:', n_patterns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape dataX to size of [samples, time steps, features] and scale it to 0-1\n",
    "# Represent dataY as one hot encoding\n",
    "X_train = np.reshape(dataX, (n_patterns, seq_length, 1))/float(n_vocab)\n",
    "Y_train = np_utils.to_categorical(dataY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_1 (LSTM)                (None, 256)               264192    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 3063)              787191    \n",
      "=================================================================\n",
      "Total params: 1,051,383\n",
      "Trainable params: 1,051,383\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(256, input_shape=(X_train.shape[1], X_train.shape[2])))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(Y_train.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the checkpoint\n",
    "filepath=\"word-weights-improvement-{epoch:02d}-{loss:.4f}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\n",
    "callbacks_list = [checkpoint]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.3350Epoch 00000: loss improved from inf to 6.33474, saving model to word-weights-improvement-00-6.3347.hdf5\n",
      "26594/26594 [==============================] - 268s - loss: 6.3347   \n",
      "Epoch 2/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.2693Epoch 00001: loss improved from 6.33474 to 6.26830, saving model to word-weights-improvement-01-6.2683.hdf5\n",
      "26594/26594 [==============================] - 267s - loss: 6.2683   \n",
      "Epoch 3/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.2528Epoch 00002: loss improved from 6.26830 to 6.25301, saving model to word-weights-improvement-02-6.2530.hdf5\n",
      "26594/26594 [==============================] - 264s - loss: 6.2530   \n",
      "Epoch 4/10\n",
      "26496/26594 [============================>.] - ETA: 1s - loss: 6.2447Epoch 00003: loss improved from 6.25301 to 6.24534, saving model to word-weights-improvement-03-6.2453.hdf5\n",
      "26594/26594 [==============================] - 274s - loss: 6.2453   \n",
      "Epoch 5/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.2421Epoch 00004: loss improved from 6.24534 to 6.24260, saving model to word-weights-improvement-04-6.2426.hdf5\n",
      "26594/26594 [==============================] - 263s - loss: 6.2426   \n",
      "Epoch 6/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.2393Epoch 00005: loss improved from 6.24260 to 6.23889, saving model to word-weights-improvement-05-6.2389.hdf5\n",
      "26594/26594 [==============================] - 266s - loss: 6.2389   \n",
      "Epoch 7/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.2325Epoch 00006: loss improved from 6.23889 to 6.23288, saving model to word-weights-improvement-06-6.2329.hdf5\n",
      "26594/26594 [==============================] - 269s - loss: 6.2329   \n",
      "Epoch 8/10\n",
      "26496/26594 [============================>.] - ETA: 1s - loss: 6.2178Epoch 00007: loss improved from 6.23288 to 6.21756, saving model to word-weights-improvement-07-6.2176.hdf5\n",
      "26594/26594 [==============================] - 282s - loss: 6.2176   \n",
      "Epoch 9/10\n",
      "26496/26594 [============================>.] - ETA: 1s - loss: 6.1895Epoch 00008: loss improved from 6.21756 to 6.19026, saving model to word-weights-improvement-08-6.1903.hdf5\n",
      "26594/26594 [==============================] - 278s - loss: 6.1903   \n",
      "Epoch 10/10\n",
      "26496/26594 [============================>.] - ETA: 1s - loss: 6.1562Epoch 00009: loss improved from 6.19026 to 6.15744, saving model to word-weights-improvement-09-6.1574.hdf5\n",
      "26594/26594 [==============================] - 277s - loss: 6.1574   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x208a6f3a588>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, epochs=10, batch_size=128, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the network weights\n",
    "filename = \"word-weights-improvement-09-6.1574.hdf5\"\n",
    "model.load_weights(filename)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.2680Epoch 00000: loss did not improve\n",
      "26594/26594 [==============================] - 387s - loss: 6.2680   \n",
      "Epoch 2/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.1648Epoch 00001: loss did not improve\n",
      "26594/26594 [==============================] - 393s - loss: 6.1646   \n",
      "Epoch 3/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.0983Epoch 00002: loss improved from 6.15744 to 6.09835, saving model to word-weights-improvement-02-6.0983.hdf5\n",
      "26594/26594 [==============================] - 380s - loss: 6.0983   \n",
      "Epoch 4/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.0300Epoch 00003: loss improved from 6.09835 to 6.02994, saving model to word-weights-improvement-03-6.0299.hdf5\n",
      "26594/26594 [==============================] - 359s - loss: 6.0299   \n",
      "Epoch 5/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.9535Epoch 00004: loss improved from 6.02994 to 5.95360, saving model to word-weights-improvement-04-5.9536.hdf5\n",
      "26594/26594 [==============================] - 363s - loss: 5.9536   \n",
      "Epoch 6/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.8710Epoch 00005: loss improved from 5.95360 to 5.87116, saving model to word-weights-improvement-05-5.8712.hdf5\n",
      "26594/26594 [==============================] - 333s - loss: 5.8712   \n",
      "Epoch 7/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.7632Epoch 00006: loss improved from 5.87116 to 5.76342, saving model to word-weights-improvement-06-5.7634.hdf5\n",
      "26594/26594 [==============================] - 331s - loss: 5.7634   \n",
      "Epoch 8/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.6398Epoch 00007: loss improved from 5.76342 to 5.63985, saving model to word-weights-improvement-07-5.6399.hdf5\n",
      "26594/26594 [==============================] - 331s - loss: 5.6399   \n",
      "Epoch 9/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.5049Epoch 00008: loss improved from 5.63985 to 5.50492, saving model to word-weights-improvement-08-5.5049.hdf5\n",
      "26594/26594 [==============================] - 341s - loss: 5.5049   \n",
      "Epoch 10/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.3570Epoch 00009: loss improved from 5.50492 to 5.35694, saving model to word-weights-improvement-09-5.3569.hdf5\n",
      "26594/26594 [==============================] - 330s - loss: 5.3569   \n",
      "Epoch 11/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.1926Epoch 00010: loss improved from 5.35694 to 5.19249, saving model to word-weights-improvement-10-5.1925.hdf5\n",
      "26594/26594 [==============================] - 331s - loss: 5.1925   \n",
      "Epoch 12/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.0186Epoch 00011: loss improved from 5.19249 to 5.01853, saving model to word-weights-improvement-11-5.0185.hdf5\n",
      "26594/26594 [==============================] - 339s - loss: 5.0185   \n",
      "Epoch 13/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.8451Epoch 00012: loss improved from 5.01853 to 4.84521, saving model to word-weights-improvement-12-4.8452.hdf5\n",
      "26594/26594 [==============================] - 338s - loss: 4.8452   \n",
      "Epoch 14/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.6732Epoch 00013: loss improved from 4.84521 to 4.67316, saving model to word-weights-improvement-13-4.6732.hdf5\n",
      "26594/26594 [==============================] - 337s - loss: 4.6732   \n",
      "Epoch 15/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.4998Epoch 00014: loss improved from 4.67316 to 4.49968, saving model to word-weights-improvement-14-4.4997.hdf5\n",
      "26594/26594 [==============================] - 342s - loss: 4.4997   \n",
      "Epoch 16/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.3350Epoch 00015: loss improved from 4.49968 to 4.33504, saving model to word-weights-improvement-15-4.3350.hdf5\n",
      "26594/26594 [==============================] - 342s - loss: 4.3350   \n",
      "Epoch 17/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.1791Epoch 00016: loss improved from 4.33504 to 4.17905, saving model to word-weights-improvement-16-4.1791.hdf5\n",
      "26594/26594 [==============================] - 335s - loss: 4.1791   \n",
      "Epoch 18/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.0297Epoch 00017: loss improved from 4.17905 to 4.02971, saving model to word-weights-improvement-17-4.0297.hdf5\n",
      "26594/26594 [==============================] - 336s - loss: 4.0297   \n",
      "Epoch 19/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.8645Epoch 00018: loss improved from 4.02971 to 3.86451, saving model to word-weights-improvement-18-3.8645.hdf5\n",
      "26594/26594 [==============================] - 337s - loss: 3.8645   \n",
      "Epoch 20/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.7214Epoch 00019: loss improved from 3.86451 to 3.72140, saving model to word-weights-improvement-19-3.7214.hdf5\n",
      "26594/26594 [==============================] - 337s - loss: 3.7214   \n",
      "Epoch 21/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.5928Epoch 00020: loss improved from 3.72140 to 3.59274, saving model to word-weights-improvement-20-3.5927.hdf5\n",
      "26594/26594 [==============================] - 347s - loss: 3.5927   \n",
      "Epoch 22/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.4729Epoch 00021: loss improved from 3.59274 to 3.47298, saving model to word-weights-improvement-21-3.4730.hdf5\n",
      "26594/26594 [==============================] - 335s - loss: 3.4730   \n",
      "Epoch 23/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.3554Epoch 00022: loss improved from 3.47298 to 3.35538, saving model to word-weights-improvement-22-3.3554.hdf5\n",
      "26594/26594 [==============================] - 346s - loss: 3.3554   \n",
      "Epoch 24/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.2327Epoch 00023: loss improved from 3.35538 to 3.23260, saving model to word-weights-improvement-23-3.2326.hdf5\n",
      "26594/26594 [==============================] - 342s - loss: 3.2326   \n",
      "Epoch 25/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.1181Epoch 00024: loss improved from 3.23260 to 3.11812, saving model to word-weights-improvement-24-3.1181.hdf5\n",
      "26594/26594 [==============================] - 339s - loss: 3.1181   \n",
      "Epoch 26/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.0241Epoch 00025: loss improved from 3.11812 to 3.02410, saving model to word-weights-improvement-25-3.0241.hdf5\n",
      "26594/26594 [==============================] - 360s - loss: 3.0241   \n",
      "Epoch 27/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.9272Epoch 00026: loss improved from 3.02410 to 2.92707, saving model to word-weights-improvement-26-2.9271.hdf5\n",
      "26594/26594 [==============================] - 377s - loss: 2.9271   \n",
      "Epoch 28/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.8311Epoch 00027: loss improved from 2.92707 to 2.83090, saving model to word-weights-improvement-27-2.8309.hdf5\n",
      "26594/26594 [==============================] - 437s - loss: 2.8309   \n",
      "Epoch 29/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.7398Epoch 00028: loss improved from 2.83090 to 2.73978, saving model to word-weights-improvement-28-2.7398.hdf5\n",
      "26594/26594 [==============================] - 433s - loss: 2.7398   \n",
      "Epoch 30/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.6583Epoch 00029: loss improved from 2.73978 to 2.65816, saving model to word-weights-improvement-29-2.6582.hdf5\n",
      "26594/26594 [==============================] - 405s - loss: 2.6582   \n",
      "Epoch 31/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.5815Epoch 00030: loss improved from 2.65816 to 2.58153, saving model to word-weights-improvement-30-2.5815.hdf5\n",
      "26594/26594 [==============================] - 413s - loss: 2.5815   \n",
      "Epoch 32/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.5238Epoch 00031: loss improved from 2.58153 to 2.52395, saving model to word-weights-improvement-31-2.5239.hdf5\n",
      "26594/26594 [==============================] - 434s - loss: 2.5239   \n",
      "Epoch 33/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.4340Epoch 00032: loss improved from 2.52395 to 2.43384, saving model to word-weights-improvement-32-2.4338.hdf5\n",
      "26594/26594 [==============================] - 394s - loss: 2.4338   \n",
      "Epoch 34/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.3886Epoch 00033: loss improved from 2.43384 to 2.38869, saving model to word-weights-improvement-33-2.3887.hdf5\n",
      "26594/26594 [==============================] - 413s - loss: 2.3887   \n",
      "Epoch 35/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.3396Epoch 00034: loss improved from 2.38869 to 2.33977, saving model to word-weights-improvement-34-2.3398.hdf5\n",
      "26594/26594 [==============================] - 380s - loss: 2.3398   \n",
      "Epoch 36/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.2474Epoch 00035: loss improved from 2.33977 to 2.24749, saving model to word-weights-improvement-35-2.2475.hdf5\n",
      "26594/26594 [==============================] - 373s - loss: 2.2475   \n",
      "Epoch 37/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.2548Epoch 00036: loss did not improve\n",
      "26594/26594 [==============================] - 370s - loss: 2.2548   \n",
      "Epoch 38/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.1616Epoch 00037: loss improved from 2.24749 to 2.16186, saving model to word-weights-improvement-37-2.1619.hdf5\n",
      "26594/26594 [==============================] - 367s - loss: 2.1619   \n",
      "Epoch 39/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.1599Epoch 00038: loss improved from 2.16186 to 2.15990, saving model to word-weights-improvement-38-2.1599.hdf5\n",
      "26594/26594 [==============================] - 363s - loss: 2.1599   \n",
      "Epoch 40/40\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.0721Epoch 00039: loss improved from 2.15990 to 2.07200, saving model to word-weights-improvement-39-2.0720.hdf5\n",
      "26594/26594 [==============================] - 363s - loss: 2.0720   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x208ae8fce48>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, epochs=40, batch_size=32, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the network weights\n",
    "filename = \"word-weights-improvement-39-2.0720.hdf5\"\n",
    "model.load_weights(filename)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" Queen ordering off her unfortunate guests to execution once more the pig baby was sneezing on the Duchess's knee while plates and dishes crashed around it once more the shriek of the Gryphon the squeaking of the Lizard's slate pencil and the choking of the suppressed guinea pigs filled the air mixed up with the distant sobs of the miserable Mock Turtle So she sat on with closed eyes and half believed herself in Wonderland though she knew she had but to open them again and all would change to dull reality the grass would be only rustling in the \"\n",
      "\n",
      "Generated Sequence:\n",
      "wind and the pool rippling to the waving of the reeds of rattling teacups and made of of upon to herself was be a left shelves to to be that to and it curious when to be that since to King two as a could you no try try the the the be winter for sharp as little a she she it the the the the be ever out of England the hookah that be otherwise of Alice the no Rabbit in no wind and was chance to lying and when as crowded up Alice minute she wind to she the happen in the it a a snail his said the the the the and was for the the and and this it it to King the in Alice far the Alice so a ever treading said the Queen digging the the Rabbit Rabbit returning splendidly do with a to a said a dead silence eat a come pun the same said I Of I you you you glad Alice and such attending so the Mouse the the severely Alice Hare to the Cat went on the Mock I on an haven't voice voice at looked to be I I\n",
      "\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "start = np.random.randint(0, len(X_train)-1)\n",
    "pattern = dataX[start]\n",
    "result = []\n",
    "print(\"Seed:\")\n",
    "print(\"\\\"\", ' '.join([int_to_word[value] for value in pattern]), \"\\\"\")\n",
    "for i in range(200):\n",
    "    x = np.reshape(pattern, (1, len(pattern), 1))\n",
    "    x = x/float(n_vocab)\n",
    "    prediction = model.predict(x)\n",
    "    index = np.argmax(prediction)\n",
    "    result.append(int_to_word[index])\n",
    "    pattern.append(index)\n",
    "    pattern = pattern[1:len(pattern)]\n",
    "print(\"\\nGenerated Sequence:\")\n",
    "print(' '.join(result))\n",
    "print(\"\\nDone.\"  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}