diff --git a/CHANGELOG.md b/CHANGELOG.md index c6acbd92b..5aa12e59b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ Most recent releases are shown at the top. Each release shows: - **Changed**: Additional parameters, changes to inputs or outputs, etc - **Fixed**: Bug fixes that don't change documented behaviour +## 0.1.10 (2019-08-02) + +### New: +- N/A + +### Changed: +- For ```Learner.lr_find```, removed epochs and max_lr arguments and added lr_mult argument + Default lr_mult is 1.01, but can be changed to control size of sample being used + to estimate learning rate. +- Changed structure of examples folder + +### Fixed: +- Resolved issue with ```utils.y_from_data``` not working correctly with DataFrameIterator objects. + + ## 0.1.9 (2019-08-01) ### New: diff --git a/examples/text/IMDb-fasttext.ipynb b/examples/text/IMDb-fasttext.ipynb new file mode 100644 index 000000000..795213249 --- /dev/null +++ b/examples/text/IMDb-fasttext.ipynb @@ -0,0 +1,200 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline\n", + "import os\n", + "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\";\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"; \n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "import ktrain\n", + "from ktrain import text" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Word Counts: 88582\n", + "Nrows: 25000\n", + "25000 train sequences\n", + "Average train sequence length: 231\n", + "25000 test sequences\n", + "Average test sequence length: 224\n", + "Pad sequences (samples x time)\n", + "x_train shape: (25000,400)\n", + "x_test shape: (25000,400)\n", + "y_train shape: (25000,2)\n", + "y_test shape: (25000,2)\n" + ] + } + ], + "source": [ + "# load training and validation data from a folder\n", + "DATADIR = 'data/aclImdb'\n", + "(x_train, y_train), (x_test, y_test), preproc = text.texts_from_folder(DATADIR, \n", + " max_features=20000, maxlen=400, \n", + " ngram_range=1, \n", + " classes=['pos', 'neg'])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from keras.models import Sequential\n", + "from keras.layers import Dense, Embedding, GlobalAveragePooling1D\n", + "def get_model():\n", + " model = Sequential()\n", + " model.add(Embedding(20000+1, 50, input_length=400)) # add 1 for padding token\n", + " model.add(GlobalAveragePooling1D())\n", + " model.add(Dense(2, activation='softmax'))\n", + " model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n", + " return model\n", + "model = get_model()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "learner = ktrain.get_learner(model, train_data=(x_train, y_train), val_data=(x_test, y_test))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "simulating training for different learning rates... this may take a few moments...\n", + "Epoch 1/5\n", + "25000/25000 [==============================] - 4s 177us/step - loss: 0.6934 - acc: 0.4893\n", + "Epoch 2/5\n", + "25000/25000 [==============================] - 4s 169us/step - loss: 0.6927 - acc: 0.5154\n", + "Epoch 3/5\n", + "25000/25000 [==============================] - 4s 169us/step - loss: 0.5657 - acc: 0.7562\n", + "Epoch 4/5\n", + "25000/25000 [==============================] - 4s 169us/step - loss: 0.3272 - acc: 0.8656\n", + "Epoch 5/5\n", + " 4288/25000 [====>.........................] - ETA: 3s - loss: 0.6739 - acc: 0.8270\n", + "\n", + "done.\n", + "Please invoke the Learner.lr_plot() method to visually inspect the loss plot to help identify the maximal learning rate associated with falling loss.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZhcZZn38e9dS3enO53O1p2ELCSQEAgQAgSQVRBEEAdmVBBGHRBnABVxG0dwfN1nxPEFfVVcAEcdFZFFmShhE5BFZEmAQEIICWFJAiF70ulOb1X3+8c51V3d6U4vqdOnqvv3ua6++pxTp6p+XanUXc95znkec3dERET2JBF3ABERKX4qFiIi0isVCxER6ZWKhYiI9ErFQkREeqViISIivUrFHaC/xo8f79OnT487hohISVm8ePEmd68d6P1LrlhMnz6dRYsWxR1DRKSkmNlre3N/HYYSEZFeqViIiEivVCxERKRXKhYiItIrFQsREemVioWIiPRKxUJEpMg1t2W4Z9l61mxpjC2DioWISJHbvquVS3+1mIdXbowtg4qFiEiRa80Ek9Slk/F9ZKtYiIgUudc3B4efLMYMKhYiIkXughseB+BXj+/ViB17RcVCRKRE7FMzIrbnVrEQESkR/3TcvrE9t4qFiEiR26+2CoDZE6pjy6BiISJS5E49sI7KsiTjRpbHlkHFQkSkyLVmnGQiznOhVCxERIreyxt3Ut/UFmsGFQsRkSL3yMpNcUeItliY2RlmtsLMVpnZld3cPs3MHjSzZ8zsOTN7d5R5RERkYCIrFmaWBK4DzgTmABeY2Zwuu30JuMXdDwfOB34UVR4RERm4KFsWRwOr3H21u7cANwPndNnHgVHhcg3wRoR5RERkgFIRPvZkYE3e+lrgmC77fBW418w+CVQBp3X3QGZ2CXAJwLRp0woeVESkmNWMSJMa5mdDXQD8wt2nAO8GfmVmu2Vy9+vdfb67z6+trR30kCIicZo6dgTzpo6ONUOUxWIdMDVvfUq4Ld9HgVsA3P1vQAUwPsJMIiIlJ5sFs6HbsngKmGVmM8ysjKADe0GXfV4HTgUws4MIikV8s3uIiBShrDsxH4WKrli4extwOXAPsJzgrKdlZvZ1Mzs73O1zwL+Y2RLgt8BF7u5RZRIRKUVZj/8K7ig7uHH3hcDCLtu+nLf8AnB8lBlERErdS2/tJDGED0OJiEiBvLi+PtbnV7EQESkBFx4b31wWoGIhIlL0RqSTlKeTsWZQsRARKXJZd2LuslCxEBEpdu6og1tERPZsSF9nISIihREUC7UsRESkB+5O1mFLQ0usOVQsRESK2KLXtgLwmydejzWHioWISBFrbs3GHQFQsRARKWovb9wZdwRAxUJEpKjd+dybcUcAVCxERIraeUcF0wIdvM+oXvaMloqFiEgRSyeDU2avPW9erDlULEREilhbJpjipyId78e1ioWISBF7cf0OgNgnP1KxEBEpYjc88goAzW3xnkKrYiEiUsROnDUegH3HVsaaQ8VCRKSI7V87kuryFKmk+ixERKQHrZksZan4P6rjTyAiIj1qaVOxEBGRXrSoZSEiIr1pactSFnN/BahYiIgUtdaMx965DSoWIiJFzd0pgloRbbEwszPMbIWZrTKzK7u5/btm9mz485KZbYsyj4hIqckUwZSqAKmoHtjMksB1wDuBtcBTZrbA3V/I7ePun8nb/5PA4VHlEREpRVkHK4JiEWXL4mhglbuvdvcW4GbgnD3sfwHw2wjziIiUHHcnGX+tiLRYTAbW5K2vDbftxsz2BWYAD0SYR0Sk5GSyQ/wwVD+dD9zm7pnubjSzS4BLAKZNmzagJ2hqzeAOI8qSuDstmSytGSedNMpTSQCy2WAo4MQAR3d0d9yDY4ytmSwJMxJmpBKGGTS0ZDAgYcF6MpG7rfPztWWy7GrNkEokSCeNZMJwB7OgSZqLl3XY2dxGSzjAWHNbhnQyQTqZoLIsiRm4Q9a9fbuIlJas+4A/kwopymKxDpiatz4l3Nad84FP9PRA7n49cD3A/PnzfSBhbl28lv+8czkV6QQ7mtrI5AqDwYh0EgcaWzKkEsbIilTwgU7wAQ1G7vPc8549607WnUzWaWnL0pLJdro9xwzSiQQtme5HjaxIJ6hIJ0knEzQ2t9HQ0m3NJJUw2rKOWbDcmunfS1FdnsIsKIYJC4pQwiBpRlV5iras09yaoTXr1De1ks2Pax2/kongtXGC+yYSRjqZ2O2MjVyxBEgkIJNxGloyZLLBa5Z7/rJUUMjKUgkqy1IkE8Ek9Q3NbWTcGVtVTjbrtGazZLLOiHSS8nSS8mSCdCp47qxDY3MbbVmnLJXAw07BVDLIkHvOinSS8lSCdCpBWTIoxrlCGuToWE+H9y1LJRiRTjKyPEVleYqqsiRV5SmqylJUlQfL5alEURxXlqEn/wtinKIsFk8Bs8xsBkGROB/4x647mdmBwBjgbxFmYe7kGs6dP4XWjDOyPMmoijTl6QQ7drXR2JIhYVBZnqItk2Vncxvu4Hj4O2g1hHnbC0IyEXxYmhnlqeDDJpkwkmakU4mglZHN0hwWkrGVZeFjBY+dzTptWaehuS3Ypy1LVXmKURVpKsuStGWDFkomLBDN4cU57k5zJktlOviwKk8nwYMPyUw2aGE0hgUn14ppas2wfVdr8NzuZMIil83SniGVNEakkyQTRnVFx8Blub/XCRZyk7EYQSsqm3VaMt7+GpH/N4YtGzwoUlVlSZKJoLBkssHr05p12jJZmlqzNLa0kXUoDwtHwmBrYwupRIJU2MpqaM4ELcO2LM2tWeqb2jALHntEwmhpy5JMJMi409yaJePeXtS2NrbQ0palNWxZBr+z4bZgvS3b/+8j1RUpJtVUUFWeYmR5itGVZYwekWZMZZpRI9LUjapgQnU5daMqmFRTQUU62f83sQxL2ayTKoIruCMrFu7eZmaXA/cASeC/3X2ZmX0dWOTuC8Jdzwdudu/uO3nhHDZ1NIdNHR3lU8gQkWvFtGW8vWDvasmws7mNxpY2GpozNIQtwOB3G29s28XG+mYamjPsaGpjzZZGtja2sqOptdvWZl11OVPGjGBiTQUz66o5ct8xnDBzfOwT3EjxyboXxfsi0j4Ld18ILOyy7ctd1r8aZQaR/kokjPJEkvIC/O/IZp36pjbW72hiQ30TG3Y0s27bLtZsaWTdtl08t3Y7dy1djzuMqyrj7bNreceBdZw+Z2JRjAck8csUyamzxdLBLTIkJRJGTWWamso0sydWd7tPU2uG+5dv4O5l63ngxQ38/ul1VJYlee8Rk7n0pP2ZGvOkNxKvoP8t7hQqFiKxq0gnOWvuJM6aO4lM1nlk5UbueGYdty5ay4Jn3+DzZxzIuUdOUT/HMJUN+9zipnauSBFJJoyTZ9fxvfMP577PvJ39akfyf+5YymnXPsRjL2+KO57EIJMtjsNQKhYiRWrauEp+/7Hj+OXFR5NOJvjwz57k7qXr444lg2xYDCQoInsnkTDefkAtd3zieA7eZxSf/t0z3PX8m3HHkkGULZKBBFUsREpAzYg0/33RUcyeOIqP/eZpbnh4ddyRZJAUy3AfKhYiJWL8yHJ+cdFRnDBzPP9513JueuJ1Ir48SYqA+8CHICokFQuREjKmqowbL5zPibNq+eIfnueHD6yKO5JELFMkp86qWIiUmIp0kp9fdBTvmTuJ/3f/Spa/uSPuSBIhnTorIgOWTBjfOOcQakak+eqCZXHHkQhldeqsiOyNMVVlfPyUmTzxyhYeX7057jgSkawOQ4nI3vrHo6cxqaaC7973UtxRJCLFMpCgioVICRtRluTi42fwxCtb+NNzb8QdRyIwHObgFpFB8OFj9yWdNL618EWaWrufOEtKVzarw1AiUgAV6SQ//uCRrNu2i2vuXRF3HCkwHYYSkYI59aA6ylIJbl28tn3KYBkadAW3iBSMmfH98w9nW2Mrv3zs1bjjSAG5o2IhIoXzroMncPLsWq65dwUb65vjjiMFolNnRaSgzIwvnXUQDS0Z/vfZdXHHkQLJuGtsKBEprJl11dRVl/PNO5fHHUUKJDh1Nu4UKhYiQ84Js8YD8Pza7TEnkUJwzWchIlG46syDAFi4VJMkDQVBB3fcKVQsRIac2upyTj2wjt89tYa2TDbuOLKXNFOeiETmzEMnsaWhhb++rAEGS13WIf5SoWIhMiSddEDQb/G/z+isqFKWmwlxyI8NZWZnmNkKM1tlZlf2sM95ZvaCmS0zs5uizCMyXNRVV/C+I6Zw/4sbyOqK7pKVmzV3SB+GMrMkcB1wJjAHuMDM5nTZZxZwFXC8ux8MfDqqPCLDzQmzxrF9VysvaCa9kpVtb1nEHIRoWxZHA6vcfbW7twA3A+d02edfgOvcfSuAu2+IMI/IsHL8zPGYwV9W6L9Vqcq1CYf62VCTgTV562vDbfkOAA4ws7+a2eNmdkaEeUSGlbrqCmbVjeTp17fFHUUGKDtc+iz6IAXMAk4GLgBuMLPRXXcys0vMbJGZLdq4ceMgRxQpXXOnjGbJmm3tHaVSWnL/bEVQKyItFuuAqXnrU8Jt+dYCC9y91d1fAV4iKB6duPv17j7f3efX1tZGFlhkqDlsSg2bG1p4eWND3FFkAHItiyHdwQ08BcwysxlmVgacDyzoss8dBK0KzGw8wWGp1RFmEhlWjp4xDkBTrpaojrOh4s0BERYLd28DLgfuAZYDt7j7MjP7upmdHe52D7DZzF4AHgQ+7+66ikikQGZPrGZURYrFr22NO4oMQDG1LFJRPri7LwQWdtn25bxlBz4b/ohIBE4/eCIPvLiB1kyWdDLubkrpj2K6REbvHJEh7rSD6tjS0MITq7fEHUX6azhclCcixeHk2XVUpBPc98L6uKNIP3Uchoo5CCoWIkNeRTrJQZNG8cu/vaahP0qMrrMQkUF15LQxACxSR3dJGS5XcItIkbjs5P0BdFZUiSm5loWZfcrMRlngZ2b2tJmdHnU4ESmM8SPLmVU3kr+t1pnppaQUr+C+2N13AKcDY4APA1dHlkpECu74meN58pXNNLdl4o4ifVSKQ5Tnkr4b+JW7L6M4Jm8SkT46YeZ4mlqzPP2aBhYsFaV4NtRiM7uXoFjcY2bVgCb3FSkhx+w3lmTCeOzlTXFHkT5q77Mogu/mfb2C+6PAPGC1uzea2VjgI9HFEpFCq65IM6tuJM9oyPKSUYp9FscCK9x9m5l9CPgSsD26WCIShRNmjueJVzbT1Kp+i1JQin0WPwYazeww4HPAy8D/RJZKRCJxxL5jaM04K9/aGXcU6YNSnFa1LRz07xzgh+5+HVAdXSwRicLsicF/2+XrNS93KVj6RnAApxhaFn3ts6g3s6sITpk90cwSQDq6WCIShenjqihPJVixvj7uKNIHl9/0DABO/MO09LVl8QGgmeB6i/UEs959J7JUIhKJZMI4YEK1ikWJeXRl/BdT9qlYhAXiN0CNmb0HaHJ39VmIlKDZE6t5UcWipBTDHOp9He7jPOBJ4FzgPOAJM3t/lMFEJBozxlexaWczy97QCY3FpKUty7fuWk59U+tut6WS8fdZ9PUw1L8DR7n7he7+T8DRwP+JLpaIRGXKmBEAnPX9R4viG6sE/udvr/LTh1bznXtW7HbbvKljBj9QF30tFgl335C3vrkf9xWRInLOvMnsX1sFwH0vvBVzGgFobGnjm3cuB2DTzubdbn/XwRMGO9Ju+vqBf7eZ3WNmF5nZRcCddJlbW0RKxy2XHgvAv93+XMxJBIJDUDn1TW3ty9PGVjKppoJxI8vjiNVJn06ddffPm9n7gOPDTde7+x+iiyUiUcp9+GxrbCWTdZLFMFLdMJY/geGOXR19FqmkMXdK/IegoO/XWeDutwO3R5hFRAZRXXU5G+qbeXjlRk6ZXRd3nGGtLdvRstiR17JoyzipIinkezwMZWb1Zrajm596M9MloCIl7E+fPAGAR1dqFNq4Pb56S/tyearjYzmTdVLJ4uge3mMKd69291Hd/FS7+6jBCikihVc3qoITZo7n7qXrdVZUzK747TPtyy+ur6exJWhdtGaypdGyEJGh7cxDJ7Ju2y5e3dwYdxTJs27rLiDXshgGxcLMzjCzFWa2ysyu7Ob2i8xso5k9G/78c5R5RKSzI6YFnadL1miOi2IwbWwlAFsbg07uoGVRHN/pI0thZkngOuBMYA5wgZnN6WbX37n7vPDnxqjyiMjuZtWNBODXj78WcxIBuOa8wwB4bm1QvDPZEung3ktHA6vcfbW7twA3EwxxLiJFItd5uui1rTS3aUKkuOWurs9doNeadZLD4DDUZGBN3vracFtX7zOz58zsNjObGmEeEenGV/4uaPDnn5Ejg2/8yDLGVJa1r7s7mayTHuqHofroj8B0d58L3Af8srudzOwSM1tkZos2btw4qAFFhrpz50/FDJ55fWvcUYatdNI4d/5UKtLJ9m0NLZmgWJTCqbN7aR2Q31KYEm5r5+6b3T03EMqNwJHdPZC7X+/u8919fm1tbSRhRYarkeUpDqir5ll1csfC3WnNOGVhUfjGOQcD8Oa24Iyo6oo+XzsdqSiLxVPALDObYWZlwPnAgvwdzGxS3urZwPII84hIDw6bWsOSNdt0vUUMcof/ysKL8UaGxeGOZ4Pv1pVlye7vOMgiKxbu3gZcDtxDUARucfdlZvZ1Mzs73O0KM1tmZkuAK4CLosojIj07ct8xbG1s5aW3dsYdZVh56a16LrjhcSA4FAVQVRYUi+sefDncXhyHoSJt37j7QrqMTuvuX85bvgq4KsoMItK7Y2aMA+CnD7/MtefNiznN8LGpvmM48txhqMqyzh/LZaniKBbFkUJEYrXvuOBisN8/vU6HogZRc97Q5OmwKOxfV9VpHxULESkaZsbM8AK9teFQExK9255e276cO9w0ekRZp31ULESkqFxzbnD18PPrNDf3YPlz3kyFucNQFenOH8tlRdJnURwpRCR2sydWU5ZKcMcz63rfWQriX07cr30514IwM2677Nj27Qkb+ldwi0gJqUgn+fDb9uXeF95iW2NL3HGGhUxe/1B+UZg/fWz78tiqzoel4qJiISLt5k6pAeCKm5+NOcnwkM2bT3X99u77imZPrB6sOHtUHJcGikhROPWgCQC0ZbK97CmF0JZXLPKXAR75t1PY0dTa9S6xUbEQkXYjy1O8/YBatjToMFSUNtY3U9/USiavQEwf1/mU2anh3BbFQsVCRDpZ9sZ2Nu1s4bFVmzhu5vi44wxJ7/zuQ2xrbGVsVRmVZUluufRYDplcE3esPVKfhYh08vGTZwLw0Esa4Tkq28KZ8LY0tNDYkin6QgEqFiLSxcUnzKCuupzNOhQleVQsRGQ3+46r5PUtjXHHkCKiYiEiu5k6tpInX9nSqQNWCufAIjkdtj9ULERkN3MmjQI09EdUcnNtA1x15oExJuk7FQsR2c0Zh0wE4Iu/fz7mJENTJuscOrmGx686lUvfvn/ccfpExUJEdjNlTHCO/wtv7qCxpS3mNENPxiGRMCbWVMQdpc9ULESkW/OmjgbgR+GMbVI42ayTLI7xAftMxUJEuvX98w8H4IcProo5ydBT39xGVXlpXROtYiEi3Zo2rmO4iZ3NOhRVSNsaWxhTWRyjyfaVioWI9Oh/Lj4agOO+dX/MSYaWLQ0tRTP0eF+pWIhIj47dfxwAO5rUsiiU1kyW+qY2Rlem447SLyoWItKjdDLBh942DUAj0RbIDx4I+oAaWzIxJ+kfFQsR2aPcsfVv3/VizEmGhu/fvxKA5W/uiDlJ/6hYiMgeferUWUDnKUBl4M49cgoA33n/YTEn6R8VCxHZo1QyweHTRvNmD9N+Sv88+eoWgJK6IA8iLhZmdoaZrTCzVWZ25R72e5+ZuZnNjzKPiAzM6BFp/rpqc9wxhoTXNpfmaL6RFQszSwLXAWcCc4ALzGxON/tVA58Cnogqi4jsncWvbQXgwz97QsN/DFNRtiyOBla5+2p3bwFuBs7pZr9vAN8GmiLMIiJ74Zrz5gHwyMpNfPrmZ2NOU/qOnj427gj9FmWxmAysyVtfG25rZ2ZHAFPd/c4Ic4jIXjp5dm37cq6VIQNjBsfsp2LRZ2aWAK4FPteHfS8xs0VmtmjjRs0LLDLY0skEX/m74Cjy5oYW7l66PuZEpSmTddwhlSi9c4uiTLwOmJq3PiXcllMNHAL8xcxeBd4GLOiuk9vdr3f3+e4+v7a2tuvNIjIIPnL8DG78p+C/52W/XhxzmtI096v3AJAqtSFnibZYPAXMMrMZZlYGnA8syN3o7tvdfby7T3f36cDjwNnuvijCTCKyF06bMyHuCCWrLZOlIbxquyKdjDlN/0VWLNy9DbgcuAdYDtzi7svM7OtmdnZUzysi0boynAa0vqk15iSlpbG1Y3iPcSU2iCBApAOqu/tCYGGXbV/uYd+To8wiIoWxz+hg/ujVGxs4LJwgSXq3K28sqNwAjaWk9HpZRCRW86YEBeLRVZtiTlJacgMHfu8D85gwqrSu3gYVCxHpp9ykSNfe91LMSUpLQziB1Iiy0uuvABULERmgTFYDC/bHrrDPoqqstKZTzVGxEJF++8xpBwDQ1FpaczLE6ZL/CU70rEiX5sduaaYWkVhNGRN0ci98/s2YkxS3HU2trNqwk9ZMlq2Nwdlj9SU6n7mKhYj02+HTgk7uz96yBB8m81xkss7Ofn7Qf/Z3Szjt2odYv71j6Lu5k2sKHW1QlObBMxGJ1X61I9uXZ1y1kCvPPJDL3r5/jImic829K3js5c3tY2L95ENHcsYhE/t035Ub6gH4zROvA3DaQRMYN7I8mqARU8tCRAbkW+89tH356rteZO3W0pynYU+eXbONHzywqtPgif059DZn0igAfvLQywD84zFT97R7UVOxEJEByR+JFuCEbz/IVxcsiylNNO5f/tZu2w6YMLKbPbvXmul8iK5mRHqvM8VFxUJEBmRSzYj2gQVzfvHYq6x8q563dpT+9DQNzW384IFVu23/yUOr21saL2/cyfQr7+Sjv3hqt/3cnT93KTajKlQsRGQYOm3OBP7w8eM6bXvndx/mmP+8P6ZEhXP6dx9uX3716rNY8pXTAdjZ3Mb7fvwYrZksp17zEAD3v7hht/u/taN5t22j1LIQkeHq8Glj+NSps+KOUVDNbRnWbdsFwMjy4DygroeQfvvk653WH+sy/Mn6sHWVSnQMR66WhYgMa5955wE8+oVTOm1bs6V0O7y3NXaMqJs/6N/EvDGdHlnZuTh8887lndY3hMXihrxDdaV6QR6oWIhIgUwZU8mXzjqoff2d330oxjR7J3/49avCIdkB/vfy4zlwYjUA970Q9Ef87pK3AfDCmzs6PcaG+uAw1IGTqtu3mZXepEc5KhYiUjD/fOJ+3PGJ4wFoas3GnGbgVm9sAODnFx3V6ZqSCaMquPvTJ3Xad/+67s+O2lDfjBnUluh1FV2pWIhIQc2bOpq5U4KrlJvbohs76pVNDSxY8sYe9/n+/SuZfuWdtGZ6LlyLX9tKayZLWybLjrBFccmvgmljqyu6v245/5Db2MqOiYzyn2djfRPjqspJJRP85V9PZsHlx/f+RxUxXcEtIgV34bHT+dytS7hn2Vucfdg+BX3sGx9ZzcMrN/HwSxsB2G98FYf0MIRGbhj1L9z2HNd+YN5ut6/aUM/7fvxYp22r//Pd7cuVPYwQO2VMJXd84njKkgkSCePq9x7Klb9/nvXbm5g6tpI3tu3i3mVvtc9bMX18Vf//0CKjloWIFNykmuBD8orfPlPQx3V3vnnn8vZCAfCeHzza6/2WrN3W7fbtu3Yf6ym/7yHXP9GdeVNHM2ef4Art3O8T/+tBslnnuKsfYHNDC3WjhsYhKFCxEJEIHDl9TPtyIee92NXDkOjdDWbYkDfo36SaEZ1uW/zaFj57y7Ns39Wy2/0+/LMnADhsSg2JRN86pHPDegAsXNoxHMiqDTv7dP9SoGIhIgVXnkryH/9wCADrtu4q2OM2tnQuFicdEAw5sjXvVNecN7d3PO+jqzbxzOsd4ztd9POn+P3T69qvxL7m3MP45cVHd3qsH3/oyD7nSiUT7Bceanpu7fb27RXp0pwVrzsqFiISiRnhh+eaAg4w+NQrW9qXH7vyHXzomGlA99/guxaQGx5ZDcDti9dS3xS0Oq57MBjg712HTOTtB3Qe6yp3KK2vvnbOwQD86m+vtW/74+Un9OsxipmKhYhEYuqYYK7u1wt4cd4zazr6HibVVHDY1GBejafzWg03PfE6a7Y0smJ9MDx4roWz8Pn1tLRl+dytS3Z73NxV2vn6e01E7hTb3KGyP3/2pJKdb7s7OhtKRCIxqaaCVMIKeiX3uKrgNNWlX3sXZtZ+ttHVd73IpSftx67WDF/8w/MAjA+vbzhxZkeL4bXNDXt8/Be/cQYAZcn+f4/ep6aCmhFptu9q5cxDJjKzrufO8VKkloWIRCKVTLDP6BG8VsBiUd/URsKgqptv7Jf+anGn/oKZdcFhsGnjKjn3yCkA/Hn57gP+5Z/aW5FOUpFO9rljO5+ZMX1c0Jq6a+n6ft+/2KlYiEhkXt/SyJ3PvcmDK3b/kB6IHz64iqx3PkSUu9jt3hfe4vzrH2/f/vjqjv6N3EWC3777xfZtR0wbzUGTRnHteYcVJBt0tGYOmTyqlz1LT6TFwszOMLMVZrbKzK7s5vbLzOx5M3vWzB41szlR5hGReHzk57vP91Aoc6eM7vG2UeEV2GfPm9xp+0XHTeeWS4/lrk+dSGoAh5x6Mm5kcJis0BciFoPIioWZJYHrgDOBOcAF3RSDm9z9UHefB/wXcG1UeURk8P31yne0L+9q2buhP3LXUpzU5awlgM+984Bu73PzJccCwfDi+d/23zN3UkGLRE6uM//5dTt62bP0RNnBfTSwyt1XA5jZzcA5wAu5Hdw9/xWtAgp39Y6IxG7y6I6L4R56aQNnHDJpwI/1bHgm1KxuBu775KmzqK5I8eCKjfzswvmkkgncvdPhqgnVFSwl+MjpaRiPvXXpSfvz+OotvOvgCZE8fpyiPAw1GViTt7423NaJmX3CzF4maFlcEWEeEYnBki8HM8xd9pzuQOAAAAxxSURBVOun9+pxfvpQcJ1E7kK6ri46fga/vPjo9hZD11Nff/yhI/ngMdP42tkHtw/PUWinHFjHq1efxXvm6jBUwbn7de6+P/AF4Evd7WNml5jZIjNbtHHjxu52EZEiVVPZMTvc3szN/a5Dgm/rXzjjwF727F5ZKsF//MOhXHjc9AFnGM6iLBbrgKl561PCbT25Gfj77m5w9+vdfb67z6+t3f14pYgUt/PmB6eufve+l9jWuPt4TH2xeWdwv9rqoTM4XymJslg8BcwysxlmVgacDyzI38HM8ifuPQtYGWEeEYnJF98dzKB381NrOPwb9w3oMXLTlkY5R4b0LLJi4e5twOXAPcBy4BZ3X2ZmXzezs8PdLjezZWb2LPBZ4MKo8ohIfEbnTRDUzQCxANy6aA3Tr7yTv67a1G1BqAwvxMufB1sGT6TDfbj7QmBhl21fzlv+VJTPLyLF4wcXHM4nw/kttja0MKaqDHdnx642/vryJj5/23MAfPDGJzhy3zHc/rHjOt3/HQfWseyNHYwbItOUlprYO7hFZHj4u8P24ZPvmAnQfihqwZI3OOzr9/Lx33Q+U2rxa1v53p9f4nO3LGm/vmJbYytj8jrLZXCpWIjIoPnEKTPbl+9eur7TcN4AE/Jmlvven1dy+9Nr+euqzQBsaWhhbFUZEg8VCxEZNBXpJP92xmwALvv1YjY3dD4z6vGrTuXhz5/SadsVNz+Du/P6lkb2Gd15xjsZPCoWIjKoPnjMvu3Lr2zqGDJ8v/FVmBnTxlVy5xUdkwZtaWjhN0+8zs7mNspT+siKi155ERlUNSPSfO3sg9vXz5obDAHyh48f377t4H1qePQLHS2ML92xFOgY8kMGn4qFiAy694fzSwBccuJ+vHr1WZ2u9AaYMqZyt0NS1RXq4I6LioWIDLqq8hSnzA5GY6gZ0XMBqBvV+TTZ3BSpMvg0raqIxOKGf5rPkrXbmD6+qsd9KtIdM+L9+qPHMKlGHdxxUctCRGKRSiY4ct+xve730RNmsH9tFSfMGj8IqaQn5j1de1+k5s+f74sWLYo7hohISTGzxe4+f6D3V8tCRER6pWIhIiK9UrEQEZFeqViIiEivVCxERKRXKhYiItIrFQsREemVioWIiPSq5C7KM7ONwDZge7ipppfl3O/xwKYBPGX+Y/bn9q7b97Re6rm72zaYuXvb1tPfUKjchXqtu24bSu+R/OVSyl3s7+2u63vKPdvdq/sfO+TuJfcDXN/X5bzfi/b2ufpze9fte1ov9dw9bBu03L1t6+lvKFTuQr3WfXlflOp7pFRzF/t7ezBy535K9TDUH/uxnL9tb5+rP7d33b6n9VLP3dPfMhADyd3btp7+hkLlLtRr3XXbUHqP5C+XUu5if293XY8iN1CCh6EGyswW+V6MixIX5R5cpZi7FDODcg+2vc1dqi2Lgbg+7gADpNyDqxRzl2JmUO7Btle5h03LQkREBm44tSxERGSAVCxERKRXKhYiItIrFQvAzE40s5+Y2Y1m9ljcefrKzBJm9h9m9gMzuzDuPH1lZieb2SPha35y3Hn6ysyqzGyRmb0n7ix9ZWYHha/zbWb2sbjz9JWZ/b2Z3WBmvzOz0+PO01dmtp+Z/czMbos7y56E7+Vfhq/xB/tyn5IvFmb232a2wcyWdtl+hpmtMLNVZnblnh7D3R9x98uAPwG/jDJvXr69zg2cA0wBWoG1UWXNV6DcDuwEKhiE3AXKDPAF4JZoUu6uQO/t5eF7+zzg+Cjz5uUrRO473P1fgMuAD0SZNy9fIXKvdvePRpu0e/3M/17gtvA1PrtPT7A3V/QVww9wEnAEsDRvWxJ4GdgPKAOWAHOAQwkKQv5PXd79bgGqSyU3cCVwaXjf20oodyK83wTgNyWS+Z3A+cBFwHtK5bUO73M2cBfwj6WUO7zfNcARJZh7UP4/7kX+q4B54T439eXxU5Q4d3/YzKZ32Xw0sMrdVwOY2c3AOe7+LaDbQwhmNg3Y7u71EcZtV4jcZrYWaAlXM9Gl7VCo1zu0FSiPIme+Ar3WJwNVBP/RdpnZQnfPFnvu8HEWAAvM7E7gpugStz9fIV5vA64G7nL3p6NNHCjwe3vQ9Sc/QYt+CvAsfTzCVPLFogeTgTV562uBY3q5z0eBn0eWqG/6m/v3wA/M7ETg4SiD9aJfuc3svcC7gNHAD6ON1qN+ZXb3fwcws4uATVEXij3o72t9MsEhh3JgYaTJ9qy/7+1PAqcBNWY2091/EmW4Pejv6z0O+A/gcDO7Kiwqceop//eBH5rZWfRxOJChWiz6zd2/EneG/nL3RoIiV1Lc/fcEha7kuPsv4s7QH+7+F+AvMcfoN3f/PsEHWklx980E/SxFzd0bgI/05z4l38Hdg3XA1Lz1KeG2Yqfcg6cUM4NyD7ZSzZ1TsPxDtVg8BcwysxlmVkbQMbkg5kx9odyDpxQzg3IPtlLNnVO4/IPdYx/BGQC/Bd6k4/TRj4bb3w28RHAmwL/HnVO5lVm5lbuU82sgQRER6dVQPQwlIiIFpGIhIiK9UrEQEZFeqViIiEivVCxERKRXKhYiItIrFQuJnJntHITnOLuPw4wX8jlPNrPjBnC/w83sZ+HyRWYW1/hYnZjZ9K7DW3ezT62Z3T1YmaR4qFhIyTCzZE+3ufsCd786gufc0/hpJwP9LhbAFynBcY8A3H0j8KaZDcrcGFI8VCxkUJnZ583sKTN7zsy+lrf9DjNbbGbLzOySvO07zewaM1sCHGtmr5rZ18zsaTN73swODPdr/4ZuZr8ws++b2WNmttrM3h9uT5jZj8zsRTO7z8wW5m7rkvEvZvY9M1sEfMrM/s7MnjCzZ8zsz2Y2IRwK+jLgM2b2rAWzLdaa2e3h3/dUdx+oZlYNzHX3Jd3cNt3MHghfm/vDYfMxs/3N7PHw7/1mdy01C2Y+u9PMlpjZUjP7QLj9qPB1WGJmT5pZdfg8j4Sv4dPdtY7MLGlm38n7t7o07+Y7gD7NriZDSNyXqOtn6P8AO8PfpwPXA0bwReVPwEnhbWPD3yOApcC4cN2B8/Ie61Xgk+Hyx4Ebw+WLgB+Gy78Abg2fYw7BeP4A7ycYpjsBTCSYT+P93eT9C/CjvPUx0D7awT8D14TLXwX+NW+/m4ATwuVpwPJuHvsU4Pa89fzcfwQuDJcvBu4Il/8EXBAuX5Z7Pbs87vuAG/LWawgmu1kNHBVuG0Uw0nQlUBFumwUsCpenE06cA1wCfClcLgcWATPC9cnA83G/r/QzuD8aolwG0+nhzzPh+kiCD6uHgSvM7B/C7VPD7ZsJJnW6vcvj5IY3X0wwV0N37vBgzokXzGxCuO0E4NZw+3oze3APWX+XtzwF+J2ZTSL4AH6lh/ucBswxs9z6KDMb6e75LYFJwMYe7n9s3t/zK+C/8rb/fbh8E/B/u7nv88A1ZvZt4E/u/oiZHQq86e5PAbj7DghaIQRzGcwjeH0P6ObxTgfm5rW8agj+TV4BNgD79PA3yBClYiGDyYBvuftPO20MJug5DTjW3RvN7C8E83MDNLl711kAm8PfGXp+DzfnLVsP++xJQ97yD4Br3X1BmPWrPdwnAbzN3Zv28Li76PjbCsbdXzKzIwgGjfummd0P/KGH3T8DvAUcRpC5u7xG0IK7p5vbKgj+DhlG1Gchg+ke4GIzGwlgZpPNrI7gW+vWsFAcCLwtouf/K/C+sO9iAkEHdV/U0DEHwIV52+uB6rz1ewlmeAMg/Obe1XJgZg/P8xjBENIQ9Ak8Ei4/TnCYibzbOzGzfYBGd/818B2CuZhXAJPM7Khwn+qww76GoMWRBT5MME9zV/cAHzOzdHjfA8IWCQQtkT2eNSVDj4qFDBp3v5fgMMrfzOx54DaCD9u7gZSZLSeYd/nxiCLcTjB08wvAr4Gnge19uN9XgVvNbDGwKW/7H4F/yHVwA1cA88MO4RfoZsY0d3+RYKrQ6q63ERSaj5jZcwQf4p8Kt38a+Gy4fWYPmQ8FnjSzZ4GvAN909xbgAwRT7y4B7iNoFfwIuDDcdiCdW1E5NxK8Tk+Hp9P+lI5W3CnAnd3cR4YwDVEuw0quD8GCuZKfBI539/WDnOEzQL2739jH/SuBXe7uZnY+QWf3OZGG3HOeh4Fz3H1rXBlk8KnPQoabP5nZaIKO6m8MdqEI/Rg4tx/7H0nQIW3ANoIzpWJhZrUE/TcqFMOMWhYiItIr9VmIiEivVCxERKRXKhYiItIrFQsREemVioWIiPRKxUJERHr1/wH14Gs4kqr7HAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "learner.lr_find()\n", + "learner.lr_plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "begin training using triangular learning rate policy with max lr of 0.005...\n", + "Train on 25000 samples, validate on 25000 samples\n", + "Epoch 1/2\n", + "25000/25000 [==============================] - 6s 233us/step - loss: 0.5778 - acc: 0.7344 - val_loss: 0.4304 - val_acc: 0.8544\n", + "Epoch 2/2\n", + "25000/25000 [==============================] - 6s 235us/step - loss: 0.3212 - acc: 0.8869 - val_loss: 0.3050 - val_acc: 0.8854\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learner.autofit(0.005, 2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/toxic_comments-bigru.ipynb b/examples/text/toxic_comments-bigru.ipynb similarity index 100% rename from examples/toxic_comments-bigru.ipynb rename to examples/text/toxic_comments-bigru.ipynb diff --git a/examples/text/toxic_comments-fasttext.ipynb b/examples/text/toxic_comments-fasttext.ipynb new file mode 100644 index 000000000..75f13ba48 --- /dev/null +++ b/examples/text/toxic_comments-fasttext.ipynb @@ -0,0 +1,226 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline\n", + "import os\n", + "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\";\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"; \n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "import ktrain\n", + "from ktrain import text" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, we will classify Wikipedia comments into one or more categories of so-called *toxic comments*. Categories of toxic online behavior include toxic, severe_toxic, obscene, threat, insult, and identity_hate. The dataset can be downloaded from the [Kaggle Toxic Comment Classification Challenge](https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge/data) as a CSV file (i.e., download the file ```train.csv```). We will load the data using the ```texts_from_csv``` method, which assumes the label_columns are already one-hot-encoded in the spreadsheet. Since *val_filepath* is None, 10% of the data will automatically be used as a validation set.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Word Counts: 196995\n", + "Nrows: 143613\n", + "143613 train sequences\n", + "Average train sequence length: 66\n", + "15958 test sequences\n", + "Average test sequence length: 66\n", + "Pad sequences (samples x time)\n", + "x_train shape: (143613,150)\n", + "x_test shape: (15958,150)\n", + "y_train shape: (143613,6)\n", + "y_test shape: (15958,6)\n" + ] + } + ], + "source": [ + "DATA_PATH = 'data/toxic-comments/train.csv'\n", + "NUM_WORDS = 50000\n", + "MAXLEN = 150\n", + "(x_train, y_train), (x_test, y_test), preproc = text.texts_from_csv(DATA_PATH,\n", + " 'comment_text',\n", + " label_columns = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"],\n", + " val_filepath=None, # if None, 10% of data will be used for validation\n", + " max_features=NUM_WORDS, maxlen=MAXLEN,\n", + " ngram_range=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Is Multi-Label? True\n", + "compiling word ID features...\n", + "max_features is 49350\n", + "done.\n" + ] + } + ], + "source": [ + "model = text.text_classifier('fasttext', (x_train, y_train), \n", + " preproc=preproc)\n", + "learner = ktrain.get_learner(model, train_data=(x_train, y_train), val_data=(x_test, y_test))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "simulating training for different learning rates... this may take a few moments...\n", + "Epoch 1/5\n", + " 47840/143613 [========>.....................] - ETA: 31s - loss: 0.4965 - acc: 0.7510\n", + "\n", + "done.\n", + "Please invoke the Learner.lr_plot() method to visually inspect the loss plot to help identify the maximal learning rate associated with falling loss.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAELCAYAAADURYGZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhU5dnH8e+dyUZWloQtYQmrBkGWACqKoIiiLbgLLlVrpbbSWtva2tqq1db6Wu0qtaJttVp30VJFcQOhIELYCQiEgBDWEJaEhOz3+8cMcQhJSCBnTmbm/lzXXJxt5vxygLlzznPO84iqYowxJnxFuB3AGGOMu6wQGGNMmLNCYIwxYc4KgTHGhDkrBMYYE+asEBhjTJhztBCIyCUiskFEckXk3nrWdxeRuSKyQkRWi8ilTuYxxhhzPHHqOQIR8QAbgYuAfGApMEVV1/ltMwNYoapPiUgmMFtVezb2uSkpKdqzZ6ObGGOMqWPZsmX7VDW1vnWRDu53BJCrqnkAIvIKMAlY57eNAkm+6WRg54k+tGfPnmRnZ7dwVGOMCW0i8mVD65y8NJQGbPebz/ct8/cgcKOI5AOzge/V90EiMlVEskUku6CgwImsxhgTttxuLJ4CPKeq6cClwAsiclwmVZ2hqlmqmpWaWu+ZjTHGmJPkZCHYAXTzm0/3LfN3G/AagKp+BsQCKQ5mMsYYU4eThWAp0FdEMkQkGpgMzKqzzTbgQgAROR1vIbBrP8YYE0COFQJVrQKmAXOA9cBrqpojIg+JyETfZj8CbheRVcDLwC1q3aEaY0xAOXnXEKo6G28jsP+y+/2m1wGjnMxgjDGmcW43Frdqa/IPUV3T8AlKTY2Sf6A0gImMMablOXpGEKyeX7SVB2bl1M4v/+VFPDJ7Pe+t2cUDEwdwoKSC3773Re360f1SmXpeL07rkkhKQgwARWWV7DhwhIyUePYdLudbz2eTV1DCHef34u6L+iEiTc5zsLSC+JhIojzO1e0jFdW0ifawt6iM9vHRRDZxX5XVNY7mMsY4z7Eni52SlZWlTjxQVlZZzfS5ufzlk9xT+py+HRMYl9mJp+ZtbnCbXinx9OmYwK5DZfztpmGktW1DZXUN8zcW0C4+mkFpySzfdpDnP9vKii8PsPNQWe177xzbm6TYKDbsKeahSWeQEHPytVxVmbVqJ88syGPtjqLa5WemJzP9hqGkt4tj/a4invhgI1EeYcfBI0RGCJMGp/H8oq3k7SsBYMqIbjxyxUBEhKrqGg4dqaRDQgzLtx1gcV4h12Z1qy2Qxhh3iMgyVc2qd50VAthbVMbI336M/6F4+PIzmDioK0/O3cQzC7bQo0Mcr0w9i2fmb2Fh7j7+euNQyitr6NcpgTeW5fOflTvZuKeYwpIKAFISoimvrKGypoayyhp+PL4fu4vKeHHxtmZl650aj4iQu/fwcesiI4Qnrx/KJWd0btJnbdpTzP3/ySE+xsOmvYf5svDYy1oDuiZxsLSSHQePAPCTS/rz2PsbmvTZY/qn4hHhi93Fte+v6ztjetO/UyKXD6n7XKExxmlWCBox94u93PrcUgAyuyTxh+sG0y4uio5JsYD3t+aP1+9lRK/2JMVGNfpZ1TXK2h2H2La/lIsHdCY6sv5LJvM3FrD9QCmb95awYFMBm3xf8qP6dKBTYiwzV+wgyiNMv34o4wd0rv3s6hrvb/CHyyqpqlGe+GAj1apMGd6NhNhIvn5mV1ZtP8iCTfuoqlZ+d80gEmOj2FZYym/fW8+CTfs4XF51TJZrs9J5cOIAYiI9eCIEVeWfC7fy0DvenkASYiJ5aNIAPBHCmH4dKauq5h8LtzDhjC4M7tYWVeXnb63h5SXbj/nchJhIDpdXcfWwdPIPlLI4b3/tunN6d+Cftw4nJtLT1L8mY8wpskLQgL1FZYz+3VzKKmuYNrYPP764f4t8bnPV1CgiIOL9Iq6s1gaLiL+dB49w2Z8XcKC0st71R7+M/ef/76pBZKTEEx/jobyqhr4dE+ptr1i1/SD/WLiFW0dlMLhb20ZzqCprdxRRVFZJRko8Xdu2OW6bJVv2szB3HzNX5LN9v/eM4fbzMrg2qxt9OyWe8Gc1xpwaKwQ+qsoLi79kfGZnlm7dz/deXgHA9y7ow4/Gu1METtWuQ0dYmFtIjSo/m7mGzkmxPHLlQI5UVHHHi8sBiBC45+LTuHVUT2Kj3P8t/Ff/zeGfC7fWzj921SCuHd6t4TcYY06ZFQKfd1fv4s6XvF+OnZNi2V1UxsiM9rxw28gm/Qbe2uUfKCUlIab2y/5ASQWV1TW1l7lak7LKaj5ct4e7X11JtSo3n92TzC5JXDk0rcl3LBljms4KAd7r8t/4x5Jjll08oBNP31TvcTEBsq2wlInT/8dB3+WtaE8ED0zM5Kqh6XgixG5NNaaFNFYIwuY5gt1F3lswu7ePY9/hckorqjmtc9IJ3mWc1r1DHJ/+eCz5B0t54oONfPLFXu57ay33vbW2dpvnbh3OmP4dXUxpTGgLmzOC8qpqfv/hRob3aM/ofqk8syCPG8/qQXKbxu8EMoF1qLSS77+ygk17io95fiIlIZoXvzWSR2Z/wTm9O/CNs3sQFx02v8cYc8rs0pAJSnuKyli3s6j29l5/7eOj+fPkIZzb9/heyw+XVxEf7WnW09vGhDorBCaoVdcod72ygnkbCnjqxqFUVNVw78w1FBSXEx0ZQZ/UBGpUGdytLftLKvhg3R46Jsbwg3H9mDKimxUEY7BCYEJQQXE5Fzwxj+Kyqka369sxgWuzujFpSFc6Jra+u6eMCRQrBCZkbd9fSlJsFFGRwvpdRWzeW8I1Wels21/KlBmLj2ln6N8pkee+OZwuycc/8GZMqLNCYMJSTY3ykzdXM29DASXlVRyprGZkRnte/fbZbkczJuDs9lETliIihMevORPwtjP84m1vn0hzcnZz8YCmddRnTDiwp3VMWPBECA9OHMCg9GS+/cIy1uQfcjuSMa2GFQITNmIiPfzya5kA3PDsYtbusGJgDFghMGFmeM/2zPnBaGKiPNzyzyX1jvNgTLixQmDCTv/OiTx/6wiOVFQzecZnVgxM2HO0EIjIJSKyQURyReTeetb/QURW+l4bReSgk3mMOSqzaxJv3zmKiqoaxv3+U3742kpKKxp/JsGYUOVYIRARDzAdmABkAlNEJNN/G1W9W1UHq+pg4C/ATKfyGFNX306JPH7NmbSJ8jBz+Q4y75/D9c8s5lADA/0YE6qcPCMYAeSqap6qVgCvAJMa2X4K8LKDeYw5zvgBnVn5wEX89JLTiPIIizYXMvyRj3hn9U63oxkTME4WgjTAfyDbfN+y44hIDyAD+KSB9VNFJFtEsgsKClo8qAlvMZEevjOmN5t+cymPX3MmFVU1THtpBY/MXk+wPXBpzMloLY3Fk4E3VLW6vpWqOkNVs1Q1KzU1NcDRTDi5elg6c388hoFpycyYn8ev313vdiRjHOdkIdgB+A9Em+5bVp/J2GUh00pkpMTzmq8bir//bwtzv9jrciJjnOVkIVgK9BWRDBGJxvtlP6vuRiJyGtAO+MzBLMY0S5toD6seGE+EwK3PLeXxORvcjmSMYxwrBKpaBUwD5gDrgddUNUdEHhKRiX6bTgZeUbsYa1qZ5DZRPHfrCACenJvL1n0lLicyxhnW+6gxJ7Dz4BHOefQT4qI95PzqYhvoxgSlxnofbS2Nxca0Wl3btmHc6Z0orajms82FbscxpsVZITCmCZ68fggpCTH8/K01VFbXuB3HmBZlhcCYJoiN8vDA1zPZWljKPa+vcjuOMS3KCoExTXTZwC5kpMTz9sqd9uSxCSlWCIxpoogI4c3vnEOUR5j20goW51l7gQkNVgiMaYb28dHMmnYuAHe/upLqmuC6686Y+lghMKaZTu+SxKNXDmTXoTJ6/3y2jXRmgp4VAmNOwnXDu3HbuRkAXPnUIrbvL3U5kTEnzwqBMSdBRPjl1zJ553vnEu2J4N6Zq92OZMxJs0JgzCk4Iy2Zuy/qx8LcQlZutwH2THCyQmDMKbpueDeiIyO4fPpCthXaJSITfKwQGHOKEmIiuevCvgD8+t11Lqcxpvki3Q5gTCi4c2wfSsqreOrTzWzZV0JGSrzbkYxpMjsjMKaF3DKqJ1GeCGbMz3M7ijHNYoXAmBbSMTGWywd35eUl25i/0cbWNsHDCoExLWjKiO4AfOMfS9i0p9jlNMY0jRUCY1rQkO7tmHHTMGKjInjoHWs4NsHBCoExLWz8gM7cdWE/FmzaZwPZmKBghcAYB1w/sjvRnghu+vvnFJdVuh3HmEZZITDGAcltorjn4v5U1SjPLNjidhxjGuVoIRCRS0Rkg4jkisi9DWxzrYisE5EcEXnJyTzGBNLto3txfr9U3lyW73YUYxrlWCEQEQ8wHZgAZAJTRCSzzjZ9gZ8Bo1R1APADp/IY44Yx/VPZcfCIdVVtWjUnzwhGALmqmqeqFcArwKQ629wOTFfVAwCqutfBPMYE3BVD0vBECO+t3eV2FGMa5GQhSAO2+83n+5b56wf0E5GFIrJYRC5xMI8xAdc2LprhPdvx0Tr7Hce0Xm43FkcCfYExwBTgGRFpW3cjEZkqItkikl1QYE9smuAy4YwubNhTzIptB9yOYky9nCwEO4BufvPpvmX+8oFZqlqpqluAjXgLwzFUdYaqZqlqVmpqqmOBjXHC5YPTiI/2cMVfF/FlYYnbcYw5jpOFYCnQV0QyRCQamAzMqrPN23jPBhCRFLyXiqzHLhNSkuOi+MXXvPdJPPreFy6nMeZ4jhUCVa0CpgFzgPXAa6qaIyIPichE32ZzgEIRWQfMBe5RVXsU04ScKSO6c9eFfXlv7W6mz811O44xxxBVdTtDs2RlZWl2drbbMYxpttKKKqbMWMyq/EP865sjGN3PLnOawBGRZaqaVd86txuLjQkbcdGR/HHyEAB+9PoqqqprXE5kjJcVAmMCKCMlnseuHkRBcTkfrbdbSk3rYIXAmAC7YkgaGSnx/PGjjQTbpVkTmqwQGBNgUZ4IvjOmN1/sLuatFXXvqDYm8KwQGOOCa4al0ys1nn9/vs3tKMZYITDGDSLCDSN7sOzLAyzOszumjbusEBjjkhtGdic+2mPdVBvXWSEwxiWxUR4mDk7j9WX5rNx+0O04JoxZITDGRVNH9wLgmfnWs4pxjxUCY1yUkRLP7edlMCdnN3uKytyOY8KUFQJjXHbjWT2oVuXlJXYHkXGHFQJjXNajQzyjeqfwn5U73Y5iwpQVAmNagQkDO7NlXwkf5Ox2O4oJQ1YIjGkFrhnWjW7t2/Dsgi1uRzFhyAqBMa1AdGQEN4zswZKt+20UMxNwVgiMaSUuyuwEwMJce9LYBJYVAmNaiV4p8XRr34YXF39pvZKagLJCYEwrISJMG9uHdbuK7EljE1BWCIxpRSYM7EJMZAQzl1v31CZwrBAY04okxUZx6cAuvLViByXlVW7HMWHCCoExrcyNZ/XgcHmVPWBmAsbRQiAil4jIBhHJFZF761l/i4gUiMhK3+tbTuYxJhgM7d6W07sk8YI1GpsAcawQiIgHmA5MADKBKSKSWc+mr6rqYN/rWafyGBMsRISbzurB+l1FLN9mjcbGeU6eEYwAclU1T1UrgFeASQ7uz5iQMWlwVxJiInlx8ZduRzFhwMlCkAZs95vP9y2r6yoRWS0ib4hIt/o+SESmiki2iGQXFBQ4kdWYViU+JpKrhqbxn5U7WJN/yO04JsS53Vj8X6Cnqg4CPgSer28jVZ2hqlmqmpWamhrQgMa45btj+xAT6eHR99e7HcWEOCcLwQ7A/zf8dN+yWqpaqKrlvtlngWEO5jEmqHRKiuX20b1YtLmQ3L3FbscxIczJQrAU6CsiGSISDUwGZvlvICJd/GYnAvarjzF+bhzZHVX408e5dgeRcYxjhUBVq4BpwBy8X/CvqWqOiDwkIhN9m31fRHJEZBXwfeAWp/IYE4w6JsVy59je/HfVTlZYtxPGIRJsv2VkZWVpdna22zGMCZhDpZUM+/WH3HZeBj+bcLrbcUyQEpFlqppV3zq3G4uNMSeQHBfF2b078EHOHrs8ZBxhhcCYIHDJGd6hLOdv2ud2FBOCrBAYEwSuGppOUmwks6z/IeMAKwTGBIHYKA/jMjvx0fo9VFbXuB3HhBgrBMYEicsGduHQkUq+//IKt6OYENOkQiAid4lIknj9XUSWi8h4p8MZY75ywWkdAXg/Zzc1NdZobFpOU88IvqmqRcB4oB1wE/CoY6mMMccRER6/5kxUYdPew27HMSGkqYVAfH9eCrygqjl+y4wxAXJWr/YAzFyR73ISE0qaWgiWicgHeAvBHBFJBKzFypgAS28Xx3l9U/gwZ4/bUUwIaWohuA24FxiuqqVAFHCrY6mMMQ0a278jeftK2FZY6nYUEyKaWgjOBjao6kERuRH4BWCdpBvjgjH9vV2xf7rJxuYwLaOpheApoFREzgR+BGwG/uVYKmNMgzJS4unePo55X+x1O4oJEU0tBFXq7eRkEvCkqk4HEp2LZYxpiIgw4YzOzN2wl32Hy0/8BmNOoKmFoFhEfob3ttF3RSQCbzuBMcYFXz+zKzUKn6y3swJz6ppaCK4DyvE+T7Ab72hjv3MslTGmUQO6JpHWtg0frLO7h8LBcwu3sHTrfsc+v0mFwPfl/28gWUS+BpSpqrURGOMSEWHc6R35+Is97Dh4xO04xmEP/ncd1/ztM8c+v6ldTFwLLAGuAa4FPheRqx1LZYw5oZvO7okqzFxmD5eFskCMQdHUS0P34X2G4GZV/QYwAvilc7GMMSfSp2MCQ7q35cP1dnkolJVVfvXs7u5DZY7so6mFIEJV/VulCpvxXmOMQy48rSOr8w+xt9iZLwjjvsPlVbXTq/KdGbe6qV/m74vIHBG5RURuAd4FZjuSyBjTZGN9PZLO22APl4WqzQVfdTBY7VCvs5FN2UhV7xGRq4BRvkUzVPUtRxIZY5oss0sSnZNi+WT9Xq7N6uZ2HOOAvIISABb8ZCzd2sc5so8mX95R1TdV9Ye+V5OKgIhcIiIbRCRXRO5tZLurRERFJKupeYwx3ruHxp7WkQWbCqiosn4gQ9HP31oDQPv4aMf20WghEJFiESmq51UsIkUneK8HmA5MADKBKSKSWc92icBdwOcn/2MYE74uOK0jJRXVzN1gD5eFsrhoj2Of3WghUNVEVU2q55Woqkkn+OwRQK6q5qlqBfAK3i4q6noY+D/AWruMOQmj+6WQ1rYN//psq9tRjAN6p8YzIqM9Is4NAePknT9pwHa/+XzfsloiMhTopqrvNvZBIjJVRLJFJLugwBrFjPEXE+nhuuHdWJhbyJeFJW7HMS2sslrpmhzr6D5cuwXU11/R7/H2ZtooVZ2hqlmqmpWamup8OGOCzFXD0gGYk7Pb5SSmpZVWVBMX06T7ek6ak4VgB+B/G0O6b9lRicAZwDwR2QqcBcyyBmNjmi+tbRv6dUpg/sZ9bkcxLai4rJIDpRWkONhQDM4WgqVAXxHJEJFoYDIw6+hKVT2kqimq2lNVewKLgYmqmu1gJmNC1vn9UlmyZT+F1jV1yFi5/SDVNcqIjA6O7sexQqCqVcA0YA6wHnhNVXNE5CERmejUfo0JV9dkdaOiuoaXl2xzO4ppIavzvQNBDkxPdnQ/jl54UtXZ1HkCWVXvb2DbMU5mMSbU9euUyKg+HXhjWT7TLujrdhzTAnYdOkK7uCiS2zg7/Iv1F2RMCLno9E5sLSy1ge1DxOGyKhJinW0oBisExoSU8/p576qbbwPbh4TD5VUkxDg/GKQVAmNCSK+UeLomx7Jos909FAo+Wr+XjXuKHd+PFQJjQoiIcE6fFBZtLnSsp0oTGDW+v7+eHZzpaM6fFQJjQsy5fVI4WFrJup2NdgdmWrkjldUAXDfc+V5lrRAYE2LO6e295/zzLYUuJzGnoqTCOyBNm2hrLDbGNFPHpFh6dIhj5vIdARnv1jjjSIX3jCDewV5Hj7JCYEwI+vbo3qzbVUT2lwfcjmJOUqmvEDjZ/fRRVgiMCUETzuiMJ0K45/VVdlYQpErt0pAx5lS0i4/mprN6sLWwlMc/2OB2HHMSSu3SkDHmVN1zcX8Aps/dzP6SCpfTmOYqKfcWgjZWCIwxJys+JpI/TxkCwGeb7Q6iYHO43HtpKM4uDRljTsWlZ3QmISaShfakcdBZk3+QuGgPXRwenQysEBgT0iI9EZzbJ4WP1u2xJ42DzN7icrq2bUNslF0aMsacossGdWFvcTnL7FbSoLJlXwntHR6Z7CgrBMaEuNF9vT2S/i/XLg8Fi+oa5YvdxZzeOTEg+7NCYEyIS46L4vx+qfzrs60UlVW6HcecQGlFFQdLvXd5pbdzvsM5sEJgTFi45+L+HCyt5KLff+p2FNOIg6UVZN4/h7N/+wkQmFtHwQqBMWHhjDTvmLd7isqpqKpxOY1pyPS5uQBUVHv/jgLRvQRYITAmbPztxmEArMo/6HIS05DNBSXHzIdEIRCRS0Rkg4jkisi99ay/Q0TWiMhKEfmfiGQ6mceYcHZ2rw5ECMxZu9vtKKYBa3YcOma+Q0JMQPbrWCEQEQ8wHZgAZAJT6vmif0lVB6rqYOAx4PdO5TEm3CXHRTE+szPP/m8Le4rK3I5j6iirrKaguJw7zu9NV99DZH07JgRk306eEYwAclU1T1UrgFeASf4bqKr/EErxgD3xYoyDbhnVE4CpLyxzN4g5ztE7utLbtWHRzy5k66OX0TYuMM8RONmJRRqw3W8+HxhZdyMRuRP4IRANXFDfB4nIVGAqQPfu3Vs8qDHhYmRGeyIjhNwADIhumqe80ttAHBMZ+KZb1xuLVXW6qvYGfgr8ooFtZqhqlqpmpaamBjagMSFERHho0hmUVFSzYps9adyalFd5exsNRJcSdTlZCHYA/qMup/uWNeQV4HIH8xhjgK+f2YXoyAhmrdrpdhTjpyxEzwiWAn1FJENEooHJwCz/DUSkr9/sZcAmB/MYY4DE2CjO7ZPCPxdupayy2u04xufoGUFMKJ0RqGoVMA2YA6wHXlPVHBF5SEQm+jabJiI5IrISbzvBzU7lMcZ8ZWx/7yXWd1fvcjmJOeroiGSxLpwRODrigarOBmbXWXa/3/RdTu7fGFO/G0b24E8fb+KTDXu5ali623EMsDrf+wxBjw7xAd+3643FxpjAi4gQxvbvyLurd7HcGo1bhZc+3wZA5wAMRFOXFQJjwtRd47xNdK9nbz/BliYQdhw84tq+rRAYE6bS28Vx+eCuvLxkO4tsKEtX1fhGj8vskuTK/q0QGBPGvnluBgAP/CfH5STh7XCFd6D6K4emubJ/KwTGhLFB6W2Jj/awae9hVluvpK45VOrtXiK5TZQr+7dCYEyYm/GNLAB+8MpKl5OEr4O+QhCovoXqskJgTJgb1ScFgLx9Jew65F6DZTg74Buasl2cnREYY1wya9ooAD5at8flJOHpaM+jSXZpyBjjloFpyXRNjuWZBVtq72AxgVNc5m0sTox19BnfBlkhMMYgIkwaksa2/aUs2brf7Thhp9h3RpAQY4XAGOOiO87vDcDkGYvtrCDADpdVIQLx0VYIjDEuSm4TxWWDugDw+RY7KwikorIqEmIiiYgQV/ZvhcAYU+vxq88kLtrDrFWNDR1iWtqWfSUkunRZCKwQGGP8tIn2cFFmJ15est1GMAuQLftK+HRjATsPlbmWwQqBMeYY3x3TB4DXsvNdThIePsjZ7XYEKwTGmGP175zIBad15PO8QrejhIWthSUAzP7+ea5lsEJgjDnO2b06kLevhD1F7l2uCBf7DlfQv1MimV3d6XkUrBAYY+pxVq8OAIx85GN63vsu76+1IS2dsr+kgvbx7vQxdJQVAmPMcer+dnrHi8vZVljqUprQtr+kgvYJVgiMMa2MJ0J4+qZhfOPsHlyblU5sVAQPzFrrdqyQVHi4nA4unxG4d+OqMaZVu3hAZy4e0BmAninxPPb+Bnre+y6JsZGsun+8aw8/hZLK6hqKyqpC+9KQiFwiIhtEJFdE7q1n/Q9FZJ2IrBaRj0Wkh5N5jDEn57ZzM+iVEg94O0h76J11VFs3FKds5nLvLbpunxE4VghExANMByYAmcAUEcmss9kKIEtVBwFvAI85lccYc/JiIj28MvUs/jxlCADPLdrKY3O+cDlVcKuoquGnb64BoFdqgqtZnDwjGAHkqmqeqlYArwCT/DdQ1bmqerQFajGQ7mAeY8wp6JgUy8Qzu/K/n45lWI92PP1pHlv3lbgdK2h9sbsIgJvO6lE7OJBbnCwEacB2v/l837KG3Aa8V98KEZkqItkikl1QUNCCEY0xzZXeLo77LjsdgDGPz2P+Rvs/2Vwl5VV8sbsYgCtcGrDeX6u4a0hEbgSygN/Vt15VZ6hqlqpmpaamBjacMeY4Q7u3475LvcXgN++udzlN8PnOv5fzkzdWA5CaEONyGmfvGtoBdPObT/ctO4aIjAPuA85X1XIH8xhjWtDto3txuLyKP328iU17iunbKdHtSK1eeVU1s9fsOuYsKr1dGxcTeTl5RrAU6CsiGSISDUwGZvlvICJDgKeBiaq618EsxhgHHL299KI/zOdASYXLaVq/n725hrtfXQVA+/hosn8xDhH3b8N1rBCoahUwDZgDrAdeU9UcEXlIRCb6NvsdkAC8LiIrRWRWAx9njGmFMrsmccUQ7zXuGQvyXE7Tuq3JP8TMFV9dFHnqhqGktILLQgCiGlz3AmdlZWl2drbbMYwxfkY9+gk7Dh7htW+fzYiM9m7HaXV2HDzCqEc/AeCyQV2Yfv3QgGcQkWWqmlXfulbRWGyMCW4zvjEMgHvfXM1+u0R0nF/NyqmdvuvCvi4mqZ8VAmPMKRvQNZm/35xF3r4Shj78oXVQV0fevhI6JsaQ+5sJ9GuFjepWCIwxLeLC0zvV3lL6j4VbXE7Teny0bg+5ew+T3q4NkZ7W+ZXbOlMZY4LS7aN7ceXQNF7L3m53EQGLcvfxrX952zSnjOjucpqGWSEwxrSoqaN7UVpRzZCHP2T0Y3OpqKpxO5Jr/vjRptrpywZ1cTFJ46wQGGNa1Gmdk7jSd0vptv2ljH/VS1kAAA4PSURBVP/DpxypqHY5VeAdqahmydb9tfNtojwupmmcFQJjTIt75MqBTL9+KNGeCLYWljL04Q/ZXHDY7VgB9Z+V3mcGHpo0gJnfPadVPDjWECsExpgWFxvl4bJBXfjg7tEkxUZypLKaa//2GcH23NLJKqus5t6Z3i6mrx/RnaHd27mcqHFWCIwxjumZEs+K+8dz59jeFJZUcOdLyymrDP7LRNU1yoVPzOPmfyypLW5lldUUHi6nuKyS4b/+qHbb1nqnkD8bqtIY4yhPhHDXhf14del2Zq/ZTZ+Om7nj/F6UV9bQzuWRuU5GZXUNfe/z9pi/uaCEjJ/NJik2kqKyqmO2+9qgLvzxusFuRGy21l+qjDFBLzoyguxfXMQVQ9L488ebyLx/DkMe/pBDpZUUlVUG1SWjlz7fVjud4Ru+s24ROKtXe/4yZUhQnA2AnREYYwLowa8PQKC287Vv/WspS7ceACD3NxNa/RfnvA17eWBWDj07xPHRD89HRBj44Bx6dIjnhdtGcKSimtgoD+3jo1t143Bd1umcMSbg9pdU8Hr2dn773lfjHn/r3Aw8HuGbozLolBTrYrr6qSpn//YT9hSXsfCnF9C1rfvjCDRHY53O2RmBMSbg2sdH8+3ze9MrNYEdB0pZtLmQZ//n7Zbi6U/zyPnVxcTHtK6vp5ydRewuKuP7F/YNuiJwIq37PMwYE9IuyuzELaMyePL6oZyZnly7POvXH5G71zum77IvD7B9v7cTu6KySgqKAz+QYVllNXNydgNw3fBuJ9g6+LSukmuMCUvRkRG89d1RFJVVctvz2Sz78gDjfj//mG3Gnd6Jj9bvAWBwt7bccX5v1u08RJ9OiVTX1HBO75RjLilVVNWwYtsBhvVod0ptDzU1yoVPfMqOg0dITYyhSyu8bHWqrI3AGNOqlFVWM3nGYlZuP9js9353TG8Ol1fRv3Mi9721tnb5D8b1JblNFGlt2zDu9E5ERDS9IXfFtgNc8ddFpCTE8O9vjaR/59bXjXRTNNZGYIXAGNPqVFbX8OLiLzkjLZl2cVF0SW7DtJeWU1JezU8u6c/aHYd4en4e/TsnMm9DQaOf1b9TIhv2FNfO3zCyO5cO7ML8jQXceUEfqqu1wecZjlRUM+ThDyirrGHZL8bRoZUMLXkyrBAYY0LWx+v3UKPQvX0cj8xez6cbvYXh2+f34rvn9yE5Lopn5uexaW8x/9u0j52Hyo77jFnTRjEovW3t/Efr9jBzRT6z13jbBSac0ZmnbhwWmB/IIVYIjDEG2FtUxtjH51FST2+ot47qyf1fyyRvXwkX/2E+VTXe78Yoj7Dh4QnNupzUGlkhMMYYn9KKKp5f9CWXD+lKdY0ya9VO3l29i5ydRcds98w3sli6dT8/Gt+PmMjW24V0U7lWCETkEuBPgAd4VlUfrbN+NPBHYBAwWVXfONFnWiEwxrS0sspqJvxpAQdKK0iMjeSHF/XjiiHpbsdqUa48UCYiHmA6cBGQDywVkVmqus5vs23ALcCPncphjDEnEhvlYe6Px7gdwzVOPkcwAshV1TwAEXkFmATUFgJV3epbF75j2RljjMucfLI4DdjuN5/vW9ZsIjJVRLJFJLugoPFbxYwxxjRPUHQxoaozVDVLVbNSU1PdjmOMMSHFyUKwA/DvlCPdt8wYY0wr4mQhWAr0FZEMEYkGJgOzHNyfMcaYk+BYIVDVKmAaMAdYD7ymqjki8pCITAQQkeEikg9cAzwtIjlO5THGGFM/R3sfVdXZwOw6y+73m16K95KRMcYYlwRFY7ExxhjnBF0XEyJSAHzpm00GDjUyffTPFGDfSe7S/3Obs77u8sbmTzR9svlbKnt9y+zYN86O/Ynn7dg3PVtTtjnRse+hqvXfdqmqQfsCZjQ27fdndkvsoznr6y5vbL4JP8dJ5W+p7CeT3469HXs79i2f/VSOfWOvYL809N8TTPsva4l9NGd93eWNzTdl+mS0VPb6ltmxb5wd+xPP27FvWoambnOiY9+goLs0dDJEJFsb6GwpGARz/mDODsGdP5izQ3DnD7bswX5G0FQz3A5wioI5fzBnh+DOH8zZIbjzB1X2sDgjMMYY07BwOSMwxhjTACsExhgT5qwQGGNMmLNCYIwxYS7sC4GInCcifxORZ0Vkkdt5mkNEIkTkNyLyFxG52e08zSUiY0Rkge/4j3E7T3OJSLxvwKSvuZ2luUTkdN9xf0NEvuN2nuYSkctF5BkReVVExrudpzlEpJeI/F1ETjhGe6AEdSEQkX+IyF4RWVtn+SUiskFEckXk3sY+Q1UXqOodwDvA807m9dcS2fEO/ZkOVOIdAS5gWii/AoeBWAKYv4WyA/wUeM2ZlA1roX/3633/7q8FRjmZt64Wyv+2qt4O3AFc52Refy2UPU9Vb3M2aTOdzGPQreUFjAaGAmv9lnmAzUAvIBpYBWQCA/F+2fu/Ovq97zUgMZiyA/cC3/a9941gO/ZAhO99nYB/B1n2i/COsXEL8LVgO/a+90wE3gOuD8b8vvc9AQwN0uwB/T/b2MvRbqidpqrzRaRnncUjgFxVzQMQkVeASar6W6DeU3gR6Q4cUtViB+MeoyWy+8ZyqPDNVjuX9ngtdex9DgAxTuSsTwsd+zFAPN7/8EdEZLaq1jiZ+6iWOvaqOguYJSLvAi85l/i4/bbE8RfgUeA9VV3ubOKvtPC/+1YjqAtBA9KA7X7z+cDIE7znNuCfjiVquuZmnwn8RUTOA+Y7GayJmpVfRK4ELgbaAk86G+2EmpVdVe8DEJFbgH2BKgKNaO6xHwNcibcAz25ouwBq7r/97wHjgGQR6aOqf3My3Ak099h3AH4DDBGRn/kKhqtCsRA0m6o+4HaGk6GqpXiLWFBS1Zl4i1nQUtXn3M5wMlR1HjDP5RgnTVX/DPzZ7RwnQ1UL8bZttBpB3VjcgB1AN7/5dN+yYBDM2SG48wdzdrD8bgrm7EBoFoKlQF8RyRCRaLwNerNcztRUwZwdgjt/MGcHy++mYM7u5XZr9Sm24L8M7OKr2ydv8y2/FNiItyX/Prdzhlr2YM8fzNktv2V34mW9jxpjTJgLxUtDxhhjmsEKgTHGhDkrBMYYE+asEBhjTJizQmCMMWHOCoExxoQ5KwTGcSJyOAD7mNjErqNbcp9jROSck3jfEBH5u2/6FhFxu58lAESkZ93ulevZJlVE3g9UJhMYVghM0BART0PrVHWWqj7qwD4b649rDNDsQgD8nODtJ6cA2CUiAR3DwDjLCoEJKBG5R0SWishqEfmV3/K3RWSZiOSIyFS/5YdF5AkRWQWcLSJbReRXIrJcRNaIyGm+7Wp/sxaR50TkzyKySETyRORq3/IIEfmriHwhIh+KyOyj6+pknCcifxSRbOAuEfm6iHwuIitE5CMR6eTrivgO4G4RWSneke5SReRN38+3tL4vSxFJBAap6qp61vUUkU98x+ZjX/foiEhvEVns+3l/Xd8ZlnhHS3tXRFaJyFoRuc63fLjvOKwSkSUikujbzwLfMVxe31mNiHhE5Hd+f1ff9lv9NnBDvX/BJji5/WizvUL/BRz2/TkemAEI3l9C3gFG+9a19/3ZBlgLdPDNK3Ct32dtBb7nm/4u8Kxv+hbgSd/0c8Drvn1k4u0rHuBqvF0uRwCd8Y6DcHU9eecBf/Wbbwe1T+F/C3jCN/0g8GO/7V4CzvVNdwfW1/PZY4E3/eb9c/8XuNk3/U3gbd/0O8AU3/QdR49nnc+9CnjGbz4Z7yApecBw37IkvD0OxwGxvmV9gWzfdE98A64AU4Ff+KZjgGwgwzefBqxx+9+VvVruZd1Qm0Aa73ut8M0n4P0img98X0Su8C3v5lteiHfAnTfrfM7RrquX4e1Tvz5vq3eMgHUi0sm37Fzgdd/y3SIyt5Gsr/pNpwOvikgXvF+uWxp4zzgg0ztmCgBJIpKgqv6/wXcBChp4/9l+P88LwGN+yy/3Tb8EPF7Pe9cAT4jI/wHvqOoCERkI7FLVpQCqWgTeswfgSREZjPf49qvn88YDg/zOmJLx/p1sAfYCXRv4GUwQskJgAkmA36rq08cs9A6SMg44W1VLRWQe3nGMAcpUte7oa+W+P6tp+N9wud+0NLBNY0r8pv8C/F5VZ/myPtjAeyKAs1S1rJHPPcJXP1uLUdWNIjIUb+dnvxaRj4G3Gtj8bmAPcCbezPXlFbxnXnPqWReL9+cwIcLaCEwgzQG+KSIJACKSJiId8f62ecBXBE4DznJo/wuBq3xtBZ3wNvY2RTJf9S9/s9/yYiDRb/4DvCNnAeD7jbuu9UCfBvazCG8XxuC9Br/AN70Y76Uf/NYfQ0S6AqWq+iLwO7zj6m4AuojIcN82ib7G72S8Zwo1wE14x9ytaw7wHRGJ8r23n+9MArxnEI3eXWSCixUCEzCq+gHeSxuficga4A28X6TvA5Eish7vOLSLHYrwJt6ug9cBLwLLgUNNeN+DwOsisgzY57f8v8AVRxuLge8DWb7G1XXUMwqVqn6Bd3jFxLrr8BaRW0VkNd4v6Lt8y38A/NC3vE8DmQcCS0RkJfAA8GtVrQCuwzuc6SrgQ7y/zf8VuNm37DSOPfs56lm8x2m575bSp/nq7Gss8G497zFByrqhNmHl6DV78Y4buwQYpaq7A5zhbqBYVZ9t4vZxwBFVVRGZjLfheJKjIRvPMx/v4OwH3MpgWpa1EZhw846ItMXb6PtwoIuAz1PANc3Yfhjexl0BDuK9o8gVIpKKt73EikAIsTMCY4wJc9ZGYIwxYc4KgTHGhDkrBMYYE+asEBhjTJizQmCMMWHu/wEoNIjN/vuyDgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "learner.lr_find()\n", + "learner.lr_plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "early_stopping automatically enabled at patience=5\n", + "reduce_on_plateau automatically enabled at patience=2\n", + "\n", + "\n", + "begin training using triangular learning rate policy with max lr of 0.001...\n", + "Train on 143613 samples, validate on 15958 samples\n", + "Epoch 1/1024\n", + "143613/143613 [==============================] - 51s 356us/step - loss: 0.1358 - acc: 0.9530 - val_loss: 0.0536 - val_acc: 0.9817\n", + "Epoch 2/1024\n", + "143613/143613 [==============================] - 51s 355us/step - loss: 0.0643 - acc: 0.9784 - val_loss: 0.0504 - val_acc: 0.9826\n", + "Epoch 3/1024\n", + "143613/143613 [==============================] - 51s 356us/step - loss: 0.0577 - acc: 0.9797 - val_loss: 0.0483 - val_acc: 0.9831\n", + "Epoch 4/1024\n", + "143613/143613 [==============================] - 51s 352us/step - loss: 0.0540 - acc: 0.9806 - val_loss: 0.0475 - val_acc: 0.9830\n", + "Epoch 5/1024\n", + "143613/143613 [==============================] - 51s 355us/step - loss: 0.0520 - acc: 0.9811 - val_loss: 0.0471 - val_acc: 0.9832\n", + "Epoch 6/1024\n", + "143613/143613 [==============================] - 51s 355us/step - loss: 0.0500 - acc: 0.9818 - val_loss: 0.0469 - val_acc: 0.9833\n", + "Epoch 7/1024\n", + "143613/143613 [==============================] - 51s 353us/step - loss: 0.0484 - acc: 0.9820 - val_loss: 0.0466 - val_acc: 0.9832\n", + "Epoch 8/1024\n", + "143613/143613 [==============================] - 51s 358us/step - loss: 0.0475 - acc: 0.9823 - val_loss: 0.0470 - val_acc: 0.9830\n", + "Epoch 9/1024\n", + "143613/143613 [==============================] - 52s 360us/step - loss: 0.0465 - acc: 0.9826 - val_loss: 0.0470 - val_acc: 0.9831\n", + "\n", + "Epoch 00009: Reducing Max LR on Plateau: new max lr will be 0.0005 (if not early_stopping).\n", + "Epoch 10/1024\n", + "143613/143613 [==============================] - 52s 359us/step - loss: 0.0441 - acc: 0.9832 - val_loss: 0.0473 - val_acc: 0.9830\n", + "Epoch 11/1024\n", + "143613/143613 [==============================] - 52s 359us/step - loss: 0.0432 - acc: 0.9835 - val_loss: 0.0474 - val_acc: 0.9831\n", + "\n", + "Epoch 00011: Reducing Max LR on Plateau: new max lr will be 0.00025 (if not early_stopping).\n", + "Epoch 12/1024\n", + "143613/143613 [==============================] - 51s 357us/step - loss: 0.0420 - acc: 0.9838 - val_loss: 0.0477 - val_acc: 0.9830\n", + "Restoring model weights from the end of the best epoch\n", + "Epoch 00012: early stopping\n", + "Weights from best epoch have been loaded into model.\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learner.autofit(0.001)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/cifar10-WRN22.ipynb b/examples/vision/cifar10-WRN22.ipynb similarity index 99% rename from examples/cifar10-WRN22.ipynb rename to examples/vision/cifar10-WRN22.ipynb index aa1a58084..08a0bce0f 100644 --- a/examples/cifar10-WRN22.ipynb +++ b/examples/vision/cifar10-WRN22.ipynb @@ -327,7 +327,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/examples/dogs_vs_cats-ResNet50.ipynb b/examples/vision/dogs_vs_cats-ResNet50.ipynb similarity index 98% rename from examples/dogs_vs_cats-ResNet50.ipynb rename to examples/vision/dogs_vs_cats-ResNet50.ipynb index 1af64c885..7e4e93ad4 100644 --- a/examples/dogs_vs_cats-ResNet50.ipynb +++ b/examples/vision/dogs_vs_cats-ResNet50.ipynb @@ -21,7 +21,6 @@ "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\";\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\" \n", "import sys\n", - "sys.path.append('..')\n", "import ktrain\n", "from ktrain import vision as vis" ] @@ -53,7 +52,7 @@ } ], "source": [ - "DATADIR = '../data/dogscats'\n", + "DATADIR = 'data/dogscats'\n", "(train_data, val_data, preproc) = vis.images_from_folder(\n", " datadir=DATADIR,\n", " data_aug = vis.get_data_aug(horizontal_flip=True),\n", @@ -238,7 +237,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/examples/mnist-WRN22.ipynb b/examples/vision/mnist-WRN22.ipynb similarity index 99% rename from examples/mnist-WRN22.ipynb rename to examples/vision/mnist-WRN22.ipynb index 12cbc7703..853b309fb 100644 --- a/examples/mnist-WRN22.ipynb +++ b/examples/vision/mnist-WRN22.ipynb @@ -367,7 +367,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/examples/pets-ResNet50.ipynb b/examples/vision/pets-ResNet50.ipynb similarity index 99% rename from examples/pets-ResNet50.ipynb rename to examples/vision/pets-ResNet50.ipynb index 81bea01c0..8af178c42 100644 --- a/examples/pets-ResNet50.ipynb +++ b/examples/vision/pets-ResNet50.ipynb @@ -683,7 +683,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/examples/vision/planet-ResNet50.ipynb b/examples/vision/planet-ResNet50.ipynb new file mode 100644 index 000000000..96c17dbd6 --- /dev/null +++ b/examples/vision/planet-ResNet50.ipynb @@ -0,0 +1,275 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline\n", + "import os\n", + "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\";\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\" \n", + "import ktrain\n", + "from ktrain import vision as vis" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "DATADIR = 'data/planet'\n", + "ORIGINAL_DATA = DATADIR+'/train_v2.csv'\n", + "CONVERTED_DATA = DATADIR+'/train_v2-CONVERTED.csv'\n", + "labels = vis.preprocess_csv(ORIGINAL_DATA, \n", + " CONVERTED_DATA, \n", + " x_col='image_name', y_col='tags', suffix='.jpg')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 40479 images belonging to 1 classes.\n", + "Found 36357 validated image filenames.\n", + "Found 4122 validated image filenames.\n" + ] + } + ], + "source": [ + "trn, val, preproc = vis.images_from_csv(\n", + " CONVERTED_DATA,\n", + " 'image_name',\n", + " directory=DATADIR+'/train-jpg',\n", + " val_filepath = None,\n", + " label_columns = labels,\n", + " data_aug=vis.get_data_aug(horizontal_flip=True, vertical_flip=True))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The normalization scheme has been changed for use with a pretrained_resnet50 model. If you decide to use a different model, please reload your dataset with a ktrain.vision.data.images_from* function.\n", + "\n", + "Is Multi-Label? True\n", + "pretrained_resnet50 model created.\n" + ] + } + ], + "source": [ + "model = vis.image_classifier('pretrained_resnet50', trn, val_data=val)\n", + "learner = ktrain.get_learner(model, train_data=trn, val_data=val, \n", + " batch_size=64, workers=8, use_multiprocessing=False)\n", + "learner.freeze(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "simulating training for different learning rates... this may take a few moments...\n", + "568\n", + "5\n", + "2840\n", + "Epoch 1/5\n", + "568/568 [==============================] - 213s 375ms/step - loss: 0.6386 - acc: 0.7609\n", + "Epoch 2/5\n", + "568/568 [==============================] - 201s 353ms/step - loss: 0.2260 - acc: 0.9303\n", + "Epoch 3/5\n", + "380/568 [===================>..........] - ETA: 1:05 - loss: 0.3707 - acc: 0.9044\n", + "\n", + "done.\n", + "Please invoke the Learner.lr_plot() method to visually inspect the loss plot to help identify the maximal learning rate associated with falling loss.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xV9f3H8dcngzASwgozQNiITAkgoAVnUSm4FUdFqYq1jta22q1Wf9ra2tYtjtq6B4iIIm4BmWHvvcIMCYSQkP39/XGvNGASEsjJyc19Px+P+8g94977ziXcz/2e7znfrznnEBGR8BXhdwAREfGXCoGISJhTIRARCXMqBCIiYU6FQEQkzKkQiIiEuSi/A1RWs2bNXFJSkt8xRERCysKFC/c55xJK2xZyhSApKYmUlBS/Y4iIhBQz21rWNh0aEhEJcyoEIiJhToVARCTMqRCIiIQ5FQIRkTCnQiAiEubCqhBs2JvFwdwCv2OIiNQoYVMIPlq2i3Mfn0Hv+z8ldX+O33FERGqMsCkE3VrGHrl/2bOzKSgq9jGNiEjNETaFoHPzOJ4Y0w+APQfz+Mu0NT4nEhGpGcKmEACM6tOamb8+C4A35m8jv1CtAhGRsCoEAG2b1OfZa08jJ7+In76+iDkb03l02hq2pmf7HU1ExBchN+hcVbigVytG923NB0t28vnqPQB8vHwXX94zjKjIsKuNIhLmPPvUM7OXzWyvma0oY/u1ZrbMzJab2Wwz6+NVltL886q+vDpuIGYwIKkx2zJyuHfi8uqMICJSI3j59fcVYEQ52zcDw5xzvYA/AxM8zPI9ZsaZXRLY/MhFvH3LYE5p1ZCJi1L50ZOzOJCTX51RRER85VkhcM7NADLK2T7bObc/uDgXSPQqy/FERBjv/3QIZ3dvzvIdmdzx5mJy8gv9iiMiUq1qygHxccC0sjaa2S1mlmJmKWlpaZ4EqBsdyctjB3BpvzbMXL+PBz9c5cnriIjUNL53FpvZWQQKwRll7eOcm0Dw0FFycrLzMs/jV/WlWVwME2ZsIq5uFHef25UGMb6/TSIinvG1RWBmvYEXgdHOuXQ/s5T0i/O60jsxnhdmbuZHT83SVcgiUqv5VgjMrB0wCbjeObfOrxylqRsdyQe3D+WPI3uwKS2bu99e4nckERHPeHn66JvAHKCbmaWa2TgzG29m44O7/BFoCjxjZkvMrEbNSG9mjB2SRJtG9fho2S5W7zrodyQREU+Yc54ecq9yycnJLiWl+mrG9owchj32FcUOJt42hP7tG1fba4uIVBUzW+icSy5tW005a6jGatukPv+5aSAQGLV01U61DESkdlEhqIAzuyQcGbn0l+8uJUuT24hILaJCUEGj+rTmr5f3ZtWug/xhcqmjZoiIhCQVgkq4Mrktd5zdmclLdvLVmr1+xxERqRIqBJV0x9ld6NI8lvs/XEleYZHfcURETpoKQSXViYrg9yN7sDU9h4kLd/gdR0TkpKkQnIAfdGlGv3aN+Ofn69QqEJGQp0JwAsyMe87rxt6sPD5YvNPvOCIiJ0WF4AQN7dyUHq0a8szXG8gtUKtAREKXCsEJMjPuvaA7W9JzeHP+Nr/jiIicMBWCkzCsawIDOzThqS83kJmji8xEJDSpEJykP47swf6cfB7/bK3fUURETogKwUnq2SaeK5Pb8ub87ezKPOx3HBGRSlMhqAK3De+Ew/HUlxv8jiIiUmkqBFWgfdMGXJHcljfmb2Plzky/44iIVIoKQRW594fdia0TxdNfqVUgIqFFhaCKxNeP5oYhSUxbsZv1e7L8jiMiUmEqBFXopjM6UC86Uq0CEQkpKgRVqEmDOlx3enumLN3Jln3ZfscREakQFYIq9pMzOxAdGcEzX6tVICKhQYWgijWPq8tVA9oyefFO9mbl+h1HROS4VAg8cOPQDuQXFfPaXI1BJCI1nwqBBzo0a8A53Zvz+tytGplURGo8FQKPjDujA+nZ+UxZovkKRKRmUyHwyOBOTeneMo6Xv92Mc87vOCIiZVIh8IiZcd3p7VmzO4s1u3WBmYjUXCoEHrqgZ0siI4yPlu3yO4qISJlUCDzUNDaGwR2bMnXZTh0eEpEay7NCYGYvm9leM1tRxnYzsyfMbIOZLTOz07zK4qeL+7VhS3oOczam+x1FRKRUXrYIXgFGlLP9AqBL8HYL8KyHWXwzsncrmjSow79nb/E7iohIqTwrBM65GUBGObuMBv7rAuYCjcyslVd5/FI3OpKrB7Tli9V72J6R43ccEZHv8bOPoA2wvcRyanDd95jZLWaWYmYpaWlp1RKuKl0/uD2REcaLMzf5HUVE5HtCorPYOTfBOZfsnEtOSEjwO06ltYqvx4W9WjFp8Q7SsvL8jiMichQ/C8EOoG2J5cTgulrp9rM6k1tQxBNfrPc7iojIUfwsBFOAHwfPHjodyHTO1doT7ru2iOPy/om8nbJdo5KKSI3i5emjbwJzgG5mlmpm48xsvJmND+7yMbAJ2AC8APzUqyw1xbgzOpBfWMwnK3b7HUVE5Igor57YOTfmONsdcLtXr18TdW4eR7cWcfx3zlbGDGxHdGRIdNGISC2nT6Jqdve5Xdiw9xCPTlvjdxQREUCFoNqdfUpzAA07ISI1hgpBNYuJiuSxy3uz52Aei7cf8DuOiIgKgR9G9GxJ3egI3lmw/fg7i4h4TIXAB3F1o7mkXxsmL9lBZk6B33FEJMypEPjkmoHtyS0oZupyTWUpIv5SIfBJzzYN6dI8lokLU/2OIiJhToXAJ2bGZf0TWbTtAJv3ZfsdR0TCmAqBjy7u24YIg/cXqVUgIv5RIfBRy/i6nN6xKVOW6poCEfGPCoHPRvdtzZb0HJbvyPQ7ioiEKRUCn404tRXRkcaUJTp7SET8oULgs/j60Qzr2pypy3ZRXKzDQyJS/VQIaoBRfVuz+2AuC7aUN8WziIg3VAhqgHO6N6dOZASfr97jdxQRCUMqBDVAg5goBnVswpdr9vodRUTCkApBDXF29+ZsTMtmY9ohv6OISJhRIaghLurVKnhx2Q6/o4hImFEhqCGaN6zL0M7N+GDpDl1cJiLVSoWgBrm4bxu2Zxxm4db9fkcRkTCiQlCD/LBnSxrUieRtTVgjItVIhaAGiY2JYnS/Nny4bKcmrBGRaqNCUMNcM7AduQXFTFqsEUlFpHqoENQwPdvE06dtI96Yt02dxiJSLVQIaqCrB7Rl/d5DLNp2wO8oIhIGVAhqoB/1aU1c3ShenbPF7ygiEgZUCGqg2JgoLurVik9W7mZ3Zq7fcUSklvO0EJjZCDNba2YbzOy+Ura3M7OvzGyxmS0zswu9zBNKbh3WibzCYt6Yv83vKCJSy3lWCMwsEngauADoAYwxsx7H7PZ74B3nXD/gauAZr/KEmg7NGjCsawJvL9hGYVGx33FEpBbzskUwENjgnNvknMsH3gJGH7OPAxoG78cDmqarhDED27HnYB6zNuzzO4qI1GJeFoI2QMlLZFOD60q6H7jOzFKBj4E7SnsiM7vFzFLMLCUtLc2LrDXSsK4J1IuO5IvVGp5aRLzjd2fxGOAV51wicCHwqpl9L5NzboJzLtk5l5yQkFDtIf1SNzqSM7o044vVe3RNgYh4xstCsANoW2I5MbiupHHAOwDOuTlAXaCZh5lCzohTW7IzM5f5mzWNpYh4w8tCsADoYmYdzKwOgc7gKcfssw04B8DMTiFQCMLn2E8FXNCrJXExURqITkQ841khcM4VAj8DpgOrCZwdtNLMHjSzUcHd7gFuNrOlwJvAWKdjIEepXyeKkX1aM23FbnLyC/2OIyK1UJSXT+6c+5hAJ3DJdX8scX8VMNTLDLXBxX1b8+b8bXy2ag+j+x7b3y4icnL87iyWChiQ1IRW8XWZskRn14pI1VMhCAEREcaoPq35Zl0a+7Pz/Y4jIrWMCkGIGNW3NYXFjo9X7PI7iojUMioEIaJHq4Z0bh7LBzo8JCJVTIUgRJgZo/u0Zv7mDLZn5PgdR0RqkQoVAjO7y8waWsBLZrbIzM73Opwc7dL+iZjBpEXHXpcnInLiKtoiuMk5dxA4H2gMXA886lkqKVWbRvUY0L4J09RPICJVqKKFwII/LwRedc6tLLFOqtEFvVqyZncWG/Ye8juKiNQSFS0EC83sUwKFYLqZxQEaJN8HF/ZqRYTBB0t0eEhEqkZFC8E44D5ggHMuB4gGbvQslZSpRcO6nNklgYkLUykq1mgcInLyKloIBgNrnXMHzOw6AjOLZXoXS8pzef9Edmbm8q0mrBGRKlDRQvAskGNmfQgMFLcR+K9nqaRc55/agkb1o3k7RSOSisjJq2ghKAyOCjoaeMo59zQQ510sKU9MVCSX9kvk05W7ydCQEyJykipaCLLM7DcEThv9KDiLWLR3seR4rkhOpKDIMUWdxiJykipaCK4C8ghcT7CbwGxjj3mWSo7rlFYN6dGqIZMWqxCIyMmpUCEIfvi/DsSb2Ugg1zmnPgKfXdY/kWWpmazfk+V3FBEJYRUdYuJKYD5wBXAlMM/MLvcymBzf6L6tiYww3luU6ncUEQlhFT009DsC1xDc4Jz7MTAQ+IN3saQimsXGcO4pzXlr/nYO5hb4HUdEQlRFC0GEc25vieX0SjxWPHT7WZ3JPFzAk1+s9zuKiISoin6Yf2Jm081srJmNBT7imLmIxR+9Extxfo8WTFq0g9yCIr/jiEgIqmhn8a+ACUDv4G2Cc+5eL4NJxd0wJIn07Hwm6wwiETkBURXd0Tk3EZjoYRY5QUM6NaVL81jeX7yDqwe28zuOiISYclsEZpZlZgdLuWWZ2cHqCinlMzNG9m7N/C0ZbEzT8NQiUjnlFgLnXJxzrmEptzjnXMPqCinHd82gdtSJjOCFGZv8jiIiIUZn/tQSCXExXJnclomLUtmdmet3HBEJISoEtcjNZ3akqNjx8reb/Y4iIiFEhaAWade0PiN7t+b1uVvJzNEFZiJSMSoEtcz4YZ3Izi/itXlb/Y4iIiHC00JgZiPMbK2ZbTCz+8rY50ozW2VmK83sDS/zhIMerRsyrGsCL8/arAvMRKRCPCsEZhYJPA1cAPQAxphZj2P26QL8BhjqnDsVuNurPOHktuGdSM/O592FGoxORI7PyxbBQGCDc26Tcy4feIvADGcl3Qw87ZzbD3DMeEZyggZ1aEK/do2YMGMjhUXFfscRkRrOy0LQBig5qW5qcF1JXYGuZvatmc01sxGlPZGZ3WJmKWaWkpaW5lHc2sPMGD+sE9szDvPWAs1rLCLl87uzOAroAgwHxgAvmFmjY3dyzk1wziU755ITEhKqOWJoOu+UFvRr14invtxAgVoFIlIOLwvBDqBtieXE4LqSUoEpzrkC59xmYB2BwiAnKSLCuH14Z3YfzGX6yt1+xxGRGszLQrAA6GJmHcysDnA1MOWYfSYTaA1gZs0IHCrSGAlV5OzuzUlsXI/X5upUUhEpm2eFwDlXCPwMmA6sBt5xzq00swfNbFRwt+lAupmtAr4CfuWcS/cqU7iJiDCuO709czdlaF5jESmTOef8zlApycnJLiUlxe8YIWPfoTwGP/IFo/u24W9X9PE7joj4xMwWOueSS9vmd2exeKxZbAzjzujIewtTmbFOZ1yJyPepEISBO8/pTLsm9Xls+lqKi0OrBSgi3lMhCAP160Rx49Aklu/IZOIiXW0sIkdTIQgTY4ck0atNPE9+uUFXG4vIUVQIwoSZccfZndmWkcO0FbquQET+R4UgjJx7SguSmtbnldlb/I4iIjWICkEY+e66goVb9zNnoy7XEJEAFYIwc/XAdsTFRPHwx6sItWtIRMQbKgRhJjYmil+N6MaKHQd5WyOTiggqBGHpukHtGdq5KQ98uIp1GnpCJOypEIShiAjj0Ut7Exlh/PLdpbrITCTMqRCEqbZN6vPg6FNZlprJpMXHjg4uIuFEhSCMXdy3DcntG3P/lJWkH8rzO46I+ESFIIxFRBiPXtaLnPxCnp+haSBEwpUKQZjr3DyOkb1b8+a8bRzOL/I7joj4QIVAuHZQO7LyCnniy/W6tkAkDKkQCAM7NOHK5ESe/Xoj//hsnd9xRKSaRfkdQPxnFjid1Dl44ssN9E5sxLk9WvgdS0SqiVoEAgQ6jh++pBedm8fywNSVHMjJ9zuSiFQTFQI5ok5UBH+5rDe7M3O55x1daCYSLlQI5Cj92zfmdxeewhdr9jJ1+S6/44hINVAhkO/58eAkOjeP5Z+fryMrt8DvOCLiMRUC+Z6ICOPXP+zGprRsbnh5PgWa2lLEd1vTs8nM8eaLmQqBlOr8U1vy0MU9WbTtAK/P3ep3HJGwd94/ZvDMNxs8eW4VAinTtYPaMaRTU/46fS3bM3L8jiMStpxz5BcWUzcq0pPnVyGQMpkZf728NwDXvzSPXZmHfU4kEp7yCgOHZ2OivfnIViGQciU2rs9/bxpI+qF8rn1xHjn5hX5HEgk7RwpBKLYIzGyEma01sw1mdl85+11mZs7Mkr3MIycmOakJz/+4P5v3ZXPPO0s1HpFINcsrDAwIGRMVYi0CM4sEngYuAHoAY8ysRyn7xQF3AfO8yiInb0inZvzy/G5MW7Gb1+dt8zuOSFjJK/iuRRBihQAYCGxwzm1yzuUDbwGjS9nvz8BfgFwPs0gVuG1YJ87o3IzfT17B+FcXkpalyWxEqsOrwTP3YqJD79BQG2B7ieXU4LojzOw0oK1z7iMPc0gViYgwXhqbzG3DO/HJyt2c949v2LIv2+9YIrWWc45DeYV8vHwXEQYDkhp78jq+dRabWQTwOHBPBfa9xcxSzCwlLS3N+3BSppioSO4d0Z13bh3MgZwCrnh+zpHjlyJSdTKy8+n358/o+afppO4/zG8vPIVW8fU8eS0vC8EOoG2J5cTguu/EAT2Br81sC3A6MKW0DmPn3ATnXLJzLjkhIcHDyFJRAzs04dcjupGWlcej09b4HUek1nl02moOlLiSuH97b1oD4O18BAuALmbWgUABuBq45ruNzrlMoNl3y2b2NfBL51yKh5mkCv10eGf2Hszj399uoWfreC7rn+h3JJEaL7+wmDrH6fTNPFzAB0t2ckm/Ntw4NImZ6/fRO7GRZ5k8axE45wqBnwHTgdXAO865lWb2oJmN8up1pXr97qJTGNyxKfe8u5RB//c5//hsnU4vFSnDr99bStffT+OpL9eXu98zX20gr7CY6we3p3diI24/qzOREeZZLk/7CJxzHzvnujrnOjnnHg6u+6Nzbkop+w5XayD0REdG8My1pzFmYFv2HMzjX1+s1+mlIqXYnZnLOympAPzt03XM3rivzH3fXZhKi4Yx9GvrXSugJF1ZLCetcYM6PHJpbzY8fAGntGrI7yev4IrnZvPZqj1+RxOpMR6dthqA138yiA7NGnDzf1KYuf77J7/sz84nIzufn5zRETPvWgElqRBIlYmKjODNmwcxtHNTFmzZz83/TeGRj1frUJGEpeJid9T4XJOX7ARgcMemvH3L6dSrE8n1L83nrflHt6DX7skCoHPz2GrLqsnrpUo1ql+Hl8cOIGXLfibM2MTzMzaRW1DEr0Z0JzZGf24SPn753lImLQqcKHl58ESKC3u1JCLCaN6wLm/fOphRT87ivknLyS0o4seDk/hw2U7uemsJURFGf4+uGSiNhdq3teTkZJeSoq6EUFBYVMxdby3ho+W7aFAnkr9c3puRvVv7HUvkpBUVBy70Wp6aSd92jWhQJ/Kowzg7DxxmyKNffu9xD13ck+tOb39kOa+wiNteW8SXa/Yetd/NZ3bgdxd9b0Sek2JmC51zpY7npq9o4pmoyAieuqYfo1a15p53lvKzNxYTGxPF8G7N/Y4mckL2Z+fz2/eXM23F7qPWt2xYl8v7J7J2TxaxMVG8vzjQEvj6l8NpVD+a6St3M3P9Pkb2bnXU42KiInn++v488vEa3l+cSny9aB65tLdnVxCXRS0CqRZ7DuZy0ROziI2J5LWfDCKxcX2/I4lU2v1TVvLK7C00rBtFTn4RZoHDoaWNu9W9ZRyf3P0DH1KWTi0C8V2LhnV5+JKe/PztJVz1/FzeuHkQ7ZrUr7azIkRO1kuzNvPK7C1c0q8N/7iq71HbDuTks3DrfvYdyuMHXRPYezCPDgkNfEpaeWoRSLVanprJNS/MJSuvkOT2jXn5xgE0rBvtdyyRcm3el81Zf/uayAhj+f3nU79O6H2HLq9FoNNHpVr1SoznwzvO4K5zurBk+wEufvpb1uw+6HcsCSOH8gp5dNoarnx+DvdPWcmkRalk5RaUuX9uQRFj/z0fgBd+3D8ki8Dx1L7fSGq8pGYN+Pl5XenVJp7bXl/ImAlzefra0xjSqdnxHyxSSfsO5fHizM1EGHRrGcdfP1nLjgOB8/vnb844st/I3q04rV1jTu/YlK4tYomKjCC3oIiHP1rN1vQcHr+yD2d3b+HXr+EpHRoSX63fk8Xlz80h83ABHZo14DcXdOe8Hi3UdyBVIq+wiKsnzGXxtgNHrf/DyB6MO6MDxcWOBVsyeHDqKjbvyyYn/+gh1ZvHxbA3K4+LerXiqWv6hfTfZXmHhlQIxHfph/K4660lzNoQGHtlSKemPHPtaTSqX8fnZBLqfjNpOW/O38YTY/qR2LgeqfsP07RBHYZ2/n7rs7ComDmb0tmw9xDfrEvj67VpdGsRx23DO3FxvzalPHtoUSGQkFBQVMzDH63mldlbiDCYfPtQT4feldot83ABfR74lKuS2/KXy3v7Hcd3On1UQkJ0ZAR/+lEPGtWP5p+fr2fUU9/yi/O6MqpPa5Kahc6peKHs5VmbeXDqKgA6NmtAx4RYbh3WkQFJTY7ssyntEC/O2szFfdswsEOTsp7KN8458gqLuezZ2QAM76bJrI5HLQKpkbZn5DDuPwtYt+cQEQYPjj760nypegu2ZHDFc3MAOLV1Q+pGR7Jw637M4Ic9WpJbWERBUTHfbkg/8pjk9o358ZAkhnVJYMGWDB7+eDUdmjVgdN/W9G3biGaxMbw5fxsb07I5pVUcPVo1pF+7xp6Nrf/F6j088OEqtmXkAHDviO6MH1Z9o3jWZDo0JCEpr7CIORvTmTBjE7M3pnN+jxb836W9aBYb43e0Wqeo2HHREzM5lFfI1DvOONI/s+9QHne+uZjZG//34d+1RSy3De/Eqp0HeWHmZgAiDIor+FEyvFsCT47pR1wVXz/y+ryt/O79FQCYBYrZxNuGEBMVWaWvE6pUCCSk5RYU8dKszTzxxXq6t2rIC9f3p3nDun7HqlXueWcpExel8uSYfvyoz/cHBkzdnxP4Vt8y7qj3fuXOTDamZbN+Txb7DuVxz/ndAPh2wz72HMxl36F8GtWP5uYzOzJ12U5+/d4yCoocbRrV46lr+tGvXdWMqXM4v4iB//c57ZvW57Vxg3SiQSlUCKRWmL5yN3e9tZjExvWZOH4I8fV1RfLJ2nHgMH+bvpb3F+/g2kHteOjinp4eRnHO8f7iHTw4dRUHcgq48+zO/CJYPE7UwdwC/jB5BR8s2clbt5zO6R2bVlHa2kVXFkut8MNTW/LvsQPZlp7DiH/NYF1wAg85MbkFRVz1/BzeX7yDyAhj/LBOnh9LNzMuPS2RWfeezYW9WvLElxv489RVJzR50aG8Qh6Ztpqb/5PCB0t2Mu6MDgyqgZ3XoUAtAgk501fuZvxrC4kw486zu3D7WZ2IitR3msooLnbc+tpCPlu1h/t/1INrT29PdDW/h4VFxTzw4SpenbuVtk3q8eilvRmQ1AQzys2SX1jMb99fznsLU4+s+9UPu3H7WZ2rI3bI0qEhqXW2Z+Tw4NRVfLZqD+2b1uee87sxqpRj2/J9uQVFXPCvmWzel82twzrymwtO8S2Lc45nv9nIXz9Ze2RddKTRtkl97j63K8O6JLA3K5f4etE0rBfN/VNW8taC7QA0bVCHMQPbMahjE4Z0aubZmUi1hQqB1ErOOd5J2c7jn61jz8E8kts35rEr+tBB1xyUaen2A1z30jyycgu5dVhH7hvRvUacWnk4v4jX5m5l7qZ0snILWbx9PwVFZX823X1uF+4+t2s1Jgx9KgRSq+07lMdDU1cdmRx8SKem3Di0A8O6JlAnSoeMvrMs9QDXvhgoAn+/og+XBefRrYmKix3vLUrljXnbOK1dYw4XFLE78zCDOjbl5jM76tv/CVAhkLCQsiWDb9al8ca8baRn59O4fjTPX59cI69+rW5zNqYz5oW5xERF8MHPhtK9ZUO/I0k1UyGQsHIwt4BZ6/fxt0/XsvPAYX574SlcmdyWutHheWHR+j1ZjHlhLnWjI3nm2tM0flOYUiGQsJSWlccNL89n1a7AxDfXnd6Om8/sSPum4dWHcMVzs1m/9xDvjR9C5+axfscRn2jQOQlLCXExTL3jDF6atZkPlu7gnQWBY843DEniqgFta+3hkb1Zufx9+jpioiPYnpHDgi37GT+sk4qAlEktAgkbuzIPc+2L89iUlg3AeT1acPOZHWtNH8LW9Gyen7GJ91JSyS8qBgJj7lzStw1/vby3rrUIc761CMxsBPAvIBJ40Tn36DHbfwH8BCgE0oCbnHNbvcwk4atVfD0+vfsH7MnK4+3523j66418sXoPNw3twK9GdAvpwckKi4oZ++8FbN6XTZ/EeO69oDsDkpoQYaYzbOS4PGsRmFkksA44D0gFFgBjnHOrSuxzFjDPOZdjZrcBw51zV5X3vGoRSFXJzivkDx+sYNKiHbRsWJdbh3VkzMB2Idep/NmqPdz838D/iYcu1nDdUjq/WgQDgQ3OuU3BEG8Bo4EjhcA591WJ/ecC13mYR+QoDWKiePzKvpzfowUvzdrMAx+u4pFpaygsKiaubjTdW8bxh5E9OLV1wxpx0VVpsnILuP2NRQDcODSJqwe09TmRhCIvC0EbYHuJ5VRgUDn7jwOmlbbBzG4BbgFo165dVeUTAWBEz1aM6NmKr9bu5dOVu8nOK2JrRg7zNmcw8slZ1IuO5IYhSdw4NIkWNWz460mLdpBfWMzE24bQv33VDOks4adGnDVkZtcBycCw0rY75yYAEyBwaKgao0kYOatbc87q1vzI8q7Mw7ybksrM9Wk8981GnvtmI4M6NOHifmkqdlAAAArqSURBVG0Y3bc19ev4998nr7CIqUt38ZdP1tAnMV5FQE6Kl3/JO4CS7dTE4LqjmNm5wO+AYc65PA/ziFRKq/h63HlOF+48pwuLtu1n4sJUZq7fx28mLedPH6wkMsK4bXgnxg5NomEVz7ZVnu0ZOdz51mIWbzsABKZjFDkZXnYWRxHoLD6HQAFYAFzjnFtZYp9+wHvACOfc+oo8rzqLxU/OORZu3c/L327m2w3pZB4uIK5uFKe1a8yApMbE169Do3rRREcaMVGRHMwtoEOzBrRpVI+mJzHF5rxN6Tz99UZW7sgkPTsfgD+O7MENQ5J0VpBUiC+dxc65QjP7GTCdwOmjLzvnVprZg0CKc24K8BgQC7wb7Izb5pwb5VUmkZNlZiQnNSE5KXDtweJt+3lp1maW78jkm3Vp5T62WWwMXVvEckaXZpzTvQUt4+sSX+/olsTUZTt5c/42kts3ITLCSNm6n23p2WxJD0zG3qdtIy7rn8iVyYl0bh7nzS8pYUcXlIlUAeccuQXF7DiQQ/qhfPIKi8nOK6Rtk/rM35zBuj1Z5BcWs2BrBtszDh95XExUBPXqROIcFDtHVm7hUc/bLDaGDs3q069dY24f3lnTc8oJ0xATIh4zM+rViaRz8zg6Nz96W8828UctL9iSQer+HFbuOMhXa/fSvVVDYqIiyCsoplNCA352dhf25+QTXy865K5pkNCkFoGISBjQ5PUiIlImFQIRkTCnQiAiEuZUCEREwpwKgYhImFMhEBEJcyoEIiJhToVARCTMhdwFZWaWBnw3nWU8kFnO/e9+NgP2neBLlnzeymw/dn15y15kP17u8vZRdmWvzD4VWafslctVkX0qm72Lcy6e0jjnQvYGTCjvfomfKVXxGpXZfuz68pa9yH683Mqu7FWVvSLrlL1mZT/2FuqHhj48zv2S66riNSqz/dj15S17kb0ij1f2799X9srvU5F1yl42P7IfJeQODZ0IM0txZYyxUdMpuz+U3R/K7o9QbxFU1AS/A5wEZfeHsvtD2X0QFi0CEREpW7i0CEREpAwqBCIiYU6FQEQkzIV9ITCzM83sOTN70cxm+52nMswswsweNrMnzewGv/NUhpkNN7OZwfd+uN95KsvMGphZipmN9DtLZZjZKcH3/D0zu83vPJVhZheb2Qtm9raZne93nsows45m9pKZved3ltKEdCEws5fNbK+ZrThm/QgzW2tmG8zsvvKewzk30zk3HpgK/MfLvCVVRXZgNJAIFACpXmU9VhVld8AhoC6hlx3gXuAdb1KWror+3lcH/96vBIZ6mbekKso+2Tl3MzAeuMrLvCVVUfZNzrlx3iY9CSdyJVxNuQE/AE4DVpRYFwlsBDoCdYClQA+gF4EP+5K35iUe9w4QF0rZgfuAW4OPfS/EskcEH9cCeD3Esp8HXA2MBUaGUvbgY0YB04BrQi178HF/B04L0ezV9v+0MrcoQphzboaZJR2zeiCwwTm3CcDM3gJGO+ceAUptxptZOyDTOZflYdyjVEV2M0sF8oOLRd6lPVpVve9B+4EYL3KWpore9+FAAwL/8Q+b2cfOuWIvc0PVve/OuSnAFDP7CHjDu8RHvWZVvO8GPApMc84t8jbx/1Tx33uNFNKFoAxtgO0lllOBQcd5zDjg354lqrjKZp8EPGlmZwIzvAxWAZXKbmaXAj8EGgFPeRvtuCqV3Tn3OwAzGwvsq44iUI7Kvu/DgUsJFN+PPU12fJX9e78DOBeIN7POzrnnvAx3HJV935sCDwP9zOw3wYJRY9TGQlBpzrk/+Z3hRDjncggUsZDjnJtEoJCFLOfcK35nqCzn3NfA1z7HOCHOuSeAJ/zOcSKcc+kE+jZqpJDuLC7DDqBtieXE4LpQoOz+UHZ/KHsNURsLwQKgi5l1MLM6BDr1pvicqaKU3R/K7g9lryn87q0+yd78N4Fd/O/0yXHB9RcC6wj06v/O75zKXnNuyq7s4ZS9ojcNOiciEuZq46EhERGpBBUCEZEwp0IgIhLmVAhERMKcCoGISJhTIRARCXMqBOI5MztUDa8xqoLDR1flaw43syEn8Lh+ZvZS8P5YM/N7rCUAzCzp2KGWS9knwcw+qa5MUj1UCCRkmFlkWducc1Occ4968Jrljcc1HKh0IQB+S+iOmZMG7DKzapvLQLynQiDVysx+ZWYLzGyZmT1QYv1kM1toZivN7JYS6w+Z2d/NbCkw2My2mNkDZrbIzJabWffgfke+WZvZK2b2hJnNNrNNZnZ5cH2EmT1jZmvM7DMz+/i7bcdk/NrM/mlmKcBdZvYjM5tnZovN7HMzaxEclng88HMzW2KBme4SzGxi8PdbUNqHpZnFAb2dc0tL2ZZkZl8G35svgsOjY2adzGxu8Pd9qLQWlgVmTPvIzJaa2Qozuyq4fkDwfVhqZvPNLC74OjOD7+Gi0lo1ZhZpZo+V+Le6tcTmycC1pf4DS2jy+9Jm3Wr/DTgU/Hk+MAEwAl9CpgI/CG5rEvxZD1gBNA0uO+DKEs+1BbgjeP+nwIvB+2OBp4L3XwHeDb5GDwLjxgNcTmDo5QigJYG5EC4vJe/XwDMllhvDkavwfwL8PXj/fuCXJfZ7AzgjeL8dsLqU5z4LmFhiuWTuD4EbgvdvAiYH708FxgTvj//u/TzmeS8DXiixHE9gwpRNwIDguoYERhyuD9QNrusCpATvJxGcfAW4Bfh98H4MkAJ0CC63AZb7/XelW9XdNAy1VKfzg7fFweVYAh9EM4A7zeyS4Pq2wfXpBCbcmXjM83w3fPVCAmPrl2ayC8wTsMrMWgTXnQG8G1y/28y+Kifr2yXuJwJvm1krAh+um8t4zLlAj8D8KQA0NLNY51zJb/CtgLQyHj+4xO/zKvDXEusvDt5/A/hbKY9dDvzdzP4CTHXOzTSzXsAu59wCAOfcQQi0HoCnzKwvgfe3aynPdz7Qu0SLKZ7Av8lmYC/QuozfQUKQCoFUJwMecc49f9TKwGQp5wKDnXM5ZvY1gbmMAXKdc8fOvpYX/FlE2X/DeSXuWxn7lCe7xP0ngcedc1OCWe8v4zERwOnOudxynvcw//vdqoxzbp2ZnUZgILSHzOwL4P0ydv85sAfoQyBzaXmNQMtreinb6hL4PaSWUB+BVKfpwE1mFgtgZm3MrDmBb5v7g0WgO3C6R6//LXBZsK+gBYHO3oqI539jzd9QYn0WEFdi+VMCs2gBEPzGfazVQOcyXmc2geGMIXAMfmbw/lwCh34osf0oZtYayHHOvQY8RmCO3bVAKzMbENwnLtj5HU+gpVAMXE9g/t1jTQduM7Po4GO7BlsSEGhBlHt2kYQWFQKpNs65Twkc2phjZsuB9wh8kH4CRJnZagJz0s71KMJEAsMIrwJeAxYBmRV43P3Au2a2ENhXYv2HwCXfdRYDdwLJwc7VVZQyI5Vzbg2BqRbjjt1GoIjcaGbLCHxA3xVcfzfwi+D6zmVk7gXMN7MlwJ+Ah5xz+cBVBKYzXQp8RuDb/DPADcF13Tm69fOdFwm8T4uCp5Q+z/9aX2cBH5XyGAlRGoZawsp3x+wtMIfsfGCoc253NWf4OZDlnHuxgvvXBw4755yZXU2g43i0pyHLzzODwETt+/3KIFVLfQQSbqaaWSMCnb5/ru4iEPQscEUl9u9PoHPXgAMEzijyhZklEOgvURGoRdQiEBEJc+ojEBEJcyoEIiJhToVARCTMqRCIiIQ5FQIRkTCnQiAiEub+H18pgNpr+mr4AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "learner.lr_find()\n", + "learner.lr_plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "begin training using onecycle policy with max lr of 0.0001...\n", + "Epoch 1/20\n", + "568/568 [==============================] - 206s 363ms/step - loss: 0.1980 - acc: 0.9299 - val_loss: 0.6752 - val_acc: 0.6485\n", + "Epoch 2/20\n", + "568/568 [==============================] - 206s 363ms/step - loss: 0.1426 - acc: 0.9492 - val_loss: 0.1381 - val_acc: 0.9508\n", + "Epoch 3/20\n", + "568/568 [==============================] - 206s 362ms/step - loss: 0.1248 - acc: 0.9547 - val_loss: 0.1074 - val_acc: 0.9616\n", + "Epoch 4/20\n", + "568/568 [==============================] - 206s 363ms/step - loss: 0.1159 - acc: 0.9577 - val_loss: 0.0996 - val_acc: 0.9637\n", + "Epoch 5/20\n", + "568/568 [==============================] - 207s 364ms/step - loss: 0.1090 - acc: 0.9601 - val_loss: 0.0977 - val_acc: 0.9643\n", + "Epoch 6/20\n", + "568/568 [==============================] - 206s 363ms/step - loss: 0.1054 - acc: 0.9609 - val_loss: 0.0987 - val_acc: 0.9629\n", + "Epoch 7/20\n", + "568/568 [==============================] - 206s 362ms/step - loss: 0.1038 - acc: 0.9616 - val_loss: 0.1052 - val_acc: 0.9622\n", + "Epoch 8/20\n", + "568/568 [==============================] - 206s 363ms/step - loss: 0.1030 - acc: 0.9617 - val_loss: 0.0936 - val_acc: 0.9650\n", + "Epoch 9/20\n", + "568/568 [==============================] - 206s 363ms/step - loss: 0.1003 - acc: 0.9627 - val_loss: 0.0967 - val_acc: 0.9651\n", + "Epoch 10/20\n", + "568/568 [==============================] - 207s 365ms/step - loss: 0.0998 - acc: 0.9628 - val_loss: 0.0957 - val_acc: 0.9651\n", + "Epoch 11/20\n", + "568/568 [==============================] - 207s 364ms/step - loss: 0.0977 - acc: 0.9635 - val_loss: 0.0918 - val_acc: 0.9660\n", + "Epoch 12/20\n", + "568/568 [==============================] - 207s 364ms/step - loss: 0.0961 - acc: 0.9638 - val_loss: 0.0953 - val_acc: 0.9655\n", + "Epoch 13/20\n", + "568/568 [==============================] - 207s 365ms/step - loss: 0.0938 - acc: 0.9646 - val_loss: 0.0961 - val_acc: 0.9649\n", + "Epoch 14/20\n", + "568/568 [==============================] - 207s 365ms/step - loss: 0.0920 - acc: 0.9654 - val_loss: 0.0908 - val_acc: 0.9660\n", + "Epoch 15/20\n", + "568/568 [==============================] - 207s 365ms/step - loss: 0.0904 - acc: 0.9659 - val_loss: 0.0916 - val_acc: 0.9664\n", + "Epoch 16/20\n", + "568/568 [==============================] - 207s 364ms/step - loss: 0.0885 - acc: 0.9664 - val_loss: 0.0904 - val_acc: 0.9662\n", + "Epoch 17/20\n", + "568/568 [==============================] - 207s 364ms/step - loss: 0.0867 - acc: 0.9672 - val_loss: 0.0901 - val_acc: 0.9668\n", + "Epoch 18/20\n", + "568/568 [==============================] - 207s 364ms/step - loss: 0.0856 - acc: 0.9674 - val_loss: 0.0902 - val_acc: 0.9666\n", + "Epoch 19/20\n", + "568/568 [==============================] - 207s 364ms/step - loss: 0.0833 - acc: 0.9683 - val_loss: 0.0912 - val_acc: 0.9666\n", + "Epoch 20/20\n", + "568/568 [==============================] - 207s 364ms/step - loss: 0.0813 - acc: 0.9688 - val_loss: 0.0901 - val_acc: 0.9670\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learner.fit_onecycle(1e-4, 20)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import fbeta_score\n", + "import numpy as np\n", + "import warnings\n", + "def f2(preds, targs, start=0.17, end=0.24, step=0.01):\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\")\n", + " return max([fbeta_score(targs, (preds>th), 2, average='samples')\n", + " for th in np.arange(start,end,step)])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = learner.model.predict_generator(val)\n", + "y_true = val.labels" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.9249264279306654" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "f2(y_pred, y_true)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/ktrain/core.py b/ktrain/core.py index d25e3cdbf..53f55df88 100644 --- a/ktrain/core.py +++ b/ktrain/core.py @@ -416,7 +416,7 @@ def reset_weights(self, nosave=False, verbose=1): - def lr_find(self, start_lr=1e-7, epochs=5, verbose=1): + def lr_find(self, start_lr=1e-7, lr_mult=1.01, verbose=1): """ Plots loss as learning rate is increased. Highest learning rate corresponding to a still @@ -425,8 +425,7 @@ def lr_find(self, start_lr=1e-7, epochs=5, verbose=1): Reference: https://arxiv.org/abs/1506.01186 Args: - epochs (int): maximum number of epochs to simulate training - If None, chosen automatically. + lr_mult (float): multiplication factor to increase LR. start_lr (float): smallest lr to start simulation verbose (bool): specifies how much output to print Returns: @@ -446,8 +445,7 @@ def lr_find(self, start_lr=1e-7, epochs=5, verbose=1): try: # track and plot learning rates self.lr_finder = LRFinder(self.model) - self.lr_finder.find(self.train_data, start_lr=start_lr, end_lr=10, - epochs=epochs, + self.lr_finder.find(self.train_data, start_lr=start_lr, lr_mult=lr_mult, workers=self.workers, use_multiprocessing=self.use_multiprocessing, verbose=verbose) diff --git a/ktrain/lroptimize/lrfinder.py b/ktrain/lroptimize/lrfinder.py index d9ec78a1b..3d01fa29d 100644 --- a/ktrain/lroptimize/lrfinder.py +++ b/ktrain/lroptimize/lrfinder.py @@ -53,7 +53,7 @@ def on_batch_end(self, batch, logs): K.set_value(self.model.optimizer.lr, lr) - def find(self, train_data, start_lr, end_lr, epochs=None, batch_size=U.DEFAULT_BS, + def find(self, train_data, start_lr, lr_mult=1.01, batch_size=U.DEFAULT_BS, workers=1, use_multiprocessing=False, verbose=1): """ Track loss as learning rate is increased. @@ -76,12 +76,8 @@ def find(self, train_data, start_lr, end_lr, epochs=None, batch_size=U.DEFAULT_B use_gen = False steps_per_epoch = np.ceil(num_samples/batch_size) - if epochs is None: - epochs = math.ceil(SAMPLE_SIZE/steps_per_epoch) - - - num_batches = epochs * steps_per_epoch - self.lr_mult = (end_lr / start_lr) ** (1 / num_batches) + epochs = 1024 + self.lr_mult = lr_mult # Save weights into a file new_file, self._weightfile = tempfile.mkstemp() diff --git a/ktrain/utils.py b/ktrain/utils.py index d6416cbc0..c12c0db59 100644 --- a/ktrain/utils.py +++ b/ktrain/utils.py @@ -100,12 +100,12 @@ def nclasses_from_data(data): def y_from_data(data): if is_iter(data): - if hasattr(data, 'classes'): + if hasattr(data, 'classes'): # DirectoryIterator return to_categorical(data.classes) - elif hasattr(data, 'data'): - return data.data - elif hasattr(data, 'y'): - return data.y + elif hasattr(data, 'labels'): # DataFrameIterator + return data.labels + elif hasattr(data, 'y'): # NumpyArrayIterator + return to_categorical(data.y) else: raise Exception('could not determine number of classes from %s' % (type(data))) else: diff --git a/ktrain/version.py b/ktrain/version.py index 278a2c069..b7e50f176 100644 --- a/ktrain/version.py +++ b/ktrain/version.py @@ -1,2 +1,2 @@ __all__ = ['__version__'] -__version__ = '0.1.9' +__version__ = '0.1.10' diff --git a/tutorial-03-image-classification.ipynb b/tutorial-03-image-classification.ipynb index 6edf9ca0a..e95ec3447 100644 --- a/tutorial-03-image-classification.ipynb +++ b/tutorial-03-image-classification.ipynb @@ -1208,7 +1208,7 @@ "outputs": [], "source": [ "y_pred = learner.model.predict_generator(val_data)\n", - "y_true = val_data._data" + "y_true = val_data.labels" ] }, { @@ -1544,7 +1544,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/tutorial-04-text-classification.ipynb b/tutorial-04-text-classification.ipynb index 5d6d4f06a..228a223e8 100644 --- a/tutorial-04-text-classification.ipynb +++ b/tutorial-04-text-classification.ipynb @@ -569,7 +569,7 @@ "source": [ "Finally, we will train our model for 8 epochs using ```autofit``` with a learning rate of 0.0007. Having explicitly specified the number of epochs, ```autofit``` will automatically employ a triangular learning rate policy. Our final ROC-AUC score is **0.98**.\n", "\n", - "As shown in [this example notebook](https://github.com/amaiya/ktrain/blob/master/examples/toxic_comments-bigru.ipynb) on our GitHub project, even better results can be obtained using a Bidirectional GRU with pretrained word vectors (called ‘bigru’ in ktrain)" + "As shown in [this example notebook](https://github.com/amaiya/ktrain/blob/master/examples/text/toxic_comments-bigru.ipynb) on our GitHub project, even better results can be obtained using a Bidirectional GRU with pretrained word vectors (called ‘bigru’ in ktrain)" ] }, { @@ -785,7 +785,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.6.8" } }, "nbformat": 4,