Flair Tutorials 7B.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Flair Tutorials 7B.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "rpikIA0Av9Ua",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "0308e4ad-80f8-41df-f702-4f51468d710d"
      },
      "source": [
        "!pip install Flair"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: Flair in /usr/local/lib/python3.6/dist-packages (0.4.4)\n",
            "Requirement already satisfied: ipython-genutils==0.2.0 in /usr/local/lib/python3.6/dist-packages (from Flair) (0.2.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from Flair) (2019.12.9)\n",
            "Requirement already satisfied: ipython==7.6.1 in /usr/local/lib/python3.6/dist-packages (from Flair) (7.6.1)\n",
            "Requirement already satisfied: segtok>=1.5.7 in /usr/local/lib/python3.6/dist-packages (from Flair) (1.5.7)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from Flair) (0.8.6)\n",
            "Requirement already satisfied: gensim>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from Flair) (3.6.0)\n",
            "Requirement already satisfied: torch>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from Flair) (1.3.1)\n",
            "Requirement already satisfied: sqlitedict>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from Flair) (1.6.0)\n",
            "Requirement already satisfied: transformers>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from Flair) (2.2.2)\n",
            "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from Flair) (3.10.0)\n",
            "Requirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from Flair) (0.1.2)\n",
            "Requirement already satisfied: deprecated>=1.2.4 in /usr/local/lib/python3.6/dist-packages (from Flair) (1.2.7)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.20 in /usr/local/lib/python3.6/dist-packages (from Flair) (1.24.3)\n",
            "Requirement already satisfied: mpld3==0.3 in /usr/local/lib/python3.6/dist-packages (from Flair) (0.3)\n",
            "Requirement already satisfied: langdetect in /usr/local/lib/python3.6/dist-packages (from Flair) (1.0.7)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (from Flair) (0.4.2)\n",
            "Requirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.6/dist-packages (from Flair) (3.1.2)\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from Flair) (0.0)\n",
            "Requirement already satisfied: pytest>=3.6.4 in /usr/local/lib/python3.6/dist-packages (from Flair) (3.6.4)\n",
            "Requirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.6/dist-packages (from Flair) (4.28.1)\n",
            "Requirement already satisfied: tiny-tokenizer[all] in /usr/local/lib/python3.6/dist-packages (from Flair) (3.0.1)\n",
            "Requirement already satisfied: bpemb>=0.2.9 in /usr/local/lib/python3.6/dist-packages (from Flair) (0.3.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->Flair) (0.7.5)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->Flair) (4.4.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->Flair) (2.0.10)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->Flair) (4.3.3)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->Flair) (42.0.2)\n",
            "Requirement already satisfied: jedi>=0.10 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->Flair) (0.15.1)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->Flair) (0.1.0)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->Flair) (2.1.3)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->Flair) (4.7.0)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->Flair) (1.9.0)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->Flair) (1.3.3)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->Flair) (1.12.0)\n",
            "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->Flair) (1.17.4)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->Flair) (0.0.35)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->Flair) (1.10.40)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->Flair) (2.21.0)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->Flair) (0.1.85)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->Flair) (0.16.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->Flair) (2.4)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from deprecated>=1.2.4->Flair) (1.11.2)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision->Flair) (4.3.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->Flair) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->Flair) (2.6.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->Flair) (2.4.5)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->Flair) (1.1.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->Flair) (0.21.3)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->Flair) (8.0.2)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->Flair) (19.3.0)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->Flair) (0.7.1)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->Flair) (1.8.0)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->Flair) (1.3.0)\n",
            "Requirement already satisfied: kytea; extra == \"all\" in /usr/local/lib/python3.6/dist-packages (from tiny-tokenizer[all]->Flair) (0.1.4)\n",
            "Requirement already satisfied: natto-py; extra == \"all\" in /usr/local/lib/python3.6/dist-packages (from tiny-tokenizer[all]->Flair) (0.9.0)\n",
            "Requirement already satisfied: SudachiPy; extra == \"all\" in /usr/local/lib/python3.6/dist-packages (from tiny-tokenizer[all]->Flair) (0.4.2)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython==7.6.1->Flair) (0.1.7)\n",
            "Requirement already satisfied: parso>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from jedi>=0.10->ipython==7.6.1->Flair) (0.5.2)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython==7.6.1->Flair) (0.6.0)\n",
            "Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim>=3.4.0->Flair) (2.49.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->Flair) (0.14.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->Flair) (7.0)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->Flair) (0.9.4)\n",
            "Requirement already satisfied: botocore<1.14.0,>=1.13.40 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->Flair) (1.13.40)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->Flair) (0.2.1)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->Flair) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->Flair) (2019.11.28)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->Flair) (2.8)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from pillow>=4.1.1->torchvision->Flair) (0.46)\n",
            "Requirement already satisfied: cffi in /usr/local/lib/python3.6/dist-packages (from natto-py; extra == \"all\"->tiny-tokenizer[all]->Flair) (1.13.2)\n",
            "Requirement already satisfied: sortedcontainers~=2.1.0 in /usr/local/lib/python3.6/dist-packages (from SudachiPy; extra == \"all\"->tiny-tokenizer[all]->Flair) (2.1.0)\n",
            "Requirement already satisfied: dartsclone~=0.6.0 in /usr/local/lib/python3.6/dist-packages (from SudachiPy; extra == \"all\"->tiny-tokenizer[all]->Flair) (0.6)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.40->boto3->transformers>=2.0.0->Flair) (0.15.2)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi->natto-py; extra == \"all\"->tiny-tokenizer[all]->Flair) (2.19)\n",
            "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (from dartsclone~=0.6.0->SudachiPy; extra == \"all\"->tiny-tokenizer[all]->Flair) (0.29.14)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mPg_PuZywZ81",
        "colab_type": "text"
      },
      "source": [
        "# Tutorial 7: Training a Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ykHn-aiBwgVm",
        "colab_type": "text"
      },
      "source": [
        "## A. Training a Text Classification Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iRv7DmVUwImV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.data import Corpus\n",
        "from flair.datasets import TREC_6\n",
        "from flair.embeddings import WordEmbeddings, FlairEmbeddings\n",
        "from flair.embeddings import DocumentRNNEmbeddings\n",
        "from flair.models import TextClassifier\n",
        "from flair.trainers import ModelTrainer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L-wj9sU9wyNe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "outputId": "b8504968-f140-4ca5-9feb-7f4dc017f445"
      },
      "source": [
        "# 1. get the corpus\n",
        "corpus: Corpus = TREC_6()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:02:29,514 https://cogcomp.seas.upenn.edu/Data/QA/QC/train_5500.label not found in cache, downloading to /tmp/tmpwtsj73kn\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 335858/335858 [00:00<00:00, 828226.41B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:02:30,265 copying /tmp/tmpwtsj73kn to cache at /root/.flair/datasets/trec_6/original/train_5500.label\n",
            "2019-12-20 10:02:30,270 removing temp file /tmp/tmpwtsj73kn\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:02:30,610 https://cogcomp.seas.upenn.edu/Data/QA/QC/TREC_10.label not found in cache, downloading to /tmp/tmp_w_l43ft\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 23354/23354 [00:00<00:00, 284761.76B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:02:31,035 copying /tmp/tmp_w_l43ft to cache at /root/.flair/datasets/trec_6/original/TREC_10.label\n",
            "2019-12-20 10:02:31,036 removing temp file /tmp/tmp_w_l43ft\n",
            "2019-12-20 10:02:31,050 Reading data from /root/.flair/datasets/trec_6\n",
            "2019-12-20 10:02:31,051 Train: /root/.flair/datasets/trec_6/train.txt\n",
            "2019-12-20 10:02:31,051 Dev: None\n",
            "2019-12-20 10:02:31,053 Test: /root/.flair/datasets/trec_6/test.txt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aogeGbxvw25M",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "e0be6e9b-5e51-4ff2-e6cc-cc82b47528bb"
      },
      "source": [
        "# 2. create the label dictionary\n",
        "label_dict = corpus.make_label_dictionary()\n",
        "\n",
        "# 3. make a list of word embeddings\n",
        "word_embeddings = [WordEmbeddings('glove'),\n",
        "                   FlairEmbeddings('news-forward'),\n",
        "                   FlairEmbeddings('news-backward'),\n",
        "                   ]"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:03:51,881 Computing label dictionary. Progress:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 4907/4907 [00:00<00:00, 240797.57it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:03:51,940 [b'LOC', b'NUM', b'ENTY', b'DESC', b'HUM', b'ABBR']\n",
            "2019-12-20 10:03:52,036 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings/glove.gensim.vectors.npy not found in cache, downloading to /tmp/tmpdsha9y7a\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "100%|██████████| 160000128/160000128 [00:02<00:00, 68410865.39B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:03:54,559 copying /tmp/tmpdsha9y7a to cache at /root/.flair/embeddings/glove.gensim.vectors.npy\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:03:54,825 removing temp file /tmp/tmpdsha9y7a\n",
            "2019-12-20 10:03:55,296 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings/glove.gensim not found in cache, downloading to /tmp/tmpxdk29xpb\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 21494764/21494764 [00:00<00:00, 67154440.17B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:03:55,780 copying /tmp/tmpxdk29xpb to cache at /root/.flair/embeddings/glove.gensim\n",
            "2019-12-20 10:03:55,800 removing temp file /tmp/tmpxdk29xpb\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:402: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:03:57,884 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4.1/big-news-forward--h2048-l1-d0.05-lr30-0.25-20/news-forward-0.4.1.pt not found in cache, downloading to /tmp/tmpy48896ne\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 73034624/73034624 [00:01<00:00, 56862612.96B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:03:59,378 copying /tmp/tmpy48896ne to cache at /root/.flair/embeddings/news-forward-0.4.1.pt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:03:59,446 removing temp file /tmp/tmpy48896ne\n",
            "2019-12-20 10:04:10,660 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4.1/big-news-backward--h2048-l1-d0.05-lr30-0.25-20/news-backward-0.4.1.pt not found in cache, downloading to /tmp/tmpuj9gfg2o\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 73034575/73034575 [00:01<00:00, 64422695.84B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:04:11,963 copying /tmp/tmpuj9gfg2o to cache at /root/.flair/embeddings/news-backward-0.4.1.pt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:04:12,029 removing temp file /tmp/tmpuj9gfg2o\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "la_oCS-dxLFF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 4. initialize document embedding by passing list of word embeddings\n",
        "# Can choose between many RNN types (GRU by default,\n",
        "# to change use rnn_type parameter)\n",
        "\n",
        "document_embeddings: DocumentRNNEmbeddings = DocumentRNNEmbeddings(word_embeddings,\n",
        "                                                                   hidden_size = 512,\n",
        "                                                                   reproject_words = True,\n",
        "                                                                   reproject_words_dimension = 256,\n",
        "                                                                   )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NcEtTmEvxv0F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 5. create the text classifier\n",
        "classifier = TextClassifier(document_embeddings,\n",
        "                            label_dictionary = label_dict)\n",
        "\n",
        "# 6. initialize the text classifier trainer\n",
        "trainer = ModelTrainer(classifier, corpus)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ld_WH0K7yBA1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "e36baaf2-dfb6-43a0-cf24-31c435365a3d"
      },
      "source": [
        "# 7. start the training\n",
        "\n",
        "trainer.train('resources/tagger/ag_news',\n",
        "              learning_rate = 0.1,\n",
        "              mini_batch_size = 32,\n",
        "              anneal_factor = 0.5,\n",
        "              patience = 5,\n",
        "              max_epochs = 150)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:08:36,654 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:08:36,656 Model: \"TextClassifier(\n",
            "  (document_embeddings): DocumentRNNEmbeddings(\n",
            "    (embeddings): StackedEmbeddings(\n",
            "      (list_embedding_0): WordEmbeddings('glove')\n",
            "      (list_embedding_1): FlairEmbeddings(\n",
            "        (lm): LanguageModel(\n",
            "          (drop): Dropout(p=0.05, inplace=False)\n",
            "          (encoder): Embedding(300, 100)\n",
            "          (rnn): LSTM(100, 2048)\n",
            "          (decoder): Linear(in_features=2048, out_features=300, bias=True)\n",
            "        )\n",
            "      )\n",
            "      (list_embedding_2): FlairEmbeddings(\n",
            "        (lm): LanguageModel(\n",
            "          (drop): Dropout(p=0.05, inplace=False)\n",
            "          (encoder): Embedding(300, 100)\n",
            "          (rnn): LSTM(100, 2048)\n",
            "          (decoder): Linear(in_features=2048, out_features=300, bias=True)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (word_reprojection_map): Linear(in_features=4196, out_features=256, bias=True)\n",
            "    (rnn): GRU(256, 512, batch_first=True)\n",
            "    (dropout): Dropout(p=0.5, inplace=False)\n",
            "  )\n",
            "  (decoder): Linear(in_features=512, out_features=6, bias=True)\n",
            "  (loss_function): CrossEntropyLoss()\n",
            ")\"\n",
            "2019-12-20 10:08:36,657 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:08:36,658 Corpus: \"Corpus: 4907 train + 545 dev + 500 test sentences\"\n",
            "2019-12-20 10:08:36,660 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:08:36,662 Parameters:\n",
            "2019-12-20 10:08:36,662  - learning_rate: \"0.1\"\n",
            "2019-12-20 10:08:36,664  - mini_batch_size: \"32\"\n",
            "2019-12-20 10:08:36,665  - patience: \"5\"\n",
            "2019-12-20 10:08:36,665  - anneal_factor: \"0.5\"\n",
            "2019-12-20 10:08:36,667  - max_epochs: \"150\"\n",
            "2019-12-20 10:08:36,667  - shuffle: \"True\"\n",
            "2019-12-20 10:08:36,668  - train_with_dev: \"False\"\n",
            "2019-12-20 10:08:36,670  - batch_growth_annealing: \"False\"\n",
            "2019-12-20 10:08:36,671 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:08:36,672 Model training base path: \"resources/tagger/ag_news\"\n",
            "2019-12-20 10:08:36,673 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:08:36,673 Device: cuda:0\n",
            "2019-12-20 10:08:36,674 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:08:36,675 Embeddings storage mode: cpu\n",
            "2019-12-20 10:08:36,677 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:08:36,963 epoch 1 - iter 0/154 - loss 1.79611766 - samples/sec: 1689.37\n",
            "2019-12-20 10:08:40,072 epoch 1 - iter 15/154 - loss 1.68195415 - samples/sec: 155.25\n",
            "2019-12-20 10:08:42,800 epoch 1 - iter 30/154 - loss 1.66373278 - samples/sec: 177.16\n",
            "2019-12-20 10:08:45,560 epoch 1 - iter 45/154 - loss 1.64211833 - samples/sec: 174.84\n",
            "2019-12-20 10:08:48,442 epoch 1 - iter 60/154 - loss 1.62511742 - samples/sec: 167.60\n",
            "2019-12-20 10:08:51,287 epoch 1 - iter 75/154 - loss 1.60974548 - samples/sec: 169.86\n",
            "2019-12-20 10:08:54,239 epoch 1 - iter 90/154 - loss 1.60068889 - samples/sec: 163.74\n",
            "2019-12-20 10:08:57,295 epoch 1 - iter 105/154 - loss 1.58265762 - samples/sec: 157.90\n",
            "2019-12-20 10:09:00,414 epoch 1 - iter 120/154 - loss 1.56362269 - samples/sec: 154.78\n",
            "2019-12-20 10:09:03,332 epoch 1 - iter 135/154 - loss 1.54777678 - samples/sec: 165.33\n",
            "2019-12-20 10:09:06,422 epoch 1 - iter 150/154 - loss 1.53369557 - samples/sec: 156.13\n",
            "2019-12-20 10:09:07,087 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:07,088 EPOCH 1 done: loss 1.5293 - lr 0.1000\n",
            "2019-12-20 10:09:10,276 DEV : loss 1.6775751113891602 - score 0.3358\n",
            "2019-12-20 10:09:10,304 BAD EPOCHS (no improvement): 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type DocumentRNNEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type StackedEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type WordEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type FlairEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type LanguageModel. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Dropout. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Embedding. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type LSTM. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Linear. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type GRU. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:09:13,530 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:13,581 epoch 2 - iter 0/154 - loss 1.63072765 - samples/sec: 10802.06\n",
            "2019-12-20 10:09:14,150 epoch 2 - iter 15/154 - loss 1.40115912 - samples/sec: 869.09\n",
            "2019-12-20 10:09:14,688 epoch 2 - iter 30/154 - loss 1.37406500 - samples/sec: 915.51\n",
            "2019-12-20 10:09:15,214 epoch 2 - iter 45/154 - loss 1.35986563 - samples/sec: 940.17\n",
            "2019-12-20 10:09:15,767 epoch 2 - iter 60/154 - loss 1.35646938 - samples/sec: 889.92\n",
            "2019-12-20 10:09:16,313 epoch 2 - iter 75/154 - loss 1.34643362 - samples/sec: 908.97\n",
            "2019-12-20 10:09:16,861 epoch 2 - iter 90/154 - loss 1.34063302 - samples/sec: 899.32\n",
            "2019-12-20 10:09:17,387 epoch 2 - iter 105/154 - loss 1.33613134 - samples/sec: 937.70\n",
            "2019-12-20 10:09:17,929 epoch 2 - iter 120/154 - loss 1.31926502 - samples/sec: 910.49\n",
            "2019-12-20 10:09:18,456 epoch 2 - iter 135/154 - loss 1.30923989 - samples/sec: 942.43\n",
            "2019-12-20 10:09:18,986 epoch 2 - iter 150/154 - loss 1.30121881 - samples/sec: 930.71\n",
            "2019-12-20 10:09:19,078 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:19,079 EPOCH 2 done: loss 1.2986 - lr 0.1000\n",
            "2019-12-20 10:09:19,560 DEV : loss 1.2602741718292236 - score 0.4844\n",
            "2019-12-20 10:09:19,586 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:09:22,849 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:22,903 epoch 3 - iter 0/154 - loss 1.34390819 - samples/sec: 10938.69\n",
            "2019-12-20 10:09:23,488 epoch 3 - iter 15/154 - loss 1.25194350 - samples/sec: 842.68\n",
            "2019-12-20 10:09:24,031 epoch 3 - iter 30/154 - loss 1.18781764 - samples/sec: 912.52\n",
            "2019-12-20 10:09:24,575 epoch 3 - iter 45/154 - loss 1.16815444 - samples/sec: 904.56\n",
            "2019-12-20 10:09:25,128 epoch 3 - iter 60/154 - loss 1.15411401 - samples/sec: 892.48\n",
            "2019-12-20 10:09:25,663 epoch 3 - iter 75/154 - loss 1.14803150 - samples/sec: 922.60\n",
            "2019-12-20 10:09:26,188 epoch 3 - iter 90/154 - loss 1.14700057 - samples/sec: 941.15\n",
            "2019-12-20 10:09:27,174 epoch 3 - iter 105/154 - loss 1.14624759 - samples/sec: 493.88\n",
            "2019-12-20 10:09:27,697 epoch 3 - iter 120/154 - loss 1.13379692 - samples/sec: 951.80\n",
            "2019-12-20 10:09:28,244 epoch 3 - iter 135/154 - loss 1.14065524 - samples/sec: 906.12\n",
            "2019-12-20 10:09:28,777 epoch 3 - iter 150/154 - loss 1.13915101 - samples/sec: 932.16\n",
            "2019-12-20 10:09:28,881 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:28,882 EPOCH 3 done: loss 1.1402 - lr 0.1000\n",
            "2019-12-20 10:09:29,361 DEV : loss 1.116384506225586 - score 0.556\n",
            "2019-12-20 10:09:29,389 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:09:32,507 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:32,555 epoch 4 - iter 0/154 - loss 1.01307487 - samples/sec: 11736.29\n",
            "2019-12-20 10:09:33,110 epoch 4 - iter 15/154 - loss 1.02549057 - samples/sec: 890.13\n",
            "2019-12-20 10:09:33,635 epoch 4 - iter 30/154 - loss 1.03900684 - samples/sec: 938.92\n",
            "2019-12-20 10:09:34,187 epoch 4 - iter 45/154 - loss 1.03732189 - samples/sec: 892.09\n",
            "2019-12-20 10:09:34,705 epoch 4 - iter 60/154 - loss 1.04328199 - samples/sec: 951.97\n",
            "2019-12-20 10:09:35,236 epoch 4 - iter 75/154 - loss 1.04507725 - samples/sec: 929.32\n",
            "2019-12-20 10:09:35,746 epoch 4 - iter 90/154 - loss 1.04186859 - samples/sec: 967.43\n",
            "2019-12-20 10:09:36,299 epoch 4 - iter 105/154 - loss 1.02839093 - samples/sec: 889.26\n",
            "2019-12-20 10:09:36,832 epoch 4 - iter 120/154 - loss 1.02418521 - samples/sec: 924.63\n",
            "2019-12-20 10:09:37,362 epoch 4 - iter 135/154 - loss 1.02204501 - samples/sec: 929.16\n",
            "2019-12-20 10:09:37,919 epoch 4 - iter 150/154 - loss 1.02775279 - samples/sec: 884.17\n",
            "2019-12-20 10:09:38,023 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:38,024 EPOCH 4 done: loss 1.0242 - lr 0.1000\n",
            "2019-12-20 10:09:38,506 DEV : loss 1.275339126586914 - score 0.5303\n",
            "2019-12-20 10:09:38,529 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:09:38,530 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:38,569 epoch 5 - iter 0/154 - loss 1.44622028 - samples/sec: 12923.20\n",
            "2019-12-20 10:09:39,110 epoch 5 - iter 15/154 - loss 0.93565492 - samples/sec: 909.38\n",
            "2019-12-20 10:09:39,650 epoch 5 - iter 30/154 - loss 0.94278682 - samples/sec: 913.18\n",
            "2019-12-20 10:09:40,202 epoch 5 - iter 45/154 - loss 0.90897857 - samples/sec: 893.15\n",
            "2019-12-20 10:09:40,760 epoch 5 - iter 60/154 - loss 0.89812875 - samples/sec: 882.83\n",
            "2019-12-20 10:09:41,297 epoch 5 - iter 75/154 - loss 0.94142250 - samples/sec: 918.60\n",
            "2019-12-20 10:09:41,843 epoch 5 - iter 90/154 - loss 0.92884376 - samples/sec: 902.30\n",
            "2019-12-20 10:09:42,380 epoch 5 - iter 105/154 - loss 0.92746417 - samples/sec: 916.88\n",
            "2019-12-20 10:09:42,918 epoch 5 - iter 120/154 - loss 0.92812748 - samples/sec: 918.87\n",
            "2019-12-20 10:09:43,463 epoch 5 - iter 135/154 - loss 0.92315875 - samples/sec: 904.82\n",
            "2019-12-20 10:09:43,996 epoch 5 - iter 150/154 - loss 0.92054065 - samples/sec: 923.26\n",
            "2019-12-20 10:09:44,099 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:44,100 EPOCH 5 done: loss 0.9179 - lr 0.1000\n",
            "2019-12-20 10:09:44,591 DEV : loss 0.9927656650543213 - score 0.6165\n",
            "2019-12-20 10:09:44,616 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:09:47,667 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:47,714 epoch 6 - iter 0/154 - loss 1.21568131 - samples/sec: 10800.84\n",
            "2019-12-20 10:09:48,312 epoch 6 - iter 15/154 - loss 0.90351686 - samples/sec: 824.61\n",
            "2019-12-20 10:09:48,856 epoch 6 - iter 30/154 - loss 0.83904269 - samples/sec: 910.50\n",
            "2019-12-20 10:09:49,390 epoch 6 - iter 45/154 - loss 0.84991259 - samples/sec: 923.20\n",
            "2019-12-20 10:09:49,946 epoch 6 - iter 60/154 - loss 0.83893036 - samples/sec: 887.52\n",
            "2019-12-20 10:09:50,496 epoch 6 - iter 75/154 - loss 0.83256795 - samples/sec: 896.33\n",
            "2019-12-20 10:09:51,052 epoch 6 - iter 90/154 - loss 0.81977587 - samples/sec: 890.32\n",
            "2019-12-20 10:09:51,595 epoch 6 - iter 105/154 - loss 0.81297516 - samples/sec: 908.40\n",
            "2019-12-20 10:09:52,128 epoch 6 - iter 120/154 - loss 0.82331189 - samples/sec: 925.06\n",
            "2019-12-20 10:09:52,672 epoch 6 - iter 135/154 - loss 0.82297933 - samples/sec: 905.50\n",
            "2019-12-20 10:09:53,226 epoch 6 - iter 150/154 - loss 0.82550436 - samples/sec: 897.59\n",
            "2019-12-20 10:09:53,323 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:53,324 EPOCH 6 done: loss 0.8230 - lr 0.1000\n",
            "2019-12-20 10:09:53,806 DEV : loss 0.6307267546653748 - score 0.7798\n",
            "2019-12-20 10:09:53,830 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:09:56,908 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:09:56,961 epoch 7 - iter 0/154 - loss 0.57329029 - samples/sec: 10564.00\n",
            "2019-12-20 10:09:57,518 epoch 7 - iter 15/154 - loss 0.82535966 - samples/sec: 891.62\n",
            "2019-12-20 10:09:58,061 epoch 7 - iter 30/154 - loss 0.78747886 - samples/sec: 910.61\n",
            "2019-12-20 10:09:58,604 epoch 7 - iter 45/154 - loss 0.78334376 - samples/sec: 914.62\n",
            "2019-12-20 10:09:59,117 epoch 7 - iter 60/154 - loss 0.77567459 - samples/sec: 961.69\n",
            "2019-12-20 10:09:59,633 epoch 7 - iter 75/154 - loss 0.75907032 - samples/sec: 955.80\n",
            "2019-12-20 10:10:00,196 epoch 7 - iter 90/154 - loss 0.78264748 - samples/sec: 882.60\n",
            "2019-12-20 10:10:00,721 epoch 7 - iter 105/154 - loss 0.76561339 - samples/sec: 939.62\n",
            "2019-12-20 10:10:01,248 epoch 7 - iter 120/154 - loss 0.75229496 - samples/sec: 938.09\n",
            "2019-12-20 10:10:01,768 epoch 7 - iter 135/154 - loss 0.75334685 - samples/sec: 956.88\n",
            "2019-12-20 10:10:02,314 epoch 7 - iter 150/154 - loss 0.73688162 - samples/sec: 903.01\n",
            "2019-12-20 10:10:02,424 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:02,425 EPOCH 7 done: loss 0.7390 - lr 0.1000\n",
            "2019-12-20 10:10:02,912 DEV : loss 1.2764678001403809 - score 0.5872\n",
            "2019-12-20 10:10:02,937 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:10:02,938 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:02,976 epoch 8 - iter 0/154 - loss 1.48475838 - samples/sec: 12875.92\n",
            "2019-12-20 10:10:03,522 epoch 8 - iter 15/154 - loss 0.82071057 - samples/sec: 901.06\n",
            "2019-12-20 10:10:04,055 epoch 8 - iter 30/154 - loss 0.77461153 - samples/sec: 927.90\n",
            "2019-12-20 10:10:04,596 epoch 8 - iter 45/154 - loss 0.73930899 - samples/sec: 912.45\n",
            "2019-12-20 10:10:05,148 epoch 8 - iter 60/154 - loss 0.71514928 - samples/sec: 892.95\n",
            "2019-12-20 10:10:05,687 epoch 8 - iter 75/154 - loss 0.71618363 - samples/sec: 914.56\n",
            "2019-12-20 10:10:06,241 epoch 8 - iter 90/154 - loss 0.69819750 - samples/sec: 889.85\n",
            "2019-12-20 10:10:06,803 epoch 8 - iter 105/154 - loss 0.69136469 - samples/sec: 877.87\n",
            "2019-12-20 10:10:07,339 epoch 8 - iter 120/154 - loss 0.68715310 - samples/sec: 919.74\n",
            "2019-12-20 10:10:07,906 epoch 8 - iter 135/154 - loss 0.69303531 - samples/sec: 869.06\n",
            "2019-12-20 10:10:08,432 epoch 8 - iter 150/154 - loss 0.68658809 - samples/sec: 939.04\n",
            "2019-12-20 10:10:08,534 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:08,535 EPOCH 8 done: loss 0.6843 - lr 0.1000\n",
            "2019-12-20 10:10:09,031 DEV : loss 0.6105888485908508 - score 0.7853\n",
            "2019-12-20 10:10:09,056 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:10:12,262 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:12,310 epoch 9 - iter 0/154 - loss 0.55822057 - samples/sec: 11287.72\n",
            "2019-12-20 10:10:12,860 epoch 9 - iter 15/154 - loss 0.54518604 - samples/sec: 897.22\n",
            "2019-12-20 10:10:13,411 epoch 9 - iter 30/154 - loss 0.68374369 - samples/sec: 895.11\n",
            "2019-12-20 10:10:13,951 epoch 9 - iter 45/154 - loss 0.66800174 - samples/sec: 912.46\n",
            "2019-12-20 10:10:14,515 epoch 9 - iter 60/154 - loss 0.64912588 - samples/sec: 882.34\n",
            "2019-12-20 10:10:15,063 epoch 9 - iter 75/154 - loss 0.63085160 - samples/sec: 899.96\n",
            "2019-12-20 10:10:15,594 epoch 9 - iter 90/154 - loss 0.62428651 - samples/sec: 930.66\n",
            "2019-12-20 10:10:16,156 epoch 9 - iter 105/154 - loss 0.64223649 - samples/sec: 874.83\n",
            "2019-12-20 10:10:16,688 epoch 9 - iter 120/154 - loss 0.62091530 - samples/sec: 927.56\n",
            "2019-12-20 10:10:17,263 epoch 9 - iter 135/154 - loss 0.63147102 - samples/sec: 857.10\n",
            "2019-12-20 10:10:17,793 epoch 9 - iter 150/154 - loss 0.61294075 - samples/sec: 928.41\n",
            "2019-12-20 10:10:17,897 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:17,898 EPOCH 9 done: loss 0.6161 - lr 0.1000\n",
            "2019-12-20 10:10:18,386 DEV : loss 0.5224140286445618 - score 0.8165\n",
            "2019-12-20 10:10:18,411 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:10:21,389 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:21,438 epoch 10 - iter 0/154 - loss 0.63147360 - samples/sec: 10310.80\n",
            "2019-12-20 10:10:22,004 epoch 10 - iter 15/154 - loss 0.52805815 - samples/sec: 874.29\n",
            "2019-12-20 10:10:22,540 epoch 10 - iter 30/154 - loss 0.53854991 - samples/sec: 920.81\n",
            "2019-12-20 10:10:23,057 epoch 10 - iter 45/154 - loss 0.52400490 - samples/sec: 953.40\n",
            "2019-12-20 10:10:23,591 epoch 10 - iter 60/154 - loss 0.55619904 - samples/sec: 922.26\n",
            "2019-12-20 10:10:24,129 epoch 10 - iter 75/154 - loss 0.56312844 - samples/sec: 918.31\n",
            "2019-12-20 10:10:24,672 epoch 10 - iter 90/154 - loss 0.55927880 - samples/sec: 911.15\n",
            "2019-12-20 10:10:25,224 epoch 10 - iter 105/154 - loss 0.56400047 - samples/sec: 893.30\n",
            "2019-12-20 10:10:25,762 epoch 10 - iter 120/154 - loss 0.55486411 - samples/sec: 917.07\n",
            "2019-12-20 10:10:26,292 epoch 10 - iter 135/154 - loss 0.56070712 - samples/sec: 931.95\n",
            "2019-12-20 10:10:26,846 epoch 10 - iter 150/154 - loss 0.56191083 - samples/sec: 892.23\n",
            "2019-12-20 10:10:26,949 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:26,950 EPOCH 10 done: loss 0.5706 - lr 0.1000\n",
            "2019-12-20 10:10:27,429 DEV : loss 0.8336689472198486 - score 0.7064\n",
            "2019-12-20 10:10:27,453 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:10:27,454 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:27,489 epoch 11 - iter 0/154 - loss 1.06534648 - samples/sec: 14026.60\n",
            "2019-12-20 10:10:28,048 epoch 11 - iter 15/154 - loss 0.57595473 - samples/sec: 880.13\n",
            "2019-12-20 10:10:28,609 epoch 11 - iter 30/154 - loss 0.53340751 - samples/sec: 878.95\n",
            "2019-12-20 10:10:29,148 epoch 11 - iter 45/154 - loss 0.53301897 - samples/sec: 913.96\n",
            "2019-12-20 10:10:29,692 epoch 11 - iter 60/154 - loss 0.54421027 - samples/sec: 913.43\n",
            "2019-12-20 10:10:30,249 epoch 11 - iter 75/154 - loss 0.53226618 - samples/sec: 887.63\n",
            "2019-12-20 10:10:30,781 epoch 11 - iter 90/154 - loss 0.52417656 - samples/sec: 932.51\n",
            "2019-12-20 10:10:31,330 epoch 11 - iter 105/154 - loss 0.51724575 - samples/sec: 902.62\n",
            "2019-12-20 10:10:31,872 epoch 11 - iter 120/154 - loss 0.52018202 - samples/sec: 913.01\n",
            "2019-12-20 10:10:32,410 epoch 11 - iter 135/154 - loss 0.51926797 - samples/sec: 923.36\n",
            "2019-12-20 10:10:32,951 epoch 11 - iter 150/154 - loss 0.51823022 - samples/sec: 917.02\n",
            "2019-12-20 10:10:33,054 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:33,055 EPOCH 11 done: loss 0.5181 - lr 0.1000\n",
            "2019-12-20 10:10:33,544 DEV : loss 0.3883811831474304 - score 0.8514\n",
            "2019-12-20 10:10:33,571 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:10:36,990 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:37,040 epoch 12 - iter 0/154 - loss 0.41910854 - samples/sec: 10153.19\n",
            "2019-12-20 10:10:37,615 epoch 12 - iter 15/154 - loss 0.50034790 - samples/sec: 858.24\n",
            "2019-12-20 10:10:38,169 epoch 12 - iter 30/154 - loss 0.45162362 - samples/sec: 892.83\n",
            "2019-12-20 10:10:38,708 epoch 12 - iter 45/154 - loss 0.48453022 - samples/sec: 913.78\n",
            "2019-12-20 10:10:39,252 epoch 12 - iter 60/154 - loss 0.47102503 - samples/sec: 905.93\n",
            "2019-12-20 10:10:39,798 epoch 12 - iter 75/154 - loss 0.47832486 - samples/sec: 903.28\n",
            "2019-12-20 10:10:40,345 epoch 12 - iter 90/154 - loss 0.48028502 - samples/sec: 902.39\n",
            "2019-12-20 10:10:40,896 epoch 12 - iter 105/154 - loss 0.47982428 - samples/sec: 894.46\n",
            "2019-12-20 10:10:41,432 epoch 12 - iter 120/154 - loss 0.49189590 - samples/sec: 922.12\n",
            "2019-12-20 10:10:41,995 epoch 12 - iter 135/154 - loss 0.49413630 - samples/sec: 878.74\n",
            "2019-12-20 10:10:42,527 epoch 12 - iter 150/154 - loss 0.49323997 - samples/sec: 926.72\n",
            "2019-12-20 10:10:42,636 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:42,637 EPOCH 12 done: loss 0.4936 - lr 0.1000\n",
            "2019-12-20 10:10:43,128 DEV : loss 0.38787397742271423 - score 0.8495\n",
            "2019-12-20 10:10:43,153 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:10:43,154 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:43,193 epoch 13 - iter 0/154 - loss 0.55041927 - samples/sec: 12896.79\n",
            "2019-12-20 10:10:43,728 epoch 13 - iter 15/154 - loss 0.37809386 - samples/sec: 920.49\n",
            "2019-12-20 10:10:44,255 epoch 13 - iter 30/154 - loss 0.38362742 - samples/sec: 934.85\n",
            "2019-12-20 10:10:44,806 epoch 13 - iter 45/154 - loss 0.41750781 - samples/sec: 894.51\n",
            "2019-12-20 10:10:45,341 epoch 13 - iter 60/154 - loss 0.41850272 - samples/sec: 922.89\n",
            "2019-12-20 10:10:45,909 epoch 13 - iter 75/154 - loss 0.43053733 - samples/sec: 867.96\n",
            "2019-12-20 10:10:46,443 epoch 13 - iter 90/154 - loss 0.41777002 - samples/sec: 922.61\n",
            "2019-12-20 10:10:46,985 epoch 13 - iter 105/154 - loss 0.42040734 - samples/sec: 910.72\n",
            "2019-12-20 10:10:47,511 epoch 13 - iter 120/154 - loss 0.42485535 - samples/sec: 940.58\n",
            "2019-12-20 10:10:48,085 epoch 13 - iter 135/154 - loss 0.43158270 - samples/sec: 860.23\n",
            "2019-12-20 10:10:48,619 epoch 13 - iter 150/154 - loss 0.43032009 - samples/sec: 927.36\n",
            "2019-12-20 10:10:48,730 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:48,732 EPOCH 13 done: loss 0.4309 - lr 0.1000\n",
            "2019-12-20 10:10:49,204 DEV : loss 0.5279505848884583 - score 0.8147\n",
            "2019-12-20 10:10:49,228 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:10:49,229 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:49,263 epoch 14 - iter 0/154 - loss 0.36434370 - samples/sec: 14770.52\n",
            "2019-12-20 10:10:49,791 epoch 14 - iter 15/154 - loss 0.48657810 - samples/sec: 932.42\n",
            "2019-12-20 10:10:50,334 epoch 14 - iter 30/154 - loss 0.46074089 - samples/sec: 909.63\n",
            "2019-12-20 10:10:50,875 epoch 14 - iter 45/154 - loss 0.44381262 - samples/sec: 909.97\n",
            "2019-12-20 10:10:51,415 epoch 14 - iter 60/154 - loss 0.43118585 - samples/sec: 914.49\n",
            "2019-12-20 10:10:51,962 epoch 14 - iter 75/154 - loss 0.43865528 - samples/sec: 900.56\n",
            "2019-12-20 10:10:52,497 epoch 14 - iter 90/154 - loss 0.44197835 - samples/sec: 921.94\n",
            "2019-12-20 10:10:53,044 epoch 14 - iter 105/154 - loss 0.44488323 - samples/sec: 900.68\n",
            "2019-12-20 10:10:53,590 epoch 14 - iter 120/154 - loss 0.43341302 - samples/sec: 901.92\n",
            "2019-12-20 10:10:54,146 epoch 14 - iter 135/154 - loss 0.42278155 - samples/sec: 886.47\n",
            "2019-12-20 10:10:54,699 epoch 14 - iter 150/154 - loss 0.41978187 - samples/sec: 891.53\n",
            "2019-12-20 10:10:54,801 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:54,806 EPOCH 14 done: loss 0.4225 - lr 0.1000\n",
            "2019-12-20 10:10:55,295 DEV : loss 0.8001553416252136 - score 0.7321\n",
            "2019-12-20 10:10:55,319 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:10:55,320 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:10:55,358 epoch 15 - iter 0/154 - loss 0.26354823 - samples/sec: 12872.13\n",
            "2019-12-20 10:10:55,900 epoch 15 - iter 15/154 - loss 0.41095859 - samples/sec: 908.97\n",
            "2019-12-20 10:10:56,448 epoch 15 - iter 30/154 - loss 0.43052864 - samples/sec: 901.58\n",
            "2019-12-20 10:10:56,993 epoch 15 - iter 45/154 - loss 0.41916862 - samples/sec: 907.42\n",
            "2019-12-20 10:10:57,543 epoch 15 - iter 60/154 - loss 0.40760523 - samples/sec: 900.06\n",
            "2019-12-20 10:10:58,130 epoch 15 - iter 75/154 - loss 0.39172816 - samples/sec: 842.38\n",
            "2019-12-20 10:10:58,674 epoch 15 - iter 90/154 - loss 0.39117961 - samples/sec: 906.70\n",
            "2019-12-20 10:10:59,219 epoch 15 - iter 105/154 - loss 0.38579301 - samples/sec: 906.64\n",
            "2019-12-20 10:10:59,776 epoch 15 - iter 120/154 - loss 0.38213307 - samples/sec: 892.61\n",
            "2019-12-20 10:11:00,322 epoch 15 - iter 135/154 - loss 0.39401824 - samples/sec: 904.83\n",
            "2019-12-20 10:11:00,866 epoch 15 - iter 150/154 - loss 0.39586358 - samples/sec: 904.15\n",
            "2019-12-20 10:11:00,972 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:00,973 EPOCH 15 done: loss 0.3931 - lr 0.1000\n",
            "2019-12-20 10:11:01,465 DEV : loss 0.315681129693985 - score 0.8752\n",
            "2019-12-20 10:11:01,492 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:11:04,543 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:04,590 epoch 16 - iter 0/154 - loss 0.49940893 - samples/sec: 11638.86\n",
            "2019-12-20 10:11:05,202 epoch 16 - iter 15/154 - loss 0.34712869 - samples/sec: 811.42\n",
            "2019-12-20 10:11:05,742 epoch 16 - iter 30/154 - loss 0.35554900 - samples/sec: 913.02\n",
            "2019-12-20 10:11:06,302 epoch 16 - iter 45/154 - loss 0.35334113 - samples/sec: 880.76\n",
            "2019-12-20 10:11:06,837 epoch 16 - iter 60/154 - loss 0.36322450 - samples/sec: 922.22\n",
            "2019-12-20 10:11:07,398 epoch 16 - iter 75/154 - loss 0.36290220 - samples/sec: 877.94\n",
            "2019-12-20 10:11:07,926 epoch 16 - iter 90/154 - loss 0.36518221 - samples/sec: 934.00\n",
            "2019-12-20 10:11:08,481 epoch 16 - iter 105/154 - loss 0.36169967 - samples/sec: 888.59\n",
            "2019-12-20 10:11:09,030 epoch 16 - iter 120/154 - loss 0.37025867 - samples/sec: 897.29\n",
            "2019-12-20 10:11:09,578 epoch 16 - iter 135/154 - loss 0.37030731 - samples/sec: 903.37\n",
            "2019-12-20 10:11:10,148 epoch 16 - iter 150/154 - loss 0.37781325 - samples/sec: 864.03\n",
            "2019-12-20 10:11:10,245 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:10,246 EPOCH 16 done: loss 0.3773 - lr 0.1000\n",
            "2019-12-20 10:11:10,736 DEV : loss 1.3124604225158691 - score 0.633\n",
            "2019-12-20 10:11:10,761 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:11:10,762 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:10,803 epoch 17 - iter 0/154 - loss 2.32767916 - samples/sec: 12634.16\n",
            "2019-12-20 10:11:11,346 epoch 17 - iter 15/154 - loss 0.39267743 - samples/sec: 908.80\n",
            "2019-12-20 10:11:11,882 epoch 17 - iter 30/154 - loss 0.37524672 - samples/sec: 921.87\n",
            "2019-12-20 10:11:12,425 epoch 17 - iter 45/154 - loss 0.36664051 - samples/sec: 908.49\n",
            "2019-12-20 10:11:12,977 epoch 17 - iter 60/154 - loss 0.38259115 - samples/sec: 893.27\n",
            "2019-12-20 10:11:13,506 epoch 17 - iter 75/154 - loss 0.37962975 - samples/sec: 932.58\n",
            "2019-12-20 10:11:14,030 epoch 17 - iter 90/154 - loss 0.37441410 - samples/sec: 945.60\n",
            "2019-12-20 10:11:14,559 epoch 17 - iter 105/154 - loss 0.36915933 - samples/sec: 934.24\n",
            "2019-12-20 10:11:15,073 epoch 17 - iter 120/154 - loss 0.36580986 - samples/sec: 966.39\n",
            "2019-12-20 10:11:15,593 epoch 17 - iter 135/154 - loss 0.36997503 - samples/sec: 955.62\n",
            "2019-12-20 10:11:16,138 epoch 17 - iter 150/154 - loss 0.37027291 - samples/sec: 907.18\n",
            "2019-12-20 10:11:16,243 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:16,244 EPOCH 17 done: loss 0.3683 - lr 0.1000\n",
            "2019-12-20 10:11:16,715 DEV : loss 0.3236328661441803 - score 0.8972\n",
            "2019-12-20 10:11:16,740 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:11:19,880 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:19,926 epoch 18 - iter 0/154 - loss 0.15112713 - samples/sec: 12727.61\n",
            "2019-12-20 10:11:20,525 epoch 18 - iter 15/154 - loss 0.38825562 - samples/sec: 821.89\n",
            "2019-12-20 10:11:21,070 epoch 18 - iter 30/154 - loss 0.38173318 - samples/sec: 905.00\n",
            "2019-12-20 10:11:21,601 epoch 18 - iter 45/154 - loss 0.36011294 - samples/sec: 928.47\n",
            "2019-12-20 10:11:22,132 epoch 18 - iter 60/154 - loss 0.35438594 - samples/sec: 926.82\n",
            "2019-12-20 10:11:22,667 epoch 18 - iter 75/154 - loss 0.35120951 - samples/sec: 928.67\n",
            "2019-12-20 10:11:23,195 epoch 18 - iter 90/154 - loss 0.34209441 - samples/sec: 933.60\n",
            "2019-12-20 10:11:23,754 epoch 18 - iter 105/154 - loss 0.34463627 - samples/sec: 881.90\n",
            "2019-12-20 10:11:24,297 epoch 18 - iter 120/154 - loss 0.33919135 - samples/sec: 908.93\n",
            "2019-12-20 10:11:24,861 epoch 18 - iter 135/154 - loss 0.33957797 - samples/sec: 877.81\n",
            "2019-12-20 10:11:25,409 epoch 18 - iter 150/154 - loss 0.34094209 - samples/sec: 899.27\n",
            "2019-12-20 10:11:25,521 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:25,522 EPOCH 18 done: loss 0.3426 - lr 0.1000\n",
            "2019-12-20 10:11:26,015 DEV : loss 0.4325767159461975 - score 0.8587\n",
            "2019-12-20 10:11:26,040 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:11:26,040 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:26,079 epoch 19 - iter 0/154 - loss 0.31163275 - samples/sec: 13414.17\n",
            "2019-12-20 10:11:26,624 epoch 19 - iter 15/154 - loss 0.31098322 - samples/sec: 907.32\n",
            "2019-12-20 10:11:27,169 epoch 19 - iter 30/154 - loss 0.34988592 - samples/sec: 906.67\n",
            "2019-12-20 10:11:27,716 epoch 19 - iter 45/154 - loss 0.34504320 - samples/sec: 905.44\n",
            "2019-12-20 10:11:28,280 epoch 19 - iter 60/154 - loss 0.34143479 - samples/sec: 876.72\n",
            "2019-12-20 10:11:28,839 epoch 19 - iter 75/154 - loss 0.33904571 - samples/sec: 887.23\n",
            "2019-12-20 10:11:29,393 epoch 19 - iter 90/154 - loss 0.33725615 - samples/sec: 896.56\n",
            "2019-12-20 10:11:29,925 epoch 19 - iter 105/154 - loss 0.33090350 - samples/sec: 929.20\n",
            "2019-12-20 10:11:30,477 epoch 19 - iter 120/154 - loss 0.33520166 - samples/sec: 895.22\n",
            "2019-12-20 10:11:31,019 epoch 19 - iter 135/154 - loss 0.33953822 - samples/sec: 912.25\n",
            "2019-12-20 10:11:31,566 epoch 19 - iter 150/154 - loss 0.33273269 - samples/sec: 905.09\n",
            "2019-12-20 10:11:31,664 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:31,665 EPOCH 19 done: loss 0.3317 - lr 0.1000\n",
            "2019-12-20 10:11:32,148 DEV : loss 0.3643990755081177 - score 0.8862\n",
            "2019-12-20 10:11:32,172 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:11:32,174 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:32,211 epoch 20 - iter 0/154 - loss 0.27509844 - samples/sec: 13302.93\n",
            "2019-12-20 10:11:32,752 epoch 20 - iter 15/154 - loss 0.30787547 - samples/sec: 909.96\n",
            "2019-12-20 10:11:33,302 epoch 20 - iter 30/154 - loss 0.31201181 - samples/sec: 897.52\n",
            "2019-12-20 10:11:33,846 epoch 20 - iter 45/154 - loss 0.30606362 - samples/sec: 905.23\n",
            "2019-12-20 10:11:34,379 epoch 20 - iter 60/154 - loss 0.31278902 - samples/sec: 925.70\n",
            "2019-12-20 10:11:34,921 epoch 20 - iter 75/154 - loss 0.30392020 - samples/sec: 910.16\n",
            "2019-12-20 10:11:35,451 epoch 20 - iter 90/154 - loss 0.30255145 - samples/sec: 930.29\n",
            "2019-12-20 10:11:35,993 epoch 20 - iter 105/154 - loss 0.30720225 - samples/sec: 909.54\n",
            "2019-12-20 10:11:36,553 epoch 20 - iter 120/154 - loss 0.31092224 - samples/sec: 879.48\n",
            "2019-12-20 10:11:37,098 epoch 20 - iter 135/154 - loss 0.31485638 - samples/sec: 905.50\n",
            "2019-12-20 10:11:37,629 epoch 20 - iter 150/154 - loss 0.31225732 - samples/sec: 928.25\n",
            "2019-12-20 10:11:37,726 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:37,727 EPOCH 20 done: loss 0.3111 - lr 0.1000\n",
            "2019-12-20 10:11:38,207 DEV : loss 0.5996942520141602 - score 0.7798\n",
            "2019-12-20 10:11:38,232 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:11:38,232 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:38,272 epoch 21 - iter 0/154 - loss 0.88506609 - samples/sec: 12633.76\n",
            "2019-12-20 10:11:38,820 epoch 21 - iter 15/154 - loss 0.36675493 - samples/sec: 897.64\n",
            "2019-12-20 10:11:39,347 epoch 21 - iter 30/154 - loss 0.32567104 - samples/sec: 935.16\n",
            "2019-12-20 10:11:39,886 epoch 21 - iter 45/154 - loss 0.33316303 - samples/sec: 916.73\n",
            "2019-12-20 10:11:40,397 epoch 21 - iter 60/154 - loss 0.32881728 - samples/sec: 965.80\n",
            "2019-12-20 10:11:40,927 epoch 21 - iter 75/154 - loss 0.31684950 - samples/sec: 934.82\n",
            "2019-12-20 10:11:41,466 epoch 21 - iter 90/154 - loss 0.31683583 - samples/sec: 913.60\n",
            "2019-12-20 10:11:42,005 epoch 21 - iter 105/154 - loss 0.30897747 - samples/sec: 915.95\n",
            "2019-12-20 10:11:42,550 epoch 21 - iter 120/154 - loss 0.30555219 - samples/sec: 907.39\n",
            "2019-12-20 10:11:43,062 epoch 21 - iter 135/154 - loss 0.30511735 - samples/sec: 964.84\n",
            "2019-12-20 10:11:43,588 epoch 21 - iter 150/154 - loss 0.30581725 - samples/sec: 937.71\n",
            "2019-12-20 10:11:43,693 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:43,694 EPOCH 21 done: loss 0.3035 - lr 0.1000\n",
            "2019-12-20 10:11:44,174 DEV : loss 0.29593807458877563 - score 0.9064\n",
            "2019-12-20 10:11:44,199 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:11:47,366 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:47,413 epoch 22 - iter 0/154 - loss 0.16307478 - samples/sec: 10617.99\n",
            "2019-12-20 10:11:48,034 epoch 22 - iter 15/154 - loss 0.25964608 - samples/sec: 793.13\n",
            "2019-12-20 10:11:48,569 epoch 22 - iter 30/154 - loss 0.26539790 - samples/sec: 923.04\n",
            "2019-12-20 10:11:49,101 epoch 22 - iter 45/154 - loss 0.27103418 - samples/sec: 925.31\n",
            "2019-12-20 10:11:49,643 epoch 22 - iter 60/154 - loss 0.25947730 - samples/sec: 908.97\n",
            "2019-12-20 10:11:50,196 epoch 22 - iter 75/154 - loss 0.27528050 - samples/sec: 892.38\n",
            "2019-12-20 10:11:50,742 epoch 22 - iter 90/154 - loss 0.27484984 - samples/sec: 902.25\n",
            "2019-12-20 10:11:51,276 epoch 22 - iter 105/154 - loss 0.27199419 - samples/sec: 923.44\n",
            "2019-12-20 10:11:51,834 epoch 22 - iter 120/154 - loss 0.27404537 - samples/sec: 886.50\n",
            "2019-12-20 10:11:52,396 epoch 22 - iter 135/154 - loss 0.27454587 - samples/sec: 879.24\n",
            "2019-12-20 10:11:52,931 epoch 22 - iter 150/154 - loss 0.28167430 - samples/sec: 921.91\n",
            "2019-12-20 10:11:53,032 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:53,033 EPOCH 22 done: loss 0.2797 - lr 0.1000\n",
            "2019-12-20 10:11:53,536 DEV : loss 0.26380354166030884 - score 0.9101\n",
            "2019-12-20 10:11:53,561 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:11:56,686 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:11:56,743 epoch 23 - iter 0/154 - loss 0.22812702 - samples/sec: 9339.14\n",
            "2019-12-20 10:11:57,349 epoch 23 - iter 15/154 - loss 0.24750701 - samples/sec: 812.72\n",
            "2019-12-20 10:11:57,896 epoch 23 - iter 30/154 - loss 0.27990873 - samples/sec: 907.12\n",
            "2019-12-20 10:11:58,483 epoch 23 - iter 45/154 - loss 0.26966543 - samples/sec: 837.37\n",
            "2019-12-20 10:11:59,028 epoch 23 - iter 60/154 - loss 0.26999472 - samples/sec: 903.68\n",
            "2019-12-20 10:11:59,555 epoch 23 - iter 75/154 - loss 0.27617401 - samples/sec: 935.62\n",
            "2019-12-20 10:12:00,085 epoch 23 - iter 90/154 - loss 0.27858785 - samples/sec: 930.00\n",
            "2019-12-20 10:12:00,623 epoch 23 - iter 105/154 - loss 0.26984825 - samples/sec: 915.01\n",
            "2019-12-20 10:12:01,171 epoch 23 - iter 120/154 - loss 0.27634761 - samples/sec: 900.38\n",
            "2019-12-20 10:12:01,717 epoch 23 - iter 135/154 - loss 0.26952461 - samples/sec: 901.68\n",
            "2019-12-20 10:12:02,275 epoch 23 - iter 150/154 - loss 0.26898254 - samples/sec: 881.16\n",
            "2019-12-20 10:12:02,377 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:02,378 EPOCH 23 done: loss 0.2700 - lr 0.1000\n",
            "2019-12-20 10:12:02,881 DEV : loss 0.5138729810714722 - score 0.8514\n",
            "2019-12-20 10:12:02,906 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:12:02,906 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:02,944 epoch 24 - iter 0/154 - loss 0.91929269 - samples/sec: 13825.29\n",
            "2019-12-20 10:12:03,480 epoch 24 - iter 15/154 - loss 0.27984388 - samples/sec: 919.31\n",
            "2019-12-20 10:12:04,015 epoch 24 - iter 30/154 - loss 0.27780648 - samples/sec: 920.78\n",
            "2019-12-20 10:12:04,566 epoch 24 - iter 45/154 - loss 0.27444189 - samples/sec: 894.69\n",
            "2019-12-20 10:12:05,101 epoch 24 - iter 60/154 - loss 0.27350388 - samples/sec: 921.48\n",
            "2019-12-20 10:12:05,628 epoch 24 - iter 75/154 - loss 0.27098500 - samples/sec: 938.49\n",
            "2019-12-20 10:12:06,164 epoch 24 - iter 90/154 - loss 0.26769602 - samples/sec: 918.41\n",
            "2019-12-20 10:12:06,696 epoch 24 - iter 105/154 - loss 0.26615713 - samples/sec: 927.92\n",
            "2019-12-20 10:12:07,223 epoch 24 - iter 120/154 - loss 0.26443070 - samples/sec: 936.59\n",
            "2019-12-20 10:12:07,757 epoch 24 - iter 135/154 - loss 0.26363141 - samples/sec: 922.71\n",
            "2019-12-20 10:12:08,281 epoch 24 - iter 150/154 - loss 0.26517249 - samples/sec: 941.28\n",
            "2019-12-20 10:12:08,379 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:08,380 EPOCH 24 done: loss 0.2648 - lr 0.1000\n",
            "2019-12-20 10:12:08,848 DEV : loss 0.44946396350860596 - score 0.855\n",
            "2019-12-20 10:12:08,872 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:12:08,873 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:08,913 epoch 25 - iter 0/154 - loss 0.36894983 - samples/sec: 12537.62\n",
            "2019-12-20 10:12:09,877 epoch 25 - iter 15/154 - loss 0.25141369 - samples/sec: 505.16\n",
            "2019-12-20 10:12:10,398 epoch 25 - iter 30/154 - loss 0.25114723 - samples/sec: 951.02\n",
            "2019-12-20 10:12:10,915 epoch 25 - iter 45/154 - loss 0.26184621 - samples/sec: 964.82\n",
            "2019-12-20 10:12:11,449 epoch 25 - iter 60/154 - loss 0.25321895 - samples/sec: 927.11\n",
            "2019-12-20 10:12:12,022 epoch 25 - iter 75/154 - loss 0.25107815 - samples/sec: 863.38\n",
            "2019-12-20 10:12:12,557 epoch 25 - iter 90/154 - loss 0.25353388 - samples/sec: 922.36\n",
            "2019-12-20 10:12:13,102 epoch 25 - iter 105/154 - loss 0.25491512 - samples/sec: 908.55\n",
            "2019-12-20 10:12:13,641 epoch 25 - iter 120/154 - loss 0.25989110 - samples/sec: 921.96\n",
            "2019-12-20 10:12:14,178 epoch 25 - iter 135/154 - loss 0.25340847 - samples/sec: 919.44\n",
            "2019-12-20 10:12:14,730 epoch 25 - iter 150/154 - loss 0.25610462 - samples/sec: 895.83\n",
            "2019-12-20 10:12:14,836 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:14,837 EPOCH 25 done: loss 0.2545 - lr 0.1000\n",
            "2019-12-20 10:12:15,319 DEV : loss 0.24541041254997253 - score 0.9064\n",
            "2019-12-20 10:12:15,345 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:12:15,346 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:15,389 epoch 26 - iter 0/154 - loss 0.10667874 - samples/sec: 11646.40\n",
            "2019-12-20 10:12:15,961 epoch 26 - iter 15/154 - loss 0.24673976 - samples/sec: 862.21\n",
            "2019-12-20 10:12:16,500 epoch 26 - iter 30/154 - loss 0.23104120 - samples/sec: 915.38\n",
            "2019-12-20 10:12:17,045 epoch 26 - iter 45/154 - loss 0.22854064 - samples/sec: 907.53\n",
            "2019-12-20 10:12:17,607 epoch 26 - iter 60/154 - loss 0.22987568 - samples/sec: 875.80\n",
            "2019-12-20 10:12:18,147 epoch 26 - iter 75/154 - loss 0.23417861 - samples/sec: 918.71\n",
            "2019-12-20 10:12:18,687 epoch 26 - iter 90/154 - loss 0.23386899 - samples/sec: 913.77\n",
            "2019-12-20 10:12:19,224 epoch 26 - iter 105/154 - loss 0.23845763 - samples/sec: 917.22\n",
            "2019-12-20 10:12:19,784 epoch 26 - iter 120/154 - loss 0.24216552 - samples/sec: 881.24\n",
            "2019-12-20 10:12:20,337 epoch 26 - iter 135/154 - loss 0.24529742 - samples/sec: 890.19\n",
            "2019-12-20 10:12:20,897 epoch 26 - iter 150/154 - loss 0.24725573 - samples/sec: 880.29\n",
            "2019-12-20 10:12:21,011 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:21,012 EPOCH 26 done: loss 0.2473 - lr 0.1000\n",
            "2019-12-20 10:12:21,506 DEV : loss 0.4233953356742859 - score 0.8642\n",
            "2019-12-20 10:12:21,532 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:12:21,533 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:21,573 epoch 27 - iter 0/154 - loss 0.18807407 - samples/sec: 12313.78\n",
            "2019-12-20 10:12:22,126 epoch 27 - iter 15/154 - loss 0.18074153 - samples/sec: 889.51\n",
            "2019-12-20 10:12:22,658 epoch 27 - iter 30/154 - loss 0.20147242 - samples/sec: 928.31\n",
            "2019-12-20 10:12:23,218 epoch 27 - iter 45/154 - loss 0.21960894 - samples/sec: 882.32\n",
            "2019-12-20 10:12:23,782 epoch 27 - iter 60/154 - loss 0.21886689 - samples/sec: 875.10\n",
            "2019-12-20 10:12:24,335 epoch 27 - iter 75/154 - loss 0.22446706 - samples/sec: 892.87\n",
            "2019-12-20 10:12:24,899 epoch 27 - iter 90/154 - loss 0.22254842 - samples/sec: 873.71\n",
            "2019-12-20 10:12:25,437 epoch 27 - iter 105/154 - loss 0.22608932 - samples/sec: 920.35\n",
            "2019-12-20 10:12:25,973 epoch 27 - iter 120/154 - loss 0.22997575 - samples/sec: 920.04\n",
            "2019-12-20 10:12:26,515 epoch 27 - iter 135/154 - loss 0.23046654 - samples/sec: 909.80\n",
            "2019-12-20 10:12:27,081 epoch 27 - iter 150/154 - loss 0.23186344 - samples/sec: 871.15\n",
            "2019-12-20 10:12:27,185 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:27,186 EPOCH 27 done: loss 0.2355 - lr 0.1000\n",
            "2019-12-20 10:12:27,676 DEV : loss 0.3273316025733948 - score 0.8862\n",
            "2019-12-20 10:12:27,701 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:12:27,702 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:27,742 epoch 28 - iter 0/154 - loss 0.44574931 - samples/sec: 12334.75\n",
            "2019-12-20 10:12:28,311 epoch 28 - iter 15/154 - loss 0.22932423 - samples/sec: 864.37\n",
            "2019-12-20 10:12:28,877 epoch 28 - iter 30/154 - loss 0.21693039 - samples/sec: 874.65\n",
            "2019-12-20 10:12:29,442 epoch 28 - iter 45/154 - loss 0.23271672 - samples/sec: 870.75\n",
            "2019-12-20 10:12:30,000 epoch 28 - iter 60/154 - loss 0.22554080 - samples/sec: 886.43\n",
            "2019-12-20 10:12:30,543 epoch 28 - iter 75/154 - loss 0.22125825 - samples/sec: 913.69\n",
            "2019-12-20 10:12:31,092 epoch 28 - iter 90/154 - loss 0.22284823 - samples/sec: 900.75\n",
            "2019-12-20 10:12:31,603 epoch 28 - iter 105/154 - loss 0.22369292 - samples/sec: 967.70\n",
            "2019-12-20 10:12:32,145 epoch 28 - iter 120/154 - loss 0.22736187 - samples/sec: 918.74\n",
            "2019-12-20 10:12:32,669 epoch 28 - iter 135/154 - loss 0.22869607 - samples/sec: 944.02\n",
            "2019-12-20 10:12:33,185 epoch 28 - iter 150/154 - loss 0.22718949 - samples/sec: 954.55\n",
            "2019-12-20 10:12:33,289 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:33,292 EPOCH 28 done: loss 0.2285 - lr 0.1000\n",
            "2019-12-20 10:12:33,771 DEV : loss 0.3238876461982727 - score 0.9009\n",
            "Epoch    27: reducing learning rate of group 0 to 5.0000e-02.\n",
            "2019-12-20 10:12:33,798 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:12:33,799 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:33,840 epoch 29 - iter 0/154 - loss 0.31054798 - samples/sec: 12121.68\n",
            "2019-12-20 10:12:34,378 epoch 29 - iter 15/154 - loss 0.21952717 - samples/sec: 914.62\n",
            "2019-12-20 10:12:34,888 epoch 29 - iter 30/154 - loss 0.23034883 - samples/sec: 967.99\n",
            "2019-12-20 10:12:35,414 epoch 29 - iter 45/154 - loss 0.21752059 - samples/sec: 936.37\n",
            "2019-12-20 10:12:35,959 epoch 29 - iter 60/154 - loss 0.21510619 - samples/sec: 903.98\n",
            "2019-12-20 10:12:36,488 epoch 29 - iter 75/154 - loss 0.20929216 - samples/sec: 932.19\n",
            "2019-12-20 10:12:37,007 epoch 29 - iter 90/154 - loss 0.21442586 - samples/sec: 949.65\n",
            "2019-12-20 10:12:37,544 epoch 29 - iter 105/154 - loss 0.20548419 - samples/sec: 918.11\n",
            "2019-12-20 10:12:38,094 epoch 29 - iter 120/154 - loss 0.19790055 - samples/sec: 895.63\n",
            "2019-12-20 10:12:38,639 epoch 29 - iter 135/154 - loss 0.19463089 - samples/sec: 904.57\n",
            "2019-12-20 10:12:39,167 epoch 29 - iter 150/154 - loss 0.19236957 - samples/sec: 937.61\n",
            "2019-12-20 10:12:39,268 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:39,269 EPOCH 29 done: loss 0.1940 - lr 0.0500\n",
            "2019-12-20 10:12:39,769 DEV : loss 0.25090959668159485 - score 0.9156\n",
            "2019-12-20 10:12:39,794 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:12:42,906 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:42,951 epoch 30 - iter 0/154 - loss 0.16047652 - samples/sec: 12214.49\n",
            "2019-12-20 10:12:43,531 epoch 30 - iter 15/154 - loss 0.17014831 - samples/sec: 848.96\n",
            "2019-12-20 10:12:44,073 epoch 30 - iter 30/154 - loss 0.17232978 - samples/sec: 909.19\n",
            "2019-12-20 10:12:44,610 epoch 30 - iter 45/154 - loss 0.16708809 - samples/sec: 918.27\n",
            "2019-12-20 10:12:45,165 epoch 30 - iter 60/154 - loss 0.16903292 - samples/sec: 890.09\n",
            "2019-12-20 10:12:45,702 epoch 30 - iter 75/154 - loss 0.17656121 - samples/sec: 918.70\n",
            "2019-12-20 10:12:46,254 epoch 30 - iter 90/154 - loss 0.17769802 - samples/sec: 893.77\n",
            "2019-12-20 10:12:46,797 epoch 30 - iter 105/154 - loss 0.18299658 - samples/sec: 907.06\n",
            "2019-12-20 10:12:47,352 epoch 30 - iter 120/154 - loss 0.18836364 - samples/sec: 887.22\n",
            "2019-12-20 10:12:47,910 epoch 30 - iter 135/154 - loss 0.18848414 - samples/sec: 881.31\n",
            "2019-12-20 10:12:48,459 epoch 30 - iter 150/154 - loss 0.18797111 - samples/sec: 896.66\n",
            "2019-12-20 10:12:48,560 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:48,561 EPOCH 30 done: loss 0.1866 - lr 0.0500\n",
            "2019-12-20 10:12:49,052 DEV : loss 0.23973481357097626 - score 0.9156\n",
            "2019-12-20 10:12:49,077 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:12:52,137 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:52,181 epoch 31 - iter 0/154 - loss 0.18743512 - samples/sec: 11203.67\n",
            "2019-12-20 10:12:52,819 epoch 31 - iter 15/154 - loss 0.17270221 - samples/sec: 772.58\n",
            "2019-12-20 10:12:53,332 epoch 31 - iter 30/154 - loss 0.17876122 - samples/sec: 963.30\n",
            "2019-12-20 10:12:53,868 epoch 31 - iter 45/154 - loss 0.16815097 - samples/sec: 920.83\n",
            "2019-12-20 10:12:54,400 epoch 31 - iter 60/154 - loss 0.16798420 - samples/sec: 928.57\n",
            "2019-12-20 10:12:54,948 epoch 31 - iter 75/154 - loss 0.16682144 - samples/sec: 899.22\n",
            "2019-12-20 10:12:55,485 epoch 31 - iter 90/154 - loss 0.17059236 - samples/sec: 918.56\n",
            "2019-12-20 10:12:56,036 epoch 31 - iter 105/154 - loss 0.17308455 - samples/sec: 894.46\n",
            "2019-12-20 10:12:56,574 epoch 31 - iter 120/154 - loss 0.17612606 - samples/sec: 917.28\n",
            "2019-12-20 10:12:57,133 epoch 31 - iter 135/154 - loss 0.17942843 - samples/sec: 887.24\n",
            "2019-12-20 10:12:57,672 epoch 31 - iter 150/154 - loss 0.18575973 - samples/sec: 914.17\n",
            "2019-12-20 10:12:57,775 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:12:57,776 EPOCH 31 done: loss 0.1865 - lr 0.0500\n",
            "2019-12-20 10:12:58,263 DEV : loss 0.23805955052375793 - score 0.9174\n",
            "2019-12-20 10:12:58,289 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:13:01,250 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:01,303 epoch 32 - iter 0/154 - loss 0.17392716 - samples/sec: 10989.56\n",
            "2019-12-20 10:13:01,905 epoch 32 - iter 15/154 - loss 0.19910815 - samples/sec: 820.02\n",
            "2019-12-20 10:13:02,450 epoch 32 - iter 30/154 - loss 0.18226102 - samples/sec: 911.13\n",
            "2019-12-20 10:13:02,990 epoch 32 - iter 45/154 - loss 0.16532122 - samples/sec: 912.38\n",
            "2019-12-20 10:13:03,520 epoch 32 - iter 60/154 - loss 0.17022317 - samples/sec: 929.58\n",
            "2019-12-20 10:13:04,077 epoch 32 - iter 75/154 - loss 0.17936823 - samples/sec: 884.95\n",
            "2019-12-20 10:13:04,663 epoch 32 - iter 90/154 - loss 0.17661071 - samples/sec: 840.41\n",
            "2019-12-20 10:13:05,245 epoch 32 - iter 105/154 - loss 0.17484708 - samples/sec: 846.37\n",
            "2019-12-20 10:13:05,832 epoch 32 - iter 120/154 - loss 0.17457177 - samples/sec: 840.29\n",
            "2019-12-20 10:13:06,396 epoch 32 - iter 135/154 - loss 0.17485580 - samples/sec: 882.61\n",
            "2019-12-20 10:13:06,950 epoch 32 - iter 150/154 - loss 0.17976964 - samples/sec: 890.72\n",
            "2019-12-20 10:13:07,049 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:07,050 EPOCH 32 done: loss 0.1820 - lr 0.0500\n",
            "2019-12-20 10:13:07,551 DEV : loss 0.32042399048805237 - score 0.8972\n",
            "2019-12-20 10:13:07,577 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:13:07,578 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:07,618 epoch 33 - iter 0/154 - loss 0.11144777 - samples/sec: 12889.11\n",
            "2019-12-20 10:13:08,160 epoch 33 - iter 15/154 - loss 0.17303723 - samples/sec: 908.31\n",
            "2019-12-20 10:13:08,696 epoch 33 - iter 30/154 - loss 0.17895658 - samples/sec: 922.62\n",
            "2019-12-20 10:13:09,226 epoch 33 - iter 45/154 - loss 0.17408665 - samples/sec: 930.81\n",
            "2019-12-20 10:13:09,783 epoch 33 - iter 60/154 - loss 0.17143668 - samples/sec: 885.40\n",
            "2019-12-20 10:13:10,327 epoch 33 - iter 75/154 - loss 0.16981781 - samples/sec: 906.34\n",
            "2019-12-20 10:13:10,870 epoch 33 - iter 90/154 - loss 0.16472032 - samples/sec: 907.04\n",
            "2019-12-20 10:13:11,406 epoch 33 - iter 105/154 - loss 0.16832654 - samples/sec: 921.16\n",
            "2019-12-20 10:13:11,959 epoch 33 - iter 120/154 - loss 0.16810762 - samples/sec: 890.09\n",
            "2019-12-20 10:13:12,519 epoch 33 - iter 135/154 - loss 0.16461493 - samples/sec: 884.69\n",
            "2019-12-20 10:13:13,053 epoch 33 - iter 150/154 - loss 0.16540846 - samples/sec: 931.49\n",
            "2019-12-20 10:13:13,158 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:13,159 EPOCH 33 done: loss 0.1657 - lr 0.0500\n",
            "2019-12-20 10:13:13,643 DEV : loss 0.2584334909915924 - score 0.9174\n",
            "2019-12-20 10:13:13,671 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:13:16,696 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:16,747 epoch 34 - iter 0/154 - loss 0.13587666 - samples/sec: 9882.08\n",
            "2019-12-20 10:13:17,365 epoch 34 - iter 15/154 - loss 0.14949608 - samples/sec: 800.28\n",
            "2019-12-20 10:13:17,940 epoch 34 - iter 30/154 - loss 0.13536526 - samples/sec: 861.88\n",
            "2019-12-20 10:13:18,482 epoch 34 - iter 45/154 - loss 0.14520833 - samples/sec: 909.11\n",
            "2019-12-20 10:13:19,037 epoch 34 - iter 60/154 - loss 0.15815675 - samples/sec: 887.45\n",
            "2019-12-20 10:13:19,582 epoch 34 - iter 75/154 - loss 0.16384212 - samples/sec: 905.68\n",
            "2019-12-20 10:13:20,129 epoch 34 - iter 90/154 - loss 0.15901671 - samples/sec: 901.37\n",
            "2019-12-20 10:13:20,690 epoch 34 - iter 105/154 - loss 0.15568076 - samples/sec: 877.76\n",
            "2019-12-20 10:13:21,231 epoch 34 - iter 120/154 - loss 0.15618824 - samples/sec: 910.38\n",
            "2019-12-20 10:13:21,766 epoch 34 - iter 135/154 - loss 0.16087195 - samples/sec: 922.32\n",
            "2019-12-20 10:13:22,303 epoch 34 - iter 150/154 - loss 0.16302378 - samples/sec: 919.43\n",
            "2019-12-20 10:13:22,415 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:22,416 EPOCH 34 done: loss 0.1636 - lr 0.0500\n",
            "2019-12-20 10:13:22,906 DEV : loss 0.2546743154525757 - score 0.9193\n",
            "2019-12-20 10:13:22,931 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:13:25,938 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:25,988 epoch 35 - iter 0/154 - loss 0.09213778 - samples/sec: 10599.60\n",
            "2019-12-20 10:13:26,615 epoch 35 - iter 15/154 - loss 0.16193626 - samples/sec: 791.06\n",
            "2019-12-20 10:13:27,147 epoch 35 - iter 30/154 - loss 0.15836809 - samples/sec: 927.79\n",
            "2019-12-20 10:13:27,687 epoch 35 - iter 45/154 - loss 0.16732966 - samples/sec: 913.61\n",
            "2019-12-20 10:13:28,240 epoch 35 - iter 60/154 - loss 0.16233281 - samples/sec: 895.51\n",
            "2019-12-20 10:13:28,758 epoch 35 - iter 75/154 - loss 0.16007052 - samples/sec: 953.05\n",
            "2019-12-20 10:13:29,284 epoch 35 - iter 90/154 - loss 0.16077664 - samples/sec: 938.33\n",
            "2019-12-20 10:13:29,806 epoch 35 - iter 105/154 - loss 0.16573122 - samples/sec: 945.93\n",
            "2019-12-20 10:13:30,337 epoch 35 - iter 120/154 - loss 0.16997155 - samples/sec: 929.78\n",
            "2019-12-20 10:13:30,863 epoch 35 - iter 135/154 - loss 0.16827539 - samples/sec: 938.35\n",
            "2019-12-20 10:13:31,416 epoch 35 - iter 150/154 - loss 0.17050910 - samples/sec: 891.74\n",
            "2019-12-20 10:13:31,524 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:31,526 EPOCH 35 done: loss 0.1698 - lr 0.0500\n",
            "2019-12-20 10:13:31,998 DEV : loss 0.2522393763065338 - score 0.9211\n",
            "2019-12-20 10:13:32,023 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:13:35,108 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:35,151 epoch 36 - iter 0/154 - loss 0.23402126 - samples/sec: 11835.92\n",
            "2019-12-20 10:13:35,786 epoch 36 - iter 15/154 - loss 0.16187771 - samples/sec: 811.52\n",
            "2019-12-20 10:13:36,343 epoch 36 - iter 30/154 - loss 0.14915111 - samples/sec: 891.29\n",
            "2019-12-20 10:13:36,888 epoch 36 - iter 45/154 - loss 0.15259077 - samples/sec: 903.09\n",
            "2019-12-20 10:13:37,441 epoch 36 - iter 60/154 - loss 0.17610897 - samples/sec: 891.31\n",
            "2019-12-20 10:13:37,998 epoch 36 - iter 75/154 - loss 0.17047428 - samples/sec: 886.48\n",
            "2019-12-20 10:13:38,530 epoch 36 - iter 90/154 - loss 0.16649385 - samples/sec: 927.03\n",
            "2019-12-20 10:13:39,068 epoch 36 - iter 105/154 - loss 0.16451453 - samples/sec: 915.93\n",
            "2019-12-20 10:13:39,605 epoch 36 - iter 120/154 - loss 0.16616329 - samples/sec: 923.92\n",
            "2019-12-20 10:13:40,156 epoch 36 - iter 135/154 - loss 0.16825624 - samples/sec: 910.90\n",
            "2019-12-20 10:13:40,692 epoch 36 - iter 150/154 - loss 0.16856313 - samples/sec: 924.86\n",
            "2019-12-20 10:13:40,792 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:40,793 EPOCH 36 done: loss 0.1681 - lr 0.0500\n",
            "2019-12-20 10:13:41,287 DEV : loss 0.2534788250923157 - score 0.9248\n",
            "2019-12-20 10:13:41,312 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:13:44,456 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:44,520 epoch 37 - iter 0/154 - loss 0.15082148 - samples/sec: 8408.72\n",
            "2019-12-20 10:13:45,102 epoch 37 - iter 15/154 - loss 0.12052452 - samples/sec: 847.92\n",
            "2019-12-20 10:13:45,647 epoch 37 - iter 30/154 - loss 0.13483130 - samples/sec: 907.26\n",
            "2019-12-20 10:13:46,204 epoch 37 - iter 45/154 - loss 0.14434024 - samples/sec: 894.28\n",
            "2019-12-20 10:13:46,749 epoch 37 - iter 60/154 - loss 0.15287328 - samples/sec: 904.58\n",
            "2019-12-20 10:13:47,296 epoch 37 - iter 75/154 - loss 0.14819813 - samples/sec: 903.94\n",
            "2019-12-20 10:13:47,859 epoch 37 - iter 90/154 - loss 0.15029696 - samples/sec: 876.24\n",
            "2019-12-20 10:13:48,419 epoch 37 - iter 105/154 - loss 0.15017519 - samples/sec: 879.61\n",
            "2019-12-20 10:13:48,955 epoch 37 - iter 120/154 - loss 0.15224297 - samples/sec: 918.75\n",
            "2019-12-20 10:13:49,493 epoch 37 - iter 135/154 - loss 0.15077893 - samples/sec: 916.65\n",
            "2019-12-20 10:13:50,049 epoch 37 - iter 150/154 - loss 0.14742243 - samples/sec: 888.13\n",
            "2019-12-20 10:13:50,153 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:50,154 EPOCH 37 done: loss 0.1470 - lr 0.0500\n",
            "2019-12-20 10:13:50,666 DEV : loss 0.25790029764175415 - score 0.9321\n",
            "2019-12-20 10:13:50,690 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:13:53,780 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:53,836 epoch 38 - iter 0/154 - loss 0.34291306 - samples/sec: 8869.76\n",
            "2019-12-20 10:13:54,441 epoch 38 - iter 15/154 - loss 0.13659122 - samples/sec: 825.56\n",
            "2019-12-20 10:13:54,962 epoch 38 - iter 30/154 - loss 0.14542019 - samples/sec: 948.18\n",
            "2019-12-20 10:13:55,476 epoch 38 - iter 45/154 - loss 0.15357215 - samples/sec: 960.14\n",
            "2019-12-20 10:13:56,015 epoch 38 - iter 60/154 - loss 0.16242590 - samples/sec: 920.03\n",
            "2019-12-20 10:13:56,541 epoch 38 - iter 75/154 - loss 0.16106439 - samples/sec: 938.44\n",
            "2019-12-20 10:13:57,058 epoch 38 - iter 90/154 - loss 0.16560139 - samples/sec: 954.65\n",
            "2019-12-20 10:13:57,593 epoch 38 - iter 105/154 - loss 0.16668150 - samples/sec: 928.83\n",
            "2019-12-20 10:13:58,126 epoch 38 - iter 120/154 - loss 0.16681341 - samples/sec: 924.73\n",
            "2019-12-20 10:13:58,642 epoch 38 - iter 135/154 - loss 0.16824529 - samples/sec: 958.59\n",
            "2019-12-20 10:13:59,179 epoch 38 - iter 150/154 - loss 0.16757130 - samples/sec: 918.94\n",
            "2019-12-20 10:13:59,280 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:59,282 EPOCH 38 done: loss 0.1691 - lr 0.0500\n",
            "2019-12-20 10:13:59,774 DEV : loss 0.2530575096607208 - score 0.9211\n",
            "2019-12-20 10:13:59,799 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:13:59,800 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:13:59,843 epoch 39 - iter 0/154 - loss 0.25545692 - samples/sec: 11687.30\n",
            "2019-12-20 10:14:00,385 epoch 39 - iter 15/154 - loss 0.15662938 - samples/sec: 907.63\n",
            "2019-12-20 10:14:00,925 epoch 39 - iter 30/154 - loss 0.16007702 - samples/sec: 912.23\n",
            "2019-12-20 10:14:01,477 epoch 39 - iter 45/154 - loss 0.15965199 - samples/sec: 896.33\n",
            "2019-12-20 10:14:02,020 epoch 39 - iter 60/154 - loss 0.15696817 - samples/sec: 915.25\n",
            "2019-12-20 10:14:02,547 epoch 39 - iter 75/154 - loss 0.15175005 - samples/sec: 940.32\n",
            "2019-12-20 10:14:03,078 epoch 39 - iter 90/154 - loss 0.15355014 - samples/sec: 934.61\n",
            "2019-12-20 10:14:03,639 epoch 39 - iter 105/154 - loss 0.15487859 - samples/sec: 876.22\n",
            "2019-12-20 10:14:04,190 epoch 39 - iter 120/154 - loss 0.15054523 - samples/sec: 895.60\n",
            "2019-12-20 10:14:04,726 epoch 39 - iter 135/154 - loss 0.15009649 - samples/sec: 921.12\n",
            "2019-12-20 10:14:05,290 epoch 39 - iter 150/154 - loss 0.14780701 - samples/sec: 872.69\n",
            "2019-12-20 10:14:05,392 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:05,393 EPOCH 39 done: loss 0.1474 - lr 0.0500\n",
            "2019-12-20 10:14:05,889 DEV : loss 0.2593787908554077 - score 0.9193\n",
            "2019-12-20 10:14:05,914 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:14:05,915 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:05,951 epoch 40 - iter 0/154 - loss 0.24821788 - samples/sec: 13933.02\n",
            "2019-12-20 10:14:06,495 epoch 40 - iter 15/154 - loss 0.15759082 - samples/sec: 907.11\n",
            "2019-12-20 10:14:07,046 epoch 40 - iter 30/154 - loss 0.15760380 - samples/sec: 895.68\n",
            "2019-12-20 10:14:07,572 epoch 40 - iter 45/154 - loss 0.15553377 - samples/sec: 936.76\n",
            "2019-12-20 10:14:08,115 epoch 40 - iter 60/154 - loss 0.15013469 - samples/sec: 907.50\n",
            "2019-12-20 10:14:08,647 epoch 40 - iter 75/154 - loss 0.15065087 - samples/sec: 927.02\n",
            "2019-12-20 10:14:09,175 epoch 40 - iter 90/154 - loss 0.15501678 - samples/sec: 933.34\n",
            "2019-12-20 10:14:09,725 epoch 40 - iter 105/154 - loss 0.15717729 - samples/sec: 894.57\n",
            "2019-12-20 10:14:10,245 epoch 40 - iter 120/154 - loss 0.16146409 - samples/sec: 946.80\n",
            "2019-12-20 10:14:10,807 epoch 40 - iter 135/154 - loss 0.15693734 - samples/sec: 876.74\n",
            "2019-12-20 10:14:11,362 epoch 40 - iter 150/154 - loss 0.15466903 - samples/sec: 889.24\n",
            "2019-12-20 10:14:11,468 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:11,468 EPOCH 40 done: loss 0.1558 - lr 0.0500\n",
            "2019-12-20 10:14:11,947 DEV : loss 0.25020018219947815 - score 0.9138\n",
            "2019-12-20 10:14:11,973 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:14:11,974 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:12,015 epoch 41 - iter 0/154 - loss 0.08608633 - samples/sec: 12047.04\n",
            "2019-12-20 10:14:12,558 epoch 41 - iter 15/154 - loss 0.14312715 - samples/sec: 906.10\n",
            "2019-12-20 10:14:13,095 epoch 41 - iter 30/154 - loss 0.14051356 - samples/sec: 922.64\n",
            "2019-12-20 10:14:13,644 epoch 41 - iter 45/154 - loss 0.13237163 - samples/sec: 896.42\n",
            "2019-12-20 10:14:14,174 epoch 41 - iter 60/154 - loss 0.12378064 - samples/sec: 930.84\n",
            "2019-12-20 10:14:14,726 epoch 41 - iter 75/154 - loss 0.12636099 - samples/sec: 894.43\n",
            "2019-12-20 10:14:15,267 epoch 41 - iter 90/154 - loss 0.12674524 - samples/sec: 910.60\n",
            "2019-12-20 10:14:15,830 epoch 41 - iter 105/154 - loss 0.12808177 - samples/sec: 875.36\n",
            "2019-12-20 10:14:16,377 epoch 41 - iter 120/154 - loss 0.12890352 - samples/sec: 899.69\n",
            "2019-12-20 10:14:16,916 epoch 41 - iter 135/154 - loss 0.12916625 - samples/sec: 915.73\n",
            "2019-12-20 10:14:17,445 epoch 41 - iter 150/154 - loss 0.13227047 - samples/sec: 932.53\n",
            "2019-12-20 10:14:17,551 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:17,552 EPOCH 41 done: loss 0.1331 - lr 0.0500\n",
            "2019-12-20 10:14:18,047 DEV : loss 0.22581934928894043 - score 0.9193\n",
            "2019-12-20 10:14:18,080 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:14:18,081 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:18,126 epoch 42 - iter 0/154 - loss 0.12646359 - samples/sec: 11572.29\n",
            "2019-12-20 10:14:18,672 epoch 42 - iter 15/154 - loss 0.14141483 - samples/sec: 901.29\n",
            "2019-12-20 10:14:19,189 epoch 42 - iter 30/154 - loss 0.14969343 - samples/sec: 953.90\n",
            "2019-12-20 10:14:19,727 epoch 42 - iter 45/154 - loss 0.14434283 - samples/sec: 915.90\n",
            "2019-12-20 10:14:20,278 epoch 42 - iter 60/154 - loss 0.14626270 - samples/sec: 898.38\n",
            "2019-12-20 10:14:20,812 epoch 42 - iter 75/154 - loss 0.13920738 - samples/sec: 925.57\n",
            "2019-12-20 10:14:21,346 epoch 42 - iter 90/154 - loss 0.13569278 - samples/sec: 925.12\n",
            "2019-12-20 10:14:21,881 epoch 42 - iter 105/154 - loss 0.13603017 - samples/sec: 929.12\n",
            "2019-12-20 10:14:22,396 epoch 42 - iter 120/154 - loss 0.13879678 - samples/sec: 960.39\n",
            "2019-12-20 10:14:22,920 epoch 42 - iter 135/154 - loss 0.13906157 - samples/sec: 942.13\n",
            "2019-12-20 10:14:23,440 epoch 42 - iter 150/154 - loss 0.14125845 - samples/sec: 947.79\n",
            "2019-12-20 10:14:23,543 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:23,544 EPOCH 42 done: loss 0.1404 - lr 0.0500\n",
            "2019-12-20 10:14:24,020 DEV : loss 0.23121050000190735 - score 0.9303\n",
            "2019-12-20 10:14:24,044 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:14:24,045 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:24,084 epoch 43 - iter 0/154 - loss 0.13006043 - samples/sec: 12746.06\n",
            "2019-12-20 10:14:24,612 epoch 43 - iter 15/154 - loss 0.17632879 - samples/sec: 931.94\n",
            "2019-12-20 10:14:25,144 epoch 43 - iter 30/154 - loss 0.15715125 - samples/sec: 927.64\n",
            "2019-12-20 10:14:25,691 epoch 43 - iter 45/154 - loss 0.14039019 - samples/sec: 901.60\n",
            "2019-12-20 10:14:26,227 epoch 43 - iter 60/154 - loss 0.14017304 - samples/sec: 919.90\n",
            "2019-12-20 10:14:26,761 epoch 43 - iter 75/154 - loss 0.13634963 - samples/sec: 923.51\n",
            "2019-12-20 10:14:27,297 epoch 43 - iter 90/154 - loss 0.13135631 - samples/sec: 922.20\n",
            "2019-12-20 10:14:27,840 epoch 43 - iter 105/154 - loss 0.12785021 - samples/sec: 910.34\n",
            "2019-12-20 10:14:28,393 epoch 43 - iter 120/154 - loss 0.13345579 - samples/sec: 891.98\n",
            "2019-12-20 10:14:28,926 epoch 43 - iter 135/154 - loss 0.13521676 - samples/sec: 924.28\n",
            "2019-12-20 10:14:29,496 epoch 43 - iter 150/154 - loss 0.13387015 - samples/sec: 865.40\n",
            "2019-12-20 10:14:29,592 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:29,593 EPOCH 43 done: loss 0.1327 - lr 0.0500\n",
            "2019-12-20 10:14:30,084 DEV : loss 0.26154661178588867 - score 0.9284\n",
            "Epoch    42: reducing learning rate of group 0 to 2.5000e-02.\n",
            "2019-12-20 10:14:30,110 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:14:30,113 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:30,153 epoch 44 - iter 0/154 - loss 0.32430965 - samples/sec: 12458.17\n",
            "2019-12-20 10:14:30,696 epoch 44 - iter 15/154 - loss 0.12044379 - samples/sec: 906.42\n",
            "2019-12-20 10:14:31,259 epoch 44 - iter 30/154 - loss 0.11384552 - samples/sec: 876.21\n",
            "2019-12-20 10:14:31,783 epoch 44 - iter 45/154 - loss 0.11967048 - samples/sec: 941.21\n",
            "2019-12-20 10:14:32,326 epoch 44 - iter 60/154 - loss 0.12803015 - samples/sec: 916.50\n",
            "2019-12-20 10:14:32,855 epoch 44 - iter 75/154 - loss 0.12716386 - samples/sec: 934.67\n",
            "2019-12-20 10:14:33,416 epoch 44 - iter 90/154 - loss 0.12656281 - samples/sec: 881.04\n",
            "2019-12-20 10:14:33,961 epoch 44 - iter 105/154 - loss 0.13113025 - samples/sec: 902.90\n",
            "2019-12-20 10:14:34,506 epoch 44 - iter 120/154 - loss 0.12619548 - samples/sec: 904.12\n",
            "2019-12-20 10:14:35,060 epoch 44 - iter 135/154 - loss 0.12476079 - samples/sec: 890.01\n",
            "2019-12-20 10:14:35,596 epoch 44 - iter 150/154 - loss 0.12623440 - samples/sec: 923.06\n",
            "2019-12-20 10:14:35,699 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:35,700 EPOCH 44 done: loss 0.1272 - lr 0.0250\n",
            "2019-12-20 10:14:36,206 DEV : loss 0.24648036062717438 - score 0.9321\n",
            "2019-12-20 10:14:36,232 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:14:39,347 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:39,393 epoch 45 - iter 0/154 - loss 0.05617930 - samples/sec: 10786.09\n",
            "2019-12-20 10:14:40,011 epoch 45 - iter 15/154 - loss 0.11714624 - samples/sec: 814.27\n",
            "2019-12-20 10:14:40,548 epoch 45 - iter 30/154 - loss 0.11857888 - samples/sec: 917.24\n",
            "2019-12-20 10:14:41,100 epoch 45 - iter 45/154 - loss 0.13502414 - samples/sec: 893.12\n",
            "2019-12-20 10:14:41,627 epoch 45 - iter 60/154 - loss 0.13071843 - samples/sec: 935.08\n",
            "2019-12-20 10:14:42,180 epoch 45 - iter 75/154 - loss 0.12734683 - samples/sec: 891.24\n",
            "2019-12-20 10:14:42,713 epoch 45 - iter 90/154 - loss 0.12953018 - samples/sec: 925.79\n",
            "2019-12-20 10:14:43,257 epoch 45 - iter 105/154 - loss 0.13080321 - samples/sec: 905.41\n",
            "2019-12-20 10:14:43,800 epoch 45 - iter 120/154 - loss 0.13207253 - samples/sec: 907.75\n",
            "2019-12-20 10:14:44,347 epoch 45 - iter 135/154 - loss 0.12990895 - samples/sec: 909.39\n",
            "2019-12-20 10:14:44,882 epoch 45 - iter 150/154 - loss 0.13287003 - samples/sec: 923.46\n",
            "2019-12-20 10:14:44,980 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:44,981 EPOCH 45 done: loss 0.1313 - lr 0.0250\n",
            "2019-12-20 10:14:45,453 DEV : loss 0.22541838884353638 - score 0.9303\n",
            "2019-12-20 10:14:45,479 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:14:45,480 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:45,516 epoch 46 - iter 0/154 - loss 0.18462600 - samples/sec: 14018.30\n",
            "2019-12-20 10:14:46,059 epoch 46 - iter 15/154 - loss 0.15063877 - samples/sec: 905.47\n",
            "2019-12-20 10:14:46,581 epoch 46 - iter 30/154 - loss 0.14053214 - samples/sec: 945.42\n",
            "2019-12-20 10:14:47,110 epoch 46 - iter 45/154 - loss 0.13759257 - samples/sec: 931.22\n",
            "2019-12-20 10:14:47,644 epoch 46 - iter 60/154 - loss 0.13104568 - samples/sec: 933.19\n",
            "2019-12-20 10:14:48,165 epoch 46 - iter 75/154 - loss 0.12561582 - samples/sec: 948.39\n",
            "2019-12-20 10:14:48,676 epoch 46 - iter 90/154 - loss 0.12056420 - samples/sec: 966.32\n",
            "2019-12-20 10:14:49,649 epoch 46 - iter 105/154 - loss 0.12153629 - samples/sec: 500.25\n",
            "2019-12-20 10:14:50,203 epoch 46 - iter 120/154 - loss 0.11995585 - samples/sec: 893.61\n",
            "2019-12-20 10:14:50,739 epoch 46 - iter 135/154 - loss 0.12050331 - samples/sec: 923.78\n",
            "2019-12-20 10:14:51,301 epoch 46 - iter 150/154 - loss 0.12132015 - samples/sec: 880.97\n",
            "2019-12-20 10:14:51,419 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:51,420 EPOCH 46 done: loss 0.1220 - lr 0.0250\n",
            "2019-12-20 10:14:51,911 DEV : loss 0.2285883128643036 - score 0.9284\n",
            "2019-12-20 10:14:51,938 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:14:51,939 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:51,981 epoch 47 - iter 0/154 - loss 0.12732781 - samples/sec: 12481.81\n",
            "2019-12-20 10:14:52,512 epoch 47 - iter 15/154 - loss 0.11999653 - samples/sec: 926.43\n",
            "2019-12-20 10:14:53,044 epoch 47 - iter 30/154 - loss 0.13716050 - samples/sec: 926.48\n",
            "2019-12-20 10:14:53,621 epoch 47 - iter 45/154 - loss 0.13826010 - samples/sec: 853.39\n",
            "2019-12-20 10:14:54,146 epoch 47 - iter 60/154 - loss 0.13922183 - samples/sec: 938.19\n",
            "2019-12-20 10:14:54,680 epoch 47 - iter 75/154 - loss 0.14154795 - samples/sec: 923.43\n",
            "2019-12-20 10:14:55,225 epoch 47 - iter 90/154 - loss 0.13828183 - samples/sec: 904.51\n",
            "2019-12-20 10:14:55,760 epoch 47 - iter 105/154 - loss 0.13569027 - samples/sec: 920.28\n",
            "2019-12-20 10:14:56,297 epoch 47 - iter 120/154 - loss 0.13745867 - samples/sec: 918.85\n",
            "2019-12-20 10:14:56,841 epoch 47 - iter 135/154 - loss 0.13658456 - samples/sec: 906.66\n",
            "2019-12-20 10:14:57,392 epoch 47 - iter 150/154 - loss 0.13222476 - samples/sec: 894.05\n",
            "2019-12-20 10:14:57,491 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:14:57,492 EPOCH 47 done: loss 0.1317 - lr 0.0250\n",
            "2019-12-20 10:14:57,983 DEV : loss 0.22080299258232117 - score 0.9321\n",
            "2019-12-20 10:14:58,009 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:15:01,047 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:01,099 epoch 48 - iter 0/154 - loss 0.08140427 - samples/sec: 10608.76\n",
            "2019-12-20 10:15:01,721 epoch 48 - iter 15/154 - loss 0.07375694 - samples/sec: 796.41\n",
            "2019-12-20 10:15:02,266 epoch 48 - iter 30/154 - loss 0.08388045 - samples/sec: 909.57\n",
            "2019-12-20 10:15:02,852 epoch 48 - iter 45/154 - loss 0.09991398 - samples/sec: 848.75\n",
            "2019-12-20 10:15:03,392 epoch 48 - iter 60/154 - loss 0.11018673 - samples/sec: 916.01\n",
            "2019-12-20 10:15:03,936 epoch 48 - iter 75/154 - loss 0.11240723 - samples/sec: 912.56\n",
            "2019-12-20 10:15:04,496 epoch 48 - iter 90/154 - loss 0.11628687 - samples/sec: 883.27\n",
            "2019-12-20 10:15:05,045 epoch 48 - iter 105/154 - loss 0.11458401 - samples/sec: 904.81\n",
            "2019-12-20 10:15:05,589 epoch 48 - iter 120/154 - loss 0.11179939 - samples/sec: 905.84\n",
            "2019-12-20 10:15:06,129 epoch 48 - iter 135/154 - loss 0.11590063 - samples/sec: 919.50\n",
            "2019-12-20 10:15:06,683 epoch 48 - iter 150/154 - loss 0.11375062 - samples/sec: 890.26\n",
            "2019-12-20 10:15:06,786 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:06,786 EPOCH 48 done: loss 0.1134 - lr 0.0250\n",
            "2019-12-20 10:15:07,281 DEV : loss 0.22296911478042603 - score 0.9358\n",
            "2019-12-20 10:15:07,305 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:15:10,560 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:10,598 epoch 49 - iter 0/154 - loss 0.07549728 - samples/sec: 13501.43\n",
            "2019-12-20 10:15:11,206 epoch 49 - iter 15/154 - loss 0.08816786 - samples/sec: 807.96\n",
            "2019-12-20 10:15:11,725 epoch 49 - iter 30/154 - loss 0.09096376 - samples/sec: 952.11\n",
            "2019-12-20 10:15:12,246 epoch 49 - iter 45/154 - loss 0.09647528 - samples/sec: 948.03\n",
            "2019-12-20 10:15:12,765 epoch 49 - iter 60/154 - loss 0.10325232 - samples/sec: 950.75\n",
            "2019-12-20 10:15:13,322 epoch 49 - iter 75/154 - loss 0.10455573 - samples/sec: 890.28\n",
            "2019-12-20 10:15:13,844 epoch 49 - iter 90/154 - loss 0.11053192 - samples/sec: 946.45\n",
            "2019-12-20 10:15:14,372 epoch 49 - iter 105/154 - loss 0.11550174 - samples/sec: 933.35\n",
            "2019-12-20 10:15:14,907 epoch 49 - iter 120/154 - loss 0.11393603 - samples/sec: 926.08\n",
            "2019-12-20 10:15:15,424 epoch 49 - iter 135/154 - loss 0.11470138 - samples/sec: 960.69\n",
            "2019-12-20 10:15:15,967 epoch 49 - iter 150/154 - loss 0.11857134 - samples/sec: 908.33\n",
            "2019-12-20 10:15:16,066 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:16,069 EPOCH 49 done: loss 0.1192 - lr 0.0250\n",
            "2019-12-20 10:15:16,541 DEV : loss 0.2335474193096161 - score 0.9229\n",
            "2019-12-20 10:15:16,567 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:15:16,569 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:16,608 epoch 50 - iter 0/154 - loss 0.06179317 - samples/sec: 12785.32\n",
            "2019-12-20 10:15:17,141 epoch 50 - iter 15/154 - loss 0.09520150 - samples/sec: 925.55\n",
            "2019-12-20 10:15:17,700 epoch 50 - iter 30/154 - loss 0.10291882 - samples/sec: 880.70\n",
            "2019-12-20 10:15:18,241 epoch 50 - iter 45/154 - loss 0.10543041 - samples/sec: 913.10\n",
            "2019-12-20 10:15:18,780 epoch 50 - iter 60/154 - loss 0.10904054 - samples/sec: 915.78\n",
            "2019-12-20 10:15:19,318 epoch 50 - iter 75/154 - loss 0.11217595 - samples/sec: 916.30\n",
            "2019-12-20 10:15:19,875 epoch 50 - iter 90/154 - loss 0.10903825 - samples/sec: 889.69\n",
            "2019-12-20 10:15:20,418 epoch 50 - iter 105/154 - loss 0.10757719 - samples/sec: 910.02\n",
            "2019-12-20 10:15:20,964 epoch 50 - iter 120/154 - loss 0.11013255 - samples/sec: 903.78\n",
            "2019-12-20 10:15:21,500 epoch 50 - iter 135/154 - loss 0.10802432 - samples/sec: 921.17\n",
            "2019-12-20 10:15:22,046 epoch 50 - iter 150/154 - loss 0.10816062 - samples/sec: 905.22\n",
            "2019-12-20 10:15:22,147 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:22,148 EPOCH 50 done: loss 0.1087 - lr 0.0250\n",
            "2019-12-20 10:15:22,632 DEV : loss 0.2560757100582123 - score 0.9266\n",
            "2019-12-20 10:15:22,656 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:15:22,665 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:22,707 epoch 51 - iter 0/154 - loss 0.06806288 - samples/sec: 12261.28\n",
            "2019-12-20 10:15:23,271 epoch 51 - iter 15/154 - loss 0.13654767 - samples/sec: 874.40\n",
            "2019-12-20 10:15:23,809 epoch 51 - iter 30/154 - loss 0.11767971 - samples/sec: 920.95\n",
            "2019-12-20 10:15:24,335 epoch 51 - iter 45/154 - loss 0.12341507 - samples/sec: 938.35\n",
            "2019-12-20 10:15:24,910 epoch 51 - iter 60/154 - loss 0.12290790 - samples/sec: 855.25\n",
            "2019-12-20 10:15:25,436 epoch 51 - iter 75/154 - loss 0.12294174 - samples/sec: 941.08\n",
            "2019-12-20 10:15:25,972 epoch 51 - iter 90/154 - loss 0.12225724 - samples/sec: 919.46\n",
            "2019-12-20 10:15:26,497 epoch 51 - iter 105/154 - loss 0.11922868 - samples/sec: 940.78\n",
            "2019-12-20 10:15:27,057 epoch 51 - iter 120/154 - loss 0.11978302 - samples/sec: 886.90\n",
            "2019-12-20 10:15:27,597 epoch 51 - iter 135/154 - loss 0.12007245 - samples/sec: 912.42\n",
            "2019-12-20 10:15:28,146 epoch 51 - iter 150/154 - loss 0.11872662 - samples/sec: 899.29\n",
            "2019-12-20 10:15:28,245 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:28,246 EPOCH 51 done: loss 0.1198 - lr 0.0250\n",
            "2019-12-20 10:15:28,740 DEV : loss 0.23627611994743347 - score 0.9376\n",
            "2019-12-20 10:15:28,767 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:15:31,711 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:31,760 epoch 52 - iter 0/154 - loss 0.05951424 - samples/sec: 10244.64\n",
            "2019-12-20 10:15:32,395 epoch 52 - iter 15/154 - loss 0.07087678 - samples/sec: 776.86\n",
            "2019-12-20 10:15:32,914 epoch 52 - iter 30/154 - loss 0.09749871 - samples/sec: 954.97\n",
            "2019-12-20 10:15:33,443 epoch 52 - iter 45/154 - loss 0.09174872 - samples/sec: 931.62\n",
            "2019-12-20 10:15:33,982 epoch 52 - iter 60/154 - loss 0.09585627 - samples/sec: 914.94\n",
            "2019-12-20 10:15:34,532 epoch 52 - iter 75/154 - loss 0.09603394 - samples/sec: 900.76\n",
            "2019-12-20 10:15:35,076 epoch 52 - iter 90/154 - loss 0.10083214 - samples/sec: 905.29\n",
            "2019-12-20 10:15:35,615 epoch 52 - iter 105/154 - loss 0.10046731 - samples/sec: 914.83\n",
            "2019-12-20 10:15:36,169 epoch 52 - iter 120/154 - loss 0.10557234 - samples/sec: 889.25\n",
            "2019-12-20 10:15:36,707 epoch 52 - iter 135/154 - loss 0.10929889 - samples/sec: 916.68\n",
            "2019-12-20 10:15:37,231 epoch 52 - iter 150/154 - loss 0.10680696 - samples/sec: 949.06\n",
            "2019-12-20 10:15:37,342 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:37,343 EPOCH 52 done: loss 0.1086 - lr 0.0250\n",
            "2019-12-20 10:15:37,827 DEV : loss 0.254893034696579 - score 0.9358\n",
            "2019-12-20 10:15:37,853 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:15:37,855 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:37,896 epoch 53 - iter 0/154 - loss 0.05332854 - samples/sec: 12146.33\n",
            "2019-12-20 10:15:38,437 epoch 53 - iter 15/154 - loss 0.10885622 - samples/sec: 910.22\n",
            "2019-12-20 10:15:38,981 epoch 53 - iter 30/154 - loss 0.10653826 - samples/sec: 906.09\n",
            "2019-12-20 10:15:39,505 epoch 53 - iter 45/154 - loss 0.12206825 - samples/sec: 940.82\n",
            "2019-12-20 10:15:40,020 epoch 53 - iter 60/154 - loss 0.11414812 - samples/sec: 961.04\n",
            "2019-12-20 10:15:40,542 epoch 53 - iter 75/154 - loss 0.11232492 - samples/sec: 945.03\n",
            "2019-12-20 10:15:41,073 epoch 53 - iter 90/154 - loss 0.10932475 - samples/sec: 926.65\n",
            "2019-12-20 10:15:41,610 epoch 53 - iter 105/154 - loss 0.10632490 - samples/sec: 919.07\n",
            "2019-12-20 10:15:42,149 epoch 53 - iter 120/154 - loss 0.10881527 - samples/sec: 914.47\n",
            "2019-12-20 10:15:42,681 epoch 53 - iter 135/154 - loss 0.10637168 - samples/sec: 928.65\n",
            "2019-12-20 10:15:43,207 epoch 53 - iter 150/154 - loss 0.10677480 - samples/sec: 941.55\n",
            "2019-12-20 10:15:43,308 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:43,309 EPOCH 53 done: loss 0.1064 - lr 0.0250\n",
            "2019-12-20 10:15:43,803 DEV : loss 0.2342548966407776 - score 0.9321\n",
            "2019-12-20 10:15:43,828 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:15:43,829 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:43,869 epoch 54 - iter 0/154 - loss 0.11774655 - samples/sec: 12212.12\n",
            "2019-12-20 10:15:44,414 epoch 54 - iter 15/154 - loss 0.13094809 - samples/sec: 903.30\n",
            "2019-12-20 10:15:44,954 epoch 54 - iter 30/154 - loss 0.11205445 - samples/sec: 914.84\n",
            "2019-12-20 10:15:45,500 epoch 54 - iter 45/154 - loss 0.11927904 - samples/sec: 901.77\n",
            "2019-12-20 10:15:46,055 epoch 54 - iter 60/154 - loss 0.11931169 - samples/sec: 887.56\n",
            "2019-12-20 10:15:46,593 epoch 54 - iter 75/154 - loss 0.11474836 - samples/sec: 916.71\n",
            "2019-12-20 10:15:47,141 epoch 54 - iter 90/154 - loss 0.10885937 - samples/sec: 899.15\n",
            "2019-12-20 10:15:47,701 epoch 54 - iter 105/154 - loss 0.10974085 - samples/sec: 880.97\n",
            "2019-12-20 10:15:48,233 epoch 54 - iter 120/154 - loss 0.10787231 - samples/sec: 926.93\n",
            "2019-12-20 10:15:48,777 epoch 54 - iter 135/154 - loss 0.10689560 - samples/sec: 905.55\n",
            "2019-12-20 10:15:49,321 epoch 54 - iter 150/154 - loss 0.10837881 - samples/sec: 907.36\n",
            "2019-12-20 10:15:49,421 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:49,422 EPOCH 54 done: loss 0.1082 - lr 0.0250\n",
            "2019-12-20 10:15:49,906 DEV : loss 0.21954071521759033 - score 0.9339\n",
            "2019-12-20 10:15:49,930 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:15:49,931 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:49,972 epoch 55 - iter 0/154 - loss 0.04718608 - samples/sec: 11992.93\n",
            "2019-12-20 10:15:50,518 epoch 55 - iter 15/154 - loss 0.08587210 - samples/sec: 906.81\n",
            "2019-12-20 10:15:51,092 epoch 55 - iter 30/154 - loss 0.08700636 - samples/sec: 858.36\n",
            "2019-12-20 10:15:51,632 epoch 55 - iter 45/154 - loss 0.08441466 - samples/sec: 912.60\n",
            "2019-12-20 10:15:52,155 epoch 55 - iter 60/154 - loss 0.08719139 - samples/sec: 942.81\n",
            "2019-12-20 10:15:52,704 epoch 55 - iter 75/154 - loss 0.09572096 - samples/sec: 898.25\n",
            "2019-12-20 10:15:53,230 epoch 55 - iter 90/154 - loss 0.10106901 - samples/sec: 938.26\n",
            "2019-12-20 10:15:53,749 epoch 55 - iter 105/154 - loss 0.10412886 - samples/sec: 949.62\n",
            "2019-12-20 10:15:54,282 epoch 55 - iter 120/154 - loss 0.10899929 - samples/sec: 925.12\n",
            "2019-12-20 10:15:54,853 epoch 55 - iter 135/154 - loss 0.10570027 - samples/sec: 865.37\n",
            "2019-12-20 10:15:55,388 epoch 55 - iter 150/154 - loss 0.10469401 - samples/sec: 921.75\n",
            "2019-12-20 10:15:55,490 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:55,492 EPOCH 55 done: loss 0.1040 - lr 0.0250\n",
            "2019-12-20 10:15:55,980 DEV : loss 0.2320759892463684 - score 0.9303\n",
            "2019-12-20 10:15:56,005 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:15:56,006 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:15:56,047 epoch 56 - iter 0/154 - loss 0.07664128 - samples/sec: 12205.16\n",
            "2019-12-20 10:15:56,586 epoch 56 - iter 15/154 - loss 0.11403818 - samples/sec: 912.90\n",
            "2019-12-20 10:15:57,114 epoch 56 - iter 30/154 - loss 0.10931419 - samples/sec: 936.00\n",
            "2019-12-20 10:15:57,645 epoch 56 - iter 45/154 - loss 0.10915445 - samples/sec: 928.05\n",
            "2019-12-20 10:15:58,240 epoch 56 - iter 60/154 - loss 0.10752816 - samples/sec: 828.51\n",
            "2019-12-20 10:15:58,776 epoch 56 - iter 75/154 - loss 0.10120364 - samples/sec: 918.49\n",
            "2019-12-20 10:15:59,325 epoch 56 - iter 90/154 - loss 0.10398104 - samples/sec: 899.46\n",
            "2019-12-20 10:15:59,887 epoch 56 - iter 105/154 - loss 0.10494750 - samples/sec: 876.31\n",
            "2019-12-20 10:16:00,427 epoch 56 - iter 120/154 - loss 0.10180764 - samples/sec: 914.54\n",
            "2019-12-20 10:16:00,987 epoch 56 - iter 135/154 - loss 0.09953183 - samples/sec: 879.05\n",
            "2019-12-20 10:16:01,531 epoch 56 - iter 150/154 - loss 0.09658538 - samples/sec: 907.19\n",
            "2019-12-20 10:16:01,633 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:01,633 EPOCH 56 done: loss 0.0981 - lr 0.0250\n",
            "2019-12-20 10:16:02,133 DEV : loss 0.28756916522979736 - score 0.9193\n",
            "2019-12-20 10:16:02,159 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:16:02,160 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:02,203 epoch 57 - iter 0/154 - loss 0.01336183 - samples/sec: 11677.88\n",
            "2019-12-20 10:16:02,737 epoch 57 - iter 15/154 - loss 0.13357922 - samples/sec: 922.97\n",
            "2019-12-20 10:16:03,278 epoch 57 - iter 30/154 - loss 0.11373694 - samples/sec: 910.55\n",
            "2019-12-20 10:16:03,814 epoch 57 - iter 45/154 - loss 0.10973641 - samples/sec: 920.38\n",
            "2019-12-20 10:16:04,360 epoch 57 - iter 60/154 - loss 0.10277449 - samples/sec: 903.11\n",
            "2019-12-20 10:16:04,898 epoch 57 - iter 75/154 - loss 0.10362894 - samples/sec: 914.82\n",
            "2019-12-20 10:16:05,448 epoch 57 - iter 90/154 - loss 0.10232967 - samples/sec: 896.16\n",
            "2019-12-20 10:16:05,996 epoch 57 - iter 105/154 - loss 0.10163337 - samples/sec: 899.98\n",
            "2019-12-20 10:16:06,532 epoch 57 - iter 120/154 - loss 0.10656017 - samples/sec: 924.27\n",
            "2019-12-20 10:16:07,077 epoch 57 - iter 135/154 - loss 0.11035494 - samples/sec: 902.51\n",
            "2019-12-20 10:16:07,620 epoch 57 - iter 150/154 - loss 0.11062676 - samples/sec: 909.72\n",
            "2019-12-20 10:16:07,720 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:07,721 EPOCH 57 done: loss 0.1115 - lr 0.0250\n",
            "2019-12-20 10:16:08,208 DEV : loss 0.23112528026103973 - score 0.9284\n",
            "Epoch    56: reducing learning rate of group 0 to 1.2500e-02.\n",
            "2019-12-20 10:16:08,237 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:16:08,240 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:08,279 epoch 58 - iter 0/154 - loss 0.09078354 - samples/sec: 12842.49\n",
            "2019-12-20 10:16:08,801 epoch 58 - iter 15/154 - loss 0.10884448 - samples/sec: 943.28\n",
            "2019-12-20 10:16:09,372 epoch 58 - iter 30/154 - loss 0.09566445 - samples/sec: 861.99\n",
            "2019-12-20 10:16:09,893 epoch 58 - iter 45/154 - loss 0.10282743 - samples/sec: 947.15\n",
            "2019-12-20 10:16:10,434 epoch 58 - iter 60/154 - loss 0.10531762 - samples/sec: 909.86\n",
            "2019-12-20 10:16:10,971 epoch 58 - iter 75/154 - loss 0.09856797 - samples/sec: 917.56\n",
            "2019-12-20 10:16:11,518 epoch 58 - iter 90/154 - loss 0.10403422 - samples/sec: 900.92\n",
            "2019-12-20 10:16:12,051 epoch 58 - iter 105/154 - loss 0.10268479 - samples/sec: 924.93\n",
            "2019-12-20 10:16:12,596 epoch 58 - iter 120/154 - loss 0.09815521 - samples/sec: 903.88\n",
            "2019-12-20 10:16:13,163 epoch 58 - iter 135/154 - loss 0.09740110 - samples/sec: 868.88\n",
            "2019-12-20 10:16:13,695 epoch 58 - iter 150/154 - loss 0.09489797 - samples/sec: 926.57\n",
            "2019-12-20 10:16:13,797 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:13,797 EPOCH 58 done: loss 0.0939 - lr 0.0125\n",
            "2019-12-20 10:16:14,282 DEV : loss 0.24901647865772247 - score 0.9431\n",
            "2019-12-20 10:16:14,306 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:16:17,473 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:17,518 epoch 59 - iter 0/154 - loss 0.21177229 - samples/sec: 11037.64\n",
            "2019-12-20 10:16:18,119 epoch 59 - iter 15/154 - loss 0.10684629 - samples/sec: 819.15\n",
            "2019-12-20 10:16:18,673 epoch 59 - iter 30/154 - loss 0.10978744 - samples/sec: 892.62\n",
            "2019-12-20 10:16:19,207 epoch 59 - iter 45/154 - loss 0.09391997 - samples/sec: 925.59\n",
            "2019-12-20 10:16:19,775 epoch 59 - iter 60/154 - loss 0.10395507 - samples/sec: 872.34\n",
            "2019-12-20 10:16:20,331 epoch 59 - iter 75/154 - loss 0.10359952 - samples/sec: 886.62\n",
            "2019-12-20 10:16:20,886 epoch 59 - iter 90/154 - loss 0.10285301 - samples/sec: 887.04\n",
            "2019-12-20 10:16:21,431 epoch 59 - iter 105/154 - loss 0.10150200 - samples/sec: 905.40\n",
            "2019-12-20 10:16:21,958 epoch 59 - iter 120/154 - loss 0.09959961 - samples/sec: 935.62\n",
            "2019-12-20 10:16:22,545 epoch 59 - iter 135/154 - loss 0.09908877 - samples/sec: 838.33\n",
            "2019-12-20 10:16:23,108 epoch 59 - iter 150/154 - loss 0.09783631 - samples/sec: 876.02\n",
            "2019-12-20 10:16:23,216 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:23,217 EPOCH 59 done: loss 0.0977 - lr 0.0125\n",
            "2019-12-20 10:16:23,713 DEV : loss 0.2385055273771286 - score 0.9358\n",
            "2019-12-20 10:16:23,740 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:16:23,740 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:23,779 epoch 60 - iter 0/154 - loss 0.02166931 - samples/sec: 13198.97\n",
            "2019-12-20 10:16:24,323 epoch 60 - iter 15/154 - loss 0.10296113 - samples/sec: 905.88\n",
            "2019-12-20 10:16:24,895 epoch 60 - iter 30/154 - loss 0.09692900 - samples/sec: 870.35\n",
            "2019-12-20 10:16:25,439 epoch 60 - iter 45/154 - loss 0.09505894 - samples/sec: 905.41\n",
            "2019-12-20 10:16:25,991 epoch 60 - iter 60/154 - loss 0.09268812 - samples/sec: 893.61\n",
            "2019-12-20 10:16:26,539 epoch 60 - iter 75/154 - loss 0.09297875 - samples/sec: 898.69\n",
            "2019-12-20 10:16:27,076 epoch 60 - iter 90/154 - loss 0.09390026 - samples/sec: 918.39\n",
            "2019-12-20 10:16:27,624 epoch 60 - iter 105/154 - loss 0.09635350 - samples/sec: 897.76\n",
            "2019-12-20 10:16:28,176 epoch 60 - iter 120/154 - loss 0.09806727 - samples/sec: 892.52\n",
            "2019-12-20 10:16:28,724 epoch 60 - iter 135/154 - loss 0.09832725 - samples/sec: 900.77\n",
            "2019-12-20 10:16:29,241 epoch 60 - iter 150/154 - loss 0.09716261 - samples/sec: 955.97\n",
            "2019-12-20 10:16:29,339 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:29,340 EPOCH 60 done: loss 0.0964 - lr 0.0125\n",
            "2019-12-20 10:16:29,822 DEV : loss 0.25435081124305725 - score 0.9321\n",
            "2019-12-20 10:16:29,847 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:16:29,848 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:29,887 epoch 61 - iter 0/154 - loss 0.08951363 - samples/sec: 12944.72\n",
            "2019-12-20 10:16:30,392 epoch 61 - iter 15/154 - loss 0.10251426 - samples/sec: 975.87\n",
            "2019-12-20 10:16:30,935 epoch 61 - iter 30/154 - loss 0.09113961 - samples/sec: 907.79\n",
            "2019-12-20 10:16:31,467 epoch 61 - iter 45/154 - loss 0.09006885 - samples/sec: 928.87\n",
            "2019-12-20 10:16:32,007 epoch 61 - iter 60/154 - loss 0.08658241 - samples/sec: 917.45\n",
            "2019-12-20 10:16:32,537 epoch 61 - iter 75/154 - loss 0.09108655 - samples/sec: 931.01\n",
            "2019-12-20 10:16:33,076 epoch 61 - iter 90/154 - loss 0.09216363 - samples/sec: 914.53\n",
            "2019-12-20 10:16:33,614 epoch 61 - iter 105/154 - loss 0.09671506 - samples/sec: 917.69\n",
            "2019-12-20 10:16:34,133 epoch 61 - iter 120/154 - loss 0.09650926 - samples/sec: 951.79\n",
            "2019-12-20 10:16:34,657 epoch 61 - iter 135/154 - loss 0.09458507 - samples/sec: 940.42\n",
            "2019-12-20 10:16:35,196 epoch 61 - iter 150/154 - loss 0.09617975 - samples/sec: 917.01\n",
            "2019-12-20 10:16:35,293 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:35,294 EPOCH 61 done: loss 0.0960 - lr 0.0125\n",
            "2019-12-20 10:16:35,783 DEV : loss 0.23267237842082977 - score 0.9394\n",
            "2019-12-20 10:16:35,812 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:16:35,813 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:35,855 epoch 62 - iter 0/154 - loss 0.10596538 - samples/sec: 11826.11\n",
            "2019-12-20 10:16:36,384 epoch 62 - iter 15/154 - loss 0.07476360 - samples/sec: 930.34\n",
            "2019-12-20 10:16:36,936 epoch 62 - iter 30/154 - loss 0.08227204 - samples/sec: 894.96\n",
            "2019-12-20 10:16:37,481 epoch 62 - iter 45/154 - loss 0.09351510 - samples/sec: 903.28\n",
            "2019-12-20 10:16:38,017 epoch 62 - iter 60/154 - loss 0.09839528 - samples/sec: 920.51\n",
            "2019-12-20 10:16:38,569 epoch 62 - iter 75/154 - loss 0.09527427 - samples/sec: 895.44\n",
            "2019-12-20 10:16:39,126 epoch 62 - iter 90/154 - loss 0.09040769 - samples/sec: 886.62\n",
            "2019-12-20 10:16:39,666 epoch 62 - iter 105/154 - loss 0.09093284 - samples/sec: 919.77\n",
            "2019-12-20 10:16:40,199 epoch 62 - iter 120/154 - loss 0.09131596 - samples/sec: 927.48\n",
            "2019-12-20 10:16:40,745 epoch 62 - iter 135/154 - loss 0.09426713 - samples/sec: 910.83\n",
            "2019-12-20 10:16:41,293 epoch 62 - iter 150/154 - loss 0.09687444 - samples/sec: 904.38\n",
            "2019-12-20 10:16:41,401 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:41,402 EPOCH 62 done: loss 0.0958 - lr 0.0125\n",
            "2019-12-20 10:16:41,889 DEV : loss 0.2343720644712448 - score 0.9339\n",
            "2019-12-20 10:16:41,913 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:16:41,914 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:41,953 epoch 63 - iter 0/154 - loss 0.01642898 - samples/sec: 13011.23\n",
            "2019-12-20 10:16:42,498 epoch 63 - iter 15/154 - loss 0.05288271 - samples/sec: 902.68\n",
            "2019-12-20 10:16:43,046 epoch 63 - iter 30/154 - loss 0.07373830 - samples/sec: 898.54\n",
            "2019-12-20 10:16:43,576 epoch 63 - iter 45/154 - loss 0.07714607 - samples/sec: 929.68\n",
            "2019-12-20 10:16:44,113 epoch 63 - iter 60/154 - loss 0.08075033 - samples/sec: 916.31\n",
            "2019-12-20 10:16:44,663 epoch 63 - iter 75/154 - loss 0.08000987 - samples/sec: 896.16\n",
            "2019-12-20 10:16:45,195 epoch 63 - iter 90/154 - loss 0.08346969 - samples/sec: 926.57\n",
            "2019-12-20 10:16:45,741 epoch 63 - iter 105/154 - loss 0.08438402 - samples/sec: 901.67\n",
            "2019-12-20 10:16:46,287 epoch 63 - iter 120/154 - loss 0.08956848 - samples/sec: 903.38\n",
            "2019-12-20 10:16:46,841 epoch 63 - iter 135/154 - loss 0.09039142 - samples/sec: 889.11\n",
            "2019-12-20 10:16:47,394 epoch 63 - iter 150/154 - loss 0.09330248 - samples/sec: 891.70\n",
            "2019-12-20 10:16:47,502 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:47,503 EPOCH 63 done: loss 0.0934 - lr 0.0125\n",
            "2019-12-20 10:16:48,018 DEV : loss 0.23589272797107697 - score 0.9321\n",
            "2019-12-20 10:16:48,044 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:16:48,045 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:48,087 epoch 64 - iter 0/154 - loss 0.06099860 - samples/sec: 11905.70\n",
            "2019-12-20 10:16:48,626 epoch 64 - iter 15/154 - loss 0.06589461 - samples/sec: 913.90\n",
            "2019-12-20 10:16:49,166 epoch 64 - iter 30/154 - loss 0.07408018 - samples/sec: 913.48\n",
            "2019-12-20 10:16:49,693 epoch 64 - iter 45/154 - loss 0.06941038 - samples/sec: 935.38\n",
            "2019-12-20 10:16:50,267 epoch 64 - iter 60/154 - loss 0.08204338 - samples/sec: 859.84\n",
            "2019-12-20 10:16:50,828 epoch 64 - iter 75/154 - loss 0.08181396 - samples/sec: 877.69\n",
            "2019-12-20 10:16:51,377 epoch 64 - iter 90/154 - loss 0.08378090 - samples/sec: 897.92\n",
            "2019-12-20 10:16:51,936 epoch 64 - iter 105/154 - loss 0.08322788 - samples/sec: 882.78\n",
            "2019-12-20 10:16:52,476 epoch 64 - iter 120/154 - loss 0.08495468 - samples/sec: 913.47\n",
            "2019-12-20 10:16:53,016 epoch 64 - iter 135/154 - loss 0.08386271 - samples/sec: 913.60\n",
            "2019-12-20 10:16:53,564 epoch 64 - iter 150/154 - loss 0.08355141 - samples/sec: 898.32\n",
            "2019-12-20 10:16:53,664 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:53,665 EPOCH 64 done: loss 0.0829 - lr 0.0125\n",
            "2019-12-20 10:16:54,151 DEV : loss 0.2463570237159729 - score 0.9376\n",
            "Epoch    63: reducing learning rate of group 0 to 6.2500e-03.\n",
            "2019-12-20 10:16:54,176 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:16:54,177 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:54,218 epoch 65 - iter 0/154 - loss 0.04079427 - samples/sec: 12429.72\n",
            "2019-12-20 10:16:54,752 epoch 65 - iter 15/154 - loss 0.06937201 - samples/sec: 922.57\n",
            "2019-12-20 10:16:55,283 epoch 65 - iter 30/154 - loss 0.07275055 - samples/sec: 930.02\n",
            "2019-12-20 10:16:55,817 epoch 65 - iter 45/154 - loss 0.07413673 - samples/sec: 924.13\n",
            "2019-12-20 10:16:56,349 epoch 65 - iter 60/154 - loss 0.07399679 - samples/sec: 926.24\n",
            "2019-12-20 10:16:56,883 epoch 65 - iter 75/154 - loss 0.07670826 - samples/sec: 922.13\n",
            "2019-12-20 10:16:57,397 epoch 65 - iter 90/154 - loss 0.08019464 - samples/sec: 960.91\n",
            "2019-12-20 10:16:57,910 epoch 65 - iter 105/154 - loss 0.08497074 - samples/sec: 960.23\n",
            "2019-12-20 10:16:58,462 epoch 65 - iter 120/154 - loss 0.08658048 - samples/sec: 896.87\n",
            "2019-12-20 10:16:58,986 epoch 65 - iter 135/154 - loss 0.08800626 - samples/sec: 941.74\n",
            "2019-12-20 10:16:59,497 epoch 65 - iter 150/154 - loss 0.09215394 - samples/sec: 965.91\n",
            "2019-12-20 10:16:59,595 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:16:59,596 EPOCH 65 done: loss 0.0929 - lr 0.0063\n",
            "2019-12-20 10:17:00,067 DEV : loss 0.2342156618833542 - score 0.9358\n",
            "2019-12-20 10:17:00,095 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:17:00,096 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:00,132 epoch 66 - iter 0/154 - loss 0.16488987 - samples/sec: 14205.04\n",
            "2019-12-20 10:17:00,658 epoch 66 - iter 15/154 - loss 0.07961566 - samples/sec: 936.95\n",
            "2019-12-20 10:17:01,209 epoch 66 - iter 30/154 - loss 0.08267941 - samples/sec: 895.33\n",
            "2019-12-20 10:17:01,761 epoch 66 - iter 45/154 - loss 0.09054751 - samples/sec: 892.46\n",
            "2019-12-20 10:17:02,282 epoch 66 - iter 60/154 - loss 0.08983236 - samples/sec: 948.35\n",
            "2019-12-20 10:17:02,821 epoch 66 - iter 75/154 - loss 0.09187712 - samples/sec: 913.87\n",
            "2019-12-20 10:17:03,361 epoch 66 - iter 90/154 - loss 0.09120992 - samples/sec: 914.96\n",
            "2019-12-20 10:17:03,911 epoch 66 - iter 105/154 - loss 0.09068986 - samples/sec: 895.58\n",
            "2019-12-20 10:17:04,439 epoch 66 - iter 120/154 - loss 0.09209331 - samples/sec: 934.76\n",
            "2019-12-20 10:17:04,987 epoch 66 - iter 135/154 - loss 0.09229359 - samples/sec: 898.24\n",
            "2019-12-20 10:17:05,539 epoch 66 - iter 150/154 - loss 0.09214129 - samples/sec: 893.94\n",
            "2019-12-20 10:17:05,645 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:05,645 EPOCH 66 done: loss 0.0917 - lr 0.0063\n",
            "2019-12-20 10:17:06,131 DEV : loss 0.22892194986343384 - score 0.9376\n",
            "2019-12-20 10:17:06,155 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:17:06,156 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:06,198 epoch 67 - iter 0/154 - loss 0.03217352 - samples/sec: 11784.37\n",
            "2019-12-20 10:17:06,757 epoch 67 - iter 15/154 - loss 0.09415306 - samples/sec: 884.87\n",
            "2019-12-20 10:17:07,307 epoch 67 - iter 30/154 - loss 0.07935870 - samples/sec: 894.78\n",
            "2019-12-20 10:17:07,842 epoch 67 - iter 45/154 - loss 0.09390328 - samples/sec: 920.73\n",
            "2019-12-20 10:17:08,381 epoch 67 - iter 60/154 - loss 0.09002180 - samples/sec: 915.37\n",
            "2019-12-20 10:17:08,925 epoch 67 - iter 75/154 - loss 0.08637077 - samples/sec: 904.90\n",
            "2019-12-20 10:17:09,496 epoch 67 - iter 90/154 - loss 0.08467701 - samples/sec: 867.23\n",
            "2019-12-20 10:17:10,023 epoch 67 - iter 105/154 - loss 0.08438076 - samples/sec: 936.31\n",
            "2019-12-20 10:17:10,578 epoch 67 - iter 120/154 - loss 0.08540565 - samples/sec: 888.44\n",
            "2019-12-20 10:17:11,109 epoch 67 - iter 135/154 - loss 0.08590137 - samples/sec: 928.94\n",
            "2019-12-20 10:17:11,656 epoch 67 - iter 150/154 - loss 0.08537053 - samples/sec: 900.90\n",
            "2019-12-20 10:17:11,751 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:11,752 EPOCH 67 done: loss 0.0857 - lr 0.0063\n",
            "2019-12-20 10:17:12,249 DEV : loss 0.22493889927864075 - score 0.9376\n",
            "2019-12-20 10:17:12,274 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:17:12,275 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:12,312 epoch 68 - iter 0/154 - loss 0.16276120 - samples/sec: 13748.91\n",
            "2019-12-20 10:17:13,291 epoch 68 - iter 15/154 - loss 0.09671065 - samples/sec: 498.65\n",
            "2019-12-20 10:17:13,838 epoch 68 - iter 30/154 - loss 0.08840082 - samples/sec: 904.42\n",
            "2019-12-20 10:17:14,379 epoch 68 - iter 45/154 - loss 0.08173321 - samples/sec: 916.14\n",
            "2019-12-20 10:17:14,917 epoch 68 - iter 60/154 - loss 0.08130327 - samples/sec: 918.59\n",
            "2019-12-20 10:17:15,447 epoch 68 - iter 75/154 - loss 0.07885346 - samples/sec: 934.41\n",
            "2019-12-20 10:17:16,029 epoch 68 - iter 90/154 - loss 0.07810634 - samples/sec: 848.43\n",
            "2019-12-20 10:17:16,581 epoch 68 - iter 105/154 - loss 0.07891393 - samples/sec: 891.10\n",
            "2019-12-20 10:17:17,111 epoch 68 - iter 120/154 - loss 0.08061482 - samples/sec: 935.10\n",
            "2019-12-20 10:17:17,673 epoch 68 - iter 135/154 - loss 0.08114632 - samples/sec: 876.23\n",
            "2019-12-20 10:17:18,220 epoch 68 - iter 150/154 - loss 0.07960253 - samples/sec: 905.73\n",
            "2019-12-20 10:17:18,323 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:18,324 EPOCH 68 done: loss 0.0802 - lr 0.0063\n",
            "2019-12-20 10:17:18,811 DEV : loss 0.2372003197669983 - score 0.9413\n",
            "2019-12-20 10:17:18,837 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:17:18,838 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:18,875 epoch 69 - iter 0/154 - loss 0.15063606 - samples/sec: 13561.18\n",
            "2019-12-20 10:17:19,409 epoch 69 - iter 15/154 - loss 0.09247697 - samples/sec: 921.35\n",
            "2019-12-20 10:17:19,951 epoch 69 - iter 30/154 - loss 0.08455407 - samples/sec: 910.24\n",
            "2019-12-20 10:17:20,489 epoch 69 - iter 45/154 - loss 0.08571669 - samples/sec: 920.05\n",
            "2019-12-20 10:17:21,017 epoch 69 - iter 60/154 - loss 0.08460271 - samples/sec: 942.11\n",
            "2019-12-20 10:17:21,549 epoch 69 - iter 75/154 - loss 0.08887474 - samples/sec: 927.39\n",
            "2019-12-20 10:17:22,069 epoch 69 - iter 90/154 - loss 0.09260286 - samples/sec: 949.51\n",
            "2019-12-20 10:17:22,597 epoch 69 - iter 105/154 - loss 0.09554181 - samples/sec: 934.40\n",
            "2019-12-20 10:17:23,135 epoch 69 - iter 120/154 - loss 0.09726560 - samples/sec: 916.39\n",
            "2019-12-20 10:17:23,663 epoch 69 - iter 135/154 - loss 0.09513982 - samples/sec: 932.75\n",
            "2019-12-20 10:17:24,204 epoch 69 - iter 150/154 - loss 0.09482547 - samples/sec: 914.34\n",
            "2019-12-20 10:17:24,302 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:24,303 EPOCH 69 done: loss 0.0941 - lr 0.0063\n",
            "2019-12-20 10:17:24,787 DEV : loss 0.23180262744426727 - score 0.9376\n",
            "2019-12-20 10:17:24,815 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:17:24,816 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:24,857 epoch 70 - iter 0/154 - loss 0.13996227 - samples/sec: 12100.63\n",
            "2019-12-20 10:17:25,384 epoch 70 - iter 15/154 - loss 0.08429036 - samples/sec: 935.72\n",
            "2019-12-20 10:17:25,915 epoch 70 - iter 30/154 - loss 0.08655751 - samples/sec: 928.78\n",
            "2019-12-20 10:17:26,437 epoch 70 - iter 45/154 - loss 0.08067139 - samples/sec: 944.37\n",
            "2019-12-20 10:17:26,988 epoch 70 - iter 60/154 - loss 0.08943079 - samples/sec: 895.84\n",
            "2019-12-20 10:17:27,531 epoch 70 - iter 75/154 - loss 0.08682502 - samples/sec: 907.88\n",
            "2019-12-20 10:17:28,074 epoch 70 - iter 90/154 - loss 0.08467542 - samples/sec: 906.19\n",
            "2019-12-20 10:17:28,611 epoch 70 - iter 105/154 - loss 0.08182743 - samples/sec: 918.21\n",
            "2019-12-20 10:17:29,152 epoch 70 - iter 120/154 - loss 0.08458449 - samples/sec: 910.73\n",
            "2019-12-20 10:17:29,683 epoch 70 - iter 135/154 - loss 0.08563995 - samples/sec: 932.49\n",
            "2019-12-20 10:17:30,232 epoch 70 - iter 150/154 - loss 0.08432149 - samples/sec: 907.99\n",
            "2019-12-20 10:17:30,333 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:30,333 EPOCH 70 done: loss 0.0833 - lr 0.0063\n",
            "2019-12-20 10:17:30,846 DEV : loss 0.23630942404270172 - score 0.9358\n",
            "Epoch    69: reducing learning rate of group 0 to 3.1250e-03.\n",
            "2019-12-20 10:17:30,870 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:17:30,870 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:30,908 epoch 71 - iter 0/154 - loss 0.13147889 - samples/sec: 13128.48\n",
            "2019-12-20 10:17:31,436 epoch 71 - iter 15/154 - loss 0.09189462 - samples/sec: 932.80\n",
            "2019-12-20 10:17:31,975 epoch 71 - iter 30/154 - loss 0.08788472 - samples/sec: 916.71\n",
            "2019-12-20 10:17:32,522 epoch 71 - iter 45/154 - loss 0.08256366 - samples/sec: 899.29\n",
            "2019-12-20 10:17:33,058 epoch 71 - iter 60/154 - loss 0.07954167 - samples/sec: 919.21\n",
            "2019-12-20 10:17:33,597 epoch 71 - iter 75/154 - loss 0.07627712 - samples/sec: 916.54\n",
            "2019-12-20 10:17:34,135 epoch 71 - iter 90/154 - loss 0.07806685 - samples/sec: 914.56\n",
            "2019-12-20 10:17:34,696 epoch 71 - iter 105/154 - loss 0.07813381 - samples/sec: 879.30\n",
            "2019-12-20 10:17:35,232 epoch 71 - iter 120/154 - loss 0.08108464 - samples/sec: 918.97\n",
            "2019-12-20 10:17:35,772 epoch 71 - iter 135/154 - loss 0.08140166 - samples/sec: 913.78\n",
            "2019-12-20 10:17:36,338 epoch 71 - iter 150/154 - loss 0.08248589 - samples/sec: 873.02\n",
            "2019-12-20 10:17:36,438 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:36,439 EPOCH 71 done: loss 0.0815 - lr 0.0031\n",
            "2019-12-20 10:17:36,921 DEV : loss 0.23686258494853973 - score 0.9376\n",
            "2019-12-20 10:17:36,943 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:17:36,944 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:36,985 epoch 72 - iter 0/154 - loss 0.06394284 - samples/sec: 12593.22\n",
            "2019-12-20 10:17:37,528 epoch 72 - iter 15/154 - loss 0.08914767 - samples/sec: 907.55\n",
            "2019-12-20 10:17:38,082 epoch 72 - iter 30/154 - loss 0.09008854 - samples/sec: 893.27\n",
            "2019-12-20 10:17:38,611 epoch 72 - iter 45/154 - loss 0.10214605 - samples/sec: 932.33\n",
            "2019-12-20 10:17:39,141 epoch 72 - iter 60/154 - loss 0.09665913 - samples/sec: 931.34\n",
            "2019-12-20 10:17:39,686 epoch 72 - iter 75/154 - loss 0.09873966 - samples/sec: 904.51\n",
            "2019-12-20 10:17:40,233 epoch 72 - iter 90/154 - loss 0.09801043 - samples/sec: 902.01\n",
            "2019-12-20 10:17:40,755 epoch 72 - iter 105/154 - loss 0.09603002 - samples/sec: 944.88\n",
            "2019-12-20 10:17:41,309 epoch 72 - iter 120/154 - loss 0.09535953 - samples/sec: 894.41\n",
            "2019-12-20 10:17:41,855 epoch 72 - iter 135/154 - loss 0.09370734 - samples/sec: 908.55\n",
            "2019-12-20 10:17:42,401 epoch 72 - iter 150/154 - loss 0.09374303 - samples/sec: 907.00\n",
            "2019-12-20 10:17:42,508 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:42,509 EPOCH 72 done: loss 0.0932 - lr 0.0031\n",
            "2019-12-20 10:17:42,998 DEV : loss 0.23366329073905945 - score 0.9376\n",
            "2019-12-20 10:17:43,022 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:17:43,024 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:43,065 epoch 73 - iter 0/154 - loss 0.02527385 - samples/sec: 12136.08\n",
            "2019-12-20 10:17:43,617 epoch 73 - iter 15/154 - loss 0.08847748 - samples/sec: 891.97\n",
            "2019-12-20 10:17:44,160 epoch 73 - iter 30/154 - loss 0.07790276 - samples/sec: 909.28\n",
            "2019-12-20 10:17:44,685 epoch 73 - iter 45/154 - loss 0.08128527 - samples/sec: 938.95\n",
            "2019-12-20 10:17:45,227 epoch 73 - iter 60/154 - loss 0.08696361 - samples/sec: 910.51\n",
            "2019-12-20 10:17:45,806 epoch 73 - iter 75/154 - loss 0.08570247 - samples/sec: 850.93\n",
            "2019-12-20 10:17:46,345 epoch 73 - iter 90/154 - loss 0.08333121 - samples/sec: 914.87\n",
            "2019-12-20 10:17:46,871 epoch 73 - iter 105/154 - loss 0.08515394 - samples/sec: 939.93\n",
            "2019-12-20 10:17:47,410 epoch 73 - iter 120/154 - loss 0.08202945 - samples/sec: 915.11\n",
            "2019-12-20 10:17:47,952 epoch 73 - iter 135/154 - loss 0.08068114 - samples/sec: 912.28\n",
            "2019-12-20 10:17:48,497 epoch 73 - iter 150/154 - loss 0.08042865 - samples/sec: 906.02\n",
            "2019-12-20 10:17:48,592 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:48,593 EPOCH 73 done: loss 0.0806 - lr 0.0031\n",
            "2019-12-20 10:17:49,072 DEV : loss 0.2385120391845703 - score 0.9376\n",
            "2019-12-20 10:17:49,099 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:17:49,100 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:49,141 epoch 74 - iter 0/154 - loss 0.14083318 - samples/sec: 12142.96\n",
            "2019-12-20 10:17:49,662 epoch 74 - iter 15/154 - loss 0.09471533 - samples/sec: 944.85\n",
            "2019-12-20 10:17:50,196 epoch 74 - iter 30/154 - loss 0.07961605 - samples/sec: 924.16\n",
            "2019-12-20 10:17:50,737 epoch 74 - iter 45/154 - loss 0.08373699 - samples/sec: 911.00\n",
            "2019-12-20 10:17:51,266 epoch 74 - iter 60/154 - loss 0.08962540 - samples/sec: 933.07\n",
            "2019-12-20 10:17:51,794 epoch 74 - iter 75/154 - loss 0.09740456 - samples/sec: 932.83\n",
            "2019-12-20 10:17:52,336 epoch 74 - iter 90/154 - loss 0.09648455 - samples/sec: 909.34\n",
            "2019-12-20 10:17:52,890 epoch 74 - iter 105/154 - loss 0.09236797 - samples/sec: 890.42\n",
            "2019-12-20 10:17:53,429 epoch 74 - iter 120/154 - loss 0.09150019 - samples/sec: 914.14\n",
            "2019-12-20 10:17:53,961 epoch 74 - iter 135/154 - loss 0.08933836 - samples/sec: 927.38\n",
            "2019-12-20 10:17:54,500 epoch 74 - iter 150/154 - loss 0.08898176 - samples/sec: 914.13\n",
            "2019-12-20 10:17:54,610 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:54,611 EPOCH 74 done: loss 0.0891 - lr 0.0031\n",
            "2019-12-20 10:17:55,097 DEV : loss 0.2426571398973465 - score 0.9376\n",
            "2019-12-20 10:17:55,122 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:17:55,123 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:17:55,161 epoch 75 - iter 0/154 - loss 0.32590449 - samples/sec: 13274.95\n",
            "2019-12-20 10:17:55,697 epoch 75 - iter 15/154 - loss 0.08132926 - samples/sec: 922.80\n",
            "2019-12-20 10:17:56,238 epoch 75 - iter 30/154 - loss 0.08469383 - samples/sec: 910.71\n",
            "2019-12-20 10:17:56,771 epoch 75 - iter 45/154 - loss 0.08702612 - samples/sec: 931.37\n",
            "2019-12-20 10:17:57,297 epoch 75 - iter 60/154 - loss 0.08803038 - samples/sec: 936.51\n",
            "2019-12-20 10:17:57,829 epoch 75 - iter 75/154 - loss 0.09060534 - samples/sec: 925.25\n",
            "2019-12-20 10:17:58,391 epoch 75 - iter 90/154 - loss 0.09113594 - samples/sec: 877.35\n",
            "2019-12-20 10:17:58,928 epoch 75 - iter 105/154 - loss 0.08839587 - samples/sec: 918.55\n",
            "2019-12-20 10:17:59,483 epoch 75 - iter 120/154 - loss 0.08732522 - samples/sec: 890.19\n",
            "2019-12-20 10:18:00,021 epoch 75 - iter 135/154 - loss 0.08776563 - samples/sec: 917.59\n",
            "2019-12-20 10:18:00,568 epoch 75 - iter 150/154 - loss 0.08819785 - samples/sec: 900.32\n",
            "2019-12-20 10:18:00,674 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:00,675 EPOCH 75 done: loss 0.0885 - lr 0.0031\n",
            "2019-12-20 10:18:01,151 DEV : loss 0.24148495495319366 - score 0.9303\n",
            "2019-12-20 10:18:01,177 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:18:01,178 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:01,216 epoch 76 - iter 0/154 - loss 0.20050614 - samples/sec: 12908.53\n",
            "2019-12-20 10:18:01,769 epoch 76 - iter 15/154 - loss 0.07831178 - samples/sec: 889.27\n",
            "2019-12-20 10:18:02,323 epoch 76 - iter 30/154 - loss 0.07529023 - samples/sec: 889.03\n",
            "2019-12-20 10:18:02,879 epoch 76 - iter 45/154 - loss 0.09210806 - samples/sec: 885.21\n",
            "2019-12-20 10:18:03,426 epoch 76 - iter 60/154 - loss 0.08861176 - samples/sec: 902.55\n",
            "2019-12-20 10:18:03,972 epoch 76 - iter 75/154 - loss 0.09663480 - samples/sec: 902.26\n",
            "2019-12-20 10:18:04,504 epoch 76 - iter 90/154 - loss 0.09578098 - samples/sec: 926.47\n",
            "2019-12-20 10:18:05,042 epoch 76 - iter 105/154 - loss 0.09466033 - samples/sec: 916.80\n",
            "2019-12-20 10:18:05,600 epoch 76 - iter 120/154 - loss 0.09011591 - samples/sec: 882.21\n",
            "2019-12-20 10:18:06,141 epoch 76 - iter 135/154 - loss 0.08745559 - samples/sec: 910.11\n",
            "2019-12-20 10:18:06,680 epoch 76 - iter 150/154 - loss 0.08972295 - samples/sec: 914.50\n",
            "2019-12-20 10:18:06,777 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:06,778 EPOCH 76 done: loss 0.0895 - lr 0.0031\n",
            "2019-12-20 10:18:07,266 DEV : loss 0.23865996301174164 - score 0.9339\n",
            "Epoch    75: reducing learning rate of group 0 to 1.5625e-03.\n",
            "2019-12-20 10:18:07,290 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:18:07,293 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:07,338 epoch 77 - iter 0/154 - loss 0.01650889 - samples/sec: 11471.73\n",
            "2019-12-20 10:18:07,875 epoch 77 - iter 15/154 - loss 0.05815968 - samples/sec: 917.99\n",
            "2019-12-20 10:18:08,406 epoch 77 - iter 30/154 - loss 0.07100433 - samples/sec: 930.52\n",
            "2019-12-20 10:18:08,941 epoch 77 - iter 45/154 - loss 0.07430328 - samples/sec: 920.57\n",
            "2019-12-20 10:18:09,479 epoch 77 - iter 60/154 - loss 0.08257932 - samples/sec: 916.57\n",
            "2019-12-20 10:18:10,027 epoch 77 - iter 75/154 - loss 0.08256329 - samples/sec: 899.43\n",
            "2019-12-20 10:18:10,572 epoch 77 - iter 90/154 - loss 0.07896488 - samples/sec: 903.18\n",
            "2019-12-20 10:18:11,103 epoch 77 - iter 105/154 - loss 0.07621105 - samples/sec: 929.80\n",
            "2019-12-20 10:18:11,641 epoch 77 - iter 120/154 - loss 0.07909314 - samples/sec: 917.15\n",
            "2019-12-20 10:18:12,168 epoch 77 - iter 135/154 - loss 0.08070043 - samples/sec: 935.71\n",
            "2019-12-20 10:18:12,701 epoch 77 - iter 150/154 - loss 0.07880686 - samples/sec: 924.78\n",
            "2019-12-20 10:18:12,801 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:12,802 EPOCH 77 done: loss 0.0795 - lr 0.0016\n",
            "2019-12-20 10:18:13,287 DEV : loss 0.23560640215873718 - score 0.9376\n",
            "2019-12-20 10:18:13,313 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:18:13,314 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:13,351 epoch 78 - iter 0/154 - loss 0.01661289 - samples/sec: 13614.09\n",
            "2019-12-20 10:18:13,886 epoch 78 - iter 15/154 - loss 0.07734126 - samples/sec: 919.28\n",
            "2019-12-20 10:18:14,416 epoch 78 - iter 30/154 - loss 0.08526378 - samples/sec: 929.96\n",
            "2019-12-20 10:18:14,951 epoch 78 - iter 45/154 - loss 0.08286473 - samples/sec: 922.37\n",
            "2019-12-20 10:18:15,480 epoch 78 - iter 60/154 - loss 0.07885799 - samples/sec: 931.13\n",
            "2019-12-20 10:18:16,011 epoch 78 - iter 75/154 - loss 0.07939604 - samples/sec: 928.06\n",
            "2019-12-20 10:18:16,549 epoch 78 - iter 90/154 - loss 0.07975692 - samples/sec: 916.57\n",
            "2019-12-20 10:18:17,065 epoch 78 - iter 105/154 - loss 0.08025610 - samples/sec: 955.37\n",
            "2019-12-20 10:18:17,589 epoch 78 - iter 120/154 - loss 0.08015361 - samples/sec: 940.80\n",
            "2019-12-20 10:18:18,131 epoch 78 - iter 135/154 - loss 0.07975064 - samples/sec: 913.54\n",
            "2019-12-20 10:18:18,692 epoch 78 - iter 150/154 - loss 0.07909069 - samples/sec: 877.79\n",
            "2019-12-20 10:18:18,797 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:18,798 EPOCH 78 done: loss 0.0801 - lr 0.0016\n",
            "2019-12-20 10:18:19,279 DEV : loss 0.23377834260463715 - score 0.9376\n",
            "2019-12-20 10:18:19,304 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:18:19,305 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:19,340 epoch 79 - iter 0/154 - loss 0.02925172 - samples/sec: 14132.94\n",
            "2019-12-20 10:18:19,870 epoch 79 - iter 15/154 - loss 0.07319340 - samples/sec: 931.58\n",
            "2019-12-20 10:18:20,439 epoch 79 - iter 30/154 - loss 0.08229905 - samples/sec: 872.09\n",
            "2019-12-20 10:18:20,986 epoch 79 - iter 45/154 - loss 0.07898503 - samples/sec: 904.35\n",
            "2019-12-20 10:18:21,508 epoch 79 - iter 60/154 - loss 0.08093011 - samples/sec: 949.00\n",
            "2019-12-20 10:18:22,041 epoch 79 - iter 75/154 - loss 0.08282877 - samples/sec: 929.74\n",
            "2019-12-20 10:18:22,589 epoch 79 - iter 90/154 - loss 0.07923505 - samples/sec: 902.52\n",
            "2019-12-20 10:18:23,136 epoch 79 - iter 105/154 - loss 0.07668089 - samples/sec: 903.29\n",
            "2019-12-20 10:18:23,692 epoch 79 - iter 120/154 - loss 0.07927132 - samples/sec: 889.30\n",
            "2019-12-20 10:18:24,231 epoch 79 - iter 135/154 - loss 0.07916509 - samples/sec: 920.23\n",
            "2019-12-20 10:18:24,772 epoch 79 - iter 150/154 - loss 0.08233887 - samples/sec: 915.31\n",
            "2019-12-20 10:18:24,884 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:24,885 EPOCH 79 done: loss 0.0871 - lr 0.0016\n",
            "2019-12-20 10:18:25,366 DEV : loss 0.23766690492630005 - score 0.9358\n",
            "2019-12-20 10:18:25,390 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:18:25,391 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:25,436 epoch 80 - iter 0/154 - loss 0.07255379 - samples/sec: 11087.24\n",
            "2019-12-20 10:18:25,982 epoch 80 - iter 15/154 - loss 0.07836992 - samples/sec: 902.43\n",
            "2019-12-20 10:18:26,526 epoch 80 - iter 30/154 - loss 0.07407963 - samples/sec: 904.85\n",
            "2019-12-20 10:18:27,071 epoch 80 - iter 45/154 - loss 0.07712403 - samples/sec: 904.59\n",
            "2019-12-20 10:18:27,592 epoch 80 - iter 60/154 - loss 0.07978673 - samples/sec: 945.94\n",
            "2019-12-20 10:18:28,135 epoch 80 - iter 75/154 - loss 0.08562798 - samples/sec: 909.15\n",
            "2019-12-20 10:18:28,664 epoch 80 - iter 90/154 - loss 0.08432089 - samples/sec: 931.99\n",
            "2019-12-20 10:18:29,206 epoch 80 - iter 105/154 - loss 0.08092234 - samples/sec: 907.95\n",
            "2019-12-20 10:18:29,770 epoch 80 - iter 120/154 - loss 0.08113556 - samples/sec: 874.22\n",
            "2019-12-20 10:18:30,318 epoch 80 - iter 135/154 - loss 0.07995271 - samples/sec: 899.36\n",
            "2019-12-20 10:18:30,848 epoch 80 - iter 150/154 - loss 0.07942018 - samples/sec: 929.83\n",
            "2019-12-20 10:18:30,951 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:30,952 EPOCH 80 done: loss 0.0788 - lr 0.0016\n",
            "2019-12-20 10:18:31,440 DEV : loss 0.23652365803718567 - score 0.9358\n",
            "2019-12-20 10:18:31,466 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:18:31,467 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:31,506 epoch 81 - iter 0/154 - loss 0.20660561 - samples/sec: 12748.40\n",
            "2019-12-20 10:18:32,042 epoch 81 - iter 15/154 - loss 0.08145597 - samples/sec: 918.52\n",
            "2019-12-20 10:18:32,587 epoch 81 - iter 30/154 - loss 0.08459189 - samples/sec: 905.36\n",
            "2019-12-20 10:18:33,124 epoch 81 - iter 45/154 - loss 0.08397586 - samples/sec: 917.31\n",
            "2019-12-20 10:18:33,674 epoch 81 - iter 60/154 - loss 0.08061814 - samples/sec: 896.26\n",
            "2019-12-20 10:18:34,234 epoch 81 - iter 75/154 - loss 0.08190704 - samples/sec: 877.69\n",
            "2019-12-20 10:18:34,764 epoch 81 - iter 90/154 - loss 0.07666611 - samples/sec: 932.79\n",
            "2019-12-20 10:18:35,302 epoch 81 - iter 105/154 - loss 0.07856946 - samples/sec: 916.39\n",
            "2019-12-20 10:18:35,843 epoch 81 - iter 120/154 - loss 0.08049018 - samples/sec: 911.84\n",
            "2019-12-20 10:18:36,389 epoch 81 - iter 135/154 - loss 0.07856986 - samples/sec: 903.07\n",
            "2019-12-20 10:18:36,920 epoch 81 - iter 150/154 - loss 0.08037078 - samples/sec: 928.81\n",
            "2019-12-20 10:18:37,021 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:37,022 EPOCH 81 done: loss 0.0800 - lr 0.0016\n",
            "2019-12-20 10:18:37,510 DEV : loss 0.2402781993150711 - score 0.9358\n",
            "2019-12-20 10:18:37,534 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:18:37,535 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:37,575 epoch 82 - iter 0/154 - loss 0.12008694 - samples/sec: 12904.56\n",
            "2019-12-20 10:18:38,111 epoch 82 - iter 15/154 - loss 0.08517170 - samples/sec: 920.39\n",
            "2019-12-20 10:18:38,632 epoch 82 - iter 30/154 - loss 0.09030739 - samples/sec: 952.54\n",
            "2019-12-20 10:18:39,151 epoch 82 - iter 45/154 - loss 0.08570535 - samples/sec: 950.80\n",
            "2019-12-20 10:18:39,679 epoch 82 - iter 60/154 - loss 0.08547728 - samples/sec: 938.89\n",
            "2019-12-20 10:18:40,214 epoch 82 - iter 75/154 - loss 0.08546216 - samples/sec: 922.37\n",
            "2019-12-20 10:18:40,730 epoch 82 - iter 90/154 - loss 0.08670696 - samples/sec: 957.96\n",
            "2019-12-20 10:18:41,263 epoch 82 - iter 105/154 - loss 0.08205953 - samples/sec: 924.53\n",
            "2019-12-20 10:18:41,813 epoch 82 - iter 120/154 - loss 0.08375389 - samples/sec: 895.86\n",
            "2019-12-20 10:18:42,344 epoch 82 - iter 135/154 - loss 0.08218401 - samples/sec: 929.51\n",
            "2019-12-20 10:18:42,869 epoch 82 - iter 150/154 - loss 0.08280324 - samples/sec: 939.69\n",
            "2019-12-20 10:18:42,965 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:42,968 EPOCH 82 done: loss 0.0835 - lr 0.0016\n",
            "2019-12-20 10:18:43,441 DEV : loss 0.2394690066576004 - score 0.9376\n",
            "Epoch    81: reducing learning rate of group 0 to 7.8125e-04.\n",
            "2019-12-20 10:18:43,465 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:18:43,465 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:43,505 epoch 83 - iter 0/154 - loss 0.01655065 - samples/sec: 12695.51\n",
            "2019-12-20 10:18:44,030 epoch 83 - iter 15/154 - loss 0.07817895 - samples/sec: 937.70\n",
            "2019-12-20 10:18:44,575 epoch 83 - iter 30/154 - loss 0.09672653 - samples/sec: 903.55\n",
            "2019-12-20 10:18:45,121 epoch 83 - iter 45/154 - loss 0.09671161 - samples/sec: 903.17\n",
            "2019-12-20 10:18:45,652 epoch 83 - iter 60/154 - loss 0.09349839 - samples/sec: 928.12\n",
            "2019-12-20 10:18:46,239 epoch 83 - iter 75/154 - loss 0.08870677 - samples/sec: 841.62\n",
            "2019-12-20 10:18:46,801 epoch 83 - iter 90/154 - loss 0.08856921 - samples/sec: 876.64\n",
            "2019-12-20 10:18:47,336 epoch 83 - iter 105/154 - loss 0.08680531 - samples/sec: 920.94\n",
            "2019-12-20 10:18:47,889 epoch 83 - iter 120/154 - loss 0.08296415 - samples/sec: 896.34\n",
            "2019-12-20 10:18:48,458 epoch 83 - iter 135/154 - loss 0.08218946 - samples/sec: 865.09\n",
            "2019-12-20 10:18:49,002 epoch 83 - iter 150/154 - loss 0.08252613 - samples/sec: 906.27\n",
            "2019-12-20 10:18:49,103 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:49,104 EPOCH 83 done: loss 0.0823 - lr 0.0008\n",
            "2019-12-20 10:18:49,605 DEV : loss 0.23677389323711395 - score 0.9376\n",
            "2019-12-20 10:18:49,630 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:18:49,631 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:49,671 epoch 84 - iter 0/154 - loss 0.02124974 - samples/sec: 13070.61\n",
            "2019-12-20 10:18:50,251 epoch 84 - iter 15/154 - loss 0.05808718 - samples/sec: 848.23\n",
            "2019-12-20 10:18:50,811 epoch 84 - iter 30/154 - loss 0.06041118 - samples/sec: 885.08\n",
            "2019-12-20 10:18:51,347 epoch 84 - iter 45/154 - loss 0.07493939 - samples/sec: 921.83\n",
            "2019-12-20 10:18:51,905 epoch 84 - iter 60/154 - loss 0.07420388 - samples/sec: 889.23\n",
            "2019-12-20 10:18:52,438 epoch 84 - iter 75/154 - loss 0.07471464 - samples/sec: 926.34\n",
            "2019-12-20 10:18:52,980 epoch 84 - iter 90/154 - loss 0.07505566 - samples/sec: 922.15\n",
            "2019-12-20 10:18:53,519 epoch 84 - iter 105/154 - loss 0.07633203 - samples/sec: 916.32\n",
            "2019-12-20 10:18:54,061 epoch 84 - iter 120/154 - loss 0.07689568 - samples/sec: 910.53\n",
            "2019-12-20 10:18:54,597 epoch 84 - iter 135/154 - loss 0.07836978 - samples/sec: 923.54\n",
            "2019-12-20 10:18:55,138 epoch 84 - iter 150/154 - loss 0.07576070 - samples/sec: 916.33\n",
            "2019-12-20 10:18:55,244 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:55,245 EPOCH 84 done: loss 0.0748 - lr 0.0008\n",
            "2019-12-20 10:18:55,745 DEV : loss 0.23768404126167297 - score 0.9358\n",
            "2019-12-20 10:18:55,770 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:18:55,771 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:18:55,810 epoch 85 - iter 0/154 - loss 0.06370291 - samples/sec: 12958.30\n",
            "2019-12-20 10:18:56,360 epoch 85 - iter 15/154 - loss 0.09007925 - samples/sec: 895.61\n",
            "2019-12-20 10:18:56,916 epoch 85 - iter 30/154 - loss 0.08005672 - samples/sec: 886.29\n",
            "2019-12-20 10:18:57,466 epoch 85 - iter 45/154 - loss 0.07806539 - samples/sec: 896.15\n",
            "2019-12-20 10:18:58,006 epoch 85 - iter 60/154 - loss 0.07856336 - samples/sec: 913.35\n",
            "2019-12-20 10:18:58,538 epoch 85 - iter 75/154 - loss 0.08636564 - samples/sec: 925.61\n",
            "2019-12-20 10:18:59,082 epoch 85 - iter 90/154 - loss 0.08371658 - samples/sec: 906.09\n",
            "2019-12-20 10:18:59,610 epoch 85 - iter 105/154 - loss 0.08240309 - samples/sec: 935.45\n",
            "2019-12-20 10:19:00,143 epoch 85 - iter 120/154 - loss 0.08420255 - samples/sec: 925.96\n",
            "2019-12-20 10:19:00,683 epoch 85 - iter 135/154 - loss 0.08374846 - samples/sec: 911.18\n",
            "2019-12-20 10:19:01,233 epoch 85 - iter 150/154 - loss 0.08362290 - samples/sec: 895.84\n",
            "2019-12-20 10:19:01,339 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:01,340 EPOCH 85 done: loss 0.0841 - lr 0.0008\n",
            "2019-12-20 10:19:01,826 DEV : loss 0.2370755970478058 - score 0.9376\n",
            "2019-12-20 10:19:01,849 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:19:01,850 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:01,890 epoch 86 - iter 0/154 - loss 0.12591937 - samples/sec: 12468.51\n",
            "2019-12-20 10:19:02,437 epoch 86 - iter 15/154 - loss 0.08923572 - samples/sec: 899.13\n",
            "2019-12-20 10:19:02,983 epoch 86 - iter 30/154 - loss 0.07874008 - samples/sec: 900.92\n",
            "2019-12-20 10:19:03,527 epoch 86 - iter 45/154 - loss 0.07497969 - samples/sec: 915.15\n",
            "2019-12-20 10:19:04,062 epoch 86 - iter 60/154 - loss 0.07740854 - samples/sec: 924.80\n",
            "2019-12-20 10:19:04,593 epoch 86 - iter 75/154 - loss 0.08183752 - samples/sec: 931.79\n",
            "2019-12-20 10:19:05,139 epoch 86 - iter 90/154 - loss 0.08194343 - samples/sec: 904.87\n",
            "2019-12-20 10:19:05,666 epoch 86 - iter 105/154 - loss 0.08000805 - samples/sec: 937.38\n",
            "2019-12-20 10:19:06,217 epoch 86 - iter 120/154 - loss 0.08095780 - samples/sec: 899.73\n",
            "2019-12-20 10:19:06,734 epoch 86 - iter 135/154 - loss 0.07997775 - samples/sec: 953.14\n",
            "2019-12-20 10:19:07,276 epoch 86 - iter 150/154 - loss 0.08273792 - samples/sec: 915.21\n",
            "2019-12-20 10:19:07,384 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:07,386 EPOCH 86 done: loss 0.0830 - lr 0.0008\n",
            "2019-12-20 10:19:07,867 DEV : loss 0.23632033169269562 - score 0.9376\n",
            "2019-12-20 10:19:07,891 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:19:07,892 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:07,932 epoch 87 - iter 0/154 - loss 0.06473455 - samples/sec: 12753.09\n",
            "2019-12-20 10:19:08,446 epoch 87 - iter 15/154 - loss 0.10684869 - samples/sec: 959.45\n",
            "2019-12-20 10:19:08,962 epoch 87 - iter 30/154 - loss 0.09641116 - samples/sec: 955.35\n",
            "2019-12-20 10:19:09,491 epoch 87 - iter 45/154 - loss 0.09084276 - samples/sec: 935.30\n",
            "2019-12-20 10:19:10,033 epoch 87 - iter 60/154 - loss 0.09311716 - samples/sec: 907.84\n",
            "2019-12-20 10:19:10,577 epoch 87 - iter 75/154 - loss 0.08835067 - samples/sec: 907.26\n",
            "2019-12-20 10:19:11,125 epoch 87 - iter 90/154 - loss 0.08672275 - samples/sec: 900.34\n",
            "2019-12-20 10:19:11,674 epoch 87 - iter 105/154 - loss 0.08700452 - samples/sec: 899.15\n",
            "2019-12-20 10:19:12,231 epoch 87 - iter 120/154 - loss 0.08864471 - samples/sec: 884.05\n",
            "2019-12-20 10:19:12,789 epoch 87 - iter 135/154 - loss 0.08787598 - samples/sec: 882.48\n",
            "2019-12-20 10:19:13,324 epoch 87 - iter 150/154 - loss 0.08676986 - samples/sec: 921.82\n",
            "2019-12-20 10:19:13,425 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:13,426 EPOCH 87 done: loss 0.0880 - lr 0.0008\n",
            "2019-12-20 10:19:13,912 DEV : loss 0.23674964904785156 - score 0.9394\n",
            "2019-12-20 10:19:13,936 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:19:13,937 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:13,975 epoch 88 - iter 0/154 - loss 0.05013047 - samples/sec: 13291.69\n",
            "2019-12-20 10:19:14,521 epoch 88 - iter 15/154 - loss 0.09142547 - samples/sec: 901.55\n",
            "2019-12-20 10:19:15,059 epoch 88 - iter 30/154 - loss 0.08845171 - samples/sec: 917.44\n",
            "2019-12-20 10:19:15,590 epoch 88 - iter 45/154 - loss 0.09172907 - samples/sec: 928.44\n",
            "2019-12-20 10:19:16,163 epoch 88 - iter 60/154 - loss 0.08173922 - samples/sec: 861.31\n",
            "2019-12-20 10:19:16,691 epoch 88 - iter 75/154 - loss 0.08420307 - samples/sec: 934.63\n",
            "2019-12-20 10:19:17,243 epoch 88 - iter 90/154 - loss 0.08203811 - samples/sec: 892.12\n",
            "2019-12-20 10:19:17,806 epoch 88 - iter 105/154 - loss 0.08014898 - samples/sec: 875.94\n",
            "2019-12-20 10:19:18,374 epoch 88 - iter 120/154 - loss 0.07959260 - samples/sec: 866.64\n",
            "2019-12-20 10:19:18,899 epoch 88 - iter 135/154 - loss 0.08331074 - samples/sec: 939.54\n",
            "2019-12-20 10:19:19,441 epoch 88 - iter 150/154 - loss 0.08152805 - samples/sec: 909.83\n",
            "2019-12-20 10:19:19,537 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:19,538 EPOCH 88 done: loss 0.0823 - lr 0.0008\n",
            "2019-12-20 10:19:20,053 DEV : loss 0.23548759520053864 - score 0.9394\n",
            "Epoch    87: reducing learning rate of group 0 to 3.9063e-04.\n",
            "2019-12-20 10:19:20,081 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:19:20,082 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:20,123 epoch 89 - iter 0/154 - loss 0.12291306 - samples/sec: 12556.62\n",
            "2019-12-20 10:19:20,667 epoch 89 - iter 15/154 - loss 0.08520904 - samples/sec: 905.66\n",
            "2019-12-20 10:19:21,222 epoch 89 - iter 30/154 - loss 0.07089268 - samples/sec: 887.08\n",
            "2019-12-20 10:19:21,767 epoch 89 - iter 45/154 - loss 0.06873955 - samples/sec: 905.77\n",
            "2019-12-20 10:19:22,311 epoch 89 - iter 60/154 - loss 0.06686918 - samples/sec: 907.86\n",
            "2019-12-20 10:19:22,847 epoch 89 - iter 75/154 - loss 0.06651983 - samples/sec: 921.57\n",
            "2019-12-20 10:19:23,827 epoch 89 - iter 90/154 - loss 0.06736638 - samples/sec: 501.04\n",
            "2019-12-20 10:19:24,345 epoch 89 - iter 105/154 - loss 0.07111918 - samples/sec: 954.77\n",
            "2019-12-20 10:19:24,915 epoch 89 - iter 120/154 - loss 0.07167714 - samples/sec: 864.96\n",
            "2019-12-20 10:19:25,462 epoch 89 - iter 135/154 - loss 0.07231142 - samples/sec: 902.21\n",
            "2019-12-20 10:19:25,997 epoch 89 - iter 150/154 - loss 0.07595448 - samples/sec: 922.79\n",
            "2019-12-20 10:19:26,092 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:26,093 EPOCH 89 done: loss 0.0772 - lr 0.0004\n",
            "2019-12-20 10:19:26,583 DEV : loss 0.2374715358018875 - score 0.9376\n",
            "2019-12-20 10:19:26,608 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:19:26,609 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:26,649 epoch 90 - iter 0/154 - loss 0.01880223 - samples/sec: 12755.91\n",
            "2019-12-20 10:19:27,188 epoch 90 - iter 15/154 - loss 0.07384928 - samples/sec: 913.24\n",
            "2019-12-20 10:19:27,722 epoch 90 - iter 30/154 - loss 0.06827967 - samples/sec: 923.51\n",
            "2019-12-20 10:19:28,291 epoch 90 - iter 45/154 - loss 0.07857043 - samples/sec: 867.56\n",
            "2019-12-20 10:19:28,825 epoch 90 - iter 60/154 - loss 0.07967299 - samples/sec: 924.50\n",
            "2019-12-20 10:19:29,364 epoch 90 - iter 75/154 - loss 0.07660739 - samples/sec: 913.68\n",
            "2019-12-20 10:19:29,887 epoch 90 - iter 90/154 - loss 0.07263841 - samples/sec: 943.95\n",
            "2019-12-20 10:19:30,418 epoch 90 - iter 105/154 - loss 0.07391580 - samples/sec: 927.69\n",
            "2019-12-20 10:19:30,939 epoch 90 - iter 120/154 - loss 0.07080824 - samples/sec: 945.39\n",
            "2019-12-20 10:19:31,480 epoch 90 - iter 135/154 - loss 0.07074423 - samples/sec: 911.07\n",
            "2019-12-20 10:19:32,010 epoch 90 - iter 150/154 - loss 0.07141415 - samples/sec: 930.00\n",
            "2019-12-20 10:19:32,107 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:32,107 EPOCH 90 done: loss 0.0715 - lr 0.0004\n",
            "2019-12-20 10:19:32,582 DEV : loss 0.2379896193742752 - score 0.9376\n",
            "2019-12-20 10:19:32,607 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:19:32,608 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:32,646 epoch 91 - iter 0/154 - loss 0.08953148 - samples/sec: 13083.18\n",
            "2019-12-20 10:19:33,166 epoch 91 - iter 15/154 - loss 0.10642403 - samples/sec: 947.83\n",
            "2019-12-20 10:19:33,710 epoch 91 - iter 30/154 - loss 0.08251287 - samples/sec: 909.00\n",
            "2019-12-20 10:19:34,241 epoch 91 - iter 45/154 - loss 0.08081163 - samples/sec: 927.55\n",
            "2019-12-20 10:19:34,779 epoch 91 - iter 60/154 - loss 0.07647137 - samples/sec: 915.62\n",
            "2019-12-20 10:19:35,301 epoch 91 - iter 75/154 - loss 0.07336795 - samples/sec: 943.19\n",
            "2019-12-20 10:19:35,837 epoch 91 - iter 90/154 - loss 0.07404576 - samples/sec: 921.13\n",
            "2019-12-20 10:19:36,371 epoch 91 - iter 105/154 - loss 0.07171259 - samples/sec: 926.56\n",
            "2019-12-20 10:19:36,912 epoch 91 - iter 120/154 - loss 0.07113345 - samples/sec: 919.39\n",
            "2019-12-20 10:19:37,452 epoch 91 - iter 135/154 - loss 0.07456264 - samples/sec: 913.36\n",
            "2019-12-20 10:19:37,992 epoch 91 - iter 150/154 - loss 0.07706444 - samples/sec: 913.70\n",
            "2019-12-20 10:19:38,095 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:38,096 EPOCH 91 done: loss 0.0773 - lr 0.0004\n",
            "2019-12-20 10:19:38,598 DEV : loss 0.23926502466201782 - score 0.9394\n",
            "2019-12-20 10:19:38,623 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:19:38,624 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:38,660 epoch 92 - iter 0/154 - loss 0.05430795 - samples/sec: 13955.53\n",
            "2019-12-20 10:19:39,212 epoch 92 - iter 15/154 - loss 0.05672248 - samples/sec: 889.57\n",
            "2019-12-20 10:19:39,747 epoch 92 - iter 30/154 - loss 0.06604706 - samples/sec: 922.96\n",
            "2019-12-20 10:19:40,309 epoch 92 - iter 45/154 - loss 0.07137642 - samples/sec: 876.17\n",
            "2019-12-20 10:19:40,847 epoch 92 - iter 60/154 - loss 0.08051487 - samples/sec: 916.57\n",
            "2019-12-20 10:19:41,386 epoch 92 - iter 75/154 - loss 0.08266895 - samples/sec: 916.42\n",
            "2019-12-20 10:19:41,939 epoch 92 - iter 90/154 - loss 0.08377563 - samples/sec: 890.59\n",
            "2019-12-20 10:19:42,477 epoch 92 - iter 105/154 - loss 0.08465390 - samples/sec: 915.42\n",
            "2019-12-20 10:19:43,034 epoch 92 - iter 120/154 - loss 0.08528843 - samples/sec: 885.45\n",
            "2019-12-20 10:19:43,571 epoch 92 - iter 135/154 - loss 0.08591167 - samples/sec: 917.43\n",
            "2019-12-20 10:19:44,118 epoch 92 - iter 150/154 - loss 0.08533547 - samples/sec: 900.06\n",
            "2019-12-20 10:19:44,214 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:44,215 EPOCH 92 done: loss 0.0864 - lr 0.0004\n",
            "2019-12-20 10:19:44,703 DEV : loss 0.23829899728298187 - score 0.9376\n",
            "2019-12-20 10:19:44,727 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:19:44,728 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:44,764 epoch 93 - iter 0/154 - loss 0.01955143 - samples/sec: 13711.08\n",
            "2019-12-20 10:19:45,300 epoch 93 - iter 15/154 - loss 0.06644669 - samples/sec: 921.42\n",
            "2019-12-20 10:19:45,872 epoch 93 - iter 30/154 - loss 0.06474803 - samples/sec: 860.47\n",
            "2019-12-20 10:19:46,387 epoch 93 - iter 45/154 - loss 0.07084445 - samples/sec: 958.17\n",
            "2019-12-20 10:19:46,927 epoch 93 - iter 60/154 - loss 0.07272215 - samples/sec: 911.88\n",
            "2019-12-20 10:19:47,482 epoch 93 - iter 75/154 - loss 0.07415967 - samples/sec: 886.78\n",
            "2019-12-20 10:19:48,038 epoch 93 - iter 90/154 - loss 0.08204715 - samples/sec: 884.88\n",
            "2019-12-20 10:19:48,570 epoch 93 - iter 105/154 - loss 0.08284821 - samples/sec: 927.23\n",
            "2019-12-20 10:19:49,104 epoch 93 - iter 120/154 - loss 0.08256519 - samples/sec: 923.21\n",
            "2019-12-20 10:19:49,666 epoch 93 - iter 135/154 - loss 0.07948622 - samples/sec: 874.58\n",
            "2019-12-20 10:19:50,255 epoch 93 - iter 150/154 - loss 0.07890638 - samples/sec: 841.54\n",
            "2019-12-20 10:19:50,350 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:50,351 EPOCH 93 done: loss 0.0781 - lr 0.0004\n",
            "2019-12-20 10:19:50,855 DEV : loss 0.23807889223098755 - score 0.9376\n",
            "2019-12-20 10:19:50,880 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:19:50,881 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:50,923 epoch 94 - iter 0/154 - loss 0.15946160 - samples/sec: 11833.27\n",
            "2019-12-20 10:19:51,496 epoch 94 - iter 15/154 - loss 0.07983370 - samples/sec: 859.22\n",
            "2019-12-20 10:19:52,041 epoch 94 - iter 30/154 - loss 0.09465089 - samples/sec: 904.63\n",
            "2019-12-20 10:19:52,557 epoch 94 - iter 45/154 - loss 0.09248325 - samples/sec: 959.25\n",
            "2019-12-20 10:19:53,108 epoch 94 - iter 60/154 - loss 0.09274236 - samples/sec: 892.64\n",
            "2019-12-20 10:19:53,665 epoch 94 - iter 75/154 - loss 0.08780793 - samples/sec: 891.64\n",
            "2019-12-20 10:19:54,203 epoch 94 - iter 90/154 - loss 0.08693469 - samples/sec: 921.47\n",
            "2019-12-20 10:19:54,747 epoch 94 - iter 105/154 - loss 0.08082981 - samples/sec: 911.13\n",
            "2019-12-20 10:19:55,289 epoch 94 - iter 120/154 - loss 0.08345003 - samples/sec: 921.11\n",
            "2019-12-20 10:19:55,830 epoch 94 - iter 135/154 - loss 0.08278786 - samples/sec: 912.90\n",
            "2019-12-20 10:19:56,346 epoch 94 - iter 150/154 - loss 0.08314793 - samples/sec: 960.48\n",
            "2019-12-20 10:19:56,452 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:56,453 EPOCH 94 done: loss 0.0835 - lr 0.0004\n",
            "2019-12-20 10:19:56,926 DEV : loss 0.23733340203762054 - score 0.9376\n",
            "Epoch    93: reducing learning rate of group 0 to 1.9531e-04.\n",
            "2019-12-20 10:19:56,952 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:19:56,953 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:19:56,991 epoch 95 - iter 0/154 - loss 0.03369948 - samples/sec: 13143.31\n",
            "2019-12-20 10:19:57,521 epoch 95 - iter 15/154 - loss 0.11171373 - samples/sec: 927.84\n",
            "2019-12-20 10:19:58,071 epoch 95 - iter 30/154 - loss 0.10264089 - samples/sec: 897.20\n",
            "2019-12-20 10:19:58,591 epoch 95 - iter 45/154 - loss 0.10172470 - samples/sec: 947.23\n",
            "2019-12-20 10:19:59,118 epoch 95 - iter 60/154 - loss 0.09568818 - samples/sec: 935.16\n",
            "2019-12-20 10:19:59,643 epoch 95 - iter 75/154 - loss 0.09012333 - samples/sec: 938.65\n",
            "2019-12-20 10:20:00,180 epoch 95 - iter 90/154 - loss 0.08860675 - samples/sec: 916.90\n",
            "2019-12-20 10:20:00,713 epoch 95 - iter 105/154 - loss 0.08665379 - samples/sec: 924.45\n",
            "2019-12-20 10:20:01,246 epoch 95 - iter 120/154 - loss 0.08283314 - samples/sec: 924.61\n",
            "2019-12-20 10:20:01,748 epoch 95 - iter 135/154 - loss 0.08351187 - samples/sec: 984.46\n",
            "2019-12-20 10:20:02,299 epoch 95 - iter 150/154 - loss 0.08294044 - samples/sec: 893.14\n",
            "2019-12-20 10:20:02,401 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:02,402 EPOCH 95 done: loss 0.0838 - lr 0.0002\n",
            "2019-12-20 10:20:02,891 DEV : loss 0.2370092123746872 - score 0.9376\n",
            "2019-12-20 10:20:02,916 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:20:02,917 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:02,963 epoch 96 - iter 0/154 - loss 0.06291617 - samples/sec: 10778.17\n",
            "2019-12-20 10:20:03,518 epoch 96 - iter 15/154 - loss 0.05328797 - samples/sec: 891.51\n",
            "2019-12-20 10:20:04,063 epoch 96 - iter 30/154 - loss 0.08142130 - samples/sec: 903.87\n",
            "2019-12-20 10:20:04,609 epoch 96 - iter 45/154 - loss 0.09394111 - samples/sec: 902.42\n",
            "2019-12-20 10:20:05,147 epoch 96 - iter 60/154 - loss 0.09556797 - samples/sec: 916.78\n",
            "2019-12-20 10:20:05,682 epoch 96 - iter 75/154 - loss 0.09231146 - samples/sec: 920.68\n",
            "2019-12-20 10:20:06,248 epoch 96 - iter 90/154 - loss 0.08805987 - samples/sec: 871.61\n",
            "2019-12-20 10:20:06,786 epoch 96 - iter 105/154 - loss 0.08689711 - samples/sec: 915.67\n",
            "2019-12-20 10:20:07,332 epoch 96 - iter 120/154 - loss 0.08556867 - samples/sec: 899.96\n",
            "2019-12-20 10:20:07,870 epoch 96 - iter 135/154 - loss 0.08259780 - samples/sec: 917.01\n",
            "2019-12-20 10:20:08,401 epoch 96 - iter 150/154 - loss 0.08414354 - samples/sec: 927.82\n",
            "2019-12-20 10:20:08,497 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:08,498 EPOCH 96 done: loss 0.0849 - lr 0.0002\n",
            "2019-12-20 10:20:08,981 DEV : loss 0.23792828619480133 - score 0.9376\n",
            "2019-12-20 10:20:09,004 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:20:09,005 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:09,042 epoch 97 - iter 0/154 - loss 0.12632705 - samples/sec: 13843.16\n",
            "2019-12-20 10:20:09,596 epoch 97 - iter 15/154 - loss 0.09259555 - samples/sec: 887.24\n",
            "2019-12-20 10:20:10,136 epoch 97 - iter 30/154 - loss 0.08073930 - samples/sec: 912.92\n",
            "2019-12-20 10:20:10,691 epoch 97 - iter 45/154 - loss 0.08535268 - samples/sec: 887.75\n",
            "2019-12-20 10:20:11,215 epoch 97 - iter 60/154 - loss 0.08207177 - samples/sec: 944.75\n",
            "2019-12-20 10:20:11,770 epoch 97 - iter 75/154 - loss 0.08198965 - samples/sec: 886.71\n",
            "2019-12-20 10:20:12,313 epoch 97 - iter 90/154 - loss 0.08206608 - samples/sec: 906.11\n",
            "2019-12-20 10:20:12,848 epoch 97 - iter 105/154 - loss 0.08460402 - samples/sec: 924.02\n",
            "2019-12-20 10:20:13,404 epoch 97 - iter 120/154 - loss 0.08631264 - samples/sec: 886.45\n",
            "2019-12-20 10:20:13,937 epoch 97 - iter 135/154 - loss 0.08910056 - samples/sec: 925.85\n",
            "2019-12-20 10:20:14,485 epoch 97 - iter 150/154 - loss 0.08629935 - samples/sec: 900.59\n",
            "2019-12-20 10:20:14,587 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:14,588 EPOCH 97 done: loss 0.0862 - lr 0.0002\n",
            "2019-12-20 10:20:15,071 DEV : loss 0.23779729008674622 - score 0.9376\n",
            "2019-12-20 10:20:15,097 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:20:15,098 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:15,138 epoch 98 - iter 0/154 - loss 0.03719714 - samples/sec: 12453.94\n",
            "2019-12-20 10:20:15,695 epoch 98 - iter 15/154 - loss 0.07607969 - samples/sec: 885.06\n",
            "2019-12-20 10:20:16,246 epoch 98 - iter 30/154 - loss 0.07790615 - samples/sec: 898.41\n",
            "2019-12-20 10:20:16,777 epoch 98 - iter 45/154 - loss 0.07910642 - samples/sec: 926.74\n",
            "2019-12-20 10:20:17,328 epoch 98 - iter 60/154 - loss 0.07379531 - samples/sec: 894.73\n",
            "2019-12-20 10:20:17,883 epoch 98 - iter 75/154 - loss 0.07616809 - samples/sec: 887.07\n",
            "2019-12-20 10:20:18,423 epoch 98 - iter 90/154 - loss 0.07803498 - samples/sec: 911.44\n",
            "2019-12-20 10:20:18,961 epoch 98 - iter 105/154 - loss 0.08177061 - samples/sec: 917.44\n",
            "2019-12-20 10:20:19,501 epoch 98 - iter 120/154 - loss 0.08195902 - samples/sec: 912.43\n",
            "2019-12-20 10:20:20,059 epoch 98 - iter 135/154 - loss 0.08396089 - samples/sec: 885.51\n",
            "2019-12-20 10:20:20,610 epoch 98 - iter 150/154 - loss 0.08288087 - samples/sec: 894.32\n",
            "2019-12-20 10:20:20,714 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:20,715 EPOCH 98 done: loss 0.0834 - lr 0.0002\n",
            "2019-12-20 10:20:21,223 DEV : loss 0.23744773864746094 - score 0.9376\n",
            "2019-12-20 10:20:21,247 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:20:21,248 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:21,291 epoch 99 - iter 0/154 - loss 0.22347122 - samples/sec: 11612.14\n",
            "2019-12-20 10:20:21,825 epoch 99 - iter 15/154 - loss 0.07445903 - samples/sec: 921.59\n",
            "2019-12-20 10:20:22,347 epoch 99 - iter 30/154 - loss 0.08882432 - samples/sec: 944.48\n",
            "2019-12-20 10:20:22,878 epoch 99 - iter 45/154 - loss 0.10258460 - samples/sec: 932.84\n",
            "2019-12-20 10:20:23,391 epoch 99 - iter 60/154 - loss 0.09042992 - samples/sec: 962.73\n",
            "2019-12-20 10:20:23,914 epoch 99 - iter 75/154 - loss 0.08940111 - samples/sec: 942.84\n",
            "2019-12-20 10:20:24,452 epoch 99 - iter 90/154 - loss 0.08797207 - samples/sec: 913.39\n",
            "2019-12-20 10:20:24,989 epoch 99 - iter 105/154 - loss 0.08779540 - samples/sec: 919.05\n",
            "2019-12-20 10:20:25,519 epoch 99 - iter 120/154 - loss 0.08667066 - samples/sec: 929.87\n",
            "2019-12-20 10:20:26,063 epoch 99 - iter 135/154 - loss 0.08833293 - samples/sec: 905.84\n",
            "2019-12-20 10:20:26,601 epoch 99 - iter 150/154 - loss 0.08923411 - samples/sec: 916.33\n",
            "2019-12-20 10:20:26,696 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:26,697 EPOCH 99 done: loss 0.0901 - lr 0.0002\n",
            "2019-12-20 10:20:27,173 DEV : loss 0.23628942668437958 - score 0.9376\n",
            "2019-12-20 10:20:27,196 BAD EPOCHS (no improvement): 5\n",
            "2019-12-20 10:20:27,197 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:27,231 epoch 100 - iter 0/154 - loss 0.00552817 - samples/sec: 14707.07\n",
            "2019-12-20 10:20:27,772 epoch 100 - iter 15/154 - loss 0.07286392 - samples/sec: 909.70\n",
            "2019-12-20 10:20:28,315 epoch 100 - iter 30/154 - loss 0.09167686 - samples/sec: 905.89\n",
            "2019-12-20 10:20:28,867 epoch 100 - iter 45/154 - loss 0.08149050 - samples/sec: 894.02\n",
            "2019-12-20 10:20:29,388 epoch 100 - iter 60/154 - loss 0.08338354 - samples/sec: 946.60\n",
            "2019-12-20 10:20:29,938 epoch 100 - iter 75/154 - loss 0.08771447 - samples/sec: 896.09\n",
            "2019-12-20 10:20:30,463 epoch 100 - iter 90/154 - loss 0.08421754 - samples/sec: 940.09\n",
            "2019-12-20 10:20:31,010 epoch 100 - iter 105/154 - loss 0.08250424 - samples/sec: 900.32\n",
            "2019-12-20 10:20:31,554 epoch 100 - iter 120/154 - loss 0.08182632 - samples/sec: 905.98\n",
            "2019-12-20 10:20:32,122 epoch 100 - iter 135/154 - loss 0.08041715 - samples/sec: 873.63\n",
            "2019-12-20 10:20:32,671 epoch 100 - iter 150/154 - loss 0.08123797 - samples/sec: 899.46\n",
            "2019-12-20 10:20:32,768 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:32,769 EPOCH 100 done: loss 0.0803 - lr 0.0002\n",
            "2019-12-20 10:20:33,252 DEV : loss 0.2367672622203827 - score 0.9376\n",
            "Epoch    99: reducing learning rate of group 0 to 9.7656e-05.\n",
            "2019-12-20 10:20:33,278 BAD EPOCHS (no improvement): 6\n",
            "2019-12-20 10:20:33,279 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:33,279 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:33,280 learning rate too small - quitting training!\n",
            "2019-12-20 10:20:33,281 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:36,264 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:20:36,265 Testing using best model ...\n",
            "2019-12-20 10:20:36,266 loading file resources/tagger/ag_news/best-model.pt\n",
            "2019-12-20 10:20:39,200 0.958\t0.958\t0.958\n",
            "2019-12-20 10:20:39,201 \n",
            "MICRO_AVG: acc 0.9194 - f1-score 0.958\n",
            "MACRO_AVG: acc 0.9141 - f1-score 0.9544\n",
            "ABBR       tp: 8 - fp: 0 - fn: 1 - tn: 491 - precision: 1.0000 - recall: 0.8889 - accuracy: 0.8889 - f1-score: 0.9412\n",
            "DESC       tp: 137 - fp: 9 - fn: 1 - tn: 353 - precision: 0.9384 - recall: 0.9928 - accuracy: 0.9320 - f1-score: 0.9648\n",
            "ENTY       tp: 79 - fp: 3 - fn: 15 - tn: 403 - precision: 0.9634 - recall: 0.8404 - accuracy: 0.8144 - f1-score: 0.8977\n",
            "HUM        tp: 64 - fp: 2 - fn: 1 - tn: 433 - precision: 0.9697 - recall: 0.9846 - accuracy: 0.9552 - f1-score: 0.9771\n",
            "LOC        tp: 78 - fp: 3 - fn: 3 - tn: 416 - precision: 0.9630 - recall: 0.9630 - accuracy: 0.9286 - f1-score: 0.9630\n",
            "NUM        tp: 113 - fp: 4 - fn: 0 - tn: 383 - precision: 0.9658 - recall: 1.0000 - accuracy: 0.9658 - f1-score: 0.9826\n",
            "2019-12-20 10:20:39,206 ----------------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'test_score': 0.958,\n",
              " 'dev_score_history': [0.3358,\n",
              "  0.4844,\n",
              "  0.556,\n",
              "  0.5303,\n",
              "  0.6165,\n",
              "  0.7798,\n",
              "  0.5872,\n",
              "  0.7853,\n",
              "  0.8165,\n",
              "  0.7064,\n",
              "  0.8514,\n",
              "  0.8495,\n",
              "  0.8147,\n",
              "  0.7321,\n",
              "  0.8752,\n",
              "  0.633,\n",
              "  0.8972,\n",
              "  0.8587,\n",
              "  0.8862,\n",
              "  0.7798,\n",
              "  0.9064,\n",
              "  0.9101,\n",
              "  0.8514,\n",
              "  0.855,\n",
              "  0.9064,\n",
              "  0.8642,\n",
              "  0.8862,\n",
              "  0.9009,\n",
              "  0.9156,\n",
              "  0.9156,\n",
              "  0.9174,\n",
              "  0.8972,\n",
              "  0.9174,\n",
              "  0.9193,\n",
              "  0.9211,\n",
              "  0.9248,\n",
              "  0.9321,\n",
              "  0.9211,\n",
              "  0.9193,\n",
              "  0.9138,\n",
              "  0.9193,\n",
              "  0.9303,\n",
              "  0.9284,\n",
              "  0.9321,\n",
              "  0.9303,\n",
              "  0.9284,\n",
              "  0.9321,\n",
              "  0.9358,\n",
              "  0.9229,\n",
              "  0.9266,\n",
              "  0.9376,\n",
              "  0.9358,\n",
              "  0.9321,\n",
              "  0.9339,\n",
              "  0.9303,\n",
              "  0.9193,\n",
              "  0.9284,\n",
              "  0.9431,\n",
              "  0.9358,\n",
              "  0.9321,\n",
              "  0.9394,\n",
              "  0.9339,\n",
              "  0.9321,\n",
              "  0.9376,\n",
              "  0.9358,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9413,\n",
              "  0.9376,\n",
              "  0.9358,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9303,\n",
              "  0.9339,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9358,\n",
              "  0.9358,\n",
              "  0.9358,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9358,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9394,\n",
              "  0.9394,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9394,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9376,\n",
              "  0.9376],\n",
              " 'train_loss_history': [1.529336421520679,\n",
              "  1.2986358524142922,\n",
              "  1.1402394945745344,\n",
              "  1.0242177035127367,\n",
              "  0.9178591844710436,\n",
              "  0.8230080120749288,\n",
              "  0.7389780701948451,\n",
              "  0.6842842260738472,\n",
              "  0.616105574679065,\n",
              "  0.5705988012350999,\n",
              "  0.5181228120799188,\n",
              "  0.4935586169942633,\n",
              "  0.4308828277053771,\n",
              "  0.4225116881070199,\n",
              "  0.3930650787403831,\n",
              "  0.3773096896611251,\n",
              "  0.368323700768607,\n",
              "  0.3426200305776937,\n",
              "  0.33166041118758066,\n",
              "  0.31108370133034596,\n",
              "  0.3034632464072534,\n",
              "  0.2797122636398712,\n",
              "  0.26997148240735,\n",
              "  0.2648399601412284,\n",
              "  0.2544774303769136,\n",
              "  0.24733825787514835,\n",
              "  0.2354767584278212,\n",
              "  0.22853828739229734,\n",
              "  0.19404230049097693,\n",
              "  0.18660498168561365,\n",
              "  0.18648051368919286,\n",
              "  0.1819825131300982,\n",
              "  0.16569361211611078,\n",
              "  0.16355806453661484,\n",
              "  0.16981016520362396,\n",
              "  0.1681046204326989,\n",
              "  0.146975345111319,\n",
              "  0.16912824138031377,\n",
              "  0.14740240554530898,\n",
              "  0.15582076716539148,\n",
              "  0.13309393561892696,\n",
              "  0.14037632419691456,\n",
              "  0.13273066887027257,\n",
              "  0.1272151455186404,\n",
              "  0.13125139037026212,\n",
              "  0.121991343893014,\n",
              "  0.13165833288198941,\n",
              "  0.11336869566834398,\n",
              "  0.11920273906321495,\n",
              "  0.10873411873706265,\n",
              "  0.11981131368643277,\n",
              "  0.10858312921671125,\n",
              "  0.10644400817063915,\n",
              "  0.10822197015989911,\n",
              "  0.10399226967680764,\n",
              "  0.09806379424287127,\n",
              "  0.11147741000954207,\n",
              "  0.09394257128625721,\n",
              "  0.09770768232546843,\n",
              "  0.0963901425530384,\n",
              "  0.09604932675017165,\n",
              "  0.09579566636050192,\n",
              "  0.09339103091798433,\n",
              "  0.0828602502678896,\n",
              "  0.09293070280706728,\n",
              "  0.09165775901698447,\n",
              "  0.08573368956129272,\n",
              "  0.0801754604462631,\n",
              "  0.09406399663973164,\n",
              "  0.08331820070017855,\n",
              "  0.08152716769534465,\n",
              "  0.09322358283226366,\n",
              "  0.08056135289522202,\n",
              "  0.08910643464172041,\n",
              "  0.08849559434048541,\n",
              "  0.08948222634854255,\n",
              "  0.07953302119556185,\n",
              "  0.08007696986972511,\n",
              "  0.08712467742333939,\n",
              "  0.0787510893900286,\n",
              "  0.07999459495647,\n",
              "  0.08351081014647112,\n",
              "  0.08228589885856037,\n",
              "  0.07483496205494194,\n",
              "  0.0840500079143744,\n",
              "  0.0829501019597247,\n",
              "  0.0880471850207148,\n",
              "  0.0822684101083062,\n",
              "  0.07719791905543247,\n",
              "  0.07153493994484086,\n",
              "  0.07732389794735166,\n",
              "  0.086392953629037,\n",
              "  0.07809114980952839,\n",
              "  0.08351764102260788,\n",
              "  0.08376598004977424,\n",
              "  0.08493878842367754,\n",
              "  0.08624270688984301,\n",
              "  0.08338373464036297,\n",
              "  0.09007016898362667,\n",
              "  0.08028428653230915],\n",
              " 'dev_loss_history': [tensor(1.6776, device='cuda:0'),\n",
              "  tensor(1.2603, device='cuda:0'),\n",
              "  tensor(1.1164, device='cuda:0'),\n",
              "  tensor(1.2753, device='cuda:0'),\n",
              "  tensor(0.9928, device='cuda:0'),\n",
              "  tensor(0.6307, device='cuda:0'),\n",
              "  tensor(1.2765, device='cuda:0'),\n",
              "  tensor(0.6106, device='cuda:0'),\n",
              "  tensor(0.5224, device='cuda:0'),\n",
              "  tensor(0.8337, device='cuda:0'),\n",
              "  tensor(0.3884, device='cuda:0'),\n",
              "  tensor(0.3879, device='cuda:0'),\n",
              "  tensor(0.5280, device='cuda:0'),\n",
              "  tensor(0.8002, device='cuda:0'),\n",
              "  tensor(0.3157, device='cuda:0'),\n",
              "  tensor(1.3125, device='cuda:0'),\n",
              "  tensor(0.3236, device='cuda:0'),\n",
              "  tensor(0.4326, device='cuda:0'),\n",
              "  tensor(0.3644, device='cuda:0'),\n",
              "  tensor(0.5997, device='cuda:0'),\n",
              "  tensor(0.2959, device='cuda:0'),\n",
              "  tensor(0.2638, device='cuda:0'),\n",
              "  tensor(0.5139, device='cuda:0'),\n",
              "  tensor(0.4495, device='cuda:0'),\n",
              "  tensor(0.2454, device='cuda:0'),\n",
              "  tensor(0.4234, device='cuda:0'),\n",
              "  tensor(0.3273, device='cuda:0'),\n",
              "  tensor(0.3239, device='cuda:0'),\n",
              "  tensor(0.2509, device='cuda:0'),\n",
              "  tensor(0.2397, device='cuda:0'),\n",
              "  tensor(0.2381, device='cuda:0'),\n",
              "  tensor(0.3204, device='cuda:0'),\n",
              "  tensor(0.2584, device='cuda:0'),\n",
              "  tensor(0.2547, device='cuda:0'),\n",
              "  tensor(0.2522, device='cuda:0'),\n",
              "  tensor(0.2535, device='cuda:0'),\n",
              "  tensor(0.2579, device='cuda:0'),\n",
              "  tensor(0.2531, device='cuda:0'),\n",
              "  tensor(0.2594, device='cuda:0'),\n",
              "  tensor(0.2502, device='cuda:0'),\n",
              "  tensor(0.2258, device='cuda:0'),\n",
              "  tensor(0.2312, device='cuda:0'),\n",
              "  tensor(0.2615, device='cuda:0'),\n",
              "  tensor(0.2465, device='cuda:0'),\n",
              "  tensor(0.2254, device='cuda:0'),\n",
              "  tensor(0.2286, device='cuda:0'),\n",
              "  tensor(0.2208, device='cuda:0'),\n",
              "  tensor(0.2230, device='cuda:0'),\n",
              "  tensor(0.2335, device='cuda:0'),\n",
              "  tensor(0.2561, device='cuda:0'),\n",
              "  tensor(0.2363, device='cuda:0'),\n",
              "  tensor(0.2549, device='cuda:0'),\n",
              "  tensor(0.2343, device='cuda:0'),\n",
              "  tensor(0.2195, device='cuda:0'),\n",
              "  tensor(0.2321, device='cuda:0'),\n",
              "  tensor(0.2876, device='cuda:0'),\n",
              "  tensor(0.2311, device='cuda:0'),\n",
              "  tensor(0.2490, device='cuda:0'),\n",
              "  tensor(0.2385, device='cuda:0'),\n",
              "  tensor(0.2544, device='cuda:0'),\n",
              "  tensor(0.2327, device='cuda:0'),\n",
              "  tensor(0.2344, device='cuda:0'),\n",
              "  tensor(0.2359, device='cuda:0'),\n",
              "  tensor(0.2464, device='cuda:0'),\n",
              "  tensor(0.2342, device='cuda:0'),\n",
              "  tensor(0.2289, device='cuda:0'),\n",
              "  tensor(0.2249, device='cuda:0'),\n",
              "  tensor(0.2372, device='cuda:0'),\n",
              "  tensor(0.2318, device='cuda:0'),\n",
              "  tensor(0.2363, device='cuda:0'),\n",
              "  tensor(0.2369, device='cuda:0'),\n",
              "  tensor(0.2337, device='cuda:0'),\n",
              "  tensor(0.2385, device='cuda:0'),\n",
              "  tensor(0.2427, device='cuda:0'),\n",
              "  tensor(0.2415, device='cuda:0'),\n",
              "  tensor(0.2387, device='cuda:0'),\n",
              "  tensor(0.2356, device='cuda:0'),\n",
              "  tensor(0.2338, device='cuda:0'),\n",
              "  tensor(0.2377, device='cuda:0'),\n",
              "  tensor(0.2365, device='cuda:0'),\n",
              "  tensor(0.2403, device='cuda:0'),\n",
              "  tensor(0.2395, device='cuda:0'),\n",
              "  tensor(0.2368, device='cuda:0'),\n",
              "  tensor(0.2377, device='cuda:0'),\n",
              "  tensor(0.2371, device='cuda:0'),\n",
              "  tensor(0.2363, device='cuda:0'),\n",
              "  tensor(0.2367, device='cuda:0'),\n",
              "  tensor(0.2355, device='cuda:0'),\n",
              "  tensor(0.2375, device='cuda:0'),\n",
              "  tensor(0.2380, device='cuda:0'),\n",
              "  tensor(0.2393, device='cuda:0'),\n",
              "  tensor(0.2383, device='cuda:0'),\n",
              "  tensor(0.2381, device='cuda:0'),\n",
              "  tensor(0.2373, device='cuda:0'),\n",
              "  tensor(0.2370, device='cuda:0'),\n",
              "  tensor(0.2379, device='cuda:0'),\n",
              "  tensor(0.2378, device='cuda:0'),\n",
              "  tensor(0.2374, device='cuda:0'),\n",
              "  tensor(0.2363, device='cuda:0'),\n",
              "  tensor(0.2368, device='cuda:0')]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qt6wbsmxyQnH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 821
        },
        "outputId": "6e130daa-cbf7-4bc7-a03f-fc3714be8adc"
      },
      "source": [
        "# 8. plot weight traces (optional)\n",
        "\n",
        "from flair.visual.training_curves import Plotter\n",
        "\n",
        "plotter = Plotter()\n",
        "plotter.plot_weights('resources/tagger/ag_news/weights.txt')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Weights plots are saved in resources/tagger/ag_news/weights.png\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiUAAAMSCAYAAAC1ZbRKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzde5RcdZ3v/fe3012dTl/S6XSHALk0\nMUASiAYIIRdGbg9HYTToMDxRvOA/KBqPOvEsHW/ncI5w0HGpYXlBj86jshwXoM/MiEGZQREY50mI\naS4SokAQEmJudGKnQy6dTvr7/PH7FRRlddLV1O69u/rzWqtXUt17//b3d9nf/u7a1VXm7oiIiIik\nrSbtAERERERARYmIiIhkhIoSERERyQQVJSIiIpIJKkpEREQkE1SUiIiISCaoKBEREZFMGLGixMwu\nNrO3JXyM95nZgiT3NbPVpR6b2YrhHHeIsXWa2ceGsN2r+mBmN5pZ63Bjy7dXav/icSiz3RPuW9zn\n/Poxs+Vm1lDGsXJxHIrnbdhrpRxDnbshtlVqHorHaUT6lUXKMcOnHDN6c0yJeJaa2fQS319d8P+S\n852fh8HGL7+fmZ1rZjeZ2VfMrLGS8ddWsrFSzOxyYBkwGVhrZh8HTgE+C3wEOABsBC4GVsd/e4Ab\ngAeARcA64EJ3f6+ZfQI4ArQC3we+AKwF9gGnAzPMrNvdtxXEcDOwi9DfR4Br43G7gXrg8bjptWZ2\nDXA3cCowB2gBvgisAv4EzDczA74K/B7ojPsuiQt5OtAGfBdYGPu9KMb5TmALsNbdH42xvR9oBjqA\nzwG/BH4KnAE8AUwBvgdcamYOHAQeBK6P4/Tr2O/J8XiPmdkqoC+O++oY28NFY/UI8J44Dj1xTOuA\nze5+T9E0LjGzNbHPjwI/B+aZ2VvcfU3BOC8DrgAc+AFwU6k5BOaY2Qfi2N0I3BLHZT+wIca1H+gt\nWj+/As4FHjKzO4GfAHOBzwNfIqyjxe5+XT4mdz8C/EXCKIj5jkFizP+8eO1cDewBfhPH+BJgd+zv\n/wa2EdbJpqJ+FI/NzYR5vs/dXzSzvwe+A/wbcBnw98Caon2WmNldvLL2rgQ+Ciw1s6Z47JLnQDVT\njlGOKZ5DRleOeRD4ZjzODuA84DPArcA9wDJ3/1BB/zsIeede4I3Al+P224DNwCTgYFxnfcDbgb8F\nTjOzlcBM4D5gkZktdPcNhfMA3Am8PhYtDe5+S4luvRP4FLAUuBz411J9H46ReKZkOSEB3wPc7O5f\nJiyKs4Gp7v51d3+AsMgsfgHscfdvAYcJyfp5M2shJOweYDyQAx5291uBBcAzwN1FyaIdOAvoJUwW\nhAn5GuEkuZGwICFM8k3AVfFrOyGpnAZ0xHieIZzE+9z928DOov6uAW4nLKyl7v4VYH382QagiZAg\n8t5E+CV3EDgJ2Bj3cXf/BiExAjwW+zkvjumuGN9sYEnc56G47Rlx366i2ArH6grgNkJyJG5bB7RT\ngrsfIPyinQwcAzYVJotoBfBHwok1m8HncHccOwPeQEjK+wgn2xWEE/Su2Gbh+in0nLvfDjQA84Eu\nd7+NV9bPUA0W42Br535CslgKvN3d/1fcfz6wPq7vRSX6UTw2u9z9R+7+Yvz5vwN/B9wB/DfgdyX2\ngbD2euP47YvfW+fuNwHnUOIcGAOUY5RjRmWOiR539zvj/+8CfkEoUF50938irKG8dYTCYTth/gcI\nRXYfYY6nFWx7ZpyjTfHxlvi4nrDG1hcVJIU2uvtqYOpx+pXI28GPRFGyBvgg4UT/dLyKWUioOHeY\n2Uozu4hQXa8kDDBAf/z3iIf3wh+I8d4HNALPEiarcGCeBVaY2Yz8N9y9m5DgG4AnC9oeAPrdPd8u\nwJsJV1c/I1xJtBFO5K0x1msIJ8JuoCE+Lp40j181hKu2VcAFwFFgYjz2XDO72symEhZgM2FB7S7o\nd19BewALzOyjwB/imLYQkubvgHVmdh3hFyXAU2b2DkJiKI4t717CvFwVY2sDDhGuMOaY2WWFO5rZ\nJEKiaI59djO7uqj9HxeMx9MMPodT4lWMxfi7Y182xrjeBbw17lu4fgbryxPAQjO7Icb4Kmb2YeAc\nM1tWom8lYzSzlYOsncL5/Rcz++9mdn2M4fy4vteX6Efx2AzE2NriXD1KeBbmNkKS/PcS+0BYIxML\nrn7JtxXj+otzYAxQjlGOGZU5Jn5/oGDbwrl91S/9mJOOEQrLrjgOvyU8qwWvzFXe02b2IULBfJRX\n54luwrMhi4v7UhTTYIXHHYRi+yrCM28VY/rsm+SY2WzC02xnAZ90974T7DJizGwC8G7C05s/dPdN\nx98j2yzcCz0F2OHud6QdT5LM7ErClTXxykfGKOWYkTPacoyZnQOcD8x290+kHc9QVWVRYmaLCE9B\nQngKcP3xtpfhMbMc4d553o/i/dVUmdlywlUZwP3uvjXNeKT6KMeMDOWYZMVn/C6ND/e6+90lthnR\nvlZlUSIiIiKjj96nRERERDJBRYmIiIhkQlnvU9Le3u6dnZ0JhSIiaerq6up2946041CeEalux8s1\nZRUlnZ2dbNgw2J81FzhyEHb/vpymRSQJHWdAffOJtwPMbEvC0QzJkPMMsGfPHg4dOpRwRCJyPPX1\n9XR0DP165ni5Jpl3dK0ZBxMmnXg7EUlWTeJv2pyqXC5HePNTEUlLbW3l8kwyGau2HtpmJdK0iEhe\nc/PQngUSkdFBL3QVERGRTFBRIiIiIpmgokREREQyQUWJiIiIZIKKEhEREckEFSUiIiKSCSpKRERE\nJBMSeZ+SQ4cPs2XrziSaFpEyTD/1JBobG9IOQ0RkSBIpSrr37OHurqG9TbSIJOftdedx+mmnpR2G\niMiQJFKUTDvpZFa9+S1JNC0iZRjXlEs7BBGRIUvm9s3BQ/zhsSeTaFpEynD6G86kuW1i2mGIiAxJ\nIkXJhJZGzr1kURJNi4i8bNtvfsr+nS+kHYbImNbYfhIzLr6mIm1V90eIikhVm3b+pXCsP+0wRMa2\nCn4auYoSERm96vUpwSLVRO9TIiIiIpmgokREREQyQUWJiIiIZEIirynp7+9lz54HkmhaRMrQ1raM\nXG5y2mEk5rfP72V7z6G0wxAZ0zqa6lk6u70ibSVSlNTVtTB16vIkmhYRedn5nW1phyAiFaS/vhGR\nUWv75h5e+vPhtMMQGdMmNOeYNqcyFwgqSkRk1JrQvpmapu1phyEypuVy7cDSirSlokRERq3W1oVp\nhyAiFaS/vhEREZFMUFEiIiIimaDbNyIyavU9v49jPX1phyEyptU05Rg/u7UibakoEZFRq75zYtoh\niEgFqSgRkVHrYFcX/Tt2ph2GyJhW2z6ZxsWLK9NWRVoREUnBhPPOSzsEEakgvdBVREREMkFFiYiI\niGSCihIRERHJBL2mRERGrYd7XmJ7X3/aYYiMaR25Wi6c1FyRtlSUiMiodUFrU9ohiEgF6faNiIiI\nZEIiz5TsO9TPA0/tTqJpESnDhbPbmdxUn3YYIiJDkkhRMrGhjqsWnJpE0yIiIlKldPtGREREMkFF\niYiIiGSCihIRERHJBBUlIiIikgkqSkRERCQTEvnrm8MHXuK5Rzck0bSIlGHm689hQsvEtMNITNeu\nLnYe2Jl2GCJjWntDOxecfEFF2kqkKBnf2MTcCy9OomkRkZedd9J5aYcgIhVk7j70jc1eBLYkF46I\npGimu3ekHYTyjEjVGzTXlFWUiIiIiCRFL3QVERGRTFBRIiIiIpmgokREREQyQUWJiIiIZIKKEhER\nEckEFSUiIiKSCSpKREREJBNUlIiIiEgmqCgRERGRTFBRIiIiIpmgokREREQyQUWJiIiIZIKKEhER\nEcmExIsSM7vYzN6W8DHeZ2YLktzXzFaXemxmK4Zz3CHG1mlmHxvCdq/qg5ndaGatw40t316p/YvH\nocx2T7hvcZ/z68fMlptZQxnHysVxKJ63Ya+V18LMlprZ9BLfX13w/5LzPdjaK7Fdp5l9zMzONbOb\nzOwrZtZYifhHI+We4VPuGR25p0TMZ5ZzDDNbcby5LhiDK83sM2b2D2ZmlYh9MLVJNWxmlwPLgMnA\nWjP7OHAK8FngI8ABYCNwMbA6/tsD3AA8ACwC1gEXuvt7zewTwBGgFfg+8AVgLbAPOB2YYWbd7r6t\nIIabgV2xn48A18bjdgP1wONx02vN7BrgbuBUYA7QAnwRWAX8CZgfJ+OrwO+BzrjvkrhgpwNtwHeB\nhbHfi2Kc7wS2AGvd/dEY2/uBZqAD+BzwS+CnwBnAE8AU4HvApWbmwEHgQeD6OE6/jv2eHI/3mJmt\nAvriuK+OsT1cNFaPAO+J49ATx7QO2Ozu9xRN4xIzWxP7/Cjwc2Cemb3F3dcUjPMy4ArAgR8AN5Wa\nQ2COmX0gjt2NwC1xXPYDG2Jc+4HeovXzK+Bc4CEzuxP4CTAX+DzwJcI6Wuzu1+VjcvcjwF8khoKY\n7xgkxvzPHwS+GY+zAzgP+AxwK3APsMzdP1TQ/w7gEuBe4I3Al+P224DNwCTgYFxnfcDbgb8FTjOz\nlcBM4D5gkZktdPcNBeFOievlfHe/Hnh9TCIN7n5Lie69E/gUsBS4HPjXUmNQrZR7lHuK55AqzT1R\n4Tw9A7SaWVPc7+QYa34c/9nddxWOM/AwsDTus83dv18i7Mvd/e/M7L3AG4DHSvWtEpJ8pmQ5cDNh\nEG929y8TJv9sYKq7f93dHyAsJotfAHvc/VvAYeA7wPNm1gJcRljI44Ec8LC73wosIEzE3UVJoR04\nC+gl/EKAkPS/RjgZbiQsPAi/SG4Cropf2wnJ4zSgI8bzDOFk3efu3wZ2FvV3DXA7YSEsdfevAOvj\nzzYATYREkPcmYA9hIZ0EbIz7uLt/g5AAAR6L/ZwXx3RXjG82sCTu81Dc9oy4b1dRbIVjdQVwGyEJ\nEretA9opwd0PAJsIJ+gxYFNhUohWAH8knECzGXwOd8exM8LCbiEkq44Y1zeBu2Kbheun0HPufjvQ\nAMwHutz9Nl5ZP0M1WIx5j7v7nfH/dwG/ICSJF939nwhrKG8d4eTeTpj/AcIvuj7CHE8r2PbMOEeb\n4uMt8XE9YY2tLypIALrd/f8Q1gqEtbIamHqc/vnxOl/llHuUe8ZK7oFXz1NeM6H4m180jnWDxLTO\n3W8CzimzLxWXZFGyBvgg4YT+dLxaWUioLHeY2Uozu4hQRa8kJHGA/vjvEXd3QoKvIZzUjcCzhEkp\nTLrPAivMbEb+G+7eDfyOsICeLGh7AOh393y7AG8mXEX9jHDF0EY4YbfGWK8hLPjdQEN8XPwLweNX\nDeHqbBVwAXAUmBiPPdfMrjazqYSF1kxIDrsL+t1X0B7AAjP7KPCHOKYthOT4O2CdmV1HuCIGeMrM\n3kFIAMWx5d1LmJerYmxtwCHClcQcM7uscEczm0RICM2xz25mVxe1/+OC8XiawedwSrxasRh/d+zL\nxhjXu4C3xn0L189gfXkCWGhmN8QYX8XMPgycY2bLSvStZIzxWQvi48Jj5uf2Vb/szWylux8jJPeu\nOA6/JVxZwitzlfe0mX2I8EvraMFxPI7H681scVFXCrcp9bjYHYRffFcRroLHGuUe5Z4xkXvifwvn\nKe9swjqqKzGOpZwop9xnZp8m5K3HB9mmIiyMiVSSmc0mPJV/FvBJd+87wS4jxswmAO8mPI35Q3ff\ndPw9ss3CvedTgB3ufkfa8ZyImZ0DnA/MdvdPpB2PVBflnpEz2nLPaFFVRYmZLeKVp7A2ufv6420v\nw2NmOcI98rwfxfuoqTKz5YSrL4D73X1rmvEMV7zqvjQ+3Ovudx9n26ro82in3DMylHteGzObR3gt\nC8BWd7+/xDbXEm5TQrg1uXek4oMqK0pERERk9NL7lIiIiEgmqCgRERGRTCjrfUra29u9s7MzoVBE\nJE1dXV3d7t6RdhzKMyLV7Xi5pqyipLOzkw0bit9C4S8NDByhr+/FcpoWkQTkcu2MG1c/pG3NbEvC\n4QzJUPMMwJEjezh27HDCEYnI8dSMq6c+V/LtZko6Xq5J5B1d3Y/S17cjiaZFpAx1dS2E92WrTv39\nPfT3/zntMETGtNra5rKKkuO2VZFWiowbN4HW1oVJNC0i8rLGxtelHYKIVFAiRcnAoaMcfmpE/7RZ\nREqon93KuKbciTcUEcmARIqSmoZaJiyYkkTTIiIiUqUSKUqOHhug51D/iTcUkUS1jK8jV6u//BeR\n0SGRoqTv6ABP7dyfRNMiUoazT52ookRERo1EipLGgZdYdvDXSTQtIuUYuIRBPhleRCRzEilKaGiF\n11+TSNMiInm/fX4v23sOpR2GyJjW0VTP0tkZ/pNgEZGR0NIxgaOt+usikTRNrB1XsbZUlIjIqNWR\nq6VFr5kRSVXOKncOqigRkVGrrU4pTKSa6IwWkVFr2x+e5KU93WmHITKmTZg4iRlnv74ibakoEZFR\n69Qz56UdgohUkIoSERm1zCztEESkghIpSnqP9PIf2/4jiaZFpAxLTllC2/i2tMMQERmSRIqSllwL\nfz3rr5NoWkRERKqU/pZOREREMkFFiYiIiGSCihIRERHJBBUlIiIikgkqSkRERCQTEvnrm4GBAfr6\n+pJoWkTKkMvlGDeuch+WJSKSpESKkv7+frZt25ZE0yJShmnTptHQ0JB2GCIiQ5JIUVJfX8/pp5+e\nRNMiIiJSpfSaEhEREckEFSUiIiKSCYncvtnXf5Rf7d2fRNMiUoY3TmqmPafP3RSR0SGRbDWxrpa/\nOWlSEk2LiIhIldLtGxEREckEFSUiIiKSCSpKREREJBNUlIiIiEgm6GX5IjJq9T3zDEf3/jntMETG\ntHETWxg/Z05F2lJRIiKjVm1HBzUtE9MOQ2RMs1xdxdpSUSIio9a2ffvo7e1NOwyRMa2xsZFZkyrz\nNiCJFCV9B/vZsnFPEk2LSBmmz22joTmXdhiJmTrQSscxfeCgSJpqBiqXY5L5QL4JdZyxaGoSTYuI\nvKz+NN26Eakmun0jIqNW164udh7YmXYYImNae0M7F5x8QUXaSqQoGThyhP5t25JoWkTKUHfyydQ0\nVO/tjVMPzKZ13/S0wxAZ0xqyfvuGgQH88OFEmhaRMgwMpB1BojpmNNPuTWmHITKmmVnF2kqkKKkZ\nP57x8+Yl0bSIyMvG1er9H0Wqic5oERERyQQVJSIiIpIJidy+OXzgJZ57rCuJpkWkDDPnL2BCNb/j\n6Za10PuntKMQGdsaO2DWRRVpKpGiZHxjE3OXVSZAEZFBzVySdgQiUkHm7kPf2OxFYEty4YhIima6\ne0faQSjPiFS9QXNNWUWJiIiISFL0QlcRERHJBBUlIiIikgkqSkRERCQTVJSIiIhIJqgoERERkUxQ\nUSIiIiKZoKJEREREMkFFiYiIiGSCihIRERHJBBUlIiIikgkqSkRERCQTVJSIiIhIJqgoERERkUwY\n8aLEzC42s7clfIz3mdmCJPc1s9WlHpvZiuEcd4ixdZrZx4aw3av6YGY3mlnrcGPLt1dq/+JxKLPd\nE+5b3Of8+jGz5WbWUMaxcnEciudt2GuljJjPLOcYhTEeb84LxuJKM/uMmf2Dmdlri350U34ZPuWX\n9PNL0v0dYjsl+16UlxI7z2qTaLQUM7scWAZMBtaa2ceBU4DPAh8BDgAbgYuB1fHfHuAG4AFgEbAO\nuNDd32tmnwCOAK3A94EvAGuBfcDpwAwz63b3bQUx3AzsIvT7EeDaeNxuoB54PG56rZldA9wNnArM\nAVqALwKrgD8B8+MvgK8Cvwc6475L4oROB9qA7wILY78XxTjfCWwB1rr7ozG29wPNQAfwOeCXwE+B\nM4AngCnA94BLzcyBg8CDwPVxnH4d+z05Hu8xM1sF9MVxXx1je7horB4B3hPHoSeOaR2w2d3vKZrG\nJWa2Jvb5UeDnwDwze4u7rykY52XAFYADPwBuKjWHwBwz+0AcuxuBW+K47Ac2xLj2A71F6+dXwLnA\nQ2Z2J/ATYC7weeBLhHW02N2vy8fk7keAv0gaBTHfMUiM+Z8/CHwzHmcHcB7wGeBW4B5gmbt/qKDJ\nwnl6Bmg1s6a438kx1vw4/rO77yrY9zQzWwnMjMdcGvfd5u7fLxH+5e7+d2b2XuANwGOl+ljNlF+U\nX4rnkFGSX4rn2czqi2L9T+B9wDbCWvwfwAvAMeBfgU8Bf4htnVE0Z9fF/X/j7n80s3cBTwL/E/j7\n+PP/p2iffN8/TViLK9z9EuD1sYBrIKzp+Wb2B3f/Q6k+D9dIPlOyHLiZkMBvdvcvExbG2cBUd/+6\nuz9AWGgWvwD2uPu3gMPAd4DnzawFuIwwiOOBHPCwu98KLCD8Eri7KGG0A2cBvcCk+O37gK8RTpQb\nCYsS4F7CQr8qfm0nTMJpQEeM5xnCibzP3b8N7Czq7xrgdsIvoaXu/hVgffzZBqCJkCTy3gTsISSD\nk4CNcR93928QkiPAY7Gf8+KY7orxzQaWxH0eitueEfftKoqtcKyuAG4jJEjitnVAOyW4+wFgE+Hk\nPQZsKkwY0Qrgj4Rf3rMZfA53x7Ezwi/TFkIi64hxfRO4K7ZZuH4KPefutxNOlPlAl7vfxivrZ6gG\nizHvcXe/M/7/LuAXhET1orv/E2ENFSqcp7xmQnKeXzSOdUX7bonzVh8fr3P3m4BzyuzTWKL8ovwy\nWvNL8TzPL4p1OfC1ONaNwJ/d/Wux738F/DPwjwX9KJyzPne/3d3/GH/+b8D/DfwmjuPeEvvkTYkx\nvxAfb3T31cBUwvq8v9IFCYxsUbIG+CDhZP90vJJZSKg6d5jZSjO7iFBhryRcyQD0x3+PuLsDAzHu\n+wgT9CzhpPeCYz0LrDCzGflvuHs38DvC4nqyoO0BoN/d8+0CvJlwhfUzwtVEG+Fk3hpjvYYwebuB\nhvh4alF/PX7VEK7cVgEXAEeBifHYc83sajObSvgl10xIHLsL+t1X0B7AAjP7KKEyXkNYvONj39aZ\n2XXA0rjtU2b2DkJyKI4t717CvFwVY2sDDhGuMuaY2WWFO5rZJEKyaI59djO7uqj9HxeMx9MMPodT\n4pWMxfi7Y182xrjeBbw17lu4fgbryxPAQjO7Icb4Kmb2YeAcM1tWom8lY4zPWBAfFx4zP7eFx6dg\n+8J5yjubsI7qSoxjofyxfJDHxe6LVzVn8crV+Fij/KL8MlrzS/E8byyK9WfAh+NxDwCTzOy/ApuB\n/wD+C6HAyPejcM5ezltmtjKu06XAvxDOj38vsU/ei3H8TomPC/PQC8D/ZWZnFY/Da2VhfCRJZjYb\nuITwS+OT7t53gl1GjJlNAN5NeIrzh+6+Kd2IXhsL96VPAXa4+x1pxyOSNOWXkTOW8ouZ/RXhGbN2\nd795xI5bzUWJmS3ilafPN7n7+uNtL8NjZjnC/fO8H8V7rKkys+WEKzMITzVuTTOewZjZPMK9ZoCt\n7n7/cba9lnA7AcIthL1JxyelKb+MDOWXsaWqixIREREZPfQ+JSIiIpIJKkpEREQkE8p6n5L29nbv\n7OxMKBQRSVNXV1e3u3ekHYfyjEh1O16uKaso6ezsZMOGDSfczgcGOHb0aDlNi0gCxtXWYjVDe0LU\nzLYkHM6QDDXPAAwMHEGvixNJl5lRU5M78YavbD9orknkHV2PHD7E1ifG6tsliGTHtHln09DccuIN\nR6n9+5+kr2932mGIjGm53GRaWxdWpK1EipL6CY2cfsHSE28oIvIaTJyoN7kVqSYj9tk3IiIV170Z\nDv057ShExrbxLdBxZkWaUlEiIqPX+BYYpzQmkqra8ZVrqmItiYiMtKYpaUcgIhWk9ykRERGRTFBR\nIiIiIpmgokREREQyQa8pEZFRa/1ze9mx71DaYYiMaR1N9Syd3V6RtlSUiMiotei0thNvJCKjhm7f\niIiISCaoKBEREZFMUFEiIiIimaCiRERERDJBRYmIiIhkgv76RkRGrY2PbGDPi91phyEypk1snciC\nC5ZUpC0VJSIyas2afTozps9IOwyRMW1cXV3F2lJRIiKj1oSWiWmHICIVpKJEREat7m37OdTbn3YY\nImNafWMtU2a2VKQtFSUiMmo1T26gcWJ92mGIjGk146xibSVSlPQd7GfLxj1JNC0iZZg+t42G5lza\nYSTm2JOP0b9jZ9phiIxptR3t1C9eXJm2KtJKkbpxA8xo7U2iaREpQ25cC1C9RcmEhQvTDkFEKii5\n2zfuiTUtIiIi1SeRouQl6+M/6jYn0bSIlGFJTQdtTEg7DBGRIUmkKGk6NoFLes9PomkRKUP9lKa0\nQxARGbJEipKahlomLJiSRNMiIiJSpfTZNyIiIpIJybzQ9VAPPHNfIk2LSBledwk0tqcdhYjIkCRT\nlDS0wuuvSaRpERERqU66fSMiIiKZoLeZF5FR65Fdj7DzgN7RVSRNkxsmc8HJF1SkLRUlIjJqnXvS\nuWmHICIVlMxn3/T1sXXr1iSaFpEyTJs2jYaGhrTDEBEZkmQ++6aujmnTpiXRtIiUIZer3s+9EZHq\nk8ybp9XU6OpMRBK3rucltvf1px2GyJg2JVfLhZOaK9KWXlMiIqPW4la9jb5INUmkKOnv72XPngeS\naFpEytDWtoxcbnLaYYiIDElCrylpYerU5Uk0LSIiIlUqmb++OXqMHT2Hk2haRMowdeJ4xteNSzsM\nEZEhSaQoGRiAfYf04jORtLU316cdgojIkCVSlDTkxvGG6a1JNC0iIiJVSp99IyIiIpmgokREREQy\nIZHbN/v6j/KrvfuTaFpEyvDGSc2056r37Yj6ntvHsX19aYchMqbVNOUYP7syL9lIJFtNrKvlb06a\nlETTIiIvqz9tYtohiEgF6faNiIiIZIK5+9A3NnsR2JJcOCKSopnu3pF2EMozIlVv0FxTVlEiIiIi\nkhTdvhEREZFMUFEiIiIimaCiRERERDJBRYmIiIhkgooSERERyQQVJSIiIpIJKkpEREQkE1SUiIiI\nSCaoKBEREZFMUFEiIiIimaCiRERERDJBRYmIiIhkQmpFiZldbGZvS/gY7zOzBUnua2arSz02sxXD\nOe4QY+s0s48NYbtX9cHMbjSz1uHGlm+v1P7F41Bmuyfct7jP+fVjZsvNrKGMY+XiOBTPW1lrZQT6\n+6rzw8yWmtn04R5zrFBeGZzR6CgAACAASURBVD7lldeUV6ab2afM7FYzW1bw/WGvlXIMde6G2Fap\neSgep8T6VZtEo8djZpcDy4DJwFoz+zhwCvBZ4CPAAWAjcDGwOv7bA9wAPAAsAtYBF7r7e83sE8AR\noBX4PvAFYC2wDzgdmGFm3e6+rSCGm4FdhP4/Alwbj9sN1AOPx02vNbNrgLuBU4E5QAvwRWAV8Cdg\nvpkZ8FXg90Bn3HdJXNTTgTbgu8DC2O9FMc53Ej6ifa27Pxpjez/QDHQAnwN+CfwUOAN4ApgCfA+4\n1MwcOAg8CFwfx+nXsd+T4/EeM7NVQF8c99UxtoeLxuoR4D1xHHrimNYBm939nqJpXGJma2KfHwV+\nDswzs7e4+5qCcV4GXAE48APgplJzCMwxsw/EsbsRuCWOy35gQ4xrP9BbtH5+BZwLPGRmdwI/AeYC\nnwe+RFhHi939unxM7n4E+IuipCDmOwaJkeJ5NrP6olj/E3gfsI2wFv8H8AJwDPhX4FPAH2JbZxTN\n2XVx/9+4+x9jOMvNbE7cpxU4aGbzgTOBHHAH8N+ATcD33P1wqT6NBcoryivFc8jI5pUXgFvM7GLg\ndYRz+WXHyyvx58Vr52pgD/CbOMaXALtjf/83Icf8iXDuF/ajeGxuJszzfe7+opn9PfAd4N+Ay4C/\nB9YU7bPEzO7ilbV3JfBRYKmZNcVjlzwHKiGNZ0qWEwbqHuBmd/8yYYGcDUx196+7+wOEAbL4BbDH\n3b8FHCYM6vNm1kIY2B5gPCFRP+zutwILgGeAu4sSRztwFtALTIrfvg/4GuGEuZGwOAHuJSz4q+LX\ndkKCOQ3oiPE8Qzih97n7t4GdRf1dA9wOnAcsdfevAOvjzzYATYRkkfcmwmI8CJwEbIz7uLt/g5Ak\nAR6L/ZwXx3RXjG82sCTu81Dc9oy4b1dRbIVjdQVwGyFREretA9opwd0PEE6IyYRfupsKE0e0Avgj\nsCPGNdgc7o5jZ8AbCAl6HyGBXgF8E7grtlm4fgo95+63Aw3AfKDL3W/jlfUzVIPFCH85z/OLYl0O\nfC2OdSPwZ3f/Wuz7XwH/DPxjQT8K56zP3W8vKEggFChfAC4q+F5zPN65hCT1AmEOcmX2s9ooryiv\npJpXzOwc4FLghyW6NmheGWTt3A/cCiwF3u7u/yvuPx9YH9f3ohL9KB6bXe7+I3d/Mf7834G/45UL\nmt+V2AfC2uuN47cvfm+du98EnEOJc6BS0ihK1gAfJJz0n45XNAsJ1ecOM1tpZhcRKu2VhCsagP74\n7xF3d2CAEP99hF8AzxJOfi841rPACjObkf+Gu3cTJqIBeLKg7QGg393z7QK8mXCl9TNCtdlGOKm3\nxlivIUzibqAhPp5a1F+PXzWEK7hVwAXAUWBiPPZcM7vazKYCvyAkkz2x3Xy/+wraA1hgZh8lXEWv\nIZxw42Pf1pnZdYQFDfCUmb2DkCSKY8u7lzAvV8XY2oBDhKuNOWZ2WeGOZjaJkDSaY5/dzK4uav/H\nBePxNIPP4ZR4RWMx/u7Yl40xrncBb437Fq6fwfryBLDQzG6IMb6KmX0YOMfMlpXoW8kYzWwlfznP\nG4ti/Rnw4XjcA8AkM/uvwGbgP4D/QkgA+X4UztlAQXwr438vNLNPxX3z5hLWQh3hl8sBQgKZWNzP\nMUZ5RXkltbxiZq8jXHDsJTzTMOS8MsjaKZzffzGz/25m18cYzo/re32JfhSPzUCMry3O1aOEZ2Fu\nIxRi/15iHwhrZGLBM2zwSo5ySpwDlWJhfGQkmNlswtNwZwGfdPe+E+wyYsxsAvBuwlOdP3T3TelG\n9NrE+6KnADvc/Y604xFJivLKyBlLecXMriQ8e0d8RmxkjjsWihIzW0R4OhLC04Hrj7e9DI+Z5Qj3\n0fN+FF/DkSozW064QgO43923phmPVAfllZGhvDK2jImiRERERLJP71MiIiIimaCiRERERDKhrPcp\naW9v987OzoRCEZE0dXV1dbt7R9pxKM+IVLfj5ZqyipLOzk42bNhQmahEJFPMbEvaMYDyjEi1O16u\nSeQdXQ8feInnHit+Px0RGWkz5y9gQkv1voVJ3/P7ONaTmb+AFRmTappyjJ/deuINhyCRomR8YxNz\nl1104g1FRF6D+s7qLbhExiK90FVEREQyQUWJiIiIZIKKEhEREckEFSUiIiKSCSpKREREJBNUlIiI\niEgmqCgRERGRTFBRIiIiIpmgokREREQyQUWJiIiIZIKKEhEREckEFSUiIiKSCSpKREREJBNUlIiI\niEgmqCgRERGRTFBRIiIiIpmgokREREQyQUWJiIiIZIKKEhEREckEFSUiIiKSCSpKREREJBNUlIiI\niEgmqCgRERGRTFBRIiIiIplQm3YAIiLDtWXLFnp7e9MOQ2RMa2xsZNasWRVpS0WJiIxaM2fOTDsE\nEamgRIoSd0+iWREZBjNLO4TErH9uLzv2HUo7DJExraOpnqWz2yvSViJFSe/RY/x67/4kmhaRMlw4\nqZn2XPU+IXrK4e00v9SddhgiY1pj7SQgw0VJPcZZujMkkrpGqvdZEoBpc89OOwQRqaBEKof9R3q5\nf9svk2haRMrw1+MvYlpuStphJOa+bRt44UBP2mGIjGlTxzdx5czFFWkrkaKko3ESHzjvmiSaFhF5\n2eXTFqYdgohUkO6xiMioteX/e4aXduv1ayJpmtA2gdPeOKcibSVSlBzcf5inHn0hiaZFpAyz559K\n86QJaYeRmFPqnmKgbkvaYYiMaVY3FchwUVJbX8PkzvokmhaRMtQ1VPebNted/5a0QxCRCkqkKMnl\ncsyYMSOJpkVERKRKVfdllIiIiIwaKkpEREQkExK5fXNs9wu89P9+J4mmRaQMjW99D7XTTk87DBGR\nIUmkKKnpmEbLDf8ziaZFRESkSiVSlFTzB4CJiIhIMvSaEhEREcmERJ4p6du/l52b7kmiaREpw5Qz\nLqdh0tS0w0hM164udh7YmXYYImNae0M7F5x8QUXaSuZTgpvbmHnBe5JoWkTkZeeddF7aIYhIBen2\njYiIiGSCufvQNzZ7EdAHTYhUp5nu3pF2EMozIlVv0FxTVlEiIiIikhTdvhEREZFMUFEiIiIimaCi\nRERERDJBRYmIiIhkgooSERERyQQVJSIiIpIJKkpEREQkE1SUiIiISCaoKBEREZFMUFEiIiIimaCi\nRERERDJBRYmIiIhkwogUJWZ2sZm9LeFjvM/MFiS5r5mtLvXYzFYM57hDjK3TzD42hO1e1Qczu9HM\nWocbW769UvsXj0OZ7Z5w3+I+59ePmS03s4YyjjXdzD5lZrea2bKC7w97rbwWZrbUzKaX+P7qgv+X\nnO/8PAw2fvn9zOxcM7vJzL5iZo2VjD/LlGOGTzlGOaZ4m1KPi/dPKtfUVqqhUszscmAZMBlYa2Yf\nB04BPgt8BDgAbAQuBlbHf3uAG4AHgEXAOuBCd3+vmX0COAK0At8HvgCsBfYBpwMzzKzb3bcVxHAz\nsCv29RHg2njcbqAeeDxueq2ZXQPcDZwKzAFagC8Cq4A/AfPNzICvAr8HOuO+S+JCng60Ad8FFsZ+\nL4pxvpPwcexr3f3RGNv7gWagA/gc8Evgp8AZwBPAFOB7wKVm5sBB4EHg+jhOv479nhyP95iZrQL6\n4rivjrE9XDRWjwDviePQE8e0Dtjs7vcUTeMSM1sT+/wo8HNgnpm9xd3XFIzzMuAKwIEfADeVmkNg\njpl9II7djcAtcVz2AxtiXPuB3qL18yvgXOAhM7sT+AkwF/g88CXCOlrs7tflY3L3F4BbzOxi4HXA\nfxZ2zMzuGCTG/M8fBL4Zj7MDOA/4DHArcA+wzN0/VND/DuAS4F7gjcCX4/bbgM3AJOBgXGd9wNuB\nvwVOM7OVwEzgPmCRmS109w2F8wDcCbw+JpQGd7+Fv/RO4FPAUuBy4F9LbFM1lGOUY4rnEOWY4eaY\nKXG9nO/u15NSrkn6mZLlwM2Ewb3Z3b9MWBRnA1Pd/evu/gBhkVn8Atjj7t8CDgPfAZ43sxbgMsIC\nHw/kgIfd/VZgAfAMcHdRsmgHzgJ6CZMFYUK+RjhJbiQsSAiTfBNwVfzaTkgqpwEdMZ5nCCfxPnf/\nNrCzqL9rgNsJC2upu38FWB9/tgFoIiSIvDcBewiJ4CRgY9zH3f0bhMQI8Fjs57w4prtifLOBJXGf\nh+K2Z8R9u4piKxyrK4DbCMmRuG0d0E4J7n4A2EQ4cY8BmwqTRbQC+CPhxJrN4HO4O46dAW8gJOV9\nhJPtCsIJeldss3D9FHrO3W8HGoD5QJe738Yr6+dlZnYOcCnwwxJdGyzGvMfd/c74/7uAXxCSx4vu\n/k+ENZS3jlA4bCfM/wDhF2AfYY6nFWx7ZpyjTfHxlvi4nrDG1hcli0Ib3X01MHWQn0M4n8YK5Rjl\nGOWYyuSYbnf/P4S1AinlmqSLkjXABwkn+qfjVcxCQsW5w8xWmtlFhOp6JWGAAfrjv0fc3QmDX0M4\n2RuBZwmTVTggzwIrzGxG/hvu3g38jrCwnixoewDod/d8uwBvJlxd/YxwJdFGmJytMdZrCCfCbqAh\nPi6eLI9fNYSrtlXABcBRYGI89lwzu9rMphIWYDNhQe0u6HdfQXsAC8zso8Af4pi2EJLm74B1ZnYd\noVoFeMrM3kFIDMWx5d1LmJerYmxtwCHCFcYcM7uscEczm0RIFM2xz25mVxe1/+OC8XiawedwSryK\nsRh/d+zLxhjXu4C3xn0L189gfXkCWGhmN8QYC+N+HfCPwF7C1Vhx30rGGK8oiI8Lj5mf21ediGa2\n0t2PEZJ+VxyH3xKuOOGVucp72sw+RPhldrTgOB7H4/VmtpjSCrct5Q7CL8KrCFfF1U45RjlGOaYy\nOaY4t6SSayyMk1Samc0mPM12FvBJd+87wS4jxswmAO8mPL35Q3ffdPw9ss3CPelTgB3ufkfa8ZxI\nvLI6H5jt7p9IOx4ZnZRjRo5yzMipuqLEzBYRnoKE8BTg+uNtL8NjZjnCvfO8H7n7kcG2Hylmtpxw\nVQZwv7tvTTOe4YpX45fGh3vd/e4S21RFX0cb5ZiRoRyTrKHkmIJtR6zPVVeUiIiIyOik9ykRERGR\nTFBRIiIiIplQ1vuUtLe3e2dnZ0KhiEiaurq6ut29I+04lGdEqtvxck1ZRUlnZycbNgz29gkiMpqZ\n2Za0YwDlGZFqd7xck8g7uvYe6eU3236TRNMiUobFpyymbXzbiTccpX77/F629xxKOwyRMa2jqZ6l\ns0u+L17ZEilKWnItXDnryiSaFhF52fmd1VtwiYxFeqGriIiIZIKKEhEREckEFSUiIiKSCSpKRERE\nJBNUlIiIiEgmqCgRERGRTEjkT4JFREbCup6X2N7Xn3YYImPalFwtF05qrkhbKkpEZNRa3NqUdggi\nUkG6fSMiIiKZoGdKRGTUeuLFJ9h9aHfaYYiMaW3j2zhnyjkVaUtFiYiMWme2ncnpfnraYYiMaTVW\nuZsuKkpEZNTKjculHYKIVJBeUyIiIiKZoKJEREREMiGR2zf9/b3s2fNAEk2LSBna2paRy01OOwwR\nkSFJpCipq2th6tTlSTQtIiIiVUq3b0RERCQTVJSIiIhIJuhPgkVk1Nq3eyeHDxxIOwyRMS3X0MCk\nqadUpK1EipJjRwd46c99STQtImVobM1RWzcu7TASUzOulto6vVeJSJrG1VaulEikKBkYcA7tP5JE\n0yJShobmOqhLO4rkNE9uTzsEEamgRIqSQzXQ1exJNC0iZVgyDtrSDkJEZIgSKUqaxtVwWVtLEk2L\nSBlyNZZ2CInasmULvb29aYchMqY1NjYya9asirSVSFFSY8b4cdWdDEUkfTNnzkw7BBGpIP31jYiM\nWlvXPcv+3fvTDkNkTJswaQKn/dUZFWkrmaLkUA9s/mUiTYtIGWZdDI3V+2LQk2s2MaVmS9phiIxp\nNTVTgQwXJQM0cfjYG5NoWkTKUO8tVO8fBEPdordW8x8XiYw5iRQl+3EeoD+JpkWkDBfiVPPH8fU9\nv49jPXpPJJE01TTlGD+7tSJtJVKUTGyo46oFpybRtIjIy+o7J6YdgohUkD77RkRERDJBRYmIiIhk\ngooSERERyYREXlPiR49yrKcniaZFpAzjWlqwnD6wTkRGh2SKkr4++jZvTqJpESnD+HnzGKeiRERG\niUSKkiM4W4/pz/RE0jYTZ0LaQYiIDFEiRcn4xibmLrsoiaZFRESkSumFriIiIpIJKkpEREQkE1SU\niIiISCaoKBEREZFMUFEiIiIimZDIX98cOnSIzXqfEpHUzZo1i8bGxrTDSExPzwYOH96edhgiY1ou\n105b29KKtJVIUdLQ0MD8+fOTaFpE5GWtrQvTDkFEKki3b0RERCQTzN2HvrHZi8CW5MIRkRTNdPeO\ntINQnhGpeoPmmrKKEhEREZGk6PaNiIiIZIKKEhEREckEFSUiIiKSCSpKREREJBNUlIiIiEgmqCgR\nERGRTFBRIiIiIpmgokREREQyQUWJiIiIZIKKEhEREckEFSUiIiKSCSpKREREJBNSKUrM7GIze1vC\nx3ifmS1Icl8zW13qsZmtGM5xhxhbp5l9bAjbvaoPZnajmbUON7Z8e6X2Lx6HMts94b7Ffc6vHzNb\nbmYNZRxrupl9ysxuNbNlBd8f9lopI+YzyzmGma043lwXjMGVZvYZM/sHM7NKxF4NlGOGTzlmzOSY\n1QX/z0yuqU2q4VLM7HJgGTAZWGtmHwdOAT4LfAQ4AGwELgZWx397gBuAB4BFwDrgQnd/r5l9AjgC\ntALfB74ArAX2AacDM8ys2923FcRwM7CL0PdHgGvjcbuBeuDxuOm1ZnYNcDdwKjAHaAG+CKwC/gTM\nj5PzVeD3QGfcd0lcyNOBNuC7wMLY70UxzncSPp59rbs/GmN7P9AMdACfA34J/BQ4A3gCmAJ8D7jU\nzBw4CDwIXB/H6dex35Pj8R4zs1VAXxz31TG2h4vG6hHgPXEceuKY1gGb3f2eomlcYmZrYp8fBX4O\nzDOzt7j7moJxXgZcATjwA+CmUnMIzDGzD8SxuxG4JY7LfmBDjGs/0Fu0fn4FnAs8ZGZ3Aj8B5gKf\nB75EWEeL3f26fEzu/gJwi5ldDLwO+M/CjpnZHYPEmP/5g8A343F2AOcBnwFuBe4Blrn7hwqaLJyn\nZ4BWM2uK+50cY82P4z+7+67CcQYeBpbGfba5+/f5S5e7+9+Z2XuBNwCPldhmzFCOUY4pnkOUYwbL\nMaeZ2UpgZjxmJnLNSD9Tshy4mTC4N7v7lwmL4mxgqrt/3d0fICwyi18Ae9z9W8Bh4DvA82bWAlxG\nWODjgRzwsLvfCiwgTNDdRcmiHTgL6AUmxW/fB3yNcJLcSFiQAPcSFvlV8Ws7IamcBnTEeJ4hnMT7\n3P3bwM6i/q4BbicskKXu/hVgffzZBqCJkCDy3gTsISywk4CNcR93928QEiPAY7Gf8+KY7orxzQaW\nxH0eitueEfftKoqtcKyuAG4jJEfitnVAOyW4+wFgE+HEPQZsKkwW0Qrgj4QTazaDz+HuOHZGWOgt\nhCTWEeP6JnBXbLNw/RR6zt1vBxqA+UCXu9/GK+vnZWZ2DnAp8MMSXRssxrzH3f3O+P+7gF8QkseL\n7v5PhDVUqHCe8poJiXl+0TjWlYgHYJ273wScM8jP5dWUY5RjlGOGlmO2xHmrj48zkWtGuihZA3yQ\ncKJ/Ol7FLCRUnDvMbKWZXUSorlcSrmIA+uO/R9zdgQFC7PcBjcCzhMnygmM9C6wwsxn5b7h7N/A7\nwsJ6sqDtAaDf3fPtAryZcHX1M8KVRBvhRN4aY72GcCLsBhri46lF/fX4VUO4alsFXAAcBSbGY881\ns6vNbCphATYTksbugn73FbQHsMDMPgr8IY5pCyFp/g5YZ2bXAUvjtk+Z2TsIiaE4trx7CfNyVYyt\nDThEuMKYY2aXFe5oZpMIiaI59tnN7Oqi9n9cMB5PM/gcTolXMRbj74592Rjjehfw1rhv4foZrC9P\nAAvN7IYYY2HcrwP+EdhLuBor7lvJGOPVBPFx4THzc1t4fAq2L5ynvLMJ66iuxDiWkj+mD/Lz+8zs\n04RfhI8Pss1YohyjHKMcM7QcU5xbMpFrLIyLJM3MZgOXECb0k+7ed4JdRoyZTQDeTXh684fuvind\niF4bC/ekTwF2uPsdaccjMhKUY0aOckxyqr4oMbNFvPLU1iZ3X3+87WV4zCxHuHee9yN3L36qccSZ\n2XLCVRnA/e6+Nc14BmNm8wj3mQG2uvv9Jba5lnALAcJtg70jFZ8MTjlmZCjHvDZDyTEF26aWa6q+\nKBEREZHRQe9TIiIiIpmgokREREQyoaz3KWlvb/fOzs6EQhGRNHV1dXW7e0facSjPiFS34+WasoqS\nzs5ONmzYUJmoRCRTzGxL2jGA8oxItTterknkHV2P9fby0oMPnXhDEUlU47Kl1La1nXjDUWrb7zey\nf0932mGIjGmNrZOYcfYbKtJWIkXJuJYWJr71LUk0LSLysmlzz047BBGpoBH97BsRkUr67fN72d5z\nKO0wRMa0jqZ6ls4u+YkBZUukKDly+BA7Nz+dRNMiUoYpp72O8Y1NaYeRmHENz5N71WeMichIqx0/\nmUE+xqj8tirSSpFxtXVMnjbjxBuKSKLq6utPvNEodiR3Ooe8M+0wRMa0vlzlSolEipKXHH6VmU9d\nEBm73jjBKnT9kk2LJzWfeCMRGTUSKUom1tXyNydNOvGGIiIiIpHe0VVEREQyQUWJiIiIZIKKEhER\nEckEvU+JiIxafc/t49g+vapeJE01TTnGz26tSFsqSkRk1Ko/bWLaIYhIBen2jYiIiGSCihIRERHJ\nBBUlIiIikgkqSkRERCQT9EJXERm1eno2cPjw9rTDEBnTcrl22tqWVqQtFSUiMmq1ti5MOwQRqSDd\nvhEREZFMUFEiIiIimZDM7ZtDPfDMfYk0LSJleN0l0NiedhQiIkOSSFGyj0YeGKjMi15EZPgu9GYm\npx2EiMgQJVKUNGNcTl0STYtIGeqxtEMQERmyRIqSfne29Q8k0bSIlGG6Q0PaQYiIDFEiRUn9hDrO\nOH9qEk2LiIhIldL7lIjIqLVlyxb27duXdhgiY1pTUxOzZs2qSFsqSkRk1Jo5c2baIYhIBel9SkRE\nRCQTknmha38/3d3dSTQtImVoa2ujvr4+7TBERIYksds3tbW6MySSNrPq/pPgl/7cR3/f0bTDEBnT\nanPjaG4bX5m2KtJKkbq6Ojo6OpJoWkTkZceODnD0iN5+QCRNlbz40dMZIjJqTezQu7CIVBO90FVE\nREQyQUWJiIiIZIKKEhEREcmERF5TMjBwlP6jPUk0LSJlqKttoaYml3YYIiJDklBR0seBl55OomkR\nKUNz89lVXZQc7Oqif8fOtMMQGdNq2yfTuHhxZdqqSCvFjdY20ta2NImmRUReNuG889IOQUQqSK8p\nERERkUxQUSIiIiKZoKJEREREMkFFiYiIiGSCihIRERHJhET++qb3SC+/2fabJJoWkTIsPmUxbePb\n0g4jOVvWQu+f0o5CZGxr7IBZF1WkqUSKkpZcC1fOujKJpkVEXjFzSdoRiEgF6faNiIiIZIK5+9A3\nNnsR2JJcOCKSopnu3pF2EMozIlVv0FxTVlEiIiIikhTdvhEREZFMUFEiIiIimaCiRERERDJBRYmI\niIhkgooSERERyQQVJSIiIpIJKkpEREQkE1SUiIiISCaoKBEREZFMUFEiIiIimaCiRERERDJBRYmI\niIhkwogVJWZ2sZm9LeFjvM/MFiS5r5mtLvXYzFYM57hDjK3TzD42hO1e1Qczu9HMWocbW769UvsX\nj0OZ7Z5w3+I+59ePmS03s4YyjjXdzD5lZrea2bKC75e1VpLu7xDbKdn3wvZH4jwbTZR3hk95p/rz\nTvH5YWZLzWz6cI9ZCbVJH8DMLgeWAZOBtWb2ceAU4LPAR4ADwEbgYmB1/LcHuAF4AFgErAMudPf3\nmtkngCNAK/B94AvAWmAfcDoww8y63X1bQQw3A7sI/X0EuDYetxuoBx6Pm15rZtcAdwOnAnOAFuCL\nwCrgT8B8MzPgq8Dvgc6475K4aKcDbcB3gYWx34tinO8kfCT7Wnd/NMb2fqAZ6AA+B/wS+ClwBvAE\nMAX4HnCpmTlwEHgQuD6O069jvyfH4z1mZquAvjjuq2NsDxeN1SPAe+I49MQxrQM2u/s9RdO4xMzW\nxD4/Cvyc/5+9+46z4yoPPv47M3N73Xu3adV2Ja26JcuSi2RbLnLB2BgH25iWQGwCBEgj5H2BvASS\nQMgLwS8JAUKAAMaAscE4xgaDIxfJxpYtybZ6l7Zoe7v93rkzc94/5q606sVa7Uo6X31Wu3fuzJkz\n7cwzZ86cgblCiNuklE+MWM9XArcAEvgh8IWjbUNgthDiw5V193ngS5X1kgHWVvKVAdKH7T8rgUuA\nVUKInwE/B+YA/wh8BXc/ukJK+f7hPEkp24AvCSGuBaYDL45cMCHEQ8fII4dvZyGE77C8vgh8AGjH\n3Rc/B7QBNvAY8GlgWyWtmYdts/dXpn9BSrlHCPFeYDPw98CnKt//12HTDC/7Z3D3xXuklNcBCyoF\naQB3n75ICLFNSrmNC5Qqd1S5c/g2RJU7R5Q7lezcLoSYXZkmDuSFEBcBswAv8BDwSWAL8H0pZZFR\ndDZqSm4Hvgg8CXxRSvlV3B1gPlAvpfx3KeVzuDuUqPwA9Esp/wMoAt8B9gkhosAK3BXsx11ha6SU\n/wpcDOwEHj+sYKgG5gFpoKoy+Gng67gHxOdxdz6Ap3B36LdXfjpwC5AmoKaSn524B2xKSvltoOuw\n5X0CeABYDCyTUt4PvFL5bi0Qxi0Mht0M9OMe9HXApso0Ukr5DdydBOD1ynLOrazT7kr+ZgBLK9Os\nqow7szLtusPyNnJd3QJ8C7cgpDKuB6jmKKSUOdydMom7828ZWTBU3APsATor+TrWNuyprDsBLMQt\ngFO4BeQtwDeBhytpd9g4LwAAIABJREFUjtx/RtorpXwA90R8EbBOSvktDu4/BwghFgHXAw8eZdGO\nlUc4cjtfdFhebwe+XlnXIWBQSvn1yrJfDTwKfG/EcozcZiUp5QMjCobfAu8EXqisx4GjTDOstpLn\ntsrnTVLKrwH1uPvnMxdyQFKhyh1V7qhy58TlDrgByj8D14wYFqnM7xKgB7esSeLu+6PqbAQlTwB/\nintQf6ZyxbIEN7rsFEJ8TAhxDW4k/THcKxaAcuW3KaWUgFPJ79O4G2I37sEtR8xrN3CPEGLK8AAp\nZR+wAXcn2jwibQcoSymH0wV4C+6V1K9wrxoSuAdtayWvd+Nu2B4gUPlcf9jyysqPhnuF9gngcsAC\nYpV5zxFC3CmEqAd+g7sD9FfSHV7u0oj0AC4WQvwFbjT7BO5O6q8s28tCiPcDyyrjbhdCvAu3EDg8\nb8Oewt0ub6/kLQEUcK8mZgshVoycUAhRhVsoRCrLLIUQdx6W/iMj1scOjr0NaytXLKKS/77Ksmyq\n5Ou9wNsq047cf461LBuBJUKIj1TyODLf03EP0AHcK6/Dl+2oeRRCfIwjt/Omw/L6K+DjlfnmgCoh\nxJ8Bu4DVwE24BebwcozcZs6IPH6ssp8uA36Je3z87ijTDOutrL+GyufhtCRu4XGDEGIeFzZV7qhy\nR5U7Jyh3Kn9eJYT4dGXaYXNw9wUPbtCaww2WYowy4a4PZTQIIWYA1+FeMf1vKWXpBJOcNUKIIPA+\n3KrMB6WUW8Y2R2+OcO8/NwCdUsqHxjo/o0kIcTXulWu1lPKLY50fZXxR5c7ZcyGVO2fLeRmUCCEu\nwy20wa3ue+V44yunRwjhxb1PPuwnUkpzrPIzTAhxO+4VGLi3MlrHMj/KhUGVO2eHKnfOb+dlUKIo\niqIoyrlH9VOiKIqiKMq4oIISRVEURVHGhVPqp6S6ulo2NjaOUlYURRlL69at65NS1ox1PlQ5oyjn\nt+OVNacUlDQ2NrJ27dozkytFUcYVIUTLWOcBVDmjKOe745U1o9Kjq1OwKG4fGI2kFUU5Bb4ZcfTw\nqPd3NGZKe1PYqXHzxKuiXJC0sBf/jPiJRzwJoxKUWD6Nvlmj3seKoignUOs10Mc6E6PI16TKGUU5\nn4xOUCKhx7RGI2lFUU5BlcfAP9aZUBRFOUmjEpQEdY0lsdBoJK0oiqIoynlKPRKsKIqiKMq4oIIS\nRVEURVHGBRWUKIqiKIoyLqigRFEURVGUcUEFJYqiKIqijAsqKFEURVEUZVxQQYmiKIqiKOOCCkoU\nRVEURRkXVFCiKIqiKMq4MCo9uiqKopwNZiGPbdtjnQ1FuaBpmo4vGDwjaamgRFGUc9ZQdxf5ocGx\nzoaiXND8kSj105vPSFoqKFEU5ZxV2zhtrLOgKMoZpNqUKIqiKIoyLqigRFEURVGUcUEFJYqiKIqi\njAuj0qZEmiZWX99oJK0oyinQq6vRvN6xzoaiKMpJGZ2gxLIod3WNRtKKopwCLRoFFZQoinKOGJWg\nRAsGCV5yyWgkrSiKoijKeUq1KVEURVEUZVxQQYmiKIqiKOPCqNy+KVgFdg/tHo2kFUU5BU2xJkKe\n0FhnQ1EU5aSMSlCiC52INzIaSSuKcgp0oY91FhRFUU7aqAQlXjPP1JZXRyNpRVFOxbRrwfCPdS4U\nRVFOyui8+yYQh4vuGpWkFUVRFEU5P6mGroqiKIqijAsqKFEURVEUZVwYnds3iqIoZ4HpODhyrHOh\nKBc2TYBXOzN1HCooURTlnLUxU6DLLI91NhTlglbtMbg8Hj4jaY1KUFIsFtmzZ89oJK0oyimYOnUq\nodD520/J4tj5u2yKciEanaDE0dhePDNRk6Iop6/W0VGnbUVRzhWjEpT4pcnsvKopUZSxFpQxQPVT\noijKuWF0enT1OlTNSI9G0oqinALDb411FhRFUU7aqAQlHk+U+vrbRyNpRVEURVHOU6qfEkVRFEVR\nxgUVlCiKoiiKMi6ofkoURTlnmaaJ4zhjnQ1FuaBpmobX6z0jaamgRFGUc1bH0w+T6do/1tlQlAta\nKFnLtDv++IykpYISRVHOWT0Nl9PrS411NhTlgpaIR5h2htJSQYmiKOesyxY1j3UWFEU5g0YlKCnl\ny7Rs6h+NpBVFOQWT5yQIRM7Mvd7x6JmXfkh6qHWss6EoF7RQuJ4br/6TM5LWqAQlvqCHmZfVj0bS\niqIoB1y/9P1jnQVFUc4g9UiwoiiKoijjgpBSnvzIQvQCLaOXHUVRxtBUKWXNWGdClTOKct47Zllz\nSkGJoiiKoijKaFG3bxRFURRFGRdUUKIoiqIoyrigghJFURRFUcYFFZQoiqIoijIuqKBEURRFUZRx\nQQUliqIoiqKMCyooURRFURRlXFBBiaIoiqIo44IKShRFURRFGRdUUKIoiqIoyrigghJFURRFUcYF\nFZQoiqIoijIuqKBEURRFUZRx4bwNSoQQ1woh7hjleXxACHHxaE4rhPja0T4LIe45nfmejjeznMdJ\nc5kQYvIJxhle1qQQ4oYzOf8341j5Oda2Ui4squw5cy60suco6/ysrevxwhjrDJxpQogbgSuBJPCS\nEOKvgQbg/wB/DuSATcC1wNcqv4eAjwDPAZcBLwNXSSn/SAjxvwATiAM/AP4ZeAlIAc3AFCFEn5Sy\nfUQevgh0467f9cB7KvPtA3zAG5VR3yOEuBt4HJgIzAaiwP8FPgHsBy4SQgjg/wFbgcbKtEuFEAFg\nMpAAvgssqSz3ZZV8vhtoAV6SUr5Wydu/AE9WvnsEcIBlwEBlGX8MfBX4fiWtZCXd1w9bzz8D/ht4\nGvgM0A7sAhYBg5U8/RD4UmW81cBfVr57EZgK5IUQtwARoAb4LPD3lbTWAHOFEB8AXgDmCyFkJa8x\n4CuVdfIkcKWU8qOH5e/5ynxnAhuB2kraf175+wngppF5lVLuEUL4gL8DdgCTKtvxWeDeEdv00Up+\nBO6+Vg38D+ATQnwQuA74VCX/t0kpn0A576myR5U9lfydVtlTmXy2EOLDQKOU8tOVdb0GeFtlG/0t\n8OnD1+355HysKbkd+CLuDvNFKeVXgbXAfKBeSvnvUsrnAAmIyg9Av5TyP4Ai8B1gnxAiCqzALTj8\ngBdYI6X8V+BiYCfw+GGFQjUwD0gDVZXBTwNfxy1gPo9bcAE8BXwBeHvlpwO38GgCair52Ym7I6ek\nlN8Gug5b3ieAB4DFwDIp5f3AK5Xv1gJh3ANvWKky/z24J8/VQEJK+Q3cQgDgxcqJdGklvVVHWc/d\nUsqfANdU0uzHPYmDW9D9FLhhxHhLgZ9LKb+Ae4ANu7kybR6oB7JSym9KKdcBW6SUPwCsyri3Sin/\nEfgFsBzolVL+uLJeD7epkndZWbY4EMTd7q242/XwvCKlLOEW3hMq38/APQkcvk3h4L72TOVzWUr5\nXWAd7n61RQUkFxRV9qiyB06z7KnoqaxrIYTwVoYFcQM4E7iIo6/b88b5GJQ8Afwp7ob/TOVqZQnu\nFUqnEOJjQohrcK84PoZ7tQJQrvw2pZQSdyfQcA/qELAbd6eQI+a1G7hHCDFleICUsg/YAASAzSPS\ndnBPWsPpArwF9yrqV7iRdQL3AGmt5PVu3JNiDxCofK4/bHll5UfDvTr7BHA57sEUq8x7jhDiTiFE\nPW7kPxX3iqFJSmkCA0KIj+EeoFTyCvCyEOL9uFcIhxse5/nKb39luQHeCXwYWDlivFXAXUKI4eUd\nzvtvcA+uftwrvJAQ4k+FEJcArUKIj3OwRu/JyvR3VtIbuS0ON7w9SyPmVcPBwlo/PK9CiI8IITTc\nqy4d92pEHmObgruvfQZ3X7NGLOvwNpZCiDuPk0fl/KLKHlX2wJsre2orNSWisn4A5uIGrHrl58C6\nPU4ezlnCPQaU84EQYgbuFcg84H9XrvrPdh4+D3xNSjl0nHEM4JvAZyoF6Zg4mbyeYPo63KvjOcCX\npJS9ZzB7inLOUGXPqXmzZc/5TAUlZ4AQ4jLcaBbcar9Xjjf+uahyRXZ95eOAlPLxsczP4S6EbaAo\nh7sQ9ntV9lxYVFCiKIqiKMq4cD62KVEURVEU5Rx0So8EV1dXy8bGxlHKiqIoY2ndunV9Usqasc6H\nKmcU5fx2vLLmlIKSxsZG1q5de2ZypSjKuCKEaBnrPIAqZxTlfHe8sua86zztRNKWzWvpPNckzstH\nvBVFUc5duT4IVR/4aNsObVsGsEwHTRNIKbHKDrHaAP6gh/79WfwhD/H6IIGIl3zKJBT34jgSy3Tw\nBdxTnGM7pPuKxGoC7jPMmjhGBpSxdsEFJdtzRV4eyqqgRFEU5TTZtoMQ4vRP7mYeafjBKiEK/ZTW\nPERPaQrd2zuY/6EPs39nCst0yKXLNC2sRghBMVcmXhtEMwSZ/iL5tElyYhjHkfS3Zylky/gCBvs2\nltANDa9fp1Sw0HWBbUvCUY2OXUOUSzZCgC/ooVyy8fh0hAApwbYcdEMDKRGVZZOOBCEQR1lU6X51\nxN8IDunJ5IjvjkaO+P5of5+O4XlJDp3v4Wke77uTEIx6mTQ7ceoTHsV5H5TYUvKjjn6+3dbDJxvr\nebI3xYpkFCkl4mh7maIoyrkuPwCODb4weALI1jWIKZef1KRde1IITZDv7qGnWwMBopTF0f10703T\n3DhEb3EyZsGktjHOwhUjXiNTLsCulTDnNkh3wov/Sl++hkDES64/i52YjQgnKG57iSFnMsIbQPca\nGP4FhKpjLLpesvk/v8rkRkEoUMITr0VYV8CmR6FuHrR3gidAcvoK8PSBXg/JKSQmhMAywfC6EUB+\nADpeg7oq0AzY9T/Q1u3WwoSrIJiAUhaGdkD8YkhOc/NfMxuG2qCUAU0HwwcIqJ0D6Q6QDhQGwRsC\n3QNGAKTtfs71gdDccQAMP0QnuPMxc2Cb7voJ14BVcj/rXvBF3HTMrDtP6bjT6p7jbygp3W08HFEJ\n4c7/WNHTMT/LY493st8dM9I6daf0SPCSJUvkuXSvt6VQ4hfdg9xeGydvO4R0jelBP21Fk6d6UwxZ\nFp9srFfBiaIAQoh1UsolY52Pc62cGTMdr0HtXKTmpfBGL7lNu7CSA1THTDxP/zEFasks+hz6nkFy\nqToark6hD7wE132GzsFqcqkSzUvqsMo2rZsG0D2CXMrE57GJpl/B2L8acfVfEdz8HUTnemS4gdSe\n3ZhTb2bidTcin/8yb9R9hWSVScOcOsy968nu2U5bC/hDPnTdwZ5+M6G4DzM1RDQq8eRb0bJd+C67\ni3LJJpoMHLlcjg2FIffk7wsjOzfj1C5BG9hAMT8bUezGXLsG3yQPdkHDH21Fi0TBG3RP9pkuCNfB\n7Fvdv6WDnHjZwdoOu+QGGMnpbpBQykKmww1qOl+H2kqXI/7owSBkqBV8UTeACMTd83E5506ve92g\nI1x7MNCQEsp5yHa7AUagEhx5Am7wYvjc8WzTDUbMvBvYSBuEDlbRXQ+HV3EMByDDtOEHaIera5yD\n35/Mee2QccRxvjvB96EamHq0znePNdtjlzXnVVAipWRztsCPOvpZkYyyPp3nDxuSTPR7jxj3K3s7\n+YO6KvpNi8vj4SO+z9k2IV0/YriinK9UUDKOjDyxvP4TaLway1dDpq2d0ORG0v1FtNd/QqBmKbnd\nkr62LIVYO+GeGqxCHjm5SLpYT03vINkajXCqH+mpo2D4SQ/tozSpCVkYwOf0UW5YRk3SJLr1G9RN\nr6LQMpPoYgftivfDK/8JjVch/VUIX5jyK7/BY/RA/y4wfFgX30fPw/9CiinQeBXxeReTmBgHS6ID\nsmRhtmWws2WkaeOpD6H5DfSYF6doU9jST2BOAi1g4BQsChv68M2sQvPrmK0ZZNnBSPqxBosITcMz\nIYSR9CP8BrLopm22ZzFqAyDByZVBgDQdjGQAadqgC+xUCT3mQ5oO0nIwEn70uA+hCTz1QZyiDY4E\nQ0MWLYRHQ9oSLWCg+Y3DNo2qZX+zLpig5Au7O5jg8xA1dJ7qS/HtuY0YJ7jn+e8t3WhCcEt1jCqP\nTszQ2ZUv8e22Xq5LRri1Jn6Wcq8oY0sFJWPEcUZc8YJZtKDl96T2tVPjvMFAax+v7W3GO3cF9O+h\nezBKb5+PK6psonYKM9HDlHgnoZtuxmlYCjufI/2GgZmrovquCYhoElHoQ+oBTPzkntiO7NqDVU4S\nTvyQkGcI2bMNcem9lBK30/dwF94ajfD1zfhnVFFqSZN5ro3Eu2aReb6d+K3TkP17sEUDxso/xYyt\nQL/0ToTPIL+hDztdQo94wZF4JobRq/xoXh1haNgZEztdwhooYvUXiSyfSObZdoyaAHrUi68pRmZV\nO576EIG5yeOstJMnbQeha4d8Lnfn3fyk3fxoQQ9CEzhFCz3mw8mV0fwGdrqENN3bMdKRCF/lQtWW\nCF0cqMeQHFpxIKX7nxDi2O06jsgoJz/uyaRxFmlhL/4ZJ3+uPG+DkoGyxTdbeyhLSdqyuTIe5q76\nU2tsI6Wkx7T4TV8KvyZYEAnyj7s7+PyMiawezDAj6OPaRPTA+L/pHeIWFago5yEVlJx9ZtFi9Tee\nwEk0UzM5Qu3UCJmBEi89vJH5TR2YRQu7ZxeL3n8Hme4BJqz5EMWSoFPegRm5mpkfuQmR3usm1nDx\ngXSHy/VjXdFLR2IPFkk9sQWnBNaAjachjBY0qHr7dNA1Cht7ya3rIbS4FumAuScFAqyBInrUizVQ\nxDslitBAC3rQQh68UyJ4aoKjvt6Uc9vxyppztqFrd6nMJ7e3sbwqwkWRANODPjynUaUmhKDO5+ED\nE6v5aWc//9rSzYMLpqELwayQn3e/sZs+02J+JMDTfWlSls1bqmOq+k5RFKByBX0ST6GkevNoXTl8\nzVW0berBaVtP2jOTeH49waWXs2V1G4X+QcpdDrcuKhCNWbDkerzBtyA0g/AjN2Et+xJOSz/xfSVi\n12qIcAzCFx8xrxOVT0ITGMkA0bfOwR4sgSbQAgbeiQdvZQcX1uKdEsWo8gNgxLz4psVxSjZO0aK4\nbYDA3KRbK6IoZ8g5WVPyVG8KU0om+TxcEgud0bQtRx5yy2dDJs8vugep9hisSEbxaxqbswXeVuvW\nluRsm6CmqSBFOeepmpLTk325A//MhNtmwbLRfIbbjiJj4p0cobCxl9LeNJ05C29nls7qADWhHMWN\n66iu3snkSR2IUDXFXp1UTwfo92JUR/HOaQYp3dqH+iD2q79iaNsMgr6XsJNLid66YKwXXVFOy3lV\nU2I5kkd7Brk4EuT22jN/G+XwNigLIkEWRA6tjvxGazc9ZplLYyGe6ktxdVUEKWFZ1cGrjD7Tor1o\nkrVtGnxepgV9ZzyviqKMPadg0/+jLWg+i54hSf1t08n+dBs6EFoxhcHndlOcV0PUu4fkfdcT/9UG\n6BrAf9USSq0L6NofpG7pWvKtteCJE4qtI3zvX4LhI7e+m9yaLszWNNKaSM1HZ6EP5bCCM8Z6sRVl\nVJxTQclzA2m+1drLF2dORBuL1jwVX5k1mcd6hvh1b4pbamL8rHOASX4vecdhRSLCyoEMLYUSzUE/\nPWaZjGXz3z2DXBIN0eDz0Bzyn9L8VGtvRRk/rP4CRuVRVrtjL5rfpBQFn7GNTHsM/WcWRm2QdFAw\nuLUH0zPAnEtr8L/xPwirhkKxj8Ss1Wh/8C9IR1LY0EvXEw6RpQlke4HAbR+p9I8BwUW14EiEX0eP\nejFiPogt5QQ9WCjKOWtcBiUbMnmKtsPuQoll8TA1Xg/r0zn+o7WXf2yeyIzgqZ3UzzRNCGYEfTT6\nvSyMBFkYCeJIyYMd/ezIl9CAP2qoxjOi1uWmasmefImuUpnnBzMA3FIdI6hr/Lo3hSUl756QoGA7\nrBzIsCASYM1Qjrzt0G2WqfUaaEIwLeAjYuhkLBtdCK6MhylLSUBXL3xWlNFQ2pPCOzWK0AWZVe3I\nskPgomo8NQH6n36VlMekry9DrquX65d14sy9ByO3g4HWQeLmJrQlV8ArX3L7pfjBLSSnXIX4gycB\nt21H8OJavI0xsB0iKw7tt0MIQWhJPcFFhz5Bci7p2r2TxMRJGB4vCNA01dWCcmzjKigZKFs8sL8P\nTQhaCiU+NW0C32rtJaALlsXD/NdFTQTHyYF5+C0dTQjmRwL88ca9PHvZrEMCEgBdCJpDfppDfq5O\nRHCk5NmBDL1mmesSEQqOw8+7B8laNtcmomzNFrkkGqQp4MOjCfRKTcnGTJ60ZTPR72VdKsdDXQMM\nli1uro5RchwuigQZLFukLbeflXzl9tHI21KW47Yj2pkvUrAdvJqgxushZzs0+NxrsKztIJHUeD2U\nHcmmbIEJPg9DlsXs0FE6PBpBSolTWWZFOZeV9qXIru3C3jlIYsVkcuu6sbrzZLd20dOxAwJ+MvRz\ntf13yDCIhY+iPfh2mHoViepmmHg5XP4hyPVA2xq47WsI48hbuUb8+Ld3jxWQlM0SpWyWYCzOq4//\ngkI2w8U33Uq8rp69r6+jmM3QvWcXANe8716EpmFbZdq3bKamsYlgNIZlmhhet7GqWSyw8rvfJFpb\nR7lUomHmbGZefiXde3dTO7UJoR2//LWsHEIY9OzbTW/nSwy2Oniruuhqr0bYE7HLDjg6hseDNxCg\naO4nPbCbSHQWtuVQ1ray+NpPsn/vSgrmdgw9gO7Tyad7idc0E43MIxhqwjAiSFlmf8fDGHoYj1FF\nKNyMEBq6HsTjiWHbJprmUbXM55gxb+gqpWSgbBP36Hx5bxc7c0VurI7y7gln5hn1s0lKSc52CBtn\n90rAlpLHe4aY5PfyZO8QtV4Ps0J+8raDXxP0li00oDHgY2e+CEBQ05gR8mM7EhtIWTYJj86GTIGQ\nrhE3dMzK8uRsh/nhAGnLBqC9aBLUNUxHogtIeAxuSEYpOZLf9afIWg62lIR0DQk4gE8TmI5EADVe\nDz5NkLFsJvm9TAl46SyVaS2Y+DWNLrN8xCsfDr+FNfx5eIghBFUenbihE/MYSCnxaAJDCAQCR0ps\nJLYEDQ585xGV3yMCv+H0bcmBaSQSNyW3G4CT6izxmMOPPfHxkj2dslXjxE9iHExfNXQdyezIUtjY\nR9aRpF/pYkjXSEyPEcmk2N+aJRLWmHn1HrRozO2evH6+2+dI7zaI1Lu3YLwjGuIPtkDV1DOWv83P\nr2Sws4NoTS1mIU+0LoZetZXejQn8wSi618P+rZtYds+7GBxYR9trA4Ti1eRSrdQ3N5HvF5SLBXpb\n99EwczZD3V2k+tpYePt0dGsy/piPlp1PYOVCeEOSno0+Ji9J4tUbMT2raZh0O60t3yXVlcegHmGY\nOJaFwE+0po76yddSKu8mEpmPEDqZ7GbMUg9C6MTC15DJrUXTdbxGA5bTh2NDODSfrev+k2h8Hh45\nncGu/RSzaaYvuYSWzesxAnnSqa2E4glKhQyG3USsah6WPYCt9SAA20njUEAIHbvsoHsEjg2aZoAA\nj1aNJoJgRbA8O4jHloCeppDrR6OKcjGLIy2E0NDwI4QPIX04sowm/EitDNhoRBFoSCmRoogjs5Ut\nIw/8FvjRZMQN5o5yHEop3V5ccQCBxEagub26HigALcBTGQdAIMTJX5wf/Rw/onO+kd8fkcfjTDtC\nKFbF1IsuOek8jet+Sr7X3svufIlus8wHGqqZGw5QlpJ6n7preqbtK5RIegwiZzhoai+avJrK4RGC\n2WH/EbfXpJQUHYlfczsb6i9bmI4btLQXTfYWTCb7vUzyeyk6DhN8HgRu7dPJMh2HlGUzWLYZKlsA\nWNIN2NxaG9AQ6MI9tMuOpCwllpSUHfe3PSLwcecPemUagUAi3SDrNF+QdXKHN5W5HDbsNOd5VVWE\nau/JVYheSEFJZqBIJHHofuqYNqkn9hC/fTp939+EHnTwTKulzYHs468w860LeOrZDiL5jdTUaVz6\niQ8hHBv00atwlo5zSO1ENrcTgYaVC7DxmZXMuWEW+eI2CvkWhOZhYsM9bN32t8xs/hzhUDNDqVfo\n7HyUmpqbGBx8CU3z4vXWYFlpPN4kkya+Byklnbs2U9vUxN69/0795A9Syr6OZecJB2cz1LeXZP1c\n9u//Cb3tW4gEL8HJTSWb30DA18S8q/8Ary+EECdXrjiOxcDAany+eiKROae1TkqFPP7QkT1xn3je\nJfL5fZTLg1h2jnBoHgM9r6PJJMFYLaXSfgLBBLrhR0oH285h2wVsOwtCw7GLaJoPXQ9gmn3Iyntu\nDCOC4XGDlJFBg2VlMM3+4+ZJaEYluLHRNC9S2gd+hNAQwoPjlCppCiTO6RUIxyhPj32RdIzhR0nH\n60lSVXVy71ZykxinQcnr6TxbsgXe05DkJ5393F2XOOK2h6IoZ8e5HJR07NhGw8zZRwzftGo/85dP\nPGRYZqDIT/9hDW/7+EI696TQdY1YTYCB59qo7cyizU6Q3T1EUDfZPaGagpnjyjmbibAfu2Uteuvz\nsPTjcPMXT2v5HMc+pF1Fx46tNMycg22Vad20gdrmKgYH1rHhsRby6RR3/M1nD9xeef2NeymX02zr\nH+CBwN/ymUYPVfEleDTBjICXV9N5lsUCtHQ/RX/nI0SqrmZm430IIciXcxhaAK+u4ThlbDtHe/uP\nyOV3szczwLbQPZQD82k3YXbIT0epzLSAF5+m0W2Wieo6tT4P76irOuay2VIe9bbtyTbWHyhbJDxu\nkJe3HdKWjQbUjrhIPVHHcMr4Ny4fCX62P82qwQwfnFQDwHvOwds1iqKMLcexAY2Xf/FTbvnzz1Au\nOESrA5RLRUr5PC0b+5i/fCKlfJkdr3QzaXYVbVsHmbuwmg0/2IIl4IqbprBvSz+1qRL5Kxrwv9RB\n+YoI69/oY9GkFL6uF4lMuQnautBrp8O0ZdB84ynn1SqXWfv4Lxjq7mL5+/6Y/NAghWyGLauexfD6\naNu6EcvYxO6en6IJP3VLF+Ar3s5LT36Zi667AyEcPOZSAnYzhbDD+/wvEo//NULA9lyR1QMZFkWD\nfLt9gLS1mIfwheTIAAAgAElEQVSKjVxZiHB9zxC78kU6S2UuCge4JhGhs1Smq+SQiL6fHbKTmmQN\nt4T8zA0H0ITAlpLCiFvRLYUSUwM+vt7Szd9sb2Oq38s76qooOpIne4cI6hq6EORtBwFUew1Slk3Z\ncW+jeippRgydsiNpK5r4NEGt18PrmTyzQ35Mx/3elO5tXtNx8GsaUUMnY9tsyRZZEg3SX7aIGjo5\n2yHpMZjk95K2bKYFfVhSEtQ0HCBr20gJrUUTgLCuETZ0gppG3KNjS7f2eF44gCEOdhnvEQIHScqy\nKTmSvO0QMTRqvR6kPLLbCOXMGpOakt/1pcjaDh1Fk49PrXvT6SmK8uadizUlv/nWd6mfcQN71z1A\nb2sXxekf4Mb3zmHt1x5F0/rR/Vdy+dumsfbJXzP7yut45Vf7uGzZBGLr2rBn1WG3ZYgE3LYGepWf\n6j+aS+tr+2moT7Hpl8+zaE4P5Hrhjm+6Vea92yHV7rYfidQfkZ/MQB+RRDVQuW2Zy9K+dRNTFyxk\nw6ofEqrPkUu30rVtAM2aRHxSEF1U45DC8DvEpxgkI2/j6ZZuqup0Xhzo553GOrZ276NaG2BT8c/o\nKVn0z5jHv81tQNeP/SRir1mm2mPwWjrPRL+XWq9Ba9HkO+29vL22irztEPfoGEIwL3z8xuvD+k0L\nqxJc/Fd7L0IIrktEmB3yH/V2a6psEfMc/drXkfKUbtEeza5KG7mYodNSMDGEIGXZ6ALCuo4hoN7n\nRRdQctzXkWRtm4zloAto8HnZkSviVG6ZOhIsKRECEoaBVxP4dY2sZdNZKh/I78g2b8equRlZO3Ss\nmqITvQrnXFHrNbiqKnLS449pTUnGsvldX4rWottuYH4kQN52jlsFqCiKcjKejU5h+pM/4NmJc7h7\najU/ru3gyefyLDG3UaiJUm5OEVz3LFooRV2ThX9xksD6NrYWfkh455VM1JuRwTByQYbaphwi287U\nwsNQWsQi+z/A90fIxDT6WvaSmDgZvXY21Lq3iaTjsP2l1cy+8hpsy+K33/oawVj8wI9ZKFDIpsgW\nXqF/8EnWem00z1sI112BrClynb8DrzfO5i1/hU49lK8lX/e3+AydPdVVzPL7aay1+WXhEvbba8iH\nmrh59jTuigepD544iKjxurc8RvZ6PTXg4wvNk057fSdHtE86mQvKYwUkcGptxo5lZPu14eU9nqO1\nVVQdW44voxqUPNjRT59ZZl44QGPAR8lxuH9fN/82e8pozlZRlAvEDQvns3vAINeU5N9rvCzr3EA6\nP8Bvbng71WEvK0p9fI8g1dOv4MGXdhMNell7UZCryjGukIOk0z52Bd+g+Pxz7H3sDSY2JOks+IhO\nWEdowWeZetVd7N++hWe+cT83fejPqGlsolwsMdDRzmtP/Yp8ahDHthnq7mLJ295BzdQmMplN7Pj9\nOuZfdzM71j2B7R2iKJchapfTHA6xIVPgrfUxnknlqNMNBmc+Q4O1GS00l82ZAs8MpPnnmZMPNFC2\nZRJ9xlRylk3oLD/Zpyhn26gFJQXbIaxrvK/x0CrOd6m2I4qinCHzJid4fu5E6oomN9R4ub7wfXIT\n/oH3+STdDdO4/jt/x21Xv5NJscv47a4WUjnJwgU+Hts/A1/3ahbOrqGc6+PGuhpW7r2GTLSWPq/k\n/4au4Pq1a9nQ8yixZJJ593yA3776Ks2vvcre7h5kXw8TPv4pcBy2vbGWqcEQv9TDLM8V6Wv7JRuq\nYP/v38KkKX/GVt+/8ERPin+rTdIU9HFjdQyAXtPi0e5B3lmfZHP2Uh7ZN8BTS6by7gmJQ6r6hxuO\nqoBEuRCMWlDyaPfggZfWKYqijIaW3iz7NJN3JnIs2PdvBLa+TmrCh1hUmsyP1iwhtHQrmF9Edmxk\ncvN6En3/xLTN/8TcBfeyQ3P4teVnmeHj15d/jocm7GChp418QfKR+iCD8+5kxdBrxK1dlJNz+f6k\nuTQYgmTzYorZIeK2hePk2ZUMsd7sYZHs4Hf7Czw6cCXvmjSN16x7ebxY5hP1ET48ufaIvC9PRFie\ncO/Dzwz5uak6ekZuaSjKuWxUgpKhssW8cICoiuwVRRlFL/3uK3zJ2kFrdTX79TaaCnOI9t3IxuRq\n7vLup2bjn1CIb0UUw1Rd+m7WZz+HP1TLjPK/sLBxOpdlfsa28t10p1K8I//3hOvupsGzjUzXBmo9\nVUjhkCluoqn4dv6hLsTXSu9kZqSIp/QjrtDvZE/L15hXfR0TJ76bTZv+ggXRhdy36E7C4SbAfaz1\nZHqhrvYaVI+vDrYVZUyMylEQ9xhcfJwGToqiKGeCNxXnDXkJk40i/b0hnMI7CMgXSHXfyNTyVDb6\ndjNzaBO6HCT33z8nPd9Pv1fwk+1387mbVhCNFdC2Pkx4zyf46Z6P8l+Xr+CfVt7PddPv4uLkNIKB\nqaTTG9H1AMHgNO7Y9nUuYg5iwiK6u5+gqfajJCdci5POMLv+s4Qnzj8kf6f7Wozy/v3k160j/+pa\npGOTvO+D+KY1HTpOdzdGdTVCP/Liz2xtJfvCC3gaGggtXYrmG9+NOYefTnEKBbTAyT0JdKrpy2IR\naTsgHfSIW0Mlbfuo608ZOypyUBTlnHXp7ASb2l7GN+NFUi/N4yW+TJ5qAtPr+D/2T/j0Lj8Pz17O\nXckID+0W6IVeXo+9h57cI/zTCwMMFoa4u/lt2D4f6197gcu++wMcKXm8Zzs/e+tjTDFqaGMCXX2t\nvHVyPctn/ilB/wQMI0LPY9/FfOkZun2/xxoYQI/F0N/zHsod+4nedNMx89zTuZt4SWfgwR9j1NYS\nfestSLOMLBUp799P6KqryL28Bi0aof7zn6O0azdtH/4wE77wBXq++lUmf+c/kdks+//6k0Tfegvx\ne+7BHkpReG091sAA2A7ZVavwTZuG3dePuXsPTi6HXlUFQuAU8kRvvhnvFPeBA+k4FDduxOrrQ9o2\nWiBAYNEl6GH3qR07nUaPRk9r+xzrUdhyZyflri6Km7eA4+CUimA7oGnokTAiEMBsacGIx0E3kKUi\n0rLR43GEYeCUijiZLE4uh7epCRzb7f3W8CBNt18SLeAHobnfWRbC4wFNR3g8ONks6BqyZCI8Hvc7\n6SDLZdDdLt6FroGmg5RI23Lf1mzobpoHHtgduWzDXbdrwwtfGSYO9oJ6eJfuBz4P95Jy+HCOMf5Z\ndKy8j2BUJwldccUZmZ0KShRFOWdFZu1gXriXtduuYsaKy5g0cyI9r6zmjbjJ1zffhvO7R+mfsIHV\n1UtIl8p86o6HeOB/vso3H/Ww6eIdaKte53fv70er07lohsXGXc288Vef588+dzXf2P4nXDbpYjZP\nLnLb/S/z6gdb6DHyLL78XjZ+++PMnbYcu7ObxN13Er7+evZZ3YRWbyXzzEpK5SJ2Syuxa66jZWAP\noWAc+7UNVF27gvV/cy9zay9CrriSYLSWwrp1pB7/FaVdu0jedx/7/+oTBBYupPrOdwDgnzWTqT9+\nkK7P/h2R66/npS/8JQ0dJSb+3WfRk0kGHngAI1lN5Prr0ONxHNMkftedB2ocpJTIctk9+QqB1dvL\n4EM/w6ipxtzXglFXh5PJEL3tNkCSfuIJci+9jDRNvE2N2P1uN+laJIoWCGAPDVH1rnvQYjGEEAwV\nh9A1HX+2TO6ZZ9zu2AsFZLGAdCRCdwMBAOExQAi0SARPfT1V732PO/xY74Wx3fdtCcPATqfRwuFD\nut2Xpomo9HY7FkzbRBc6+lHefOxIB0c6GNrJnWadSpf12oj32kgpsaR1IN6Rw/9GBAiHfx4eNjz9\nMFF5D9jR+lN5s7RTeBfPiYz5u28URRkfzsXO04rFDlp3f5cdq4M0XHodlyxeTHbVKtq+8iXa5i4g\nfHGC2lW7kYvnEzc9eCZMoNzWilMqEVy8mJZf/pTEihuJz5zHll9+n5isoe6yRbywchUri88zO3sR\nM6fNosWIovU/Ti4YYsnPN/Lft0SwrvxzptQnKPu24tf97Evv48MLPkzr5pfxPPY/CClIl1Kktm0k\nOm0Wj9S18K43QmgL5tLyzqX8eu+vmZOYw7tnv5tvrfs6s2rm8uHZ92ILieH1H/VE3dXfymP3f5T+\nWXW01Ot8ftnn8epeEv4EOwd3Mj0+HUMzWN2+mqsnXX3cdSctCzuTwag6ss8oy7YwDnunT7m7B4DN\nP/8OkbxkT9sGhqq8hIWPRLiO7XoPvisuJVY9iZ5cF5rXvWXkN/w0RhuZHZ9JSZYZKg3h032kzBTz\nkvOOmT/bsQ852TvSYefgTvam91K2y2TLWcp2mdpgLZa0kFJSskuU7BIRbwRHOng0D6ZtkjEzxHwx\nLMfCb/jJmll8hg9d6FiOhUf3uAGAY2FoBhKJIx30Ee/zEUIcMcyjebClfdQTuxBuAOA4brAx8p1W\nwy/4HGn4xO5IZ8QLQAWaph34+0BgcXi3a+LgO2xGBh7Df0spDyyTlPKIXtuO95LQkxmnyl/FxbUX\nnzCNA2mN13ffKIoyfpyLQclIGzZsoKGhga1bt5Lfv5/gxImsXLmSz/zJnzD0m6dI3n0Xux5+hOqq\nKsKXLMJTX49TKtHxqU/ha5qGHosSWr6czFNPkfjgB9mec3h1ezdTN6/hn8xGJtTvo8P6NZeWNBqa\nPs/68ldpT3dz38J305ppZWXLSpKBJD7dxy1NtyDaOrlowiJqklP4x61fY1psGpQt3rvg/fx676+Z\nVTWLF/a/QH+xn7ZMGxFvhCV1S9jSv4UvL/8y96+7n08s/sSBE8zD2x9m28A2PnvFZxFCsC+1jzWd\na0ibaTpzndQEatjUv4kpkSlMikxia/9WFtctJhlI8mzbszRGG/nDuX+IQNCd72Z/dj81gRqebXuW\n+dXz8et+5lXPoy3dxl8//9csn7Sc6kA1zVXN7B7ajRAC27HZ2LeRkl1ifnQ2b4tfRbyqHj0ex3Is\nNKGxuW8zU2NTiXqjSCkxHZPWdCtP7nmSulAdCX8CicQjPPQV+hgsDRLxRvDpPhrCDWzt30rEG2H7\nwHYaY414NA/9xX5KVombG29menw6Xt2tHdGERq6cI2gE1btwziFnPSix7SKZzCZKpS4sK4sjTSbU\n34mun96OIx3p9v8rJdIBbAdpS6QtD/5tOWBLZOUzUrpVXnL49dC4aTjy4PeKcp7zzYijh0+uevtc\nD0ps2+ZHP/oRN910E4899hjTpk3DNE3syi2A22+/nVdf/D0zJk2ketq0A9P1f+97JO+777hpr97Z\nyyt7B9jXn+e915a4YsIVtGXa+OjPf849827h2uYprN7VwYa2Ak31JhdNqGPJlAYAPLrGxt6NxH1x\nZDnJlGSQ7V0ZZtUf7Ja7O9vPv67/NwJOMzfObOZXex7Hr/u5ZvI15Mt5HtnxCHc238n1U67Hbxy7\na/nV7au5auJVB8rZlnQLuXIOgFe7XuWBzQ8wt3oui2sXUxeqY0PvBu6YcQcPbn2QKZEppM00+XKe\n9855L8lAEr/hZ03nGprjzQghqA+5/U450jljVfYFq4BA0JnrRCBoCDccCDrg5F/mp5w7znpQYuXy\npLdtx6vVogkPjizTZz6F6fRjiCgSm2rvCjwicUQ10lF3QAEIgdBE5X3yAqELtyFS5bcwBOhaZXil\nYdHwdMPvoR9OQ6/8VhTlgHM9KAEolUr4fD4GBgbYuHEjixcv5re//S3lcpn+/n4GBwe55ZZbsCyL\nBQsWEAgE3AaS2olPsEcrmx5/o4MHX2oBAQsnxbh1QQPPbO1mem2YdKFMLOglFvBwzcwaHny5hdda\nh5BS4jU0bp5fT9G08Xt0NE3w1I7XyRd85PI+ps74PTdPu5rf7thIu/ky75z5TqbHp9MYawTg2W09\nvLCrj8/eNveU1s9wAHC0wKY7183rva9z/eTr8egHu2Pvy5YQQG+2hKFpzKgNn9I8FeVwZ/3dN0Yo\nSGLxokOGhTl4JSKlpLPzEUAgcXAcE014sO08EhuvJwEIhNArwyTBwFQSiSsplXoxjCiWlcLrrUFK\nm0xmA8VSF5HgHAKBRrea0S5i23m83sRoLKKiKOOQr/LoayKR4JprrgHgzjvvZHBwkK1btxKPx1m5\nciWpVIqOjg6SySTLly8/qbSPdrV++8IGrpyepD9nMrPOrfm4eHKc76zag6ELypbDL9e3c/GkOI+s\na2dyVYAPLZ/G5Kogf/7Qa7xtYQP/6xcbWDG7lo9edwVvtA0R8Xv45CND+JaGMZwrmK4t5dK6qYR9\nPtLFMql8me+s3sPlTUk+99+buHZWLdfNPrJzNseRaCMuvgZzJlWhAGXbYXdvlhd29pEtWdRF/dww\np5ayGeG59Q0kyRANePjd5i5qIj5+8kobV89wXzIY9hu83jbE5KoAliOZURvmiQ2dxAMeipbN1TNq\nmJwI0NKfpyropW0wj8/Q6EwVSYS8zJ8Yo1i26c2UGMqXmT8xiuW4bwU2TuHx6aG8SSzgUTUo56Ex\nb1PiOBZC6CfcudKZTaSG1qLpAXTNfdTLKqcAQSy+mGBgKtnsdgqFFqR0EEJH14PYds5tYCQdHFlG\n03yVBjsC286haX6E0NE0nzuutPF44oDAslKARjx+KYYRxTR7cJwSodBMNO3EL39SlHPJ+VBTcjyF\nQgHbtuns7OTHP/7xgeHvete7mDx5Mrqu4/P5zsiJTkpJqlDG79H5g2/+nrBP5yt3LSQZ9hLxu40q\nu9Ml6mN+/uuFvdxyUT0TYgeflvnB7/dx07x6JsYDfOaXG1neXM3iqQkeXtvGnt4cH7lmGk3VIVbt\n7OW3m7qZVR+hsTrI9bPdl+SZlsOXfrOVkuUQ8uoUyjZ9GZOrZ1ZTKjsMFcosmhJn4aQ4Xaki33th\nLxdPjrFoShU/XtNK+2Ce+65q4tpZRwY7j65vp1h22NefY09vjv/8w8VommAwZ/Li7j5eax1izoQo\nmoApiSCOhFjAw9bONIN5k0TIS9BrIIC2wTxSwrauNH9xw0wifoOIz0BKWLN3gP1DBa6YlqBjqIgm\nYP7EGBv3p1jfMkhnqkjJcvjEjTN5Ze8AQsCEmJ9dPVm8hhvgzJ8YY2oiSFe6SMTvwe/R8Bk6qUKZ\nVTt6sRwHy5YYutt41NAFpbL7zmBdcxt3Fso2Xl3DlhJDc2veNSEOtAAYubtIefDJXb9Ho2xLbEce\nGHayu9apjDse1IR9LKsErifjrN++KRQK7Ny5Eykl8XicZDJJKBQa86j28OpXKW3AfebccYpomnuV\nZdt5pJR4PFGkdBgYeBHbzuP3TwChkU69ga77kdJG03w4TgkhdBzHRGgeBLp7q2g4bVlGCAOBQEob\nIUZWUI14Pv2IYbLSCls7MNznm4BhhCtpVFpUOyWQNkLz4vfV4/We/M6hKMPO96BkpL179+I4DoOD\ng7z++uu0t7czf/58rrvuOt544w2WLl1KYBQ68TpdH/3xOvb0um1DnvrLQ2t2HEfyo5dbWLmth2nV\nIcq2Q3/W5NNvnU1VyEvnUJHJiQDbuzJMiAWoj/mPUYty6o/Wliwb3xnouduyHR59bT896SJ+j05/\nzuSypgRNyRBr9vYzsy5C+2CBTNFiciLA1c01AAzkTFZu7eaq5mqqgl7aB/NMSYQOBCUv7OyjN1sk\nEfLRky6iCYHlOET9Hq6ZVUPQe+jNgmLZvZ32ZkkpKVkOhiZOqQboQnHWg5JSvkzLpn6klOQKGTK5\nFEWzABJ0QyMU8xGK+5BIAoEAgUCASCSC1+vF6/Wi6yeuObkQSSkpFvdXanQchNAAgaZ5EMLAcUyK\nxf2Y5YFjPL7lBjqH/j6lHBxnmuEAyq2FkjgjgqlTTesCMsr7eaJqGV7vyb0E80IKSkYaGBhACMHT\nTz9NMpkkk8nQ1NREc3MzwWDwrOXjeJ7d1sM1M2vcpnLH2Gee3d7DsulJciWbsu1QFz12g1hFGUtn\nvU2JL+hh5mXDbweecMh3lmkz2JVnsCuHlFBMF8lYBbr0FKVSCaFLQlUeNN098IZrN4aftR5p+OD0\neDx4PJ4DQc3w38PDtZNoxHYuEEIQCEw67jih0PSzlBvlzToTnRYpb14i4bY7u/XWW1m1ahW33XYb\n999/PwANDQ3oun5gnLFytDYjR4xTudVyJmouFGWsnPUeXQ2vTs2UCDVTDj4OJ6UknzYxPBqlvEX3\nvjSOefCxXSkhkvATqw0QivkOeXLGcRwsy8I0TUzTpFwuk8vlGBoaOvB5eB7AYbdv/j97bx5kx3Hf\neX5+Wce7+77QuBoEQBIkSPAAD1GkRFmyblGyNVpdG7bHti57d+y1HZ61NV5pIiTtODZmLHs0Edba\no5A9WlnSSKu1KIdlSSNSFCmKIEFKICUeAAECII6++/XxrqrK3D+q3uvXjb4AdKMbYH4iKl69qryq\n3qtffTPzl5nzZsFr6t4REVpaWshkMrS2tjYc6CyW1cK2Bm4scrkcb3nLWwDYuXMn3/zmN7nmmms4\nffo09913H4VCgb6+vmVSsVgsF8OGmGZeRMi1xi/9VNajpWtuX67RhplileJwmdNHJtBRvfVkbph4\nyC+4bhpRaXK+Q64zRabgEYUaL+Xgp10QqIaaUi2iVAsp1SJmqrOf5SBCRxHhqTJhrUhQPkIU1JKm\n9nO7HRzPw0tl8XMtOJ5PPHJobe/Z+XK5OU5ZVoe7d3XRmbeC+nz51V/9VV7/+tfzta99jVQqxQMP\nPMCrXvWqOaKkVqshIniedXq3WFaLNRElI+OTfO+Hj8fj/x0HUU48D4Byku8KIw7KcVCpDJLKznG6\nWgrjxqVe7AVrjMGEQTxp2owmOjWOLkegBBNqTC2eZtdzFL7rkHIF33FIeYqU59DiKDzXQYmDcXy0\nr1GZHjL5WNgYA8oR/IxLKuOiHKFarVAql5gojlOdrqIjg+s7OI5K4sTe10opVDJHiqhYjAXVeIpi\nUbF7togiiiKiwBDWInRk5qzpJCrxFhEhijQ6jKcwFlEYMQ251PApab5PydQtjU+IvUAakZr2TZJX\n0nWmlGrsN6enJJn/BcFxVHJeMNqgo6Y1GRrTHq8BSyW6VA/JMoVZWVlXJZE1w4/aACtKzhcRoa2t\njQ9/+MMcOHAAx3EwxjA5Ocn09DQnTpygVCrR39/Ppk2baG1t5fTp0/T396930S2Wy5o1ESVt+Qzv\neO3NKKXQUYTWETqMiKIQE0VEUYSOQnSkqZaKVGdOL5zQEqsSzqn6L9QMUD+WJt4WWulwwTzB1AxR\nPbwIUU0zfiaKBQWC1oYwiAirEYiLn8nieh6eEVKOg59LEQZgauCl4pYZHen4JR0vNglGMAj5rI84\nqrHSASJ4vouf8fCzHq7n4rgKEqdRow1axzPbur5KzjWVHTO7eFPj08yeN8lO03mtNUbreMZcM3uv\n4nzi9RJ0FIur5kUtDbFIM8aA1oShBonXVXAchZNMbGfM7DoPq+HeOn/pqfq1LN4StMCJRf4GZvET\nKyxP84n19xnx/A3RGHpZc80116C15sCBA3z729/m5ptvZnh4mLvvvpsTJ07wla98hY9//OP85Cc/\n4fbbb2fLlqX9viwWy+KszeRpnke+/cqdtKw+A6QxhigMqZVLhLUqGIiikLBWQ0fxUtr1sJIs4mS0\njkWAMRgdYkzQOIaOV5UMKppayWCGNcbMho8TaBoIf8EXcK6gi8sYD2E2Wp9fX48xjRkx5/jp1H2C\nmvaXpfnabH/TRdPZfRNkbEvJxdDa2grAjTfeSF9fH8YY0uk07e3tDAwMMDAwwD/90z/R1dXFl770\nJT7ykY/Qnixyd/z4cbZv377iqdKNMZw9e5ZNmzYtGiaKIhzn/JxZH330UbZu3UpPTw+u6zYGDkRR\nhOu61Gq1OX5zWuvGAIFisYjv+5w9e5a2tjba2tqsP5RlzVjTalSkI0phiVJQohyWG6s3tqXaGgsp\nTQfTjFfGmaxNknbSVKIK5bBMoIM5w1rTTprABEQ6mpOHII3lnBWKvJ+nNdXKSGmESlSZk0Y9XPOx\n+fFTbopIR2ijiczcvJpXbpx/fEEExJFGPEPcTWPUAitKJueXYqGVJRcr08VQL0dzmVayfz5caLzV\nyGe21cacc6zOQvf6cqTXj9gYg1ovfzZt2sSmTZvOERgdHR1s27aNG2+8kUKhwMmTJxkZGWFkZISH\nH36YX//1X2dycpK+vj7y+cWnaC+Xyxw6dIijR4/S2trKW9/6VgCCIMBxHJRSTE1N8Vd/9Vd87GMf\nY3h4mGuuuSZpwYwFRrN/y8zMDNVqlWq1ysTEBH19fXz5y19m586dPPnkk+zduxelFI7jMDQ0xE03\n3URvby/lcpmvfvWr7N27ly1btvD0008zPj5OtVqlt7eXarXKjh07aGlp4dixY2zbto1UKsX4+Dg7\nduxgenqaU6dONVbI7enpYevWrWQyGWq1GmNjY4yOjtLV1UV/fz9hGOI4TkMYPfbYY+zZsweAoaEh\ngiCgr6+P8fFxRIRarUa5XOb06dPs3bsX13UpFAoEQcDU1BQTExO4rsu2bdt46aWX6OvrI5PJkM1m\nmZqawhiD67oL/hbNYmw+dg2eS8OazFNSrBZ5+MRDKAM5N0tGUmSdND4uU9VJipVxTBSRcdLkVZYW\nv0DBzVGNqqRUirSbwm3MmBr/Caq6giPu7JoMTd0r9U9jDDPhDJO1KdrT7WTczOyfaI4vxLwp+JKH\nWhtNNaziiUM89VnyWmp0W5hGeACUQhwn9pVRAkoljh9NomeBci74icx+XWGcRi6LXdsK9u1DZqnz\nSp2nZLX5m7/5G06dOsUHP/hBfN/n/vvvZ2RkhHe/+93s2LGDVCrVEA8vvfQSjz32GPv37+ehhx6i\nXC7ztre9jZGRESqVCi0tLRw5coSjR4+yb98+Tp48ya5duxgaGsL3ffbu3cvTTz9NEAQEQUBbWxvZ\nbLzw6eDgIC0tLaTT6cZU+sVikRdffJGbb74ZEaFarTI4OEi1WmV4eJjTp09TrVa59957qVQqpFIp\nyuUyu/zxAZcAACAASURBVHbtQkQYHh5GKUU2m+W5555j37591Go1ZmZmCIKA06dPE0URu3btIpfL\nEQQB5XKZM2fOUCwWKZfLXH311Xiex+joKJVKhRMnTpDL5Roiau/evaRSKYrFIv39/biuy4EDB9i3\nbx8iQj6fx3VdtNYcOnQIrTXd3d1kMhmUUvT29iIiHD58mN27d3Py5ElGR0dJpVKopIW7Uqngum6j\npaiOiDT8h5qno4iXLoka/nUwK1KaR3bW9+vx62Hr5xY6dj6stLVtoTgL5VUvw1JhVkI+n+eqpkUu\nl+PSrxI8McHMjx4C1wNn1skVJfFLXFTyEk+OKRW/0Oc7ZzaKdm6XwPzPud0G88LOd7KYv98QEk0v\n/fokRc0bc4+bSEMUxiN/TNz9YrRZPN/5ZW3y9WjsrDTs/DyWuL5zfuPmr4v9/vO7UVayv9G5WB+P\ni7nGdfIvyb36LtwVzrFhRcnq0VzjPnbsGN///vfp6OhgenqaO++8k127dvHMM8/wwAMP8M53vpMf\n/OAHzMzM8OEPf5h0Ok0QBBw8eJCuri5GRka47rrrKJVKHDlyhDvuuIOvf/3rdHd3097ezp49e0in\n00xNTZHJZHjxxRfZs2fPkrX+pRgeHqa7u3u1b8mS1MtaKpU2zIR1lrXjkosSKpNw9AGIgtljDV8I\n3Zw9y7s+mqaw9e8LtH7U82iesVTqLReJ4DHR3JaO+ektmPdl8LJdEWt0LZeDGNnIrPX92/FayK1s\n2QErStaOAwcOsGPHDp588knOnj1LPp+ns7OTe+65p9F1UavVluzeaebll1+2DrWWy5ZLPqMr6Ra4\n7p1rkvSKMUnrRV2oxGN5Z4WK5cpjrVoj1qyV4xK0nsiVMZvx5c7tt98OwBvf+MYFm+Drs1GvFCtI\nLFcqV+54QREQO93yK4q1EptWxFpWCevDZbEszXl134jIMHB87YpjsVjWke3GmEvrTLAA1s5YLFc8\ni9qa8xIlFovFYrFYLGuF7XC2WCwWi8WyIbCixGKxWCwWy4bAihKLxWKxWCwbAitKLBaLxWKxbAis\nKLFYLBaLxbIhsKLEYrFYLBbLhsCKEovFYrFYLBsCK0osFovFYrFsCKwosVgsFovFsiGwosRisVgs\nFsuGwIoSi8VisVgsGwIrSiwWi8VisWwIrCixWCwWi8WyIbCi5DwRkXtF5F1rnMdviMhNaxlXRD67\n0HcRee+F5LvaiEiniLxhgeMLlnvesbtEZOtC55rjiUhBRP5CRD51offbYlkLrJ25NFyknfmkiLQt\nFab5nLU3K8Nd7wJcLojILwOvBjqBR0XkD4F+4N8B/waYAZ4B7gU+m3xOAB8FHgRuB34C3G2M+TUR\n+WOgBrQBXwT+A/AoUAR2A9tEZMQY83JTGT4NDBL/bk8CH0jyHQFSwM+SoB8QkfcA3wI2A9cCLcCf\nA38AnAJuEBEB/gJ4FhhI4r5KRDLAVqAD+Ftgf3LdtyflfD9wHHjUGPNUUravAv8IBMBrgFHgYeBu\n4AxwK/BxY8ywiKSA/wN4AdiSXNMDwG82Xd//C+xNyvhqoAv4PpASkd8GXgf878B1IvJ2Y8y3m36u\nq4ES0CMiHwZuM8Z8iHN5A/BN4MfAZ4CfLhDGYrlkWDtzWdkZgF8TkduS67X2ZhWwLSUr5z7g08A/\nAZ82xvxH4AlgL9BnjPmcMeZBwACSbACjxpi/BirA3wAviUgL8HpiY5IGfOAxY8xfAjcBh4FvzTMU\nXcD1wCTQnhz+HvCfiY3OJ4kfKoDvAJ8C3plsp4kNyg6gOynPYaAHKBpjPg+cnXe93wb+nvghv8sY\n85+AA8m5J4A8UGgKP2iM+TIwDPwA+EvgruTc14B/BvYAGGOqxMZtU3J+F7GRnH99zff9B8n3wBjz\nt8DB5B7/YgFDUWfEGPN/EwuUxTDJZrFsBKydubzszJeIhcYerL1ZFawoWTnfBj5G/JD/aVKD2U9c\nazkjIr8rIq8lroX8LnENBmJFD1AzxhhAE9/37wE54EXih735j/oi8F4R2VY/YIwZAQ4BGeDnTWlr\n4geoni7Am4lrVvcT1yo6iB+UE0lZ30P8gA4BmeR737zrrT88irjG9gfAHUAItCZ57xGRd4tIX1KO\nheLO+S4iHxURBRwBHOKakFnk+ur3/U+T+x425VO/XiMi72Zh6mEXMwLfB36F2NB+eZEwFsulxNqZ\ny8vO6KYw1t6sAhL/fy2WxRGRXcTNmNcD/zapgVyqvHuJazF7gP/TGDN8qfK2WCyXDmtnLGBFyYZG\nRG4Hrku+/sIYc2Cp8K9URMQn7veu82VjTG1eGHsvLZYFsM/GyliJnWkKa+/pBWJFicVisVgslg2B\n9SmxWCwWi8WyIbCixGKxWCwWy4bgvOYp6erqMgMDA2tUFIvFsp4cPHhwxBjTvd7lsHbGYrmyWcrW\nnJcoGRgY4IknnlidUlkanJw6yfNjz/OG7W9AG81EdYKMmyHjZta7aJZXECJyfL3LAOdnZw4OHuTs\nzFlMMgqzxW+hI93B1sJWWlOta1lMi8VygSxla+yMrutANaqSclK8OPEitajG0yNPs7WwlU/8+BPk\nvBwpJ0XWzQLwvmvfR8EvLJOixfLK5NbeWxv7xhimg2lGy6McGj7ERHUCgHiyTsi5OXa17WJLYUvj\nmGVxtNbUajWmp6dJp9MAOI7T2MIwBGBiYoJKpTLnngZBQBAE+L6PiDQ2iH+PelzXdXEch0wmQyaT\nwfM8oigiiiKMMXO2enkAoigC4t88iiK01mitqQ/caP5ej9uclmV1yeVyXHXVVauSlhUlq8yZ6TPk\n/BwtfgszwQwPnnyQuzffje/4ZNwM9794P6enTzNWGePuzXeTdtOk3TSv6n8Vd266c86DHeqQLzzz\nBd61613kvTwPnXqIN21/E4+efpRyVGZ7YTu72net49VaLBsHEaHgFyj4BQZaB+ac00ZTDsscHj/M\nT4fj2b2zXpb9vfvJe3kc5axDidePiYkJoiiiWq02XurFYhGtdcMGKaXwfZ9cLkexWEREGoIhiiIc\nJ75nLS0ttLa2znnZ+76P67oEQXCOuAAacaMoIgxDZmZmGBkZIQiChuhpFjNKKUQE3/eBWMwYY1BK\nNbZ6ms3h658LpWXZmFhRsopEOuLRM4/ygxM/4E0Db2KkPMKbB97M5w99ns35zbjiclXbVbxj5zsI\ndICnvDnx5z8ornJ5/7Xv5wvPfAFPeVzfeT1//vifc1f/XYQ65LGzj6FEcVXb6ijUiyXUITPBDGk3\nzTde+AYFv0CoQ7TRFGtF2lPtlMISbak2BOH1219PLarhKY+0m17v4luuYJQocl6Om3pu4qaeeC20\nYrXIz4Z/xmh5FCWK2/puoz/ff8nLVq/JN4uBpQiCgCiKKBaLVCoVpqenGy0LItL4rKfd3EpRFwWF\nQgHP80in04gIjuOwZcuWxot9tUilUquanuXK57zmKdm/f7+xPiUxkY7OqV198sef5HVbX8dd/Xfx\nP07+D9488ObGuefHnmdTfhMtfst556WNRklsqGpRDd+JawvFapFHTz/KC+MvcGvvrdy56c6LqvE9\n9PJD7O/dz+d++jm2FbahRFGNqvTl+rhj0x2cmT5DZ6YTT3l85bmv4CiH9lQ7N/fezLGJY5ycOkl7\nOl5O4g3b30DOywGxIQxNOEeEnZo+xdPDT5NxM1SjKtWoSmQiHHHQRmMwaKPZkt9CT7aHlJOiPd2O\n7/iNe2FZXUTkoDFm/3qX43zszOTkJJVKpdFcH4YhnufR1tZGJrNynyxjDI+cfoRitUixWuSeLfew\ntbD1Qi9hWSqVCqOjowwODlKr1RpljaKoUZuHuBui3lWilGp0V6RSKTo7O8lms+RyOTzPmyNGLJaN\nzFK2xoqSC+Tvfv53PDf2HJ+5+zOICNpovvLcV/jAng8sH3mVqYQV/uG5f2CoNMRgaZD3X/t+buu7\nbcGwpaDEWGWsIWwmq5M8Pvg4aSfN1e1X8+TQk2xv2c5rtrymEefszFmeGnqKrkwXw6VhirUi9+28\nD9/xCXXI/S/ez629t3JV61WrahSNMQyVhhguD1MJK0xUJ5iqTQHgOV4jTP3+u+KilCLS0Zx0mkXd\nnPQT50iFauxfSgSZk+/874uFgdmyC0vf7zv776Qj3bGy8lyGomR0dJRarTanCT8MQyYmJiiVSg2R\nopQil8sxMDCwbEsEwCOnHmGoNETKSdGX6+Pq9qvJ+/kl49RbPEZGRshkMhw9epRCocDExET8H9Ua\n143/o57n0d3dTVtbG657+TRYN3w2DASRZroakk/F5fecWDQZoP5aMRiMib87SnCVoNTF2whjDNqA\nNgZt4jx0cizSZsnzWpvGfvP7r+HzAogs/2xtBDaKBk15ip7Cylu7rShZhvo9WOkL9bmx53jk1CP8\ny0v/wp/c8SfkvByHxw+zt2sv21u2r2VRF8UYw2BpEFe5PH72cSphhW0t2/jZ8M/ozfYS6IDxyji9\n2V46M51UoyqPn32c12x5Dbf03HJF9KkHURA3RYtja4wXwOUoSlZKtVplcnKSEydOoJRiYGCA9vb2\nZeMFUcCp6VO8MP4CxfEiRhncjItooTJRoVausbNlJ7VqjYJfIOWn6O7uZnp6mu3bt1MulykUCisS\nQs1obeIXowjHRmY4PVFmcLICxC9d1xG0Btdp8kGL4jhO00u/2byLxN8NBkEa3+fTfLz5MTIGQm1w\nlTQERj7tMlMNASGI9JyXebxfT0OItCHUmtVCiaBEcFTiRyKCkvi4JJ+Omt2vn5d54ZrvUyyo1qOK\ncv5cWn/dpTMrpD2u7l35gAwrShagFJQaNf2vPf81bui+gWvaryHtpjk+eZwHTz7Iv977r+d0l9T5\nxyP/yH077+PIxBF+619+izcOvJGCX+D3bvm9dbqaczkzfYbB0mCj/9xiWY7LUpSMHYW2ASiPw9QZ\n6L1+2eqjMYZnn32WMAzp6emhtbWVTCaD1pozZ84wNjZGFEWUSiVSqVSj9aW3t5dyuRx3FxnNli1b\nyGQyHBg6QCaV4cWJF/GUh+/4yUtfEIQ9HXuYmWnn4Mmz5L0WBPDdpHumyf7Gtfm4fL6rCCKDAB05\nn+v7W+gupKzYtlwRLGVrLp92w1XmEz/+BMVqkT2de+jJ9jBeGeevD/01O1p28OiZRwl1SC2q8cWf\nf5EP3/jhRrwfnPgB5bCMiLC7fTc/fO8PKYUlHNlYLQ2b8pvYlN+03sWwWNYWreHpr4HjQfsOePxv\nId0G5THwczB2DFo3x5+3/Dp07UJEuO6665ienmZycpIjR44QBAEAmzdvZvfu3Xiet2Knz9cNvA6A\nOzfd2ThWrkWcGCvx5Ilhvn78MKF/lD09PQ1H1BDozfVydfvVTAfTBNGs4/toZZQbu2+kFJTwlLds\nt5HFciXxihUlr936Wu7ZfA9HJo5wS88t/Oa//Ca/c9PvcGTiCH+0/494euRpnh55mmPFYxw4cwAl\nil+M/oK0m+Z9176vkY6INBw6LRbLJaZrV7zV2XzL3PNBOW5FKWyCZ74BL/wz5LrhxveS11Pke3vo\n71+dETeDkxV+cnSUqUpIS8ZjoDPL+27bgcjCo+NOTZ/iWPEYKSdFxs0wWhlFELYWtvLgyQfxlEdN\n16iGVbTRTNYmuav/Lna3716V8losG5FXZPdNqEO+f/z7vHnH7OiYUlAi62Ub3yMd8WeP/Bkf3fdR\nnhl5hvHqOE+cfYLfu+X3zpkDwWK5Ergsu28uhBM/gSf/HvpugJHDsPfdUJ2Ctm3Qe93y8YHDg1M8\nPzhFEGmCKLah3fkUN2xppSt/4cNgjTaQ+D0YY6iVQ1zfwUm6ew4OHuTw+GF8x8cRh7ZUG325PvJ+\nnryXR7SQVlkwoBwhrGmiUGO0QUcGrQ1BJSKsRUSRQYeaMNCEQTTXbSApg4jguEIUmoZTuXJiZ1Vx\nYj8NcYTYZUZm40KjG63R41T3NxFQKk5HEp8PSRxQ4u4pg9EgzWle2N288JgX46+xjg4p6zUxnOs7\nFDpWx9H1FdVScnr6NN888k3eMvAWOjOdc841CxIARzl85p7PANCf7+fvfv53fHTfR60gsVgud7bd\nGW8Qi5GZYTB9cPYQHP4u6JBxv59qYStdLRl0300cGy0zNlNjaKpCsRywrSPL66/tJePPdvGEtYip\nsQqjxSCZaEwTBRodaSZHKvgZlyjU6FBjiEVDFMzOT1J/QWttUEowBlIZlzBJA4G83szNbEYUzNRm\nUOIwWC1yPJqgEpYxYqjoMkoJOjLgGRxXMGJADEYMXkbh+g6RhERE+BmXIFvDc1yUKCITgREUCsIQ\nE0bgRLiiiZw0EQo0CAqCkKtbrseXFJEOmaxOgYAnHhk3Q9pJU/BakNo0xkmDcgBBT42i023ooD6p\nGpD400QmpBSWGauMxUOixUGJwlUu2mi0idDTQ4RRDSMKLRAqFy2CDspERhM5LoiDEUBHzKocARMB\nOnHi0WA0xoTJKB096+ezkB4SmXdCEs/gMHYIotmR1zQJlBWIhTleyQsGWCb+shksf+4CNWBnSyev\nvfOdFxZ5Hpe3KCmPg5eDsAKlUYhqoFxIFWB6EFo2g5uGsMJMbZq/+PEn+NC2N/Fff/RnfGjzG2D4\nOLRuifuic92wiIe8q1x+64bfOveE1lCJp7LGz4Prx3+sMz8FJwV+FmolqE3Hfd6iYHo4fkC8TFw2\nLx1/quSncNOAieOnW+J4y6Gj5GFPMGb22EIu9mvhLGfMxhmfZrEkGGMoBxHHRmaS77C5LUN7LnZe\nn4hSfOeIy0Q5IOXexPbstYzpHOkXnkGCEtUoIFv5Kh3pkJTjsttzccQgLxlOHhtAZ7pRmdjnw/FU\nXFvUEeK6KCV4eQ8RoX93O9VSgOMpXG+V/M9mRuD0U1Aai7+LxLbF8SCsQd9eGH4+to+iwPEhCjA6\nJDQODi5K12A6is/pILZDxiSfetZu4VDzW3FLE6hgGpQHykXrkJ8d/ybTAi5Ch5PFCAQ6YsoEDEVV\nJsIyxvEgCmbfeV4GCSpgQkxdLGAAwVUOBZWix8ujkFhkAKGJUKJwMLjtm1FuBteEOEbjhDUcHeGk\ncrgonGAG0fFssuKkYlsYBXEeykOUFx9TbrI5OCoWZQpZ2KHYmFjQzCqo+onZdOrXkow4aoiYOZ+L\nscS5SxnvQuJ4q7dO28YXJUEZTh6AqbOzL1hRgIlvRFCOBUG2k5IYwqBKi9GQ64KXn+Do9EmyXp5P\nHP3vfGzvb3N19438zta76PFbYvEydhQGn4byBOh4aBsQ//nEifPRYbwfVWMRVEckTgPiGlcUxPE2\n3RQ/0NNDscjI98RGwmjovzkWP0EFwjKE1TiujteCIKzG4XSQpJkcb6hcOVdo1I1R04ONcmJh0hwG\nFmiXNMz5oy31h6wLj5U2EVqRcnGsRlPsztfFz8IVyoFjY5wplpcMk3Idtndm8RzBaMODPztLWI2Y\nGi6RRrh5aztOxscYKE179HsOfa+/l7be7JLpcvIAPPf3kNsBpRFIbYUz47GjLcC+98H4S437n1Jl\niAw4hYUrQFrHFSsdxratXtGoFONnuToJZw7BzFAsCtwU7PplyM1t9aVWiuOffhKufXscTiS2JY6L\nAHOqOiusUPgLHFPAzcvGtFhWzsYRJVODMPJ8/CI3TU1gbjp+kV/12mWTeOzEA0ROxGBpkHduvY2g\nbQv/z1OP8fiJx3nnzney76o3gQibmyN17Y63S41dANhiuWi2RIq2YJGWh7qmq4GZKaEl7iK5rSOP\nl3JpvX0L6dwKWiIXY+vt8Qbxiz0K4tZSgMnT8Mhfxn4rpw7CxHFo3RaPCBp+Ftq2xyJDeUkrRRi3\n9rrpOI2pwbilNQog0x6fTxVg4O64dXcp/ERMbb517nFnEXNvKw+WDcTaiJLyBBz+XtOBeR5U9dp5\nc+091wldV8P2V8/tijgPZsIZjowfYaI6wUuTL/GNw9/g7Ve9nY/s+wgd6Q47xt9iucLo39W23kWI\nEZkVJAAt/XD37y8c9pq3xHOqtCwx6icK426XlB0ObHllsTaiJNMGN75nTZJejo/s+whPDT7Ff/vF\nf+OPb/vjcxxaLRaLZV0RWVqQQNyq4VhBYnnlsXG6b1YBbTQZN8Ndm+/irs13rXdxLBaLxWKxnAdr\nIkqCYJLR0QfPOZ5K9ZHNbsf3ey6qK2WhFXqfGXmGlGOXybZYLBaL5XJlTUSJ57XQ13ffnGPGaKrV\ns8yUjjI29ggguG6elpYbSaV6V5x2Jazwuac+xx/d9kcEUcB3XvoOrnI5MXmCN2x/wypficViscxS\nn0DMYrGsDZes+0ZEkU73k07P9qWG4RTF4k8ZHf0R6fQm2ttfhSywxPxQaYhvvfgt3n7V2/nsk5/l\nhq4b+MYL3+CLP/8iWwpbGs6sFovFAslKryZExG2IiIVWA9c6JAyLRNEMlcoZjAmp1UaZMwQ/jt1I\nb+7xRUuwQLj68Pt5w/BXfH5+OItlY+D7XXR0rI7LxLr6lLhugc7OewAol09x9uz/BxhSqT7G6CBC\ncXDwIEeLR3nzwJv50i++xGfu/gzGGP7qqb/ig3s+yDt2vsOuPWOxvEIZn3icauXMgudEHIxpmqsH\ngzYBSmZHyYhycd0WXCdLJrMVEZf29rtsa4jFsk5cMlHy2JnHeOjlh/i1636N3lwv07VpPn/o8/zK\n7l/h0JmnGJ46y7uuegf/8MR/JdX+PB3uJK1ellRtit+55r20FK5hf18yVb7A79/y+9ZwWCyvcNrb\nblvvIlgsllVkTURJNDnJ9A8fAmA6mObAmcfY3X41H0xv43uP/Bl5L482mn/Vt5+nHvscd265i3R6\nJ//98U/xP+15N97pGZwIxHMJRBiLRni+ZwJ0mSIeY6FDCBgEbeLVBlw3j++24/ltKHXu3IP1TqH6\nvKeuCK5Ah+fS6bu4IlS1oao1NW3o8FzaPAcFlLXGF4UjoESoT4oMMBVpxoOQYhjXyHKOos1zSeaC\nheRz/kwtTWtULYkBoqaZPRvrZDSnvcTMn4sJt8XyXex4vRyGeImH+r24UFl4yeOtkn698CteKK21\np91z8BdZPsFisVg2GmsiSsZTPv+wZzuBjjhWPMY1+z6EyW8CIHPXL6OTFTAPimDMW/lx8sbovOkO\nHm1KxwBKa/IT4+SfP06k02wJa+wLq7iuwgEUAmhCM0RNjRP4M7g7t6IyaaLpacz0NGGlhlaC1AJ8\nncedThEQEeIwY7oYcnwCAykTkRJFWoecUi6/EIVJpcj6HuWpaTSJEKK+7JKhNZOms6VApzEQBszU\nAl6YmlmwR7lZSJDcg0URQTJpxHVxlYNyFEYpRCm0NpgoBK0xkUZJHB4dxYtnaY1JNkTOmdK6WaiY\n5rf1Im9uI4LCJGJMkjsOeqG48ZKfiFJxvsnW+M782dOTL6bps747u7NwuHoKpilc83oU887PCTd/\ncapLvLrmpcrt1huuw+9ov0S5WSwWy8WxJqIkqzRnz36Ft+14G39w9a9cfIKbOmHPrhUF1eUyM489\nhp6ewe3ajdPejurJQRRB2iGUMlWvSNbxiaIKqcoJOgkwJoqdbE38Eu8iXnMmmh7CmIjUnq0EwThK\nXPxUN7nsTjyvg8nBg0yNPo7rtNBSuAE/24vbfvGzxxqt0TMzmCDAhGGyLkaEieKF9sR1EMcB122s\ndimeF69t4ShwnLgMxsTipJGwWXZ/QdGw0rhaJ2IpFkd1gRSXvakcDQ0jcwXNOZ8y59B88TPvZOND\nlkpz3m9zTtgrCJW7sv2txs+colQsEoUhOgyoVcpEYTgboHldlyXWeBERtNb46QytvX209W3C8+dO\nMbBaI2+CaoUoDBFRREEN1/eTWeprlKemqJZmyBQKKMfFS8fLwZeKEwTVCoJQnppEOS6O5+J4Hq7n\no9zElDcJbpM8+2FQw+i4vdbo+B4suBRW83O80HPffGgRuzC7u0CloenwYvHnlmG5cjVHW07mL31+\n1eskq5ygWe1qzCqXL9PSypZrr1+VtNZGlHhZPn7Hx8l6yyxotQaoTIbCvfcuet4HmkvVzu3LpmmM\noVYbwfc7gXh/euYFgtooufwutvT9FkEwxtTUz5mqHcYMmkWa+QXlpHBUmigqo1QKUR5BbQzlpPDc\nNjy/A9fJ4zhpSuYElfA0OqrgugVS2R58vxtjAoJgFGMitK5SrQ3FyYcgyiOT3oxrWnGcDEqlwQFj\nkhUyUckIp/hzdl+A2DEwHmVQP+cg4lj/HcuGxEtnKLgeynVxXBcvncH1zn89G60jMBDWqkwMnuWl\npw4S1qpzRUxd1Mw/BgsveLnQMWNwUykc18MYg+N5hLUaIuC4HplCC5mWFirTU+gwIqiUEeWQLhRI\n5/IYA619mzBRRBgEREGNKAgIq9U5+Ta6eZUincsjykmK09xeWxf8zS2ezXdl9suscG8+u3Ar60K2\nYm4eKz8vLJDvnPjN+Z6T7PxMlj692h2qq20yV9kGr2pXtFq9tNbM0XU9BMlaISKkUt2N76lU7zlz\nq/h+F52dSy8aaIxG6ypaV1Aqi9YVtAnwvXa0rhEE4wTBBOXgZXRUIp3eSlfnL6GUTxhOEQSjlMvH\nEeXhex2JWPBob7+zkUcUValWTxME41SqsaCBWKwIEituo5NRCQZj4s4okxyLh1AqTHJsNuxyNA9T\nXG7I43JprGTIox0WuRI6Ol6diOkrk3x7x6qko5LJGP1Mlp6Bq+gZuGpV0rVYLOfHFTXN/EZHROE4\nGRwnXiLYaZqBtn68eR6XZjyvBc9rIZvdsWQejpNaNozFYrFYLBsR65ZvsVgsFotlQ2BFicVisVgs\nlg3Bhui+qXu1G20IqhHVckhYi3B9h6AaUauEBJX4Mwr0gmm4noOXdpKRsQYdxWlpbRb3D6r7oemm\ngR6LjBR1fYVSEo9Y1QblCMpVKIn9NFJZD0zd491gjEHrxCtcx97TC4xWBQHHEZSjEGn2OD9nJ3Zg\nHGztRAAAIABJREFUI3YqEhGUI+fMYr2YF7pJJkuJfUrq302Ts77B6PruEr7eSzltL3Ju0dSWSOt8\n81/e+/6VSd9VraRz5+/4abFYLOvBmoiSaing+DOj8ZdzXppLjNAT8FMufsbF9RXlqQAv5ZDJe7R0\nZvDTDo6nzvHuNsYQBppaOR4KqJKXvJdyUKvkFRzUomRYXZx+XfjEQ+ygOhOCEOeXfMbDXefuNy61\n6R7oyBCF54qt2cuM1VJdEMGs8Donjoodc5tf0s2e83UP/LoIa3jpzzs+J955sJwAPPfwEnmcZ/ZL\nFvcVOnrI821jqMViuXxYE1GSynpcfXvfWiS9ICKC5zt4vrNmeZyT9rzKZyZ/7iyyFotlbakeKxIV\nq+eeaJ690FGojIt4CqfFR2VdiAziOYiniGaCeI7B7OxDbYII3HMrQBaLZW3ZEN03FovFciGkdrQu\nG8YEGl0OMUFENF4lODMDjmBqEUQGlXExoUFXwkYXrriKqFjFaU1hQo3KeZhyiHgK8R1UzsPrz62L\naAlHRlAtLSh/ZRUhE0WYMGTmRz8iHBsDbZBUChMGeP39ZPbtI3j5ZZxCgeqxl3BaW6kdfZH869+A\nk48n39O1GrVjL5HaMYD4flOXu0aUmjOxnIkixHEwtRp4XjyJolKNpvE5M0rPm5BOV6tg4vKJCOHo\nKHpqCl0qoatV0tdcQzQ5iS6VEM9Dl8uEg4PJRJNh81XHzcar1q3bpHKTSSkvqPV1PbuZ6+Vu/t7M\nRZTN7e4id+edywdcSVqrkso8yuUyhw8fXoukLxppnljoPPYvJM7FpAWJf8oC21LXM/9zpUZztYzr\naqSzkcqymumsB+l0Gtd9Zdc9xFM4XvwCdzszK45njMFUI8QRwmINryuDrkWYWkQ0UWXmsbOIE3eV\nilLgAGEsclTBwe1KgRaQiNKBxyn80utm0w5DZJnfJRwdxSkUGP3CFxDPQ2WzlH92CH/nVfELuVTC\n69sExqCrFUy5gtPRkczu7KLLFcRRmDBCPJfCG9+IUyjMlkFrwsFBZh75Mf72bQRnzuAPbCccGqLl\nrW9l8rvfxVSqiOuA4+L19zP+9a+jMlnQEbpaRfl+LFKCEByFOC4mCOI49edGa3Dc2Hmv8TKf15dd\nt2tKEM/DVGsAOC2FeFbuXA43l2P64Ydx29uRdBoThKh0ivTevTi5HLJCkWbZ2Mj5OAju37/fPPHE\nE2tYnLWl+aU+/yW/GvurmRbMFTHzBU3zNS2Uxvxjq4VN7/Ji06ZNZDIrexGLyEFjzP41LtKybDQ7\nE03PMPXd72LCALTG7e0lf889BGcHEc8lHB6hdvRFqoePoFrbEC8NboHg9BQEVZzOblQmS1ScBMch\nmhglGh+k5w8/itfTgihJlo+Iu4umHngAp6WF4v3347S20fXhDxFNTOD29ADEy0kA0cQETlvbouVe\nrWnxLZbVZilbsyZVKF0OqTw/tnzA+fZ+pZN51qMb4iVr6w6bTaNnDKBSDuIIkvdQ9ZE8eY9po3GU\nkEu5OCJkPEUt0riOwnUWdwysBBFpb3X9VrQ2BFrHTqxmdpRO/dYoAUcJvrNw//ZChicWIyQr+sb7\nOnm5Nn9PlsNI9uPvxhhqkaYSaCJtMCSjiOrlmlfGSMdxI920GYPWs+klA32oO+s2ynnOtTSfMwse\nnx/vchcNa01rj2LlbQOXHz+ZmOZ0NZhzTAEtrkOL61BwHXp9l1Z3ZUslBIND1I4dJTh7Nm4l8H0w\nhvxrX4PT0YEoReXZZyl++9s4+Tw4Dk5bG4U3v5nW+5avqc/8+Mekrr0XXQ6Z/PYPCccriGtwshUk\nrTBhKyYUvP4WCm/5EOI4lH8+BZ5H9eRI3OrTlkI8hS5rzIlh3M4M4ivcrviXnt9iuhKMNqs6VbjF\ncqGse0vJ2WKFQy9PUA4Wnsq83tpXf5GqpsWkmsPA3BEtACrQKGNwyxEqMjieIl+OyImgtaESabQ2\n1LRpTNii62ktMPbUUYowCSDJC1bmCSnTvBbDvDTMAmpLMLhKYZxkHIpJknIElKAFao5QSjvx8OFA\nI4ki0CLolCJ0VNx9mtwEIyBOvH5y/X6punAzBlMtocszcZO2iQimiigFnhf346ZyOdq6exA0nudT\n13tBeRrXT+M4DspxEh9CQSnBEUEp0NUKnufhpfw47+R3EWZ/t7lLbcy9J3PPNR+Xc47PpmeN6Wpw\npbSUhNowFUVMhvE2WAuZSPwNTBgSnDpFanSUHTqkr1omW38ZC7idXaR278Lp6Fixz8bFYIyhdvw0\nupJBzwS4nWn8zQV0NcRZwHneaEM4WsbUNCrt4LSlCIfL6GpENFZpmubAxL4jjjq3slefOsDE0wtQ\n7z2p1zTqvh/1czoJ6wgm0OAqCDU4gvIdVIuPU4g3lGBCHZ93FaYSoathXA4lmCBCzwTomSBOC1AF\nHyKDCXUcNzKcYyoNcddO0l2GNixgoi3rhMr7pHct3mo3n0veUlIsBzz4/NCy4YLIsLktw6t3dZFL\nXV793pMjQ7R09axKWvW5TIDZh1EbTGQg0kwNDjN+5GWqMyUi4v7afFs7PQM7MVWNqcTr2NSNTVQL\nmDwzTHmyiI6iuS9tEbxUCi+VxkQlUEI61wUIUS3ux60Vp6kePosohW5aYTjj+/FqrFrHLS7MDj+u\n5+D4HjrS6DCksQLwQk5hdUPZrCiXcsK6Ulnjy9x0z17ym67ctW+CU6cIR0bQ5TK6VEaXS5BUMtqA\nNgzbmm+yEjI33EDlpus5NlPmZ5GhFJ07HL9ruso1OaEvtbZzvIgIqYHN5xyv+8CcE14JXvfcdcW8\nvmQl6O0tq16+5TBBRDRZIypWqZ2cip1UXYW4ChNoJO2iUg46CkAn/j0tKbz+POLGgklP1uKRTvXN\nWfihMJGO7aRiVjRZrjjWZkiwrnJremLVljJeLbSOGgtvTY2NUOjoWjDc+JlTRGHIxNkzbLthH2eP\nHCbT0kL3tgHGz55m6NiLjJ16GeU43PD6N5FtaWV6bJSh40c5evBxenbsxOgIt2np8/rLu1YukS60\nYPRcQxjWqjiuh44iHM9DhyGIEAUBrd099NxxDZlCCyJCFIYUhwZ57ueP4KXT1EolHM9DOQ5KKcRx\n6N67g76+6xEljWu2WK44RHDa2/H6+5FMFpVJI87y/3cf2Jfy2bfI+bPVgOdnKjw8PpWsrQ2tnkuP\n77It7dPmXV6VqLVCPAe3M3NeDsTzcVpTywcibv3FmrIrnjV5stK5PMXBs/QMXIWfjv+sOoqYHh+d\n07owcuIlXnzycTZfvYeubQM89+OH0FEIxlDo7qFj0xa8dJpCZyweFmqmDyoVXn72GfIdnXRu2Yaa\nZ5CmRkc48vijOJ7HyMnjeH4KP5PFGEM6l0e5LlEYkM7maOnuZfjES7R0dZPK5th87XUcP/QUubZ2\nJoeHGDr2Iqlcnk27r+GaV92DjiKeffhBwlqNfEcnjufx2v/5NxvLqC+E1vEkbIudj/1B9JJCwnFd\nOvo309F/bg3LYnkl4fUvvIDlxdKX8ua0kmhjmAwjhmohB4ozTIVxd7MmbvArOIq9hSxbUp7tTrRY\nLoI18ykJKhWe/Odv4fopwqBGqThBrq2dnfvvYOz0ywy/dJSurdvZuf9Onn/0R1Smp9iyZy9To8Ps\nvPUOxk6f4sUnfkK+o5PpsVEc12X7jTfT2rsJ1/M48sRjFAfPkMrm6Nmxk7BW5fihn5JrbyeVzaOj\nkFKxSCqb5fp730BYreKl043yGa0ZPXWSXHsHpYkJ0vk8lelp0vk8ubb2876RFsvlzpXiU3KpMcYw\nE2menCwxVAsaLStVbcg4ih2ZVOxIDuzOpsg5DiWtiYwh5ygCbahoQ7vnoKygsbwCWMrWrKmja1Cr\n4rpe7HypNTPFcV7+xTNs2nUNrb19jRpFWKtx4uc/46qbb1syvecffZhnHvgubX393PBLb6Rn4Kpz\nwhhjmBwepDw5Sb6zi3x7x4rLa7G8krGiZPUJteFYuYqbOJy/MFOhFGmyjsIRYTqKcBCyjqIYRov6\nbtbtdGjAUxKPeDPgJg7s9RFHAFWtqQ9MDMzsOLa6G5eCxppddfNv6tsaDCM2yQCFej6rLbvqaS71\nJlvofJPf75zvlvOn23e5u72wfMCEdRMla0FQreCl0ssHtFgs54UVJZcvoTYUwwgRSClBiIWKr+Sc\n1pfIxMP2hebRccm2Ri01OnGGt11bFliH0TdriRUkFovFMhdXCZ3+ysy5I/EQ/kuJ7ZayrJTzaikR\nkWHg+NoVx2KxrCPbjTHd610Ia2csliueRW3NeYkSi8VisVgslrVi8TnVLRaLxWKxWC4hVpRYLBaL\nxWLZEFhRYrFYLBaLZUNgRYnFYrFYLJYNgRUlFovFYrFYNgRWlFgsFovFYtkQWFFisVgsFotlQ2BF\nicVisVgslg2BFSUWi8VisVg2BFaUWCwWi8Vi2RBYUWKxWCwWi2VDYEWJxWKxWCyWDYEVJRaLxWKx\nWDYEVpRcJCJyr4i8a43z+A0RuWkt44rIZxf6LiLvvZB8z5cF8l9xviJyn4hk5qcxP30R2SYi/1ey\nbbmY8loslxJrZ1aHi7QzC5Z9sbDW3lwY7noX4HJFRH4ZeDXQCTwqIn8I9AP/Dvg3wAzwDHAv8Nnk\ncwL4KPAgcDvwE+BuY8yvicgfAzWgDfgi8B+AR4EisBvYJiIjxpiXm8rwaWCQ+Hd8EvhAku8IkAJ+\nlgT9gIi8B/gWsBm4FmgB/hz4A+AUcIOICPAXwLPAQBL3VSKSAbYCHcDfAvuT6749Kef7gePAo8aY\np5KyfRX4RyAAXgOMAg8DdwNngFuBjxtjhpN8rhWRjwADxpg/SfJ9DHhHUt6PA38yP5+EW4CHgB0i\n8rvAdmPMHy/ws70b+M/J/r9KfheLZcNi7cyGsjMpEflt4HXAb2DtzZpgW0ounPuATwP/BHzaGPMf\ngSeAvUCfMeZzxpgHAQNIsgGMGmP+GqgAfwO8JCItwOuJjUka8IHHjDF/CdwEHAa+Nc9QdAHXA5NA\ne3L4e8QPQQ34JLExA/gO8Cngncl2mtig7AC6k/IcBnqAojHm88DZedf7beDviR/yu4wx/wk4kJx7\nAsgDhabwg8aYLwPDwA+AvwTuSs59DfhnYE9T+KEkXxERPzmWBXRyPTcskk8zx40x/4XYUC6GWeKc\nxbLRsHZm49iZwBjzt8BBYBPW3qwJVpRcON8GPkb8kP9pUoPZT1xrOSMivysiryWuhfwucQ0GYkUP\nUDPGGOKHQRE/6DngReKHo/nP/CLwXhHZVj9gjBkBDgEZ4OdNaWvih6eeLsCbiWtW9xPXKjqAEnAi\nKet7gF3AEJBJvvfNu16TbIq4xvYHwB1ACLQmee8RkXeLSF9SjoXizvkuIh8VEQX0JDUYMcbUknDX\nERtVJ9ka+bAw9TwXMwTfAP5X4H8Bvr5IGItlI2HtzMaxM7rpU2HtzZog8f/VYlk5IrKLuAnzeuDf\nGmOq61wki8VyhWHtzCsTK0ouI0TkdmJVD/ALY8yBpcJfqYhIB3GzNsQ1wS8vEOaXgHqN74Ax5heX\nqnwWy+WMtTMxK7EzTWGtvVklrCixWCwWi8WyIbA+JRaLxWKxWDYEVpRYLBaLxWLZEJzXPCVdXV1m\nYGBgjYpisVjWk4MHD44YY7rXuxzWzlgsVzZL2ZrzEiUDAwM88cQTq1OqywyjNSYMQWvCkRGiYhFx\nPVQuh1PIo0slUA61Y8cIh4cQz0M8D12uoHJZnNY2nEIeb9s2TK2GqVTQpRJOWxtOa+tsPsZgajWi\nsTEwJt4cB10qIUqB6yGug7guOA7i+Tj5HMYYCENM0+bk80TT06hsFvF94jmLLJaFEZHj610GeGXa\nmWOlKjuyS013YbFcOSxla674GV3rL+vayy9TOXQIXamC0ahMBmMM4riYKEQcBxNGYAzieYDBRBFo\nA0pAm0RouLi9vTgtLehSmWB8nGppBpXNYaKQ1M6dZO+4HYIAEwRINouemSGamEBPTjL94A8R30Ol\n06hcjurhw+hyZbbAIojn4Xa0g0i8aY1ksqCjuIxRiIkiTBBighp6ZgZEzRUrroeenkLl8+hyGVOt\nLXqPZuddmi1DcvNWECf5XDJOU/qLhptXhqXyFInjzxdZpj4tAStIaxnqeVzM8cWudaHjzdey2PHm\ncysUmLm7XoXb0bGisJbV4Xi5yvbMXIHx5OQMra7Dzmz6nPDjQcinjp5ma9rnf9veS6u3cc1yZAyO\nrdxY1pCN++9fASaK0OUytSNHqB45AqLI3HIzbnc3tePHqTz9NCaKEMcldc3VFN7yFpTvL5/wauD7\nSJKXk8/j5PMApK+7bqlYFotlA/DUZImso/j8ySE+srWHTx45xXv6OmhzHZ6bqTAdRbS6Di2uw1+f\nHOZXe9pRAlNhxLMzFd7T18E7uuMW0Io2/PYzL/HBTZ0UXMXbutto91weHp+ipg2PFaf5L3u2M1gL\nuH+4iCtQijSv62hZsvXkYgRCJdIcmipxU0sWXy3vWliONC+Vq3z17Bi/vaUbX4QfT0zT4bm8pmOx\nCZYtlvPnvIYE79+/36x3s2o0NcXkd76Dt6mf2onjuF1dpHbvxh8YAK0pHThAODqG19dLdv/+dS2r\nxXI5ISIHjTHr/tCsp50ZC0K+dHoUbQx51+F1HQWGaiEtrsMLMxV2ZFI4AldlUogIh0sVNvkep6oB\nN+QzuEr48plRjIGj5Sq+CJ2+y3v7Oii4Dn92+GXCxORen8+wK5vi+nyGguvMKceZao3HJmZ4V287\nxSDkdDVgTz4DwEwY8UKpyj+cGeVPr9rEl06P8uxMhX+/s5uXpoZ5eiag06lRkwKuRFyXdXh4osKR\nWpoOz2Mg43OqPE1/9Bxj0oWoLLe097M7I0Ti843BCc5UKnT5KWoGOtyIsHKGbi9kPPIIasP4yrAv\nE3LK9HDa9HFXexu7MhptIhwVC6koKv3/7L15tB3Hedj5q6re7vr2B+DhYSUBguAqiRRJSZSszbKs\nzRo53hSvieNF44nj5MRnksxMcuxMEjt2nDmZE3uWOGPHa2zJsixLlkRRiy1LFEmRkrgAJAESwMPb\nl/vu1ltVzR99730rgAcSD3gA+4dz0X27q6uq+92u+uqrr76PRvM5sAbH7WMqMrSSJeajNiVpscJn\nvDRAWYRYq5HSR0oXIRzStI7nDeN5QwjhkCQ1tG5h0bjOANq0MLqNMTHWpmjdRkofa1Os1dlDFAKB\nAOSqfUHm2HX1PtlWqCyNkAhkbyuEg5SZ9tyYGGMSLs97/BYExy0Il+KVan+vAPYC9+2oIoXC/k3P\nbcbF2prrQijRjQbLf/FJdH0Z1d9P9d3vJp2cxD9y5KrXJSfnRuXVLpRYa/nIM2f454f3MOI5W9Ig\nXIjEWB5bbnJPtYQjN3YmjVRTXieIrOc3z8yQWMv5KGFf4PHT4/3M1Z7kI88lPFiuM+IYno98fmK4\nTsvAx5aq7AkKvGfQ4dthAd8s0baKhhG8tiwR8QQCQT01NGyBu/a+jSiaYbY9x8NzszSsjzUJrym0\n2O8brDU4wKm0D6dwiNdUi0TRDJXK7QghsNYQx7PU60/z6SVBaH0MEmyCIyDFpVTYjSsE9aTNsCMZ\nDPoYK1SZT1ImwwZJ2mQuldxe9hE2pShTzrUT9hTK1KJF2kmdqdjygeECNVvg203DQtzGkR6u9Ki4\nLm8fLOCoIsbEJChaRtLQhqqCihI0taEkbWeGNZviNVZjrEX1jpmOMGN7341JiYwhtZrUJAgEBeXh\nKXfL06druGyfYNvvQ8xir4iwo1SRQmHfltNf10JJ6+tfJzl/nup3f3fH1uPGI9EJFks7bRM4Aa50\nmWnNUHAKVLwKUsjM/mWb53KNNWircYRDYhJSk5LaFG10tt/53kpaGGtITMJCuECkIxKT9PJxpIM2\nes0PvithX+wFuFIvyHYhEGtGCju5rl3uH7ufwWBrNiWvNqHkTDtiyHX4zPwyfzaziCckbx+q8AN7\nhra97EuhdcTMzCdBSKyJ+VqzwFdbJSrBHt4zOsxrBq75IqkryolmZlfX0oZ9gcdkFDPgOkigoCQP\nzS8z6DrcVSky6KpeWzgZxXxxoY4jsrdRCUHFUZSUZDnVLKeaslK0TBamZnV/lwlWdsO2e04ArhC4\nUvSs5yJjSYx5RW1xN6+LneciaVaXvFmai53frNb2EudXp7nQ+RHP4U0DW5/Gu1hbs+NsSqy1NB5+\nGFOvk0zPUHzda+n7wAeudbWArNOWQvLC0gucrZ9FCsmxwWMsR8tMt6Z7HfpCtLChw+p2uNrqXucm\nEAghcGUmbJXcEu20TWpShgvDtNM2jbhBalOMNbjS3dBxb6ZO26zzXF3melYLDFJKtNF4ykMJhSMd\nHOHgSAcls++BCvCUR8ktcbjvML7j44iVn1KiExzpIFeNNK21TEQJ9VQTSElBSfodhRKCqTihrQ2R\nMdRSTVsb2saSrm5A2PwFXC+svVzh7VIv4/WKkfl8/2bExvDnM0t8o97iOwYr/PKRcb5Zb/Gm/vLL\nztNa25tK0LqBEApjEpaWvoYQDlq3ARAye1eULGBMiLUWKV2MTTAmRusmnjvM6Oh7UMrHWsv3AG9J\nNYM72Aj2lXBLaa0B8LC39j6/d/fmgvUe39sRQmTOlWNH/cLDkydpf+MJ5L13UT10tCe5ru5orLWk\nNgULrnJ5ev5pDlYPshwvU4tqnFw8yXBhmONDx5lqTjHZnGQ5XmakMIKvsrnO0eIo2mom6hOca5zD\nWovv+EghkZ05xm6ZAoGxmWTcFUr2VfZx98jdCCF4au4p+vw+jg0ew5EOraTNA6UHAK7aEtwn6y3q\nqeZ8lOAIQagNSoiehtFZ9ew0mcc8ASTW4gqBBoy1OEIgO888BGKbiStBR7iIjcGxAmUElszQzjRj\nhIiRrITnzM6BK8WakclY4NHnKBbTlInIsJhotLXs8V2KSuJJyU1Fh5JSFKTE3UTtnZNzJfjY9BI/\ntneYnxCCosp+3/uCrRvBR9E0SbrM8vKTGBODzUbiUgW4Tj9KFUmSBaT0GBh4AM8bRkqvN30ghCRN\n6x07Cq9nE9G1cVhNtx25UQWSnJzVbMuvfDle5kvnvgSwZoRuO/N1vY6/2aL819/CiGzZbTrcT+O1\nB6moM4SnTmKsoZW2qHpVrIUUy/Ohh7aGmvGIrOTNA/18YvoRhv0CxwuG+8feyL89Pcvb48cYDarc\nOngro8VRztXPZeVjmWxOAnCg7wAPjD1wUeFhKkoYcBWpsdS14XwYc7RcwJeCZ5ohpngHg0WfId/j\n98/P8+hyzA/uaeFJwW2lAsta4wlBXWuWEs3ZMGYqSkispeooLJAaCyJTFYbG4AjBoYLPckfQUJ1z\ngux8UUlSa4lN1uEfLQUUpOStgxUGHGfNHPbVmPbJybme+Oj0IrdXCpe06VhPmtaZnXuIJF7A90dx\nvSF2jb4PpbbuXyR7F7P30XFWtFhKFS6rLjk5NyrXxKbENJssffzjqGoflXe+A+lf+qX+Dy9O4QrB\nHt8lNJbDRZ+7KkV+7cUpHugvE2rDE/UWfY7iPSP9PDS/zNkw5j0jfXyznqlNi0pSTzV3Voosp5oz\nYcSJZshdlSIA379nEE8IDPCFhTrfrLeoOIpQGwpKMuq5DHsOE2HMiWbI+0b7SYzlW402qbXcWgp4\n40CF55ohy6nmq7Vmb9mgBEY9h8PFgKojkd2GaRNtQGQMZ8OYEdfZ0T4Lcm4sXg02JSeaIX81V+N/\nOrDrkmmNSQnDCVrt00TRNI4qMTj4Rlx3YFvqlpPzamHHGLraOGb5M5/FpgmVt78dVbn0fLe1lpk4\n5ZFak9dUi4yvU7GmxvY69vVagaUk5UwYc0spwO9MQbS14aUwos9R7PZcDJl/tK8sNTjRbPNiO2Zf\n4HFLKeA7BivIXMuQ8yrhRhdKtLX8+9NT/MLB3RedGoyiaebnv4ixKYG/m1LpFgqFvVe8Pjk5r1au\nuaFrMjFB86tfxUQRfe973yWFkU/MLPFAf5nZOOHzC3UU8BPjw5su0VutaVg/TdHvOvSv0zQUlORY\naUVVqgAl4C2DFd6SOwHKyblh+dRsjZ8YH76gQNJqvcjc/MMUCwfYs+dDCHF50zs5OTmvnG0RSkyr\nRTI9TXz6NHphAWf3Hvo++MEsdssF+MZyi6ko5qlGSElJPj1Xo+JIPrJ/dDuqmJOT8yoiMZalVDPi\nbXQr0G6fYWb2M5SKN7Fv/Ec3NTbNycm5OmyLUCJcF7O8TPF1r1sTbG4zjLX8i+cmaGrDD+wZ5BcO\n7sqnTHJycq4ofzq90HP73qXVeon5+YcplY6wf9/fyw3Cc3J2ANsmlBTuuuuiaay1/JtTk8wlKT+1\nb3TDOvWcnJycV8pSkvJnM0u8rlrsGY2322dYXHoE1x1gfPxHc2EkJ2cHcU2Wdjxaa/LbE3P8zL4R\nbu+sfMnJycm5krzQCvn9yQXeM9zHHZUi1lpm5z6DFC57dn+IWGe+RZKOX58vPjdLI0yJUkOtneAq\nwaHhEnv6CgyWPAZLVymYZ07Oq5irLpT87VKD55ohv3J0nNJl+gnYQGsBvDI4eWOxnVhr0WmKThKE\nADcooJOENImJWy2sNUjHAQvWmt7WGpttOw7wsBZrVr5f6Ji1BnrX0ssDY7CszruTtutgb5VLe2sy\nJ1Wry8Halxez4jrmwJ2voVi9+BTqjcaJZsjn5peJjeF/PrSbVuMpzp9/BoSkr/p6npwsMj8xydnF\nFo4UBK7CdyT7BoscPTDASNnHcyRhomlEKc9O1jk5XacRplgs2kA5cHj9wUF2961oeK21LDRjlsOU\nA4NFhIDldkpf8cYMj5GTsx1cNaHkbBjzV3M19gceP7J3+PIujlvw1EehuhfiBoQ1kC7UzmSCycBB\nQMDCC3DfT0H/QXgFwbTWY4xGyp1jia/TBCEl1ljSOCZqNYhaLZpLi8TtFjpJ1l6QRaJa2X8xksTK\nAAAgAElEQVQZKMfBcT2MMSRR2PvuFYuZa/o0zdTgIou8KYRAyI6De9mJ0rnqI5Vad0wiZCdyp1zJ\no5dGrvveSS8QINe50V+VL4I1x3JuLJJY43or76axlq8tNfjZfcMsLn2NyYm/Qvh38tjMGwkTTZQa\nHjxS4NY9VQYuofkIXEXgKt50ZK0fJWst7UTz1VPzfOnkLL4r0SYL+NZf9Cj7Dt84s4gQUPQc6mGK\nI1dCP2gDA0WXoufQTlLasaEZpbiOQBsYHyjQX3Q5t9DGUQJHSobKHrfuqa6pg2kkyIKDCVNMI0HX\nIkxssldciczXgSV7PxyJLLvY1EBqEa5EtxJsmK6NpWBBuBLhKYSvECp7d7LguaITXHdlK9Z9zwR/\nsndedq6RnfdPdtOKTuDe/H3M2ci2CiXaGJ6YnOZrc0sUdcL3jPRR9Lw1nby1Fp0kNJcWsVpTOPkn\naLdCtLxIvDCBXyzgTT9B/cF/xeLzpyhUyuzefze1lkUNPUAaRUihOf/CKaS7j+JDf0opmcbrGyHF\nYb7tM1jSDOy7mXriEhy8B5MmWCGZPv08aRxj0jSrb5p0QliD43q9EXwSRSjHwWiNcl2UUmitUY6D\ntRY3CChW+7FG4wYFmosLZH2lBCkxaUrUbq0dqXf3u8LCan8xvZc721pWxWYRAqkcjNFZ3Bw/wC8W\n8YslhvcdICiVcbxcc5Tz6uCFx2Y49sAemqnmb5YanAlj3h6c4ZGn/pwXlo/huG/DcyRvPjpyxaZf\nhBAUPYe3HdvcAZsJU+4uBthIo2sR1lG4u4qoqofwFTa1zJ9bJq1FBLtLFPt9PM9BdwSL6XPLhKbF\n3b6D9RW6GTF7epkvfW2yV0ZqDE0pqAqB9iWJp0jLLloJhLWgwUqoFlz2VgI8C6IeIRyZCSxhgqi6\nhH0OUkmUFCTasNRKCNsJNkwQy5rMkZOho5Bc0UB2BB5PCQJHZV727aqIW9qCsQib9QNhrDvaz+yY\ntKCk6H0cKXEkKCmREhKd5dQdWhq4cACsi8k265rVbhhNwUpTvOZ4d19slu2alrgz4Ll6XPGyrqBQ\n6FV9xu66Mitlt8V52nStxh898W2EgCPlIveMDOEVCrTry7Try4SNOlbrTg0EjrA4foBz6rO0h+6E\nxjSl0XH8gTEWJieQhT4Gxg/QN7qbsFFn6oXnKA9kAZq8YpG43WZ4/0Ec16W1XKO9OEvYjnE9l8Gq\nw8LZl6gvLhLQJJ59EVWoYqNlRveOUYimYfgI7H8A+vZmWpioke1Dtj/5JIzfC+0FMBo7dxLmX8C2\nFhG7biVsNIhEAbH7dpL6IuX9x7JYOVpjjUEqB79U2hA4Dmsvukw6J+dqcj06T/ujr5zhRDvEepJx\nNc2oeJJUHePI3tdx21h120bjJkpJ50JMM0E3O5pJYzPNgCNwBgKEK1F9PsKVxBMNTCvFNBOEJ3H6\nA9RgQDrbQtciQCDLLqrqoao+YBGuwjQSVHVrwpQxNothpVbidy23U15aaKKN7Ux9QjatmSlSCq7C\nYkmNxVOSauBS9BWyEyVXdnrnTNmxcqz7WKPU0IrTTnDRTsfZ0U72Ym9JQdl3ECITQAC0sSTadD4r\n+3FqSI0lcBRCZJovYzvlX6RbvtSf2drOtG5HeWSs7T0LY9edt5sHOu3ms35/daDT7VL+XEY3vbX8\nrnCo0ZLvcNPI1oNZ7hiPrpuSRvCpX8yEgPs/At5VNny1FppzcOoLEC6BX4GkDVJB3ISgH/rGoT4F\nhQHQEYzeCgOHsl/gwikojUBzFs5/A9IYBg5A0Ac6huXJbMpJJ1neI8egsiubfvKK0F6E2ZPQvx/K\nuzZOO4U1UH6WLm5AYzrLszwKI7fA4otQOwdCZkHBlJel698PXgmkA24RHB+WzoJJs/pak11rUvDL\n4ATZPXql7Fj//ux+Ibs+jbJnYg2kYVZXa8AprNS5vZSVl7SyOkgnq7tbyJ5FGmb5CJl9oPMWi43b\n3jmy/ITK7qHr0Kr39q9Ka01WTic4Wu/vu/YPvvHvv6VzWzl/DbhUK1gZA3drK9uuR6HksZcWOPfF\n57HRX3Pr649yy13v3ZBGp5mdk3Ilz3zlPEZbjr9pjCcfOkvYSBAq61Lufsd+pBI43tqpWtNKSGbb\npPPtbHpDCoSvcAYDZMlFltzO1GNOTs5WuOYeXS/I9NNw6mF4x/+20gFebYSA8gjc+Xde3vWDh7Ot\nX8n2rYWzX8s6YKcABx5Yube4CbPPwksvgNGZ8BNUM03N1Leg/UV6naxJs7p55ayjLQ1nZQwcgv1v\nyOxpZk9kAtPYa7M00sm2fgUWT4NOwWpozWf5VfdmQktzJhMKDrwhE0aieiZEjN+b1VEqWDqTfazN\nhALHz+osZLY/+c0sXdJeEQKCvqwcrwRuR7jRESQhKDcrywk66TtDks220NUVZ/tdQSmNVq7tpsl2\nun/M7P422P+s6zA2dORia+e2dJ7t4eXKP6UR4MZdbn+4+jyFO17Cl99Ha3FjqImJE4t8/ZOnqS+E\nHLp7hKDkcvCOYT71m9+i3O/zxu89QhymLM20+ZN/9yg337OLA7cNMuArkskmkNlYuHtKlLYQLycn\nJ+eVcW00Je0l+NKvZh3Vm//p1deO5OTkbOB61JSs5qO/+hj3f/AmihWPUr+PtZb/++e/xHs+cif7\njg9mBtYdjUZjMaI8sGLAahNN+5kFFs7WOffsIknJpeVI9h0bZN/xQWqzbWZfqmOxJJHm0J3DDO+r\n9PLLycnZOjtLU7J0Br79p/Dmf5J9v4RAYoyh2WzSaDQoFAoYY9Ba9z6tVot2u40QgkKhgBCCJEmw\n1qKUwnVdHMeh1WplK0eSBCEE1lpKpRJpmhIEAdZa0jTFGIPpGLh2r63Vahedm+6e830fKSWtVgsA\n13UpdlandJempmmKlJJisUiapnSFwu553/dxHIcgCFBK4TgOURStyaN7b57nodTOWRWUk3Mt+cDP\nv4aHf+9ZdGoQQnDg9iGOPbCb4fEySq2dFi0P+JgoJX6pntl0KEHh1iHG7xxh9O0pC5NNGgsRQsKz\nfzvJ2JF+xo8NMLK/Qhprzj+3xDc+8xJxqBncXeSW+/dco7vOybmxuLqaktNfzmwwjnwnVPegtWZq\naoowDHtCw/rOXwhBqVSiVCoRhiFSShzHQSnV69wLhUK2TK/d7gkTQgi01iRJQpqmFAqFnpDSFUqa\nzWZPYJFS4rouUkpkZ/lpHMekaUpfX9+GenWfW/e4MYYoitBaUy6Xs1U7SUKr1VqjUnYcB2MMrVYL\nx3HWLJMFiKKINE0JwxCtNXEc9+4PQHbsN7rnjDFcC7rPcPX2Ylzod7YVQ8TVz/pyfq9Xgm79tlru\nZvdzsWsv5/4v55ouhw8fplQqbSnt9a4pATDa0KzFFPs8Pv87z/DOH79t7flYEz49jzUW6SvcvWWc\n/s701lb82KxLY61l5sU6p56YRSrB0dfvQipJserh+tmAIU00WDbYquxIjIbl89lU78sxwu8+H51C\n0szyg2z61aSZbV0aZdPM3VWGVmfHjc72rcn20zCbWpYqmxqGbCo4bW+0ON3K323tAdbao623LbvA\ntO2VaH8uVtf1qzA3O776/HbU50JlXYzSCBx+y2UUeY01JdZalk4/yflnn8GM3485PQPMIIRgz549\n9PX19TQKL5eu8LKeQqGwSeosfbmcWQsHweZz7t5Fltau7xiklGvKEkLged4F86hWq5sez8nJeflI\nJakMBrD4Eg++JYHHfxdrJeGEi7FFRFonGGsjCwVIBZxRcIasw7QaENlWeR0bKpHZasXNzJYqbmaG\n38oDkyCsYZfy2XVYQ9Liyc/0Y4xkftYysssSRYpyKSaMfQoDZXRhDKEUylWUqor+XWVqzzxByU4y\n0JdkNmSqU163Q4fMFqs1lxnIj96KdYsszAvmJiJ0cxEhFYX+Mo7vUBlwCFQTJVLCxSWEzXyXSAm+\nbxFiM4PtzjHpZEbsZ/42Ew6EWjm3zvbKGovAbDQ6z/4QHUN71fGVorLpeulmz1F5nfxlVqZXztII\n1dnK7J47dmq2vUTU1ki/QBQ7OH7mkC5qZe4cvIKDVFmdpCN702rW2MzQWYDjKpJIk0Q6W9IsyNJ1\ntt02XTlZP5QmBuVkxy12xeyNjmbbWIzu+GXprEjq+kbKHtG6FTndFUTrTNiyFT9r8++WsfJls1/7\nRbiCs4pbGQRJJa6Y5dq2CCVRFDE/P8/i4iLp0iTMP0f/0ChH3/njuG7u3TAnJ+cK8dJXoDbBGuNn\nIbClPdiZIk3nnQhX4r+tH1Xefv89d3XHfl0NZnegpROoT8L889goc3BYX1bMn3co33ScWnQnk/WE\n5lQbJTVeudTroUr9PoNjJYpVn6gZ88LfPk/aDhkcMhy5dwTZfzdpFJMuzZHGKUuzEbPxCMZK/P7b\nsUKSxlkHG7VSEGBSi3IEjaWIQsXDpAbXzzptsSQQ4qbOiiLb808ipMBou5JOZNqfCw20RWu1Lfpa\n541CgNG2t2rJ6Kw+QM9bs5AhQoQdOdHil1x0GlEoG+KwDUBQzjTf9fkQrQ1SCUxqMaajXZUCpbJ8\n09jgBgrXVx0BZkW4WPHBkq3WsoDjSLRe0UT3BIrORirRc+mw4p26k2fvYWSf3pHOkmRWCTeZs8i1\n+cMquUJcbDH0RrYsv2xBy7LVvIKiS1C6Mn37tggljm7jvvgFjqgIb88heN0PrajfcnJycq4UB96w\n5qvVhvbTC9iFlOCOwasiiGzKeq2vcrNl9v37EWQN70DnA3AhH9dGG8JmytlnFkhjjVdwuOO7bkOu\ns5Fx3AJOOWtjy4ev4H3k5FxltkUoUUGFkfu+L1ND5uTk5GwzNtG0n57HppbCrYPIGyTeTNc+5Zb7\ndl/rquTkXBW2R2q4jDgx1lpMM2HxT56j+o79uHvLxGfq+AeqpAsh7afnEa4kuGWQ8Jl53LEy7u4i\n0s8FnpycVzvR6RrpYoiQguDWIaR/HRiT5uTkXJBr1rPrZkJ8ukZ8voEzWKD//TcRPrtA87FpnIGA\n+MVlhCepvGkv4fNLxC/WKNw7zKmH/jP9p+8ncMbxD1bxxisdgyCbxa15hWTLdmu4bn/vmDEpUl75\nR5Wt0FlE6xaOU8VxKnmQqpycy8A/1Id/KJu20I0m0dlJkskp9NLSSiJr1q5UEAKb6pW5eymzc90p\nF5tFrhZKrdioaI1QDjbNbDKEUp2YMBbhOJlR6gZD/cxDsVASWS6jl2qrTq2tD9ZgU41wFML18I/c\nnJ3yfWycYNotpO+Tzsxg2m2E5yFLJbz9+7M0SbJix7I2qAs2irBRhIljbJytchGeh6pWMc3mmpAf\nMgiwxoDW2b2azrNbk6foGXey+tO753Wr1ozJyrDZqhahJFabziqb7BlidJamm9Zk50wcIzdZLLCS\n32Wy6pqeIemKv/jVCS8/74uUdaOjBgYpvvY1VySvqy6U2MTQfHSK+Gyd0j27qb5tfxYkCii/YYwo\nnmNx4Ssk6TJJvMji2QqxmCcpLCFPB+x7848yP/UlGuIxkmcj0udrBGIPci+IeoDqD7BuBIDr9HXC\n2CcIodC6hZAuUjgYE2OtxtgEzx1CdNyXJ8kSrjdAEi8gpItAEMdzuO4ACIk1CVJ6lEpHSZIFkqSG\nMRHKKRH4ezAmolK5A2NCwmiSOJolTes9gSmLkyCzPavxvCGk9EmSJYyJe8/J2Bgp3J6BlEBgbYoQ\nbiefLEJWN19jEkRXcLK6l27Vk4deRKv1gs966/pNrO03/2uuyjNLb9OVhsYaC6nFxqYTjItO4y+y\nDyvG/aaV9jqLyzPrWl+jjQ3BVvJbfV034vCNwOjRd1IYuHFV/62vf51kahoAWSri7t5NcPttqP7+\nHSHgdztY3WjgDGz0Wm2TBJTKon5rjVAKE4bEp04BoGvLCNdBFovYOMa76WZUpYxptTBhRHTyJCaO\nEY6LULLXEdpVHa30PUQQZIJMsYhQEhNFJOfPI8vlTMCis1JlaQmkQrhOdtxx1ubZtdLsGKPS8ZvU\nLWuNd2YhoBv9W2bBSbE2u2epsvxdl16E8M5zQCmQEqEUwnGy9KsFH2uza9cLgRcTAlYvG179u9hw\nTGw4tW0BbW4krmAMt6sqlESnaiTTTYqv3UXx3t3IjjCSJEss1R5DIGk0nmF09N04TgXPGyYMJ7HW\nEARjvUZm/NAPZRkehLQRQ2ppf3sOVfUw5+NsqVZi8EaLOGMl9HJMcPDChrZRNINS2XJex6lc8j6M\nSWk2T1KuHCfwd6N1iDEhUTSNlEMsLn4Fx6kQBGMUCwfWaF12KrqZoBdDkALpK0w7RTcSpK86AcNW\n0ZFFutbq3cZIKAmOQLgqsyh3JKKkkEMKHNkLfW5Tm8VvFwKbZMsuVZ+f5eGrHdGZ5FwfFO+991pX\n4aIIpUCpTQUSALFqNWJXOJBBQHD8+EXzlcXM6aQ3vvcK1TQnZ2dwdfyUaEPriVlU1aN0/x6iaJJv\nP/nzDA6+CaWKSOnTV70bKT2Ght7c01oABMHFPSU6Hev6yptWXk7dTJBFh3SuTXyqhjNcoPGV8z37\nFf9AFWe4QLoYEdzcj5P0YxsaWXI3PBGbGpCC+FwdIQXeeAUpHSqVlUZDqQClAly3H6stztII6UQL\nmxpiG5IEs9jUID2FM1pESIEsOoiCg401QkmS6RZ6OcLGGqst3r4K7siKt1u9HGHancikYZqF+BRk\nGgdtEW5HE5MYZMHJBIbOGvoNCo21u9kgJHBwBnywZMJIwcHbl4Vf9w5U84BjOa8KtNY954kAjUaj\n54U5iiKazSZpmqI7Ux4HDx7E933aaZswDVmOl9lV3EW9XWduaY7T50/TbDcRUjA6PMrg4CBhHNIK\nW9TmazjWoeyW6fP7es4BU5vSiBsUnAKloAQ287fk+37mNDGJacdt+kp9OI7Tcxjped4a549dLiTk\nr55i6XqPfiW+onJyrgTb7tHVtBKaj88QHOnHDkRMTPw+hcI+du163xWxAblcTCsLNZ5MNBCuzDrg\nQKEqHulcSLoUghA4gwE2MaANJtR4+yvoRgKpIbhlkGSqSTxRR1V8TDvF21siXYgwYYq3r4K3r9Lr\nyG2iEa7KtBELYSYctVJMI0EWFFZb3NFiFm3Uz9b9xxMN0tlsLX6mSfAQrsQZKaIusR5cNzMNR3da\nLCdnK9wIHl0vRLvd5vz587iuS6PR6IWi2MwzsRCC1KQshUukbgoKmlGTRCS4BRehRBb2wViWZ5bR\nicZTHgpFyStRi2p4jsfAwAA3772ZwcogWmtePP8iS4tL+J5PwS8wODSIdjTz4TzTrWwKylpL4AQM\nBoPMt+eJoggkpGFKmqaZICHB93zCdoiwoufHQyc6a1uMQdhNpiDXDEpsb5pSILLwHZ2pU4HAVS6e\nzAZ8gRMwUhwhUMHL0mJuhxfmrXp3Xu0te/WxV/J9q2kuh2t9/SvNo1AoMDY2djllXX2PrmktInx6\nHlXxcG6XzDQ/gVossX//T6KUf+kMtglZdJFFd40Woou/hfX9NjE0H5vCHS1SefM+hOqMbmbbFA/2\nbapREG6m+VEl95ICRa8uB/vwLzLldDG2WkZOzo1GrVbj3LlzxHHca2SllPi+z969e2m321SHqkyE\nE5ypnSG1ae9aYw1SZFoGRzqMlcYYCAZwhEOf30fBKWxsuC8+y7KGgZs2n8I5zM5yLGKtJdIRrTQL\nkdFMmpyqnWI2me0JMV3W219tZo+1/pr1AhFAwSkghUTbTAPlCAclFb7yiXVM0S1S9apUvSoVr4Kv\n/C2FtrDrbWzWnb/Y95eb5nK51tdfiTyuZAy27fHoWl9g+tSncA+VAYuTlNm16704Tnk7iruqCFdS\nvn+tRCiEwB3NIx3n5FxtXnrpJWq1bFWLtZZqtcrRo0d7nqOttZypn+Fs/Swnp08C4DU9jvQf4R0H\n34ErcwF+PUIIAicgcDLH4UOFIfZX929bedZa2mkbi0WKLPCosQZtNe20ja98WmmL5WiZU7VTLMfL\nxDq+dMaXKnedsLQZl2skv1XWC2kXOna9MFwY5r49912RvLZFKPErg+x/zYe3I+tLEiaawM19FeTk\nvBo4cODApsfP1s/y+PTjtM/NcHDPEY7tv53hwoX8puZcS4QQFN3NB3V9fqYtHmCAveXcqPfVwHXn\ngSxODdpYWnFKOXD49sQyZxda2dLfjnCppCDVFtOdI+4c62r7otSwuxow1h+gDQSupL/gUeqs/DDW\n4qrcHiMn53pivj3P4zOP04gbjJXHeNeed/Dx3/slnpj8Mrf/7/8BCvDEZ/6SI69/gMf+8uPoJOGt\nP/qTAMTtFl4h6xgnTjzD0N59BJ1o33Bl5uxzcnIuzTUVSuYnGkglkEpQm22z+3AfSkm0tSTWMrUc\ncmKqzlQtZKCU+QxRUhBNt/GFYHmuzZ6+gNcdHCCJNANjRQZ2XzpMu7WW6YU2M40IlKAVp5yea1IP\nV88vrwg53fYo0ZaurNJVtd053sfUfJvaXItC0aNqBYP7yuyuBtQmW0TthGLVw5os0JMxFtsRmKy2\nJLEmjQ2FiouUgjQxJJGmUHaJwyyipXIkphMYan2U7hV3AGJNpElrLVKKC0bCviCdVTq9e89cqlzg\nOa7ku7peF9q/aJlcuJzrlh3Qj+07Nkihco3iv1wFnpj+BmcaZwEYDAZ549gbKbpF4rDN5//Lb3Hn\n297F3LkzfOPTn2D3kVsI68t89aN/xNt+7B/w7S98jvMnn8EvlXnkY3/M+eee5V0//Q/55uc+jeN5\npElC38gog2PjHH3gTTzy8T/BLxSpju5m5vTzFPsGOP7gd9Cq1ejfvadnKJuTk/PyuTpLgo1lbqLB\n7Jk6RluScCUkd//BMo+/tIhWED45BVKgIkNQcvFcyajnUq2njB4oEDZTGouZ8DK8r0xlMMBoS3Mp\nIii7LJxv8s2Hz+J4ipF9FVrLMcZYdCcEdbueoByBTi3KlQQys1oH6PMdjHHQSVa3wb2VXvjrxakW\nAMYY0CudvjGW5x6ZplpyuWWsQm05oikNM49O89UoRfV7qEBiJjVIQIpeREgrBEJlH+lL9FLmC0Q4\nmcdDMx8iPInwBBgQnuh18Os7fejch1oZ0V2obdySkLAKKQSuyspOTSZMdY/LTrhu24mC2fWtdCmj\nqQvVQQiB7ETLFD3/at39zrmOt0jRqcP6fNbPAW88v77QV3b9Riv8S5S3SZr1qS67zIvkb7wbW+On\nnp3nwVvuYWB3ZudltOavfvP/wPV93vB9P8T06Rd48Ad/lKe++BBCSO5613uYeu4EQkqOv/ltPPnZ\nTzHz4gt818/+IyafP8HZp77Fgz/0Y0ilsNaik5j5ibP87Z/8Aa/97g8QlMosz85w8z33kYQhX/uz\nP6axME+xfwApFYNjezn6wIM4l4iGnsQRAoGzibfSGwljNDpOcPyNhqnWmMyO5DLCkuTc+GzLkmCd\nGmbP1KnNtnuj96HxMkNjJYy21BPNQ8/OYIyl5Du89dgIRW+jfBSHaU+LkIQap7Nc9lKjkTTRLE61\nKA/4SCV7Tr4uFlo5DlOUkig3M7San2gCmUDVv7uYdYKO7AkqNzq2oyky1pIaixDgStnRvlhM55yx\ntidECFYJFpch+XTL6go0ZrWQ09k3HUHQknmn7h5bn8+a7xvKWX3uwsslL3XtZtevzfsS9brEK3ex\nem6sx8XLGusvbNnG6npdEvythz/DHW/9TpIo5GP/9l9x8+vfwM333kd1eHQba7k5SRTyxd/9f7n5\nnvtxC0X23nLrhjS1mSm+/Pv/H/tuu4Pjb34bjutlnkyBdqOe+QzxfebPnWVk/8Es3zBESHnNhBhr\nLQsTZ5k+9TxuELD3luNMnXoOk2ocz8s0s0phtKZdX8ZojVKKqN2m2NdHEoY9D7DScTKFrLUrLv67\nPlriCOW4mb+U9SOw7jVCYLTGGoNUCjqDlG76Xr5dPyzGrDm/6Yjockdrq69ZPUJcf2x12i4Xq8fl\nlL2DKPUPsP/2u7ac/mJtzbYIJUmsadUiqkOFNUtkm1HKl5+bZbmd8qHXjaNeJR18Ts71wPUqlEw8\n+zSP/eWfsefIMQ7d/TqGxvdf02mU5tIi5555Cq9QYPal05T6Bzj+4FsRUrJwfoInP/uXvOn7f5iF\n8+c49Y2vI6UiajUpVPtoLi2iHIe+kV0gBLXpSUr9AyxOnadQqdK/aw9D4/sZGt/P4uQEynWpDA6T\nxZuRJFGI4/s4rkdjYR43CPALRYSURK0WjcV5qkMjJFFIu1Fn8uSzmTYjSXD9gGJ/P82lRYJiGeW5\n1OdmEUKiXJfywBDjx2+jVasxf/YlhvcfxCsWScIQgDSOcYOAQqWaT2PlXJSrLpS04pTnphsstGI8\nJZmtRz3j0bfcMkI1yJfh5eTsNK5XoQQgDtvZ0nw/2KZavTystdTnZznxlS/TrC0RlMrc98Hv2ziV\ncQF7FGtMT4sCMPncCaJWk6XpKcL6MnuPHae5tJhpEHUWsiFs1JFSUR4aJm41idotHM9DJymDY+Ms\nTJylf88YhXKFvtHdBOXMVYNOExbOTzC87wCt2hLWGPxyGde7dn6lcm5MrrrzNEdKSr5i/2A/iTHc\nf3go14rk5ORsG15QuNZV2BQhBNXhUe59/4cywYmNHka76Ta9fp3b9z1HbnnFddp/+52bHleO25su\nKvVv7ugtJ2e72RahxHMkN49eOrBdTk5OzquFnSo45eTsJC5r+kYIMQu8tH3VycnJuYYcsNaOXOtK\n5O1MTs4NzwXbmssSSnJycnJycnJytosb24lBTk5OTk5OznVDLpTk5OTk5OTk7AhyoSQnJycnJydn\nR5ALJTk5OTk5OTk7glwoycnJycnJydkR5EJJTk5OTk5Ozo4gF0pycnJycnJydgS5UJKTk5OTk5Oz\nI8iFkpycnJycnJwdQS6U5OTk5OTk5OwIcqEkJycnJycnZ0eQCyU5OTk5OTk5O4JcKEzE6KsAACAA\nSURBVLlBEEJ8hxDie17Gdf9SCNF/herw/ZfKXwjxG1eirJycnGvPdrc769uLzdqYnBsL51pXIOeV\nIYR4J/BGYAh4WAjxz4Aq8O+AfwA0gW8DCngD0Af8KvBhIOpc+xtCiH8KxEA/8F+BXwN+21r7F51y\n/j3wSeAHgf8OGOB13Wustf8SeEAI8Qzww51ylzrV/BEhxL3APwOOCyF+wFr7h9v0SHJycraZq9Xu\nAGNCiJ8CbgX+CVkb8xmyNuYw8J+A7wdqwDettV/a3jvP2W5yTcn1z/uBf00mMPw2cB6YAw4Bu621\n/8la+wXgPdbaXwL+FHgzcNRa+38Cj3XyeTuZEBEAHvA3qxoGWGlITgFvBb68+hohhNdJ927gPwN/\nvura/wZ8rFOnp3OBJCfnuudqtTuhtfa3gBNkQghAobOdAx4Avg74wOA23GfOVSYXSq5//gL4GbKX\n+8fJXswWcAaYFEJ8RAjxFuCTQoh/AXwI+BJwQgjxA8DdnXw+C5SAF8hGLgZACPEhIcRu4K+BA8Df\nAIestfHqazrfAT7dqc8HgLRzzHQ+EqgJIX58Ox5ETk7OVeNqtTtBR1NynGxABHCQTAMjO9t+skHT\nse284Zyrg7DWXus65NxACCGKwN8lazj+m7X26Wtbo5ycnJyc64VcKMnJycnJycnZEeTTNzk5OTk5\nOTk7glwoycnJycnJydkR5EJJTk5OTk5Ozo4gF0pycnJycnJydgSX5TxteHjYHjx4cJuqkpOTcy15\n7LHH5qy1I9e6Hnk7k5NzY3OxtuayhJKDBw/y6KOPXpla5WBTQ3y+gbevgk0M0lPXuko5r2KEEC9d\n6zpA3s7sBKzWCJW1R9Za0BqsBSEQTtZtmChCL9WwSYyNE4SjQCqE62LjKMtICEAgBFhjMI0GNtXo\npSWEo1BDQwghQClMo4EIArAgCwFIiXAcZBBgtc7y0hoRBMggQNfrmOVlhOdlaZXCJgkoldW1g3Ac\nZKmEjSJMuw3GgJTZRwhIEnAchJRYY8GaNXVf92S298FfkPX12FnFCNfFGRjYevqLtDW5m/ltovn4\nNEIKgmODyGDjY9bNhOYjU/g39dH4m/OgsxehcNswznBhQ/qcnJxXL10hwUQRzb/5G+JTpzBRhLt7\nN1ibCQ7WgrHIQoBNNTaJEa6L8Hx0fRlVKmWdtpAIR2GtxSZJ1unHSdZJG43wPGwUIVw3O4bIhAyp\nEFKAytozIQVqaBjhKITng9FYbbBxjPC9VZXv/ieQ5RLSD3DH90Kaks7PY1INOkX19WGiCJuk6HYL\nayw2TTCtViYgCYlQEhOG2ChClsqovio2inqCk3BdrMnaUiGyHtYmnTw8D1koZPdkbZbOWITnYtO0\nJ6wIITrPk54glmW4ckvdvK8GV81txysoRvVVL0souRg3vFBirWUx1Qy6V+9WrbGQWggky587Q3Dr\nIMFN/ZhWgig4CCFoPT5N5S3jCCnw91cBSKabhCcXKfV5CHfrWhNrbNZYdL9ri1ACqy16KcQZKmAT\njXBVVgdfIdSNYU6krSUxlsgYYmvRFoy1WXvS/XS+w0o7k52za773zmN7A631edhN8uglZO3u+i1c\nuoG5VLtwuedvKxeoOrkGbifR/Q3YJCF69lmi51+geO89yFKJ+kMPIYMC6cwMwnVRAwOYZiPTFBhL\n/OKLDPzQD+IfvQVvfO81vpNXjjNyzWcLc3YY151QMhsnjHjuRdN8bHqRhtYIBM8229RSzT8+uJuD\nBX9D2qcabb6y2OBkK+R/vWmMspIknUbDk5ffcSfTTZY+cYq+7zqIN16hcPsw4clFlh8+i2kmyIKD\nTQ3B0YE1ggSAu6uE6g+ofeU8jq9wBgs4IwVk4CB8BcbSfHSKwu3DJNMtbKixxhKfreMeriJu7sc8\nOo1NLaad4gz4qD6f6IVaVoAE4Spsmo0QrLZYYUFKyq/btaE+ttPJOzIbOcTWshClnGqFTLZihCuz\ngQT0ru3msFnnaa1FCIGgO266cDrYfDSyOv+uctWVAk8IPClRAmRHXSxWpemW21PIChB0jq1KywXS\nCylW0q+5Rqxct3okdYF6X+i+Nrvm5R5fjS+3kipnO7BpijWGdHqa9hNPED3/Asm5cwS33QbWYpoN\niq+/j+D4rcRnzhC/cIq+D7w/u05r3NHRa30LOTlXnR0nlFhr+ez8MsfLBfb6Lh+dXqRlDAcDn4ko\nxpOS+TilqCRNrQmkpKEN3zPaT0Mbvl5rcqDg8cGBAf7rxBy/dPNe2sbyOxNz/MjeYYqrNATWWh5f\nbvLe0T7OtAv81tlZfCmIjaWWan7uwCh/OrXIGwfK3F4uYICJMOYPJheoOIq9gcsHRteqrMLnlhj4\n8DFUsKLiLBwbpHBskLY2pNaSWotSa0evzVTz2fllFlNN8WiJyaUWwzZEPldDa4ttJEhX4u0pkT4z\niSq5xFhk2UHcVUEshdhPPIt75zCi5BIlmsOVAhUlsbt9ImM5HyUkxgCqJwwYQMWG+Osvre3pLdjE\ngBSoiodejvCEZFBK9irFa9yuypM1vaOBTIWsRKaNEZ3p3a6gsaqTlGUPBJhGkml2bHYtJlNHrxeS\nkJn2Z0Of3r2Z7ep/u8/lOsS5uR/K3qUT5lw2JopIZ+dwRoaRvs/ypz9N9PwLmYajWiF67jm8g4dQ\n/X2owSGGfvKtCCmRxeLmGb7xjVf3BnJydiCX5Wb+nnvusdtlgFZPNX8+s4QAHhys8ORyi4Uk5b7+\nMr4UNFLN7ZW1L3N35P3lhTpnw5iikowHHvf0lTbk/0yjzR9OLXAg8FhINMOew8GCx6GCz4GCj7WW\nXzx5jnv6Snzf7kEWkpT/+NI0PzU+whcW67S0oZFq9gUeDw5UGPVd/nqxzqlWhAXeGkmeqLU40Y4Y\n2l/Fk5JACpraUHEUqbX4UhBISVsbmtpQWCUguULwpoEyA6ummbr3dyESY3HEqrnTVdM42lqmooSG\nNgiyEfNu38W/DO2PNRa0RTdiVNlDuFd2yiddigCLqvhgTFZ/R/UWqq++d9sRVlDiqs7nvpoQQjxm\nrb3nWtdjO9uZV4ppt2k99jjJubPEZ89SuOMO0pkZ0oUF/MOHqX73dyMch3R+HjUwgHgZ2tacnBud\ni7U111wosdbyZ1MzfPzFL/KLt9zNrYNHr2j+q1lONefCmJuLPmfCmKef/CTvf+BDl7wu7BihBhew\nwzDW8n999TTvaksOfseBjSP8nJzrgFwouTDLn/0s8fPPo5eWqL773bh79+b2EDk5L5OLtTXXbPrG\nWMvn5pd5oRXx2mJENP8xnpqT2yqUVCzcWgoQQnBzMeDmU38E932wY2G+ivYSFPp7Xy8kjHRpfW2S\nD6celbeNb0e1c3JyrjKm2aT+8BcIn3oKXV+m8o53MPDDP4Iqb9TC5mwkNSlSSKTI2k5jDdpotNUo\noXBVZhcY65jUpNSiGkW3yHw4T8Wt0EpbtNM2kNltxTqm4mXHm0mTSEe40qWZNDHW4EinY4xuV7ZY\njDWZMbqlt+9Ih9SkG+osEFhsb3spxCXmdDfLo3tNt5zLYSt1utJstY5DhSHu23PfFSnzmggls3HC\nR6cXef9oP9853MfT80/zgZs/QMHZvqWwVhvqXzxHcOsQ3t5ydnDqWxA3SOoOptrAV/3wzJ/DE78H\nP/yxrWcuBZUHxyGNwNloTJuTk3P9UPuLT1L/3OcY/YV/RN9734M1ZsdNw2ijkUIS6ai3hHUpWqLq\nVTm9fJrx8jhTzSlSkzIfzqONZiAYYK49RyttAVmHs6+yj7P1s6QmJdRhL39XuvjKRwpJK23hK59Y\nxwAooXCkQ6xjXOVirMGVLtpmiwu6wojp+vuA3rHu8cQkWGvxlIe1ltHiKI2kQb/fz7n6OQb8AYpu\nsZdnwSlQj+uUvBJDwRCBExDrmJKbCYmrO3kpZMdQvfNPrN0mJsFTK3ZWl5omz7m6XDWhxFrLC+2I\nh+aXGXAdfnJ8BNn5IUw1p3jfTe/j82c+T5iGBE5wRctufmMGvRB2VqIswUiElwrae38WvjnJ1Mlv\nEN7yLL4fc4i7EX4VJh6Hva/l/FO/SlQscujQRzbm++gU7u4S8cnzcM8ofPnX4f6fhsKVWa+dc+PT\nHdWt3l+9zLh3btUIsJu2l0fnXG9/1feSW8KRO86efceRLi6y+Ad/gJ6bp/Jd72LPL/9yTyuy3QJJ\nLapxunaa5XgZV7oooagndcpumfON88Q6RkqJQDDbnmV3cTcnFk9greXoQKZZLrpFym6ZZtJkX3Uf\nj0w9gq98dhV3cXzoOFJIFtoL3Nx/M77yEUKgreZc/RzvPfxeUpviyouvarxRWC2QwNX1OZJzaa5K\na/Xwl3+XqQNv51C1yt8dG6LUnl+zfrKZNHsS7689+mv88/v/+YUzS0KsThBBBdtewvol5LqXKV0I\nsanBxhoroDE/Qe2mr3Nw7/dT/9ILnPirX+emfa9Bjr2Juc8/QvnDIwzP/D3c5BNM73Fx3/73mfna\nP6Tt/AbVFz9J6Y6/TxhO8x+f/C/8wmt/BqwkfKSOXHyKJz83zbP7PsV7P3eYAacAD/0SvPfXt+U5\nXi6JSbatoYl0xGRjksnmJGPlMUaLo6QmpeJVsNaSmAQlFHPtOc41zjHfnic2MSWnRDtt91SrUsgt\nq0u7aS6kUny5KtHVatv1129Wr0vV43LpjuJ6eYqVvNef655f8x16o8Dufvf8nSN30uf3XZF63mjo\nWo3pX/kV/JuPoBfmGf65n8M0m6/YCVRqUh6ffpxm0sRi6fP7OFc/R7/fz0NnHuJQ3yFKbonFcJE+\nv49W2uL+PfdzpP8Ibd0GC0cHjrIULfH63a8HoJ228ZWPklvzOXPb0G0bjg0Gg2u+Ozjc1H8TAK54\ndQgkOTufbRFKFsIFppvT3Fo9yCN/+BHU4XfyP5gp/P6xLMHHfor0w/8dRzoshAt88dwXed9N7+N9\nN70vm+uLm5CEUBrakLf9f97ON2/SDN/6P7L4wu8R1OvsHvtekiBg4LafBiA+s8y5U3+A1x7HBAN8\ne+Af0176O4TL/wtDqs7x5B0snz9LNPQspQeXGD01SevEkwRvuYvAjtFuzjNd+wDzJ/8No40Af2KG\nmeiPGU2+xZkTv8Pc9KcYnB3k64VvEbzhvVRMgYfki9y1+7Xo2Qblc79LnCzieyNIGaBNG4FEqSLG\nxmAt2oSMDL8DISSO08//3957R8dx3Xm+n4qdG0A3cgaRSTCLURQpURapYImj4CfJsq2R1155kt/u\nGc/6jO2d2fXMWt63b9Yz3gnHIznKlmXJsoKVsyiRohjEDBIEARI5x85daf8oAASYSZFicH3Owenq\n6lu3bjW66377/pIk2RFAhhElFmsmkezCsgwsU7MFnGUhK0EsU0cQZFQ1jCi5MYwEsuQjluxna9fb\nSALopkYkHUPyVOD1lCMIFrppokgusj3Z+BU/ATWAJEoMxAdoHWtFlVQUUUE39RmTWlJP4lN9ZLoy\nGUwMohkaAH7VT6GvkMrMSpqGm9g/sJtocgCvBGltlIAEumWR6coix1tAeWYBiigR1SK4xQCCYGHo\nUQRRQRBkBOSJyzSxp31zKt3z5D5RVEGQJiZmAQHRzvI4mTFEELGTsUzGIcP0FGbHViCsafvPYKs9\nrSP4ub92/uc63ZlOfZzvKkmSd6GxDIOhx36Md9EifKtWoeTlASCq5x4+nTJSPHHgCSRBIugK0j7e\nzh2Vd5AyUuwe2I1bdrMobxGmZfK3K/52yrxwJjLdx/zavMopwogdHK4yLoooCblDbOraxNBHLzMq\nZbIit4EPn3sS782dNHgrcMkevvbm13hs3WO8cuQVvrnkm1PHKpJCT+9O3P0GWdesOaHv5JJ7KW38\nNWrvTyi0vLTkuknHuxhK9eNL34PV76Il9giB4CKC1xXw+y1/hR7+Cum9z4LspbD6bojvI7LnZgq+\nsBS1OEBciyN4ukhaWcRfaqdX0cgtWMLY/g6eKY9RceApMubfT0Jfjn9oHdm70gwtLEFIww1z/wiX\nK4f3e3axzzCRZLij34Ka+7B8OViWPrWSE3n3XZSCAty1tZhmipGRrYBFKtWPKCpYloEgyASD8wkE\n5iFOW3ZPpQeRJS+CIGMYcTRtlHiqj/0j7XRGjmIKHlaWfI6KrFqOjHdQ6CsgFT9AMtGFIEhYmFiW\nSTTdQjKZoE9PYlg6QTXIykAYUbCIpIfxe3z2ZD+BZYkk9BHSqT5K3D4kcfKmncDSmzHHD1MtCwiq\ngpJVhywHkOUAqpqNZelYloVppTH0OIIgkClnIAgigiAhSX4sS8c07dBgm5niQph6DqaZPiZaLBMm\nrokpc8Wx5zPqVkxMAOJxKdJmTgznlqLs1JPK+aQ6O9dznH0/oujkKJnEsiyGfvTviF4PyUOHcFVU\nkHn36aPvDNOgN97L+53v45JcGJaBIipEtSjvdbzHmpI16KbOvbX3okgKsXRsSvAD1IZqP41Lc3C4\narho5ps9fX2MNsVZGLF4izdYkOvlqcf/iRFZpaehimJfFi2jLXx/6/d5oP6BqeM0Q+NH7z7HNaMr\nWT9/BEXJoq/nBXy+amQ1k7b0FmoX/xeElrchXE1+9RL8uofMgQM07foXrD1u8iu/QPaNi9jet52K\nBY+wZ3APhblzaRNl5hlpuO1/4bluFNwqm7s3c2j4EHO7Mig7Ws8jlT9lmdzA55Z8kdK8OubPSlD+\n1vegr4te33/giR0/pEYsYpf7MN+8/ufIokzfI49QtriKlqd/ysityzA/8wDWhz9GDy3D0F2kWttI\nN+0h845bie/ejVJUDFhkuRcx3tJEXNMJxiGYW4ycm4PkyaRtvI2WsRb2DuxluVlO7RENK5lADARp\nadnOaEila1k5NaEaPl9xD/sG91EZngswtSTryVgMGYvP+n+Wf0E/Acfh+P86XELGX32N3p88inDT\ndWQuWcrvE5v40hf+mvbxdnb272RR7iJ+uv+n1GTVENfj+BU/C3IX8MvGX1ISKEEWZebmzCWgBBhN\njeJX/dxVfRduyT1DPE5f3XBwcDh3Lkqekt7hYZ5/5sdkVmUytOcgvcII65bN46PnXsT06RhdGayt\n87HN7Sc418sXlv0ArTeGIAmY6Gx89hHSoVZyCuspqZqH/8m/oKuyCLXiZvL1QrwL/wRMA6bZV42P\nfkvv6wLezxUSLLmG33U9xxM7/pmnrEIGUklSi77KPr+B8GEbmYvqeKz9lwTVIItzF5EwktTtUWjM\nGeSWhg1s/u0ThOQMFhWsI3v9LGIfvE/s1TcQ/QUkizvo2XA/IXeImqwa9OFhIm+8iTKnjnhOgOF/\n+TdaPr8C4ZmXWFW1BNWn4UrtgwX3IsQGMQUPA/tGGYkN0pbuxUqlkasrGVCT+EeSpPt6cQ/HiC+p\nZ/lYDh6Xj8H0CC9WjRPTYxT4CqjOrGb5eC6R114lY8MGlOJiRLebdFsbsc2bEb1efKtXo3V3o7W3\nY+mG/X4JAoLbjZydg1JchBmNkm5tRc7JQS0vx9I00u3tiF4vgsuFlJGBHLLt0JZp2uYFUSR14ABm\nMoUZjWCmUkh+O5rJmsjnIsgyalkpSkHBCZ8NyzQxxsYwx8Ym0mmbdubXyWJfomjX+TCMiYJZ5lQx\nMkGW7XOYxlRVUETpWHVSSZwoSmNNs4Sc4vM92bc5YcqxrGnHThbDmWbumf7auXCuFplzNuGcvr1a\nMeusw1ivtjwlk1EV/f/4j/SZo/xdxV4+U3ojUZJsqNzAjr4dtI238fD8h3n1yKuUBctYVrCM/UP7\nMUyD1rFW1pWtOydfDgcHhzNzaZKn7Xma/xrZw4OzH+TP3vwzsr3Z/Nwzm2eOiMzNyaPz0C+JBGPM\nW/3HZHkbiHzURiSikRDbSPXPZatrOxuq/yPV60tg0w9IVyxHHulCzCyH0uWYcY3IB11krCsHIPby\n23hb/yvCtX8KCx/g15v+jvsy5iD07aNlLEifkUube4i9rdsIF5Xw4G3/GdkQSP7zY7xZJlCcVcpH\nv3kCn+DhTx99AkPX2Pn0C+TuP4oiRFFLV6Bv/0f8d3yZZMcg/tWr0YeGSB89SuiLX5wq853s7uLD\nX/0Dlfd+mQNmF+vL12NpKSxZQRREPtz1Ez7u/pD6OfezMHsumZILQfWBKNEf7yfHk2MX6mpsxD13\nLunWVlzV1fTGegm7w3REOpiVOQuwHfUSe/ehDwxgxmMoBQX4b7gBY3iYxM6dqGVlyDk5SJnHfr2Z\n8Tj68DBaZyeix4Orqgp9YIDU0aMIsoJaXoYZj2Mlk+iDQxhjY1NiAVEEw8BVU4Mgy0jhsF1CfDxi\nf5hkaaKmR4xUSytmLHbMoXnSj0MUkYIZSBlBkCQEWZmqLoplTlTqnBQaoh35MClUdN0+ZrLE+ORx\nhn5MrAjCNCdq4ZRWE7vqqACCOM3KIxw7ftL8c8rXzsRZtDmbJp8wMkDOz0d0n10029UkSizTZPDf\n/o3YYC9PlnSStXg5DzU8RMpIoUoqLslZunNwuFR86qIkkUjQu+kJwkvuJhDIAAFeb3udtZ5cxk0P\nI7EXGN/8PjmeCjxLvk48soWxnU1YqVq2flzIyroov7Be5it5f0pVwyHw50FuPWz+Iaz4c0xPNvGt\nvQiqhHd+Dggw+LP95Dw0B23HT9H9eWze+3NuXP8DyCjmwAfv0rZ3F+OpMZSiMNWBcooiCVIHDnDA\nSNLX1UrXtblU9mQwOjTEA3/5HZTCQrqffJLBozIZ/gLGxCHK75hNsLgKY3SU+M6dpFuPELztVrt8\n+El4r+M92iPtuGU3B4cOUhwoZlXRKqp1E9q3QGwA/LnQuQ3m/j8gyiAp9v6ya0H1n5jYzcHhInG1\niBJNSzHw6L8zvPEdfvGnVXx35XcveJoBBweH8+fTXymJDvDx269RoS9DCrtRajKQFIvm5u/jyglT\nUfFniL9+gLHeHo6UP0zx2GLGAypNnVHWzt9MojGTAS2HZq2HAfUjqueu4No198LPbsO4/xV6/u73\nZH1uJboHYv1j5NSXMDQ6QEZVHt/54DssHulhUcV66hY8CMDHLz+PqRvUr76Bj373G6IbN7Lw81+i\n9eB+hsZG8FhpwlqcHDUPlyBALE7WFx4gffQog4MBBvw95JRXoCUTVC+7Fkk+e1ec8fQ4aSNNtif7\nhPcId9BOtpaOw2gbGGlIRaBnD6TGwZ0BszdAoAASI7a5SpTB0GZknHVwuBBc6aJEMzR+88G/4tq6\nj46FRRQUVLOh5k4ncsXB4TLj008z78/BSJagLA7xxiuvM7LjMEtrZMKDt9ETlaFCRfjcz/Ef3UPo\nZ0MMi2MYt85i2bVFuB9di/rFV0nmlXPgpadYuW0Dnbv307TwKLVrv026fQBzrIWR4VK+L/2cz0SW\nM/rOm/TJh6j7wWHu/pvv8NHWZ6iYdQfjr7/Ozm2bifg8LEkJRBubKWppxvO1P+epH3yPqiUrWHj7\nnYiiSGnDfARRZOinPyPznrvp+/u/J+/b3yYjGKT5X3/A0jvv4Y1//xcaN77NXX/938/6rQiqwVO+\nR1OoXnslaJLyVfbjWBccftMWK4IIscEJ84Fkr6SULD1LU4KDw9VNOjLGUwef4vpDKjn3/y2uYqfk\ng4PDlchFWSkxEzoDh7rZ0bWP65avon3LjxnPXM7hxkPkpgIMWjHu+5MH6Hh0H7GuA0SjYQJ31TN7\nSQB++xCRwj9n+OhRnutsZk1kLT3pHfy+4QO+WHwnNcOluGtzaGr6gBxfNkqbSGTeLJ57/28I1zRw\nX829pJqaMONx2kYGGRJMSjt6KbrrbuS8fESPG6WkhFf++R9Y9/DXkU+Rl0AfHETOzj5h/4H336H+\nuhvO+j27KGgJezWleycs/Y9gavaKS3zYXklxn0IIOTichit1peTFLT8n7+9/TuG628n5o7uRi4vA\ngva9uwhk5xIuLplqq6VTyIqKlkoiqyqi48Dq4PCp86mvlIgemdx5JcxyJRgYeomS+Wt4+t+fp6Sg\ngIV33MlHP3qNw1t7GWlvZe94B27hdWo/ymHkcJJE6CY8/X20HTrE/Ows0iM7qb1xNf3PbkTp/AmJ\n+Q+Sde/1DLS8Stmwl3SiDPdzb/PQ6od43Uwz/uJLeObPQ8oKMWLEKZ+/mOplK08YY2FN/SkFCXBS\nQQLgDgSJj43izbiE5hPFA6XLIFwJO34Chg6WYfugmBrU3gbBEyNfpogN2e3HOiGrHMa7IK/Bfq1v\nH4x2gJ6AzDIYPmKbihSvvS8xap+/5pYT/V0sC+JD4Mu2t1PjtlBKjUNGCXhDxyJZYv22Y6ukQnLM\nFlWyB2QVZLctri7WKtDkGLDsekVaAoyUPR5rIspo+vb0CJfzTHA2cfD5j/d8ySy1V+KuUp56/FHM\nwyKuO/+WxoPvY/3zm7h9A3j8LlbcvYHn/+F71K1chSQriJKElkriDWbSsuMjMnLzyS4tJyu/kPyq\nGnpbDhEuKqHncBMjPd14gkGC2bkMdbaTX1lN3qwqTN2g5/AhktFxsksrCITCJCLjuP1+EpFxgtm5\nE47YJq07t6G43CQiEbzBINllFbh9frRUEtV98ep8OThcyVzUNPPJo79jJNVAbjF8bt0txD7YSPeh\nMQquKyG9r5vuIhHTlUci1s7mYZ3WPovC8BgL/8NdRJIJbn3wj9n4fx4nb9VCQju+zNjtEqWDBTzZ\n/BRPujbxwr0voI+niLwewSgvJrOvD2H5MjLWraP/aCv+5PhJBQnA3BvXn9c15ZRV8Nz///fUr7qe\nOWtuvLQ3F182LPnKzH2WBS/9JVSvs1dMunfZzrOKxzYBTRYN9OWAJwTtH0KwEPY+bWfSDc2C0uV2\nm0gvzJ5niwY9aQsTd9AWMbt/bfdrWbZ5Ccue3DNK7PaCAO5M8IbBFYCuHbZfDACCbb4SJHs8Lr8d\nOaMnQE/bAsHQTy1KzjrLqjDx/CT9TPYtu+zrklTbZ0ews8TakTfiKaJtzkEsBh1wMQAAHO1JREFU\nnZOwuhj9fvqVRT9NctrbiYdDRIbfp+q+B8jxmUhmgEMf9NG0VaNh7dcQJTfhohAls0PIikQimqZ4\n9mLCxXn0H2lhoP0o4x/1Iysq6XgcLZVk3mduRpJk+tuOULl4KaO9vRzctJHoyDCBcDalDfM5snM7\nisuFnk6TiseQVReGrqG6PRi6RmnDfMb6eimorkVLJmna/D6GlsbtD9jh7oKAaRgkoxFk1YUky6Ti\nMVvoRCMoLjeyy0VWfiHjA31kFRZjpNNER4bJKigiVFj0ievyaKkksqLO6Mc0DYCJJIcCpmlMbQPE\nx0anxJ2ua4iiiChJSIpi3wZSSQTRbq+lUlimiWnaIf72tolpGJjGxHkA2eUmGY0gShKy6mLyczu1\nkn9CiL+AKIkTfUyUZhDEia/rROLFib/TbYuTqQhmIEwE351YduIEzvQ1PJfv/9n++DiHr/R5VRY+\njx9BgiiiuC6MM/lFMd+kU6NsffVRlPZs2NdBLFhC+OYb4fA+yu+6Ho+i88Jjz7HmwVv4/TPP4FZl\nFo8dobtrBHXFWpo+3oJaUcsXvvJVnnn8NzR4ZnGwVaAwMEr92vn8j/7/zbeWfYuAGpg6Z1tbG4FA\ngO7ubhoaGmj+aDOIAtVLVpzXG3M6TNPgo2efwpeZxbwbb+bw9o8orpuDeyJfx3T2vfsmBVW1HN6+\nheK6OfQfbbFvePJFrDVhGtC53S4MmFEEqlNu3eHMXInmm0OvPcbr8nXENYt5xZlsPTKMWxHxqhK7\nOsYAKJBl7i7OxkqZaGmdzFwvYwMJJEUkndAJhNxYFrh9Ct6gQl5FBqJ0fMbfS4OeTjPW30cgO5vR\n3h4MXSOYnUtkaJD+Iy1IijI1yZqmiSiKmKaJZVl2Eb+J/ZOVjidztwiind9YUhSMCXGAZU0JBWCq\njf1konSDIKC4Pbi8XhKRcWRVxTItTENHT6enJifTNMCyUFxuBFG0xyKKCKI0tW2LGDv3j55O4fYH\nMA0DQ9NmvgmTZS+m/T8mhY4kKxN5hawZj9aM5+ZEKiJzYpF02nPTnFkAc2Jl1H44/dx4xgl/uqD6\nBOLlzMLo9K+f38f43A7yBIIU1tSdfe+ftvlGG9bp2hqkpjifynuKCd50E2/8ZD/hjFyUjiZizc0s\nvX0FOTk5rDp0iL74OOX33UDVnZU8/vxm1IoaGmaVMT7Yz+3rP8uO373PkoZaPviglTd/+DAb/vqv\nMOIGz738HLfccgsul4uxsTFKSkpobm7GNE10LU3dtTPT1JumSVdXFyUlJacY+Yk0NzdTXV09Y58o\nSqy4+372v/cWlmly4P13OPDBu9z+n745o11PcxNaMsFwdwezFl5DIhKhbN4i9r79BgvW3Xr+b/CZ\nECXbvOPgcJUTDrr58xXHUrnf3GCH54/G09wxX+ftg334XDKvDUaRBAGPR6RYNRgIWliaxr03lBAM\nuoindbSxNOmkwbtPNJFbGkDXTErqQ/izXEiKiKJ++v4nsqpO+cTkls+a2u/LzCK/svpUhzk4XLFc\nFFHiK8hmw7e+jKyKyIr9Rb7m1nJiY2lGHnsEtbyMXMEivn07uddeix7K4MMjh3D1N6O1NFJ2292I\nWGx+6les+8rXSaUSGKkhEoUDGHouHb94hZayVrKzs2lra6O4uJjh4WFEUcTlcvHiiy9SmelHEASS\nySRNTU3Mnz+f/fv3E41GKSwsRJLOfIPRNI3Dhw+zZ88e/H4/69fPNPmEi0p475c/Ye1DD9O+d9fU\n/mQyiZ6Ic3TPxyxY/1k8/sCM4zr276Z562aql57ctOTg4HB2hHPy7Si1jKIZ+zO9KplelT++tgIA\n07ToGk2Q7XfRPZYg5FUZjqd5fn8PA9E0YZ9KNKUTcMsky1SUAhcrK0L0NI/R2zKKIAq2y9R4Gl+G\nSjKmYegWvgyV+Hgat0+hpD6Erpn4s1woLglJdnIMOTicKxfNp8Ttm2meyMr3kZXvI/Hgg4z86lf0\nfOvbKEVF5H3rW3h7O1EGMhju7WbWgsUsWbKUviMtCKLEjlefI5YRp/P93cSyU1juPI4cbGT9bXdj\n9nSQSCR46623qK6uZqiznQULFiDLMlvef4+cugZisRi9vb0Eg0Esy6KmpoYPP/yQJUuW4HLNzOqo\naRqyLGMYBrIsc/DgQebNm8fw8DA9PT0nXGN+VQ055RVIskJ8fAzTMGg+dIjBvh56j7Ry55ceQjyJ\n+KlfdT0HN20kFY/j8l69TogODhedWWth1y9h0ZdO20wUBUpC9netMsc2s2b5VCpz/FMmjbGEhiwK\neFWJ/++1Jna0jZDtV1E8IiUhL0nNoKwuSFVu4IT+TdOi69AIHr9K295BDN0iFddx+xUEAfS0gSAK\nGLqFKIIgCgiiQDqh4/LKWCb4Qy60pIE3w4WW1IlH0iguCT1loGvmlElJEAW0lIEoCYiiMC2Bsb1h\nGsfOMWVBsCws00KUxBnL+ZZlt59sI4oCbr+CpIhYhkUypk21m0SSRWRVJCvfOzUeSRYRJftRkoWp\nc9uuG5feDOZw5XBRHV1PhruuFt/yZUjBIIF1N6Hk5eJPJ/BlZjHQ1krtytV4MzLxZWZRPn8hhqbh\n0k20a0S0tj5ml9QTLyqg4+PtxNsPI1QtpHZuHQFMPnjycTZ849vU1dUR72rjvffeQ1VVVq9ezdat\nWwkGg4TDYRYvXsyzzz7LfffdNzWuaDTKpk2bKCgoYNu2bYTDYaqqqigsLKSoqIgjR44wMjJCVlbW\njOuZ9A2Ju3387298neLqWmbX1lJzy2fZum0buq4jCAIVFRWEw2FcLheqx0vdtatp+XgbsqI4KyYO\nDueLKNpO25E+COSdVxeTk2aG59gPqW/efMw+blkW7x4aoDTkZSiaZtPhowgC1OQFWFCSSe9Ykt7x\nJJVFfkI+lexiW/QYmomknHy1xDRMBFHANC0kyS6bMNQdI1jhJjaWwp/loqAqE9MwcXkvov8Zx2oE\nAVimLURM097nCdg+K5NtLMsinTSwTIuR3vhUnSjDMDF1C0M3MXS7ptRke/sc9rmmF+8+WVDbNNeV\nU4z1WB+WeZI+J9uZx/Ux6fN+Jk6ln87GL+QPGG9ApbgudEH6+tRFiSDLZGzYgH/tWqSA/YsjEM4h\nMzeP9V/7f/EEghP77LDSkd5uhg/tp03y4hMsAhkVDB7eSt/uF5l30zdoatuNTzZ443e/YtbipSSi\nETz+AH6Ph+vmL0ZVVbKysvB4PMyda1fR9Xg8zJ49m3g8jtfrpbu7m/b2dm688UYeffRRrr/+empr\naxGneaTfdtttvPzyy1x33XX4/X5kWWbr1q14PB6ys7NxBzPJDmczu76eeTfcBEBOfj66riOKIoOD\ng3z88ceUlJRQVFSE6vFycNN7rLzn8xze/hGlDfMY7+8jVFzCxl//gmUb7kGQFdra2sjKyqK7uxtd\n10kkEuTl5VFTUwPA0NAQHo8H78SKy+HDh8nLyyMQOPHXnIPDVUn1Otj2YzsS7SKUZRAEgRtqc6ee\nL5sVxrIsjgzGeGFXNwG3TFnYx7tN/cTTBvG0QcinYExMjJIgkBNwYZgW+RluvKqEKAgkdYMcv4uQ\nTyWS1hGzVFSPjKiKyNLkdVx8P5bpKxmCKOAJnJgqYbKNIAi4PPa0UVCZcdHH5vCHx8UryHcKth0d\nZkl5iJFYmrRhkhc8fRjR4W1beOEfvsfqb/wNe956jfv/4i9Jxyxe/Kefk1GwkOatr7Dkjnpmr1hF\n48a3CBUWU1zfwPhAP8WzG6b66ezspLCwEFEUGYik6OkfINrXzsqVK9m8eTPpdJrrr7+eSCRyygk9\nmUzy29/+lmuuuYba2lp2796Nqqp0dXVRWFiIz0jTf6SFpRvuOeX1dHd309nZicvlwjR05s1fwP4t\nm2jcvQtBkkmOj1G5ZDn73nuLinkLqamfg+L3k5WVhaoq9B89gqm6adyzC8njY9asWUQiESzLIpVK\nEQqFGBwcpL6+nsyJQnymaTIwMMDIyAiJRAK3200qlQKOuyFNeOrn5OQQDodRVXWGMHO4urkSo2+m\nGD4CA01Qe/PFGdQnIJE2iKd1VFlkf/c4pmUhCQI+l0xzf4R42sCyIORTGY1rSKIdceFSRMyJ+7Om\nWxiWhSqJWIAkQixlUJDhJsOjkDZMekaT+N0ymmFHmZiWhVuxBVDQLSOKAoORFIIgkO1XcSsSlgVe\nl4RXlVAlEUUW6R9PktYtJFEg06sgiwJuRZroCwzTQpZExuIaKcNAEgQk0TYlTW0LArIoEEvruGQJ\necLM5JhyHOBSVQk+BV97fAdVuX4USUSWBP7shqrTth/t62Wwow1vYQnNhw6x5vrrAYiNpYgMJ2nc\n+AG+TAF3YC41SzJo37cbTyBIqLAYf8j+RZPUTHZ2jBBN6mxsHuCashA/+6CFe/P6Wf+ZtbQebae8\ntJjsUyRMm040GuWdd94hEAhQXV1NUVERhmHYoXiGTtueXcxatOSs3otoNEpjYyM5OTmUlpQgyTKm\naUxlmdzz1mtoyQTB7FyK6ufw+o9+yOzVaxnu6kRxuRkb7GPNAw+dEF5sWRaNjY3E43FM08QwDMrL\ny8nKysLtdpNOp3G5XFNLq7quoyjK1PbIyAg9PT0IgjAVLjjpRCyKIqFQCJfLhf8kIdAOVy5XtCgB\n2P5TmPs5O+/NVU5at20XCc1gLK4hS/ZqjGaYKJKINJlTRDMwDIu4pmNaEPapWBYMRlPE0waSKJBI\nG8TSOmndJKkZFGR4kCVbRAxG0hiWRUozSOkmpmUhCgKGaRFwy7gVCcO0MC37zzCZeq6bFh5FIq2b\nGJMhuhxnypngdCabU70+ue/4187Ul8OFJ8fvYmXVmefPSS4bUWKaFt99sZEMj8J/vqmGF3Z3s6oq\nm5Dv1JlVj+dQX4SavGMrGemkzgv/tJPS2WGW3j6LbS+8iiDEWfzZOxEEgc0tgzy9vZOu0QTXVmYj\nSwL7u8dYVhFmX9cYmf27oGguNy+sYHFZ1gnn+8WHR1k3O5+8oGuGym9ubqaysvKiriRM2nG7Dx2k\nbe9OKhZcMyMMUEsmeeXZ3xOdtYyOkTizC4L0jScJuGUWhESaWtrQs0vxqgrRlM41ZVlk+VR8E8vH\nfZEkezvHCPtVhmOa7egmCKyty0U8LqHQ5ErM5KpLPB5Hm8gnoOv6lIOwOJEL4Uzvy3Q79qXmbMbx\nScb6Sa/zkxxfXl4+Zdo7i/Nc2aJES8Cep2Dxgxd+UA4ODheMTz9PiWGreEU6NjENRVN8+Wfb+G93\nzGEgYpsObm3I5/uvHOQ7n5094/jTTVhPbevgO5+dzRuNfdxYl4smwO1fX0jb3kEADm07yOr71yMI\nApGkxq+3dvDD+xbQPZakKNMz1b9pweaWQfo3xrn7jxbx7M5O4mmd66qPFco7tLWX3+7oZGf7KHfM\nL2RNTQ7RtE7AJZ+Qu+RiIAgCB3vHCRdU0B3x0zFsMdB+mJBPZTiWpirXTyKziPrRPSz0+9HdHm6d\nU8rml19k54EUK1YtoXvXh5iixPyGBra9t5OYZqElk+jpFF6fl1UrFtH08RYCPi+FNXWMRxM8uS2F\nz2Uv7aZ0A1EQmF0YZHZBEEEQKCsrI5bS0QyTTO/ZC8rLFdM0P9Hx5yLsL+SxZ3P82YS+XzUoHiha\nDI3P29W1HRwcrjguiiiJpwy2HBli/Rw7kVH3aIJH32/lf94zj9q8wJTgkCWRlG5ysHecuvxjReT+\n8und/Lc75tA7lqQs7EURRRKaQVNfhG1Hh2nsHuerv9jODbU53NyQz71LSgFo3TnAaG8fH70wSFGd\nxWv7+6gvCJCMakSbxtjU0cm191QjCAKSAKuqsnn5jR6Ge2K4JYnnd3VTke2jOMv+Zdny8QAPLS8j\nbpj8ZlsHI/E0332xkQeWlfJX6+tIagZu5eLc9PvGk/xmWwdZPpVYSmdNTQ6SKKBKItkBF4oksL97\nnPW3rGSkp4velmYGt7zOzuZsCosKue4OO3la2axZxEZH6D/ayk3r1oAg4PbZOVxMw6CrqZHVt92C\nlkjQeXA/qmUxNxFH1l3klJQx0NGGkU7TvjnG1vEESmk9QjpBuruVvLq5xN2Z6Cbopp3f0O+SyPAo\npDQTtyqR43dRlesnmtIZjqUJuhWCHhmPImFatkP7UCxN0CPjki/NBOr4zVxF5DfYtZY2/RCWPWyX\nEXBwcLhiuGjmm2d3duJRJJZVhPnN9g7uWVxMtv/EG0RSM/jllja+cp2drXAsrrH6f73Dt26t4wdv\nNLNuTh4V2T4O9UXoHEmwpiaHNxr7uHtRMSurwmw7OsydC4vpOjTCYEeUcEOA997sZJOV5I4FRayf\nk8/L/7qHqmvySCd0OzOjSyIQdrPr9XaK67LY+WYHvgwV98pc3ukaZl1JCCthEOmIohZ6uf0zs+iP\nJPnF5jZumZuPZlg8v6sLv0vmjxYWMRxLs6R8ZjiUZVm0D8eJJHX2dY2R7XexpDxE0CPbvhqm7UiW\n1OyViL7xJG809qGbJh5FIqWbfHFF2TlN1LqmISufPHwwGYuCBb2tzeRXVuPy2mnqe5oPMtTVQbio\nlILqWo7s3E4iMs5Yfx+Vi5eSVViEqLjsfA+YjPT2kFB8tPRH8fu8ZMgGKdnDeEJjPJ5C1FNIHh+Z\nLpGYbtvJT2Y3hmO245NxttaN6X0JgoBLFjFMC8M8eccn6/f4lM8nbzP9NWFq2y6nIUw8nt2YPymL\ny7LOejXrijffTCcVteszlSyDgnkXZmAODg4XhE/dfAP2zbtzJEH3aBfvHOzna2sqT9rOrUiEfOqU\nyWZj8wCrqrLZdnSEp7+2gh9tbOG1/b10jSb4m8/OoSjTQ2nIS2nYS3GWl+1H7SJv4YogvgIvLzX2\nIvkUFh2McfMXCtj20hEari+mdHYI07Bo2tJL294h0kkdQzNpWFNMbnmQviPj7Ph9G3d8sZrGF4+S\nGkxyaL6Pewbt2gq5ATffWH8snXV+0I3fLfPY+60cGYyxr2uMhyayR3aOxDncHyWlmwjA6pockprB\nE1vbkUWB0UQan0vG75IRBduzvSLbx92Li1EkuwiU5zxSWl8IQQLg9tmOguXzFs7YX1hTT2FN/dTz\nSYdeXdNo3fERvS3NaKkkoigiu9yECouQR/qoF0W0kSSK20MyGiFDkkAQUN0eUpEuhMgxE8rkqoVl\nWXaRsOmebNNVyam82aa3O5a96djziddMCzTTRJyIFjiuE06W1MBi5u6TOt9Ne+3E463pQzgPTlZs\nkNPuE/MXwVVgYjtnXH5Y+lU4+DKMHIW62+zyCw4ODpc1FzVPSftwnJG4xiN3zT1tu1k5fpr6ItTl\nBzFMi395YBEPP76d3KCL797RQPdYYsqkAjC7MHhCH3s6x9jeNkyO38Uta8rZ2NOEZVr0to4zf22J\nbbKRBWavKqRuRT562kRx2Tep3LIgmXleZFUkeiSGsSCTexcWkZYE6EuSiGh4gzNv7PkZdijzf/pM\nDZph8uTWdsaTGo9/2EbQoyAJAp9fVjrjmD+5/uqMCpAVhZrlqy71MBwcTqTuVhjrhO0/gXAVVN5w\nqUfk4OBwGi6aKFk2K8SCkkxKQ94TIjmOZ3ZBkBd2d/Pk1g4qc+2JO+RzTZkupguS4/GqEpGkxv98\n9SC1+QHq84P4s1xULc5l60tHWLiuFNUz8zJFSUT1zPQjUN0ylYtyefsXB/iTrzZM+b1EBYnNzxzm\nMw/NdMadjiKJfO6aEv7P283MyvZz9+Li016vg4PDp0hGsb1qMnjYDht2B2HWDeC9MBkoHRwcLhwX\nTZQUZHjOuq0qi7y+vxfDtPjCcnt1YX7x2WULXFWdzcOP7+Av1laxoCRzKlV01eJcdr3ZQSB0+uRs\nM8bhllm0vmxG5I8/y0UqoZOKa6dN9+xWJO5fWorsOE06OFyeZFfZf/FhaH0H0nHQk+AK2CLlPNPU\nOzg4XDg+9eRpp+LDliFWVIbP69ijgzG8qkTuGbLDni+GYXJ4Wx+1ywvO6TjTMBElR6Q4XBlcVY6u\n54KegtZ3ITZgP5dUu56OpIAnBIF8cGc4PikODheIS+Loeq6cryABKM/2XcCRnIgkiacMl9jyXAtL\nbqtgpC9G35FxelrG8ARUQgVexvoThIv9GLoJFpTMDuH2Koz2xxnpjSMIUDonjKyICGcwcZ0MLW1X\nCpUc4ePgcP7ILqhZf+y5aUCk1/7Ox4egfQvE+m2xkooAAhRfA6IMiWHIqbOFi4ODwyfmshElVxJD\nXVE6D45QvSQPb4aL9sYh0kkDWZWYd0MxnoBKMqpRsyyfvtZxJFkkFdd4+nvbCBcHmLOqkILKDPrb\nI7z3RBOCCEtvn0Ug5GagPcJQV5SS2SF2vNJGuMiH6pYpqQ9x+ON+imoy8QRUJFlk//tdWBbMua4Q\n1e38Kx0cLgiiBBlF9nawEPKPc9TXkjByxC5Fm1UOvXuh+XVbpEyuPPuyoXAhGGkIFjl5zx0czhJn\nJjtLFJdEKqHj8sjsebuD3PIgBz/sweWViY+nkVWJ2mXHfi1N+rIUVmdO7fvSI9fOcPqtyHTh9ink\nVwTZ8Vobbq9MMMeD6pFp/KCbykU5FNVkMdIbY9/GLmqW5tF1aARZlRjuiTF3TTGiJLD190eYtSCH\n4e4ovkwXhTVZpOIazdv68GW6UN0yqbiOoZuIkkAyquEPuTA0ExBw+2S0tIGiSpimhSiJBEJuLMsi\nM8+Lch7hyQ4OVy2KG3KPhcaTVX5im9ggdO+y2x7dZIsSLQGy2xYqogSCZGeh9WRBOgZaHPLmQLga\njJT9uvwHGM7t8AfNZeNTcrmjpQ0+fq2NwspMEtE0NUvz2fVmO3OvL6bxg260tMGidWXn3b9hmMRG\nUgSzz95BePqxBzb1UDo7xEhvnN7WMWRVZMFNpUiSyPhQApdXmSo5blkWhm4iKxKWaRGPpHF7FVIJ\nHcUlkU7qJCJpDM1ipDdGOqmjumUsy0JLGXiDLqIjSURJRHGJGLpF/coCJNkxI13J/MH6lHzaGJrt\nrwK2GIkPgeyxhcpAE4x12GJFT9krL6Y+IWRk+0/12j4vOXW2r8v0VRjTsNvK0/zrnFUah8uMy6Yg\n35XOUFeUN3/WyNov1ZNTEpixPyPXg3yRUs5fbkRHkniC6pQvSzqp07ytD1kRMQz78+TxKwSzPQiC\nwHBPzPar4STJxkxrhj/N9I+jIDCVD8yymFplsrCO5Qib3t+ZPspne28+WU6yM7U52bk+WVmbC0JJ\nfQhP4A8wo+vVTHLMFjJ9++0IImHajwFBsAWPbtcXw5qs6yRMtJv2xZlMNGgZ9qrM9D5OmBesY8eY\nhi2gRAkUry2gBMle3Znse7KfyWOmn3P6tiAeG5swbd/Un2Q/itK0NtOOkV3Hzjd5TtOwr9sy7XFb\n5nF9imfOYHhKIXeK/Res/QUYz6UYuyDZq4JnyRXh6HolECrwcddfLT7BnBEuujqTop0Kf9bMD5/q\nlplzXRF62varARjpjaFrJlgWpXNCn9jnxbKs4+6nzq8/hz9Q3Bn2X2jWpTm/OW2y1+K2WcrUQZpe\nRmQydfG07MrHb089mhPiyJy5b1JcmBroEyLDnBQb2GJKTzElmKZ+vUjTBI1wTIRMChXLYKZAOo6T\nCpZTiJiL1vasd16AMZyi/bm09YahdPkp2p8bjig5BwRRcPwrToM87b3Jyr+wEVHCZPEYBweHS8tU\nLiYJpAmB5OBwgTgn840gCANA28UbjoODwyWkzLKsnEs9COc+4+Bw1XPKe805iRIHBwcHBwcHh4uF\nEy7h4ODg4ODgcFngiBIHBwcHBweHywJHlDg4ODg4ODhcFjiixMHBwcHBweGywBElDg4ODg4ODpcF\njihxcHBwcHBwuCxwRImDg4ODg4PDZYEjShwcHBwcHBwuCxxR4uDg4ODg4HBZ8H8BbC5MtnqL7HoA\nAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 576x792 with 22 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-354Uihy1Rae",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "4ae2bfb1-4558-4326-bb89-d402777ece7f"
      },
      "source": [
        "from flair.data import Sentence\n",
        "\n",
        "classifier = TextClassifier.load('resources/tagger/ag_news/best-model.pt')\n",
        "\n",
        "# create example sentence\n",
        "sentence = Sentence('France is the current world cup winner.')\n",
        "\n",
        "# predict class and print\n",
        "classifier.predict(sentence)\n",
        "\n",
        "print(sentence.labels)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:25:50,788 loading file resources/tagger/ag_news/best-model.pt\n",
            "[NUM (0.8993986248970032)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9W_l3cvi2T3F",
        "colab_type": "text"
      },
      "source": [
        "## B. Multi-Dataset Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AWY5V2GD1yNX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from typing import List\n",
        "from flair.data import MultiCorpus\n",
        "from flair.datasets import UD_ENGLISH, UD_GERMAN\n",
        "from flair.embeddings import FlairEmbeddings, TokenEmbeddings\n",
        "from flair.embeddings import StackedEmbeddings\n",
        "from flair.training_utils import EvaluationMetric"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7JMK7mP22vMW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 619
        },
        "outputId": "de1e3265-5308-4623-c533-30d82dac740c"
      },
      "source": [
        "# 1. get the corpora - English and German UD\n",
        "\n",
        "corpus: MultiCorpus = MultiCorpus([UD_ENGLISH(), UD_GERMAN()]).downsample(0.1)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:49,955 https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-dev.conllu not found in cache, downloading to /tmp/tmp8qbi4rvj\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "1668224B [00:00, 81161784.41B/s]         "
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:50,010 copying /tmp/tmp8qbi4rvj to cache at /root/.flair/datasets/ud_english/en_ewt-ud-dev.conllu\n",
            "2019-12-20 10:28:50,013 removing temp file /tmp/tmp8qbi4rvj\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:50,467 https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-test.conllu not found in cache, downloading to /tmp/tmpfjv_k0o3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "1662046B [00:00, 74138832.96B/s]         "
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:50,522 copying /tmp/tmpfjv_k0o3 to cache at /root/.flair/datasets/ud_english/en_ewt-ud-test.conllu\n",
            "2019-12-20 10:28:50,526 removing temp file /tmp/tmpfjv_k0o3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:51,804 https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-train.conllu not found in cache, downloading to /tmp/tmp3tsypmpb\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "13303560B [00:00, 121538497.39B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:51,947 copying /tmp/tmp3tsypmpb to cache at /root/.flair/datasets/ud_english/en_ewt-ud-train.conllu\n",
            "2019-12-20 10:28:51,965 removing temp file /tmp/tmp3tsypmpb\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:52,302 Reading data from /root/.flair/datasets/ud_english\n",
            "2019-12-20 10:28:52,302 Train: /root/.flair/datasets/ud_english/en_ewt-ud-train.conllu\n",
            "2019-12-20 10:28:52,303 Test: /root/.flair/datasets/ud_english/en_ewt-ud-test.conllu\n",
            "2019-12-20 10:28:52,308 Dev: /root/.flair/datasets/ud_english/en_ewt-ud-dev.conllu\n",
            "2019-12-20 10:28:58,564 https://raw.githubusercontent.com/UniversalDependencies/UD_German-GSD/master/de_gsd-ud-dev.conllu not found in cache, downloading to /tmp/tmpjye0_yqy\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "882822B [00:00, 65446354.52B/s]          "
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:58,610 copying /tmp/tmpjye0_yqy to cache at /root/.flair/datasets/ud_german/de_gsd-ud-dev.conllu\n",
            "2019-12-20 10:28:58,613 removing temp file /tmp/tmpjye0_yqy\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:58,953 https://raw.githubusercontent.com/UniversalDependencies/UD_German-GSD/master/de_gsd-ud-test.conllu not found in cache, downloading to /tmp/tmpq5i6o31q\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "1177197B [00:00, 75082831.55B/s]         "
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:28:59,000 copying /tmp/tmpq5i6o31q to cache at /root/.flair/datasets/ud_german/de_gsd-ud-test.conllu\n",
            "2019-12-20 10:28:59,004 removing temp file /tmp/tmpq5i6o31q\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:29:00,312 https://raw.githubusercontent.com/UniversalDependencies/UD_German-GSD/master/de_gsd-ud-train.conllu not found in cache, downloading to /tmp/tmp6ne8a96p\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "18886219B [00:00, 127353957.93B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:29:00,508 copying /tmp/tmp6ne8a96p to cache at /root/.flair/datasets/ud_german/de_gsd-ud-train.conllu\n",
            "2019-12-20 10:29:00,529 removing temp file /tmp/tmp6ne8a96p\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:29:01,029 Reading data from /root/.flair/datasets/ud_german\n",
            "2019-12-20 10:29:01,030 Train: /root/.flair/datasets/ud_german/de_gsd-ud-train.conllu\n",
            "2019-12-20 10:29:01,033 Test: /root/.flair/datasets/ud_german/de_gsd-ud-test.conllu\n",
            "2019-12-20 10:29:01,035 Dev: /root/.flair/datasets/ud_german/de_gsd-ud-dev.conllu\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t0sWcK1Y24qY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "fa32aca3-e9d5-4bfd-ff94-95d2320f559e"
      },
      "source": [
        "# 2. what tag do we want to predict?\n",
        "tag_type = 'upos'\n",
        "\n",
        "# 3. make the tag dictionary from the corpus\n",
        "tag_dictionary = corpus.make_tag_dictionary(tag_type = tag_type)\n",
        "print(tag_dictionary.idx2item)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[b'<unk>', b'O', b'PRON', b'ADJ', b'AUX', b'VERB', b'ADP', b'PUNCT', b'ADV', b'CCONJ', b'PART', b'PROPN', b'NUM', b'DET', b'NOUN', b'SCONJ', b'SYM', b'X', b'INTJ', b'<START>', b'<STOP>']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NFrghKgc3NA_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 183
        },
        "outputId": "388cad3b-9461-4927-d864-9911c7b27f97"
      },
      "source": [
        "# 4. initialize embeddings\n",
        "embedding_types: List[TokenEmbeddings] = [\n",
        "            # we use multilingual Flair embeddings in this task\n",
        "            FlairEmbeddings('multi-forward'),\n",
        "            FlairEmbeddings('multi-backward'),\n",
        "]\n",
        "\n",
        "embeddings: StackedEmbeddings = StackedEmbeddings(embeddings = embedding_types)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:32:06,530 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4.3/lm-jw300-forward-v0.1.pt not found in cache, downloading to /tmp/tmpbx_tw_sr\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 172513724/172513724 [00:01<00:00, 86554145.89B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:32:08,709 copying /tmp/tmpbx_tw_sr to cache at /root/.flair/embeddings/lm-jw300-forward-v0.1.pt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:32:09,122 removing temp file /tmp/tmpbx_tw_sr\n",
            "2019-12-20 10:32:11,034 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4.3/lm-jw300-backward-v0.1.pt not found in cache, downloading to /tmp/tmpols8lb_u\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 172513724/172513724 [00:02<00:00, 74201534.54B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:32:13,549 copying /tmp/tmpols8lb_u to cache at /root/.flair/embeddings/lm-jw300-backward-v0.1.pt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:32:13,913 removing temp file /tmp/tmpols8lb_u\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j2S2-i6F3oqQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 5. initialize sequence tagger\n",
        "\n",
        "from flair.models import SequenceTagger\n",
        "\n",
        "tagger: SequenceTagger = SequenceTagger(hidden_size = 256,\n",
        "                                        embeddings = embeddings,\n",
        "                                        tag_dictionary = tag_dictionary,\n",
        "                                        tag_type = tag_type,\n",
        "                                        use_crf = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XoK1aekY4Anm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "fa036f4b-4ca0-4910-b774-9ea78a7cba1b"
      },
      "source": [
        "# 6. initialize trainer\n",
        "\n",
        "from flair.trainers import ModelTrainer\n",
        "\n",
        "trainer: ModelTrainer = ModelTrainer(tagger, corpus)\n",
        "\n",
        "# 7. start training\n",
        "\n",
        "trainer.train('resources/taggers/example-universal-pos',\n",
        "              learning_rate = 0.1,\n",
        "              mini_batch_size = 32,\n",
        "              max_epochs = 150,)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:34:58,067 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:34:58,073 Model: \"SequenceTagger(\n",
            "  (embeddings): StackedEmbeddings(\n",
            "    (list_embedding_0): FlairEmbeddings(\n",
            "      (lm): LanguageModel(\n",
            "        (drop): Dropout(p=0.1, inplace=False)\n",
            "        (encoder): Embedding(11854, 100)\n",
            "        (rnn): LSTM(100, 2048)\n",
            "        (decoder): Linear(in_features=2048, out_features=11854, bias=True)\n",
            "      )\n",
            "    )\n",
            "    (list_embedding_1): FlairEmbeddings(\n",
            "      (lm): LanguageModel(\n",
            "        (drop): Dropout(p=0.1, inplace=False)\n",
            "        (encoder): Embedding(11854, 100)\n",
            "        (rnn): LSTM(100, 2048)\n",
            "        (decoder): Linear(in_features=2048, out_features=11854, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (word_dropout): WordDropout(p=0.05)\n",
            "  (locked_dropout): LockedDropout(p=0.5)\n",
            "  (embedding2nn): Linear(in_features=4096, out_features=4096, bias=True)\n",
            "  (rnn): LSTM(4096, 256, batch_first=True, bidirectional=True)\n",
            "  (linear): Linear(in_features=512, out_features=21, bias=True)\n",
            ")\"\n",
            "2019-12-20 10:34:58,074 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:34:58,079 Corpus: \"Corpus: 12543 train + 2002 dev + 2077 test sentences\n",
            "Corpus: 13814 train + 799 dev + 977 test sentences\"\n",
            "2019-12-20 10:34:58,081 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:34:58,081 Parameters:\n",
            "2019-12-20 10:34:58,082  - learning_rate: \"0.1\"\n",
            "2019-12-20 10:34:58,083  - mini_batch_size: \"32\"\n",
            "2019-12-20 10:34:58,083  - patience: \"3\"\n",
            "2019-12-20 10:34:58,085  - anneal_factor: \"0.5\"\n",
            "2019-12-20 10:34:58,085  - max_epochs: \"150\"\n",
            "2019-12-20 10:34:58,086  - shuffle: \"True\"\n",
            "2019-12-20 10:34:58,088  - train_with_dev: \"False\"\n",
            "2019-12-20 10:34:58,088  - batch_growth_annealing: \"False\"\n",
            "2019-12-20 10:34:58,089 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:34:58,090 Model training base path: \"resources/taggers/example-universal-pos\"\n",
            "2019-12-20 10:34:58,091 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:34:58,092 Device: cuda:0\n",
            "2019-12-20 10:34:58,092 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:34:58,094 Embeddings storage mode: cpu\n",
            "2019-12-20 10:34:58,095 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:34:58,939 epoch 1 - iter 0/83 - loss 69.26076508 - samples/sec: 304.37\n",
            "2019-12-20 10:35:05,058 epoch 1 - iter 8/83 - loss 54.42609787 - samples/sec: 41.97\n",
            "2019-12-20 10:35:10,962 epoch 1 - iter 16/83 - loss 47.61938050 - samples/sec: 43.49\n",
            "2019-12-20 10:35:18,153 epoch 1 - iter 24/83 - loss 43.99043945 - samples/sec: 35.69\n",
            "2019-12-20 10:35:24,673 epoch 1 - iter 32/83 - loss 40.78706215 - samples/sec: 39.38\n",
            "2019-12-20 10:35:31,181 epoch 1 - iter 40/83 - loss 37.99252840 - samples/sec: 39.44\n",
            "2019-12-20 10:35:36,999 epoch 1 - iter 48/83 - loss 35.55068148 - samples/sec: 44.14\n",
            "2019-12-20 10:35:42,773 epoch 1 - iter 56/83 - loss 33.81946336 - samples/sec: 44.48\n",
            "2019-12-20 10:35:49,965 epoch 1 - iter 64/83 - loss 32.24067981 - samples/sec: 35.70\n",
            "2019-12-20 10:35:56,198 epoch 1 - iter 72/83 - loss 30.83912740 - samples/sec: 41.22\n",
            "2019-12-20 10:36:03,659 epoch 1 - iter 80/83 - loss 29.63828056 - samples/sec: 34.40\n",
            "2019-12-20 10:36:05,290 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:36:05,291 EPOCH 1 done: loss 29.4169 - lr 0.1000\n",
            "2019-12-20 10:36:12,651 DEV : loss 11.181921005249023 - score 0.7225\n",
            "2019-12-20 10:36:12,665 BAD EPOCHS (no improvement): 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type StackedEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type FlairEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type LanguageModel. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Dropout. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Embedding. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type LSTM. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Linear. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 10:36:13,863 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:36:14,142 epoch 2 - iter 0/83 - loss 18.27656746 - samples/sec: 931.55\n",
            "2019-12-20 10:36:15,834 epoch 2 - iter 8/83 - loss 17.09182252 - samples/sec: 155.91\n",
            "2019-12-20 10:36:17,348 epoch 2 - iter 16/83 - loss 17.37812065 - samples/sec: 171.35\n",
            "2019-12-20 10:36:18,648 epoch 2 - iter 24/83 - loss 16.43968861 - samples/sec: 200.23\n",
            "2019-12-20 10:36:20,101 epoch 2 - iter 32/83 - loss 16.02409054 - samples/sec: 178.29\n",
            "2019-12-20 10:36:21,477 epoch 2 - iter 40/83 - loss 15.62083384 - samples/sec: 188.76\n",
            "2019-12-20 10:36:22,955 epoch 2 - iter 48/83 - loss 15.34706361 - samples/sec: 175.42\n",
            "2019-12-20 10:36:24,530 epoch 2 - iter 56/83 - loss 15.01506966 - samples/sec: 164.50\n",
            "2019-12-20 10:36:26,048 epoch 2 - iter 64/83 - loss 14.88080632 - samples/sec: 170.90\n",
            "2019-12-20 10:36:27,644 epoch 2 - iter 72/83 - loss 14.68894113 - samples/sec: 162.41\n",
            "2019-12-20 10:36:29,087 epoch 2 - iter 80/83 - loss 14.49139281 - samples/sec: 180.21\n",
            "2019-12-20 10:36:29,468 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:36:29,470 EPOCH 2 done: loss 14.5416 - lr 0.1000\n",
            "2019-12-20 10:36:30,060 DEV : loss 8.465563774108887 - score 0.7895\n",
            "2019-12-20 10:36:30,074 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:36:31,318 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:36:31,656 epoch 3 - iter 0/83 - loss 11.30260181 - samples/sec: 760.83\n",
            "2019-12-20 10:36:33,443 epoch 3 - iter 8/83 - loss 11.68857098 - samples/sec: 145.11\n",
            "2019-12-20 10:36:34,936 epoch 3 - iter 16/83 - loss 12.17649101 - samples/sec: 173.80\n",
            "2019-12-20 10:36:36,460 epoch 3 - iter 24/83 - loss 12.08102161 - samples/sec: 170.38\n",
            "2019-12-20 10:36:37,860 epoch 3 - iter 32/83 - loss 12.02867343 - samples/sec: 185.83\n",
            "2019-12-20 10:36:39,538 epoch 3 - iter 40/83 - loss 12.02219661 - samples/sec: 154.46\n",
            "2019-12-20 10:36:41,008 epoch 3 - iter 48/83 - loss 11.87339148 - samples/sec: 176.58\n",
            "2019-12-20 10:36:42,536 epoch 3 - iter 56/83 - loss 11.79550445 - samples/sec: 169.92\n",
            "2019-12-20 10:36:44,028 epoch 3 - iter 64/83 - loss 11.73329437 - samples/sec: 174.12\n",
            "2019-12-20 10:36:45,423 epoch 3 - iter 72/83 - loss 11.66714467 - samples/sec: 185.90\n",
            "2019-12-20 10:36:46,982 epoch 3 - iter 80/83 - loss 11.60569786 - samples/sec: 166.46\n",
            "2019-12-20 10:36:47,258 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:36:47,259 EPOCH 3 done: loss 11.5680 - lr 0.1000\n",
            "2019-12-20 10:36:47,871 DEV : loss 6.524747848510742 - score 0.8345\n",
            "2019-12-20 10:36:47,886 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:36:49,169 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:36:49,327 epoch 4 - iter 0/83 - loss 9.65458679 - samples/sec: 1664.83\n",
            "2019-12-20 10:36:51,066 epoch 4 - iter 8/83 - loss 10.98426618 - samples/sec: 150.23\n",
            "2019-12-20 10:36:52,540 epoch 4 - iter 16/83 - loss 11.00101757 - samples/sec: 176.18\n",
            "2019-12-20 10:36:54,261 epoch 4 - iter 24/83 - loss 10.98830910 - samples/sec: 150.51\n",
            "2019-12-20 10:36:55,829 epoch 4 - iter 32/83 - loss 10.97502252 - samples/sec: 165.23\n",
            "2019-12-20 10:36:57,218 epoch 4 - iter 40/83 - loss 10.67235807 - samples/sec: 186.98\n",
            "2019-12-20 10:36:58,713 epoch 4 - iter 48/83 - loss 10.61145099 - samples/sec: 173.49\n",
            "2019-12-20 10:37:00,096 epoch 4 - iter 56/83 - loss 10.60798096 - samples/sec: 187.78\n",
            "2019-12-20 10:37:01,370 epoch 4 - iter 64/83 - loss 10.38886308 - samples/sec: 204.08\n",
            "2019-12-20 10:37:02,816 epoch 4 - iter 72/83 - loss 10.37339729 - samples/sec: 179.59\n",
            "2019-12-20 10:37:04,256 epoch 4 - iter 80/83 - loss 10.22885620 - samples/sec: 180.33\n",
            "2019-12-20 10:37:04,542 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:37:04,545 EPOCH 4 done: loss 10.1854 - lr 0.1000\n",
            "2019-12-20 10:37:05,135 DEV : loss 5.8639678955078125 - score 0.8458\n",
            "2019-12-20 10:37:05,149 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:37:06,461 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:37:06,826 epoch 5 - iter 0/83 - loss 10.47763252 - samples/sec: 711.08\n",
            "2019-12-20 10:37:08,284 epoch 5 - iter 8/83 - loss 10.91794178 - samples/sec: 177.71\n",
            "2019-12-20 10:37:09,737 epoch 5 - iter 16/83 - loss 9.99184421 - samples/sec: 178.75\n",
            "2019-12-20 10:37:11,348 epoch 5 - iter 24/83 - loss 9.93122618 - samples/sec: 160.74\n",
            "2019-12-20 10:37:12,866 epoch 5 - iter 32/83 - loss 9.92745372 - samples/sec: 170.88\n",
            "2019-12-20 10:37:14,337 epoch 5 - iter 40/83 - loss 9.62890298 - samples/sec: 176.10\n",
            "2019-12-20 10:37:15,778 epoch 5 - iter 48/83 - loss 9.60751630 - samples/sec: 180.26\n",
            "2019-12-20 10:37:17,307 epoch 5 - iter 56/83 - loss 9.49888836 - samples/sec: 169.65\n",
            "2019-12-20 10:37:18,886 epoch 5 - iter 64/83 - loss 9.47183503 - samples/sec: 164.28\n",
            "2019-12-20 10:37:20,388 epoch 5 - iter 72/83 - loss 9.47795019 - samples/sec: 172.58\n",
            "2019-12-20 10:37:21,858 epoch 5 - iter 80/83 - loss 9.46905272 - samples/sec: 176.52\n",
            "2019-12-20 10:37:22,128 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:37:22,129 EPOCH 5 done: loss 9.4091 - lr 0.1000\n",
            "2019-12-20 10:37:22,707 DEV : loss 5.972289562225342 - score 0.842\n",
            "2019-12-20 10:37:22,721 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:37:22,723 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:37:22,901 epoch 6 - iter 0/83 - loss 8.95880318 - samples/sec: 1447.10\n",
            "2019-12-20 10:37:24,564 epoch 6 - iter 8/83 - loss 9.11131393 - samples/sec: 155.54\n",
            "2019-12-20 10:37:25,892 epoch 6 - iter 16/83 - loss 8.76296119 - samples/sec: 195.91\n",
            "2019-12-20 10:37:27,326 epoch 6 - iter 24/83 - loss 8.96934824 - samples/sec: 181.09\n",
            "2019-12-20 10:37:28,790 epoch 6 - iter 32/83 - loss 8.93455890 - samples/sec: 177.14\n",
            "2019-12-20 10:37:30,402 epoch 6 - iter 40/83 - loss 9.09161781 - samples/sec: 160.76\n",
            "2019-12-20 10:37:31,830 epoch 6 - iter 48/83 - loss 8.93318603 - samples/sec: 181.67\n",
            "2019-12-20 10:37:33,337 epoch 6 - iter 56/83 - loss 8.91900985 - samples/sec: 172.25\n",
            "2019-12-20 10:37:34,860 epoch 6 - iter 64/83 - loss 8.85779337 - samples/sec: 170.38\n",
            "2019-12-20 10:37:36,328 epoch 6 - iter 72/83 - loss 8.74361344 - samples/sec: 176.78\n",
            "2019-12-20 10:37:37,761 epoch 6 - iter 80/83 - loss 8.74043769 - samples/sec: 181.13\n",
            "2019-12-20 10:37:38,066 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:37:38,067 EPOCH 6 done: loss 8.7441 - lr 0.1000\n",
            "2019-12-20 10:37:38,661 DEV : loss 4.980047702789307 - score 0.8703\n",
            "2019-12-20 10:37:38,673 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:37:40,100 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:37:40,348 epoch 7 - iter 0/83 - loss 8.23031235 - samples/sec: 1040.96\n",
            "2019-12-20 10:37:41,926 epoch 7 - iter 8/83 - loss 9.18127579 - samples/sec: 164.23\n",
            "2019-12-20 10:37:43,303 epoch 7 - iter 16/83 - loss 8.52679850 - samples/sec: 188.75\n",
            "2019-12-20 10:37:44,642 epoch 7 - iter 24/83 - loss 8.21979301 - samples/sec: 194.06\n",
            "2019-12-20 10:37:46,198 epoch 7 - iter 32/83 - loss 8.19198080 - samples/sec: 166.75\n",
            "2019-12-20 10:37:47,621 epoch 7 - iter 40/83 - loss 8.19807241 - samples/sec: 182.48\n",
            "2019-12-20 10:37:49,129 epoch 7 - iter 48/83 - loss 8.24811704 - samples/sec: 172.46\n",
            "2019-12-20 10:37:50,604 epoch 7 - iter 56/83 - loss 8.25754030 - samples/sec: 175.87\n",
            "2019-12-20 10:37:51,969 epoch 7 - iter 64/83 - loss 8.23376444 - samples/sec: 190.71\n",
            "2019-12-20 10:37:53,638 epoch 7 - iter 72/83 - loss 8.23734928 - samples/sec: 155.17\n",
            "2019-12-20 10:37:55,251 epoch 7 - iter 80/83 - loss 8.25344080 - samples/sec: 160.80\n",
            "2019-12-20 10:37:55,552 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:37:55,552 EPOCH 7 done: loss 8.2429 - lr 0.1000\n",
            "2019-12-20 10:37:56,138 DEV : loss 5.0074687004089355 - score 0.8631\n",
            "2019-12-20 10:37:56,151 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:37:56,152 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:37:56,368 epoch 8 - iter 0/83 - loss 8.24047279 - samples/sec: 1197.60\n",
            "2019-12-20 10:37:57,934 epoch 8 - iter 8/83 - loss 8.30115975 - samples/sec: 165.16\n",
            "2019-12-20 10:37:59,548 epoch 8 - iter 16/83 - loss 8.20198497 - samples/sec: 160.59\n",
            "2019-12-20 10:38:01,088 epoch 8 - iter 24/83 - loss 8.00085224 - samples/sec: 168.62\n",
            "2019-12-20 10:38:02,662 epoch 8 - iter 32/83 - loss 8.23067941 - samples/sec: 164.77\n",
            "2019-12-20 10:38:04,100 epoch 8 - iter 40/83 - loss 8.17605514 - samples/sec: 180.15\n",
            "2019-12-20 10:38:05,638 epoch 8 - iter 48/83 - loss 8.06437376 - samples/sec: 169.28\n",
            "2019-12-20 10:38:07,001 epoch 8 - iter 56/83 - loss 8.04220616 - samples/sec: 190.61\n",
            "2019-12-20 10:38:08,509 epoch 8 - iter 64/83 - loss 7.99521533 - samples/sec: 172.35\n",
            "2019-12-20 10:38:09,859 epoch 8 - iter 72/83 - loss 7.93466217 - samples/sec: 192.36\n",
            "2019-12-20 10:38:11,354 epoch 8 - iter 80/83 - loss 7.91984476 - samples/sec: 173.31\n",
            "2019-12-20 10:38:11,622 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:38:11,623 EPOCH 8 done: loss 7.9025 - lr 0.1000\n",
            "2019-12-20 10:38:12,193 DEV : loss 4.684683322906494 - score 0.8776\n",
            "2019-12-20 10:38:12,206 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:38:13,599 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:38:13,740 epoch 9 - iter 0/83 - loss 4.92710543 - samples/sec: 1844.34\n",
            "2019-12-20 10:38:15,288 epoch 9 - iter 8/83 - loss 6.74842623 - samples/sec: 167.23\n",
            "2019-12-20 10:38:16,860 epoch 9 - iter 16/83 - loss 7.05773833 - samples/sec: 165.19\n",
            "2019-12-20 10:38:18,368 epoch 9 - iter 24/83 - loss 7.08279362 - samples/sec: 172.17\n",
            "2019-12-20 10:38:19,746 epoch 9 - iter 32/83 - loss 7.25534231 - samples/sec: 188.56\n",
            "2019-12-20 10:38:21,151 epoch 9 - iter 40/83 - loss 7.28500249 - samples/sec: 184.76\n",
            "2019-12-20 10:38:22,601 epoch 9 - iter 48/83 - loss 7.35965964 - samples/sec: 179.04\n",
            "2019-12-20 10:38:23,992 epoch 9 - iter 56/83 - loss 7.42034939 - samples/sec: 186.66\n",
            "2019-12-20 10:38:25,664 epoch 9 - iter 64/83 - loss 7.44682730 - samples/sec: 154.90\n",
            "2019-12-20 10:38:27,135 epoch 9 - iter 72/83 - loss 7.45908644 - samples/sec: 176.48\n",
            "2019-12-20 10:38:28,840 epoch 9 - iter 80/83 - loss 7.47912092 - samples/sec: 152.06\n",
            "2019-12-20 10:38:29,120 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:38:29,123 EPOCH 9 done: loss 7.4621 - lr 0.1000\n",
            "2019-12-20 10:38:29,698 DEV : loss 4.571529388427734 - score 0.8758\n",
            "2019-12-20 10:38:29,710 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:38:29,711 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:38:29,898 epoch 10 - iter 0/83 - loss 6.77807617 - samples/sec: 1377.90\n",
            "2019-12-20 10:38:31,254 epoch 10 - iter 8/83 - loss 7.12093321 - samples/sec: 191.47\n",
            "2019-12-20 10:38:32,697 epoch 10 - iter 16/83 - loss 7.01142263 - samples/sec: 180.26\n",
            "2019-12-20 10:38:34,126 epoch 10 - iter 24/83 - loss 7.03485512 - samples/sec: 181.66\n",
            "2019-12-20 10:38:35,582 epoch 10 - iter 32/83 - loss 6.98291032 - samples/sec: 178.31\n",
            "2019-12-20 10:38:37,275 epoch 10 - iter 40/83 - loss 7.08934445 - samples/sec: 152.98\n",
            "2019-12-20 10:38:38,927 epoch 10 - iter 48/83 - loss 7.21821034 - samples/sec: 156.77\n",
            "2019-12-20 10:38:40,326 epoch 10 - iter 56/83 - loss 7.21576888 - samples/sec: 185.61\n",
            "2019-12-20 10:38:41,809 epoch 10 - iter 64/83 - loss 7.28338361 - samples/sec: 175.11\n",
            "2019-12-20 10:38:43,264 epoch 10 - iter 72/83 - loss 7.28328400 - samples/sec: 178.22\n",
            "2019-12-20 10:38:44,734 epoch 10 - iter 80/83 - loss 7.23488447 - samples/sec: 176.71\n",
            "2019-12-20 10:38:45,102 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:38:45,105 EPOCH 10 done: loss 7.2881 - lr 0.1000\n",
            "2019-12-20 10:38:45,702 DEV : loss 4.601739406585693 - score 0.877\n",
            "2019-12-20 10:38:45,717 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:38:45,718 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:38:45,933 epoch 11 - iter 0/83 - loss 8.70719051 - samples/sec: 1197.10\n",
            "2019-12-20 10:38:47,382 epoch 11 - iter 8/83 - loss 7.47643206 - samples/sec: 178.85\n",
            "2019-12-20 10:38:48,936 epoch 11 - iter 16/83 - loss 7.26776412 - samples/sec: 166.82\n",
            "2019-12-20 10:38:50,354 epoch 11 - iter 24/83 - loss 7.26257210 - samples/sec: 183.04\n",
            "2019-12-20 10:38:51,901 epoch 11 - iter 32/83 - loss 7.32549095 - samples/sec: 167.74\n",
            "2019-12-20 10:38:53,408 epoch 11 - iter 40/83 - loss 7.24328122 - samples/sec: 172.03\n",
            "2019-12-20 10:38:54,719 epoch 11 - iter 48/83 - loss 7.12212392 - samples/sec: 198.15\n",
            "2019-12-20 10:38:56,420 epoch 11 - iter 56/83 - loss 7.11503856 - samples/sec: 151.97\n",
            "2019-12-20 10:38:57,842 epoch 11 - iter 64/83 - loss 7.09200459 - samples/sec: 182.57\n",
            "2019-12-20 10:38:59,327 epoch 11 - iter 72/83 - loss 7.07559994 - samples/sec: 174.79\n",
            "2019-12-20 10:39:00,979 epoch 11 - iter 80/83 - loss 7.06939439 - samples/sec: 156.66\n",
            "2019-12-20 10:39:01,294 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:39:01,295 EPOCH 11 done: loss 7.0451 - lr 0.1000\n",
            "2019-12-20 10:39:01,869 DEV : loss 4.338400363922119 - score 0.8819\n",
            "2019-12-20 10:39:01,882 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:39:03,216 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:39:03,436 epoch 12 - iter 0/83 - loss 6.56368637 - samples/sec: 1170.29\n",
            "2019-12-20 10:39:05,049 epoch 12 - iter 8/83 - loss 7.10369963 - samples/sec: 160.56\n",
            "2019-12-20 10:39:06,559 epoch 12 - iter 16/83 - loss 7.03696731 - samples/sec: 171.67\n",
            "2019-12-20 10:39:08,055 epoch 12 - iter 24/83 - loss 6.94990343 - samples/sec: 173.50\n",
            "2019-12-20 10:39:09,447 epoch 12 - iter 32/83 - loss 7.13607831 - samples/sec: 186.99\n",
            "2019-12-20 10:39:10,751 epoch 12 - iter 40/83 - loss 7.01233768 - samples/sec: 199.26\n",
            "2019-12-20 10:39:12,320 epoch 12 - iter 48/83 - loss 7.01080422 - samples/sec: 165.18\n",
            "2019-12-20 10:39:13,924 epoch 12 - iter 56/83 - loss 6.92985609 - samples/sec: 161.50\n",
            "2019-12-20 10:39:15,415 epoch 12 - iter 64/83 - loss 6.93880225 - samples/sec: 174.83\n",
            "2019-12-20 10:39:17,116 epoch 12 - iter 72/83 - loss 6.97879584 - samples/sec: 152.27\n",
            "2019-12-20 10:39:18,643 epoch 12 - iter 80/83 - loss 6.92198747 - samples/sec: 170.37\n",
            "2019-12-20 10:39:18,934 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:39:18,935 EPOCH 12 done: loss 6.9096 - lr 0.1000\n",
            "2019-12-20 10:39:19,521 DEV : loss 4.4909186363220215 - score 0.8764\n",
            "2019-12-20 10:39:19,534 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:39:19,535 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:39:19,699 epoch 13 - iter 0/83 - loss 7.60832787 - samples/sec: 1600.79\n",
            "2019-12-20 10:39:21,081 epoch 13 - iter 8/83 - loss 6.58605385 - samples/sec: 187.46\n",
            "2019-12-20 10:39:22,670 epoch 13 - iter 16/83 - loss 6.67756277 - samples/sec: 163.55\n",
            "2019-12-20 10:39:24,130 epoch 13 - iter 24/83 - loss 6.69873127 - samples/sec: 177.68\n",
            "2019-12-20 10:39:25,664 epoch 13 - iter 32/83 - loss 6.54773354 - samples/sec: 169.54\n",
            "2019-12-20 10:39:27,370 epoch 13 - iter 40/83 - loss 6.59942144 - samples/sec: 151.91\n",
            "2019-12-20 10:39:28,730 epoch 13 - iter 48/83 - loss 6.58750126 - samples/sec: 190.94\n",
            "2019-12-20 10:39:30,238 epoch 13 - iter 56/83 - loss 6.56654673 - samples/sec: 171.76\n",
            "2019-12-20 10:39:31,875 epoch 13 - iter 64/83 - loss 6.64885448 - samples/sec: 158.45\n",
            "2019-12-20 10:39:33,354 epoch 13 - iter 72/83 - loss 6.63610093 - samples/sec: 175.36\n",
            "2019-12-20 10:39:34,754 epoch 13 - iter 80/83 - loss 6.63221518 - samples/sec: 185.68\n",
            "2019-12-20 10:39:35,047 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:39:35,048 EPOCH 13 done: loss 6.5951 - lr 0.1000\n",
            "2019-12-20 10:39:35,636 DEV : loss 4.4225993156433105 - score 0.8796\n",
            "2019-12-20 10:39:35,649 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:39:35,650 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:39:35,821 epoch 14 - iter 0/83 - loss 6.81556320 - samples/sec: 1518.14\n",
            "2019-12-20 10:39:37,373 epoch 14 - iter 8/83 - loss 6.37592978 - samples/sec: 167.36\n",
            "2019-12-20 10:39:38,780 epoch 14 - iter 16/83 - loss 6.39885832 - samples/sec: 184.47\n",
            "2019-12-20 10:39:40,404 epoch 14 - iter 24/83 - loss 6.27557083 - samples/sec: 159.51\n",
            "2019-12-20 10:39:41,851 epoch 14 - iter 32/83 - loss 6.19680402 - samples/sec: 179.72\n",
            "2019-12-20 10:39:43,407 epoch 14 - iter 40/83 - loss 6.37198814 - samples/sec: 166.64\n",
            "2019-12-20 10:39:44,811 epoch 14 - iter 48/83 - loss 6.42068851 - samples/sec: 184.84\n",
            "2019-12-20 10:39:46,485 epoch 14 - iter 56/83 - loss 6.46003637 - samples/sec: 154.54\n",
            "2019-12-20 10:39:47,889 epoch 14 - iter 64/83 - loss 6.47737109 - samples/sec: 184.88\n",
            "2019-12-20 10:39:49,421 epoch 14 - iter 72/83 - loss 6.50648117 - samples/sec: 169.21\n",
            "2019-12-20 10:39:50,821 epoch 14 - iter 80/83 - loss 6.45758587 - samples/sec: 185.26\n",
            "2019-12-20 10:39:51,254 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:39:51,255 EPOCH 14 done: loss 6.4929 - lr 0.1000\n",
            "2019-12-20 10:39:51,835 DEV : loss 4.074848651885986 - score 0.8894\n",
            "2019-12-20 10:39:51,848 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:39:53,164 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:39:53,346 epoch 15 - iter 0/83 - loss 6.49517488 - samples/sec: 1420.05\n",
            "2019-12-20 10:39:54,765 epoch 15 - iter 8/83 - loss 6.05388514 - samples/sec: 182.52\n",
            "2019-12-20 10:39:56,131 epoch 15 - iter 16/83 - loss 5.88115650 - samples/sec: 190.09\n",
            "2019-12-20 10:39:57,708 epoch 15 - iter 24/83 - loss 6.02517967 - samples/sec: 164.21\n",
            "2019-12-20 10:39:59,210 epoch 15 - iter 32/83 - loss 6.11489673 - samples/sec: 172.65\n",
            "2019-12-20 10:40:00,758 epoch 15 - iter 40/83 - loss 6.16147676 - samples/sec: 167.51\n",
            "2019-12-20 10:40:02,552 epoch 15 - iter 48/83 - loss 6.37763942 - samples/sec: 144.13\n",
            "2019-12-20 10:40:03,994 epoch 15 - iter 56/83 - loss 6.31912879 - samples/sec: 180.19\n",
            "2019-12-20 10:40:05,556 epoch 15 - iter 64/83 - loss 6.29994893 - samples/sec: 166.09\n",
            "2019-12-20 10:40:06,974 epoch 15 - iter 72/83 - loss 6.32597731 - samples/sec: 183.32\n",
            "2019-12-20 10:40:08,391 epoch 15 - iter 80/83 - loss 6.28431275 - samples/sec: 183.37\n",
            "2019-12-20 10:40:08,657 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:40:08,658 EPOCH 15 done: loss 6.2450 - lr 0.1000\n",
            "2019-12-20 10:40:09,249 DEV : loss 4.040278434753418 - score 0.8952\n",
            "2019-12-20 10:40:09,261 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:40:10,732 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:40:10,974 epoch 16 - iter 0/83 - loss 6.64808083 - samples/sec: 1066.77\n",
            "2019-12-20 10:40:12,440 epoch 16 - iter 8/83 - loss 6.40014479 - samples/sec: 176.76\n",
            "2019-12-20 10:40:13,966 epoch 16 - iter 16/83 - loss 6.18790301 - samples/sec: 170.22\n",
            "2019-12-20 10:40:15,539 epoch 16 - iter 24/83 - loss 6.28128042 - samples/sec: 164.84\n",
            "2019-12-20 10:40:17,161 epoch 16 - iter 32/83 - loss 6.28709079 - samples/sec: 159.73\n",
            "2019-12-20 10:40:18,568 epoch 16 - iter 40/83 - loss 6.24958607 - samples/sec: 184.34\n",
            "2019-12-20 10:40:20,002 epoch 16 - iter 48/83 - loss 6.27539743 - samples/sec: 181.06\n",
            "2019-12-20 10:40:21,560 epoch 16 - iter 56/83 - loss 6.26266237 - samples/sec: 166.36\n",
            "2019-12-20 10:40:23,042 epoch 16 - iter 64/83 - loss 6.28356419 - samples/sec: 174.99\n",
            "2019-12-20 10:40:24,446 epoch 16 - iter 72/83 - loss 6.25290298 - samples/sec: 184.80\n",
            "2019-12-20 10:40:25,861 epoch 16 - iter 80/83 - loss 6.20792334 - samples/sec: 183.68\n",
            "2019-12-20 10:40:26,202 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:40:26,203 EPOCH 16 done: loss 6.2283 - lr 0.1000\n",
            "2019-12-20 10:40:26,806 DEV : loss 4.067651271820068 - score 0.894\n",
            "2019-12-20 10:40:26,820 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:40:26,822 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:40:27,017 epoch 17 - iter 0/83 - loss 6.13757181 - samples/sec: 1323.70\n",
            "2019-12-20 10:40:28,451 epoch 17 - iter 8/83 - loss 5.88755274 - samples/sec: 180.70\n",
            "2019-12-20 10:40:29,860 epoch 17 - iter 16/83 - loss 5.94177905 - samples/sec: 184.57\n",
            "2019-12-20 10:40:31,236 epoch 17 - iter 24/83 - loss 5.91725475 - samples/sec: 189.34\n",
            "2019-12-20 10:40:32,606 epoch 17 - iter 32/83 - loss 5.80798140 - samples/sec: 189.76\n",
            "2019-12-20 10:40:34,183 epoch 17 - iter 40/83 - loss 5.82774557 - samples/sec: 164.27\n",
            "2019-12-20 10:40:35,899 epoch 17 - iter 48/83 - loss 5.91884469 - samples/sec: 150.93\n",
            "2019-12-20 10:40:37,477 epoch 17 - iter 56/83 - loss 6.01353701 - samples/sec: 164.38\n",
            "2019-12-20 10:40:38,946 epoch 17 - iter 64/83 - loss 5.99859943 - samples/sec: 176.46\n",
            "2019-12-20 10:40:40,457 epoch 17 - iter 72/83 - loss 6.02863037 - samples/sec: 171.87\n",
            "2019-12-20 10:40:41,991 epoch 17 - iter 80/83 - loss 6.02224576 - samples/sec: 168.96\n",
            "2019-12-20 10:40:42,268 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:40:42,269 EPOCH 17 done: loss 6.0373 - lr 0.1000\n",
            "2019-12-20 10:40:42,862 DEV : loss 4.020432472229004 - score 0.8923\n",
            "2019-12-20 10:40:42,876 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:40:42,877 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:40:43,062 epoch 18 - iter 0/83 - loss 6.77072191 - samples/sec: 1398.01\n",
            "2019-12-20 10:40:44,631 epoch 18 - iter 8/83 - loss 5.76850478 - samples/sec: 164.79\n",
            "2019-12-20 10:40:46,122 epoch 18 - iter 16/83 - loss 5.80547465 - samples/sec: 173.84\n",
            "2019-12-20 10:40:47,595 epoch 18 - iter 24/83 - loss 5.78059164 - samples/sec: 176.02\n",
            "2019-12-20 10:40:48,969 epoch 18 - iter 32/83 - loss 5.87486238 - samples/sec: 189.38\n",
            "2019-12-20 10:40:50,447 epoch 18 - iter 40/83 - loss 5.88548185 - samples/sec: 175.71\n",
            "2019-12-20 10:40:51,931 epoch 18 - iter 48/83 - loss 6.00668457 - samples/sec: 174.92\n",
            "2019-12-20 10:40:53,689 epoch 18 - iter 56/83 - loss 6.06956970 - samples/sec: 147.55\n",
            "2019-12-20 10:40:55,173 epoch 18 - iter 64/83 - loss 6.08310128 - samples/sec: 174.89\n",
            "2019-12-20 10:40:56,570 epoch 18 - iter 72/83 - loss 6.02846095 - samples/sec: 185.97\n",
            "2019-12-20 10:40:58,071 epoch 18 - iter 80/83 - loss 6.00641702 - samples/sec: 173.05\n",
            "2019-12-20 10:40:58,355 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:40:58,356 EPOCH 18 done: loss 6.0139 - lr 0.1000\n",
            "2019-12-20 10:40:58,942 DEV : loss 4.0549163818359375 - score 0.8946\n",
            "2019-12-20 10:40:58,954 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:40:58,955 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:40:59,121 epoch 19 - iter 0/83 - loss 5.17207479 - samples/sec: 1559.56\n",
            "2019-12-20 10:41:00,622 epoch 19 - iter 8/83 - loss 5.86316453 - samples/sec: 172.51\n",
            "2019-12-20 10:41:01,959 epoch 19 - iter 16/83 - loss 5.77197712 - samples/sec: 194.36\n",
            "2019-12-20 10:41:03,411 epoch 19 - iter 24/83 - loss 5.61461584 - samples/sec: 178.96\n",
            "2019-12-20 10:41:04,820 epoch 19 - iter 32/83 - loss 5.62163580 - samples/sec: 184.28\n",
            "2019-12-20 10:41:06,388 epoch 19 - iter 40/83 - loss 5.64338077 - samples/sec: 165.61\n",
            "2019-12-20 10:41:07,879 epoch 19 - iter 48/83 - loss 5.69131752 - samples/sec: 174.15\n",
            "2019-12-20 10:41:09,279 epoch 19 - iter 56/83 - loss 5.74510055 - samples/sec: 185.33\n",
            "2019-12-20 10:41:10,871 epoch 19 - iter 64/83 - loss 5.77209664 - samples/sec: 162.87\n",
            "2019-12-20 10:41:12,359 epoch 19 - iter 72/83 - loss 5.81831772 - samples/sec: 174.23\n",
            "2019-12-20 10:41:13,990 epoch 19 - iter 80/83 - loss 5.81747255 - samples/sec: 158.90\n",
            "2019-12-20 10:41:14,342 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:41:14,343 EPOCH 19 done: loss 5.8320 - lr 0.1000\n",
            "2019-12-20 10:41:14,934 DEV : loss 4.086897373199463 - score 0.8984\n",
            "2019-12-20 10:41:14,948 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:41:16,532 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:41:16,738 epoch 20 - iter 0/83 - loss 5.98235321 - samples/sec: 1252.17\n",
            "2019-12-20 10:41:18,200 epoch 20 - iter 8/83 - loss 6.05781730 - samples/sec: 177.75\n",
            "2019-12-20 10:41:19,653 epoch 20 - iter 16/83 - loss 6.11805167 - samples/sec: 178.61\n",
            "2019-12-20 10:41:21,312 epoch 20 - iter 24/83 - loss 5.99347651 - samples/sec: 156.92\n",
            "2019-12-20 10:41:23,061 epoch 20 - iter 32/83 - loss 6.10889621 - samples/sec: 148.28\n",
            "2019-12-20 10:41:24,747 epoch 20 - iter 40/83 - loss 5.99494136 - samples/sec: 153.61\n",
            "2019-12-20 10:41:26,254 epoch 20 - iter 48/83 - loss 5.98916180 - samples/sec: 172.18\n",
            "2019-12-20 10:41:27,617 epoch 20 - iter 56/83 - loss 5.96649641 - samples/sec: 190.42\n",
            "2019-12-20 10:41:29,058 epoch 20 - iter 64/83 - loss 5.94897951 - samples/sec: 180.25\n",
            "2019-12-20 10:41:30,490 epoch 20 - iter 72/83 - loss 5.89368390 - samples/sec: 181.06\n",
            "2019-12-20 10:41:31,831 epoch 20 - iter 80/83 - loss 5.84584673 - samples/sec: 193.95\n",
            "2019-12-20 10:41:32,102 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:41:32,103 EPOCH 20 done: loss 5.8204 - lr 0.1000\n",
            "2019-12-20 10:41:32,687 DEV : loss 4.146292209625244 - score 0.8885\n",
            "2019-12-20 10:41:32,701 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:41:32,703 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:41:32,870 epoch 21 - iter 0/83 - loss 6.30325127 - samples/sec: 1549.93\n",
            "2019-12-20 10:41:34,485 epoch 21 - iter 8/83 - loss 5.82507009 - samples/sec: 160.15\n",
            "2019-12-20 10:41:36,045 epoch 21 - iter 16/83 - loss 5.90366780 - samples/sec: 166.39\n",
            "2019-12-20 10:41:37,388 epoch 21 - iter 24/83 - loss 5.97582770 - samples/sec: 193.18\n",
            "2019-12-20 10:41:38,807 epoch 21 - iter 32/83 - loss 5.88548332 - samples/sec: 182.93\n",
            "2019-12-20 10:41:40,306 epoch 21 - iter 40/83 - loss 5.88938426 - samples/sec: 173.11\n",
            "2019-12-20 10:41:41,941 epoch 21 - iter 48/83 - loss 5.93272255 - samples/sec: 158.49\n",
            "2019-12-20 10:41:43,440 epoch 21 - iter 56/83 - loss 5.87786641 - samples/sec: 173.10\n",
            "2019-12-20 10:41:44,750 epoch 21 - iter 64/83 - loss 5.76971981 - samples/sec: 198.54\n",
            "2019-12-20 10:41:46,165 epoch 21 - iter 72/83 - loss 5.71425136 - samples/sec: 183.19\n",
            "2019-12-20 10:41:47,706 epoch 21 - iter 80/83 - loss 5.67898103 - samples/sec: 168.26\n",
            "2019-12-20 10:41:48,023 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:41:48,024 EPOCH 21 done: loss 5.6550 - lr 0.1000\n",
            "2019-12-20 10:41:48,625 DEV : loss 4.1827497482299805 - score 0.8937\n",
            "2019-12-20 10:41:48,638 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:41:48,639 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:41:48,801 epoch 22 - iter 0/83 - loss 4.38133669 - samples/sec: 1590.63\n",
            "2019-12-20 10:41:50,260 epoch 22 - iter 8/83 - loss 5.46196151 - samples/sec: 177.54\n",
            "2019-12-20 10:41:51,862 epoch 22 - iter 16/83 - loss 5.42742686 - samples/sec: 161.76\n",
            "2019-12-20 10:41:53,482 epoch 22 - iter 24/83 - loss 5.62486238 - samples/sec: 159.99\n",
            "2019-12-20 10:41:55,024 epoch 22 - iter 32/83 - loss 5.65050129 - samples/sec: 168.33\n",
            "2019-12-20 10:41:56,474 epoch 22 - iter 40/83 - loss 5.65635986 - samples/sec: 178.83\n",
            "2019-12-20 10:41:58,098 epoch 22 - iter 48/83 - loss 5.66473019 - samples/sec: 159.53\n",
            "2019-12-20 10:41:59,473 epoch 22 - iter 56/83 - loss 5.58200301 - samples/sec: 188.89\n",
            "2019-12-20 10:42:00,943 epoch 22 - iter 64/83 - loss 5.55955905 - samples/sec: 176.51\n",
            "2019-12-20 10:42:02,406 epoch 22 - iter 72/83 - loss 5.53812813 - samples/sec: 177.74\n",
            "2019-12-20 10:42:03,784 epoch 22 - iter 80/83 - loss 5.55890392 - samples/sec: 188.49\n",
            "2019-12-20 10:42:04,093 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:42:04,094 EPOCH 22 done: loss 5.5386 - lr 0.1000\n",
            "2019-12-20 10:42:04,674 DEV : loss 3.8814477920532227 - score 0.8963\n",
            "2019-12-20 10:42:04,689 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:42:04,690 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:42:04,886 epoch 23 - iter 0/83 - loss 6.48697758 - samples/sec: 1312.58\n",
            "2019-12-20 10:42:06,504 epoch 23 - iter 8/83 - loss 5.57019493 - samples/sec: 159.90\n",
            "2019-12-20 10:42:07,863 epoch 23 - iter 16/83 - loss 5.59155812 - samples/sec: 191.20\n",
            "2019-12-20 10:42:09,464 epoch 23 - iter 24/83 - loss 5.61563593 - samples/sec: 161.94\n",
            "2019-12-20 10:42:10,811 epoch 23 - iter 32/83 - loss 5.54305653 - samples/sec: 192.61\n",
            "2019-12-20 10:42:12,238 epoch 23 - iter 40/83 - loss 5.52444827 - samples/sec: 181.69\n",
            "2019-12-20 10:42:13,930 epoch 23 - iter 48/83 - loss 5.59562488 - samples/sec: 153.04\n",
            "2019-12-20 10:42:15,369 epoch 23 - iter 56/83 - loss 5.58060132 - samples/sec: 180.46\n",
            "2019-12-20 10:42:16,965 epoch 23 - iter 64/83 - loss 5.59909935 - samples/sec: 162.22\n",
            "2019-12-20 10:42:18,323 epoch 23 - iter 72/83 - loss 5.57798681 - samples/sec: 191.13\n",
            "2019-12-20 10:42:19,908 epoch 23 - iter 80/83 - loss 5.53840210 - samples/sec: 163.38\n",
            "2019-12-20 10:42:20,277 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:42:20,278 EPOCH 23 done: loss 5.5574 - lr 0.1000\n",
            "2019-12-20 10:42:20,863 DEV : loss 4.094915866851807 - score 0.9007\n",
            "2019-12-20 10:42:20,876 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:42:22,231 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:42:22,424 epoch 24 - iter 0/83 - loss 5.40989447 - samples/sec: 1339.83\n",
            "2019-12-20 10:42:23,944 epoch 24 - iter 8/83 - loss 5.29075299 - samples/sec: 170.50\n",
            "2019-12-20 10:42:25,650 epoch 24 - iter 16/83 - loss 5.43207741 - samples/sec: 152.05\n",
            "2019-12-20 10:42:26,985 epoch 24 - iter 24/83 - loss 5.38871708 - samples/sec: 194.48\n",
            "2019-12-20 10:42:28,399 epoch 24 - iter 32/83 - loss 5.44985349 - samples/sec: 184.03\n",
            "2019-12-20 10:42:29,870 epoch 24 - iter 40/83 - loss 5.51883370 - samples/sec: 176.35\n",
            "2019-12-20 10:42:31,345 epoch 24 - iter 48/83 - loss 5.49204007 - samples/sec: 175.62\n",
            "2019-12-20 10:42:32,968 epoch 24 - iter 56/83 - loss 5.46934477 - samples/sec: 159.64\n",
            "2019-12-20 10:42:34,427 epoch 24 - iter 64/83 - loss 5.44508858 - samples/sec: 177.89\n",
            "2019-12-20 10:42:35,875 epoch 24 - iter 72/83 - loss 5.46238099 - samples/sec: 179.06\n",
            "2019-12-20 10:42:37,182 epoch 24 - iter 80/83 - loss 5.48546166 - samples/sec: 199.49\n",
            "2019-12-20 10:42:37,467 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:42:37,468 EPOCH 24 done: loss 5.4614 - lr 0.1000\n",
            "2019-12-20 10:42:38,058 DEV : loss 3.880080461502075 - score 0.9024\n",
            "2019-12-20 10:42:38,072 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:42:39,456 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:42:39,704 epoch 25 - iter 0/83 - loss 5.06822491 - samples/sec: 1046.59\n",
            "2019-12-20 10:42:41,047 epoch 25 - iter 8/83 - loss 4.85340516 - samples/sec: 193.59\n",
            "2019-12-20 10:42:42,632 epoch 25 - iter 16/83 - loss 5.26335856 - samples/sec: 163.68\n",
            "2019-12-20 10:42:44,066 epoch 25 - iter 24/83 - loss 5.22930397 - samples/sec: 181.14\n",
            "2019-12-20 10:42:45,634 epoch 25 - iter 32/83 - loss 5.19444275 - samples/sec: 165.89\n",
            "2019-12-20 10:42:46,993 epoch 25 - iter 40/83 - loss 5.14758952 - samples/sec: 191.17\n",
            "2019-12-20 10:42:48,587 epoch 25 - iter 48/83 - loss 5.13162700 - samples/sec: 162.57\n",
            "2019-12-20 10:42:50,185 epoch 25 - iter 56/83 - loss 5.15486315 - samples/sec: 162.53\n",
            "2019-12-20 10:42:51,732 epoch 25 - iter 64/83 - loss 5.25207182 - samples/sec: 167.50\n",
            "2019-12-20 10:42:53,165 epoch 25 - iter 72/83 - loss 5.22293201 - samples/sec: 180.99\n",
            "2019-12-20 10:42:54,639 epoch 25 - iter 80/83 - loss 5.27015236 - samples/sec: 176.09\n",
            "2019-12-20 10:42:54,947 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:42:54,948 EPOCH 25 done: loss 5.2872 - lr 0.1000\n",
            "2019-12-20 10:42:55,529 DEV : loss 3.879271984100342 - score 0.9012\n",
            "2019-12-20 10:42:55,543 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:42:55,544 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:42:55,722 epoch 26 - iter 0/83 - loss 4.75257540 - samples/sec: 1459.14\n",
            "2019-12-20 10:42:57,267 epoch 26 - iter 8/83 - loss 5.18465323 - samples/sec: 167.97\n",
            "2019-12-20 10:42:58,651 epoch 26 - iter 16/83 - loss 5.17828854 - samples/sec: 187.78\n",
            "2019-12-20 10:43:00,228 epoch 26 - iter 24/83 - loss 5.14444570 - samples/sec: 164.21\n",
            "2019-12-20 10:43:01,661 epoch 26 - iter 32/83 - loss 5.33015118 - samples/sec: 181.13\n",
            "2019-12-20 10:43:03,386 epoch 26 - iter 40/83 - loss 5.37939940 - samples/sec: 150.11\n",
            "2019-12-20 10:43:04,844 epoch 26 - iter 48/83 - loss 5.28249912 - samples/sec: 177.93\n",
            "2019-12-20 10:43:06,336 epoch 26 - iter 56/83 - loss 5.29032579 - samples/sec: 173.98\n",
            "2019-12-20 10:43:07,780 epoch 26 - iter 64/83 - loss 5.36425832 - samples/sec: 179.71\n",
            "2019-12-20 10:43:09,090 epoch 26 - iter 72/83 - loss 5.34923711 - samples/sec: 198.37\n",
            "2019-12-20 10:43:10,899 epoch 26 - iter 80/83 - loss 5.36440853 - samples/sec: 143.16\n",
            "2019-12-20 10:43:11,179 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:43:11,180 EPOCH 26 done: loss 5.3232 - lr 0.1000\n",
            "2019-12-20 10:43:11,767 DEV : loss 3.921248197555542 - score 0.9033\n",
            "2019-12-20 10:43:11,780 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:43:13,177 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:43:13,503 epoch 27 - iter 0/83 - loss 7.21477413 - samples/sec: 797.01\n",
            "2019-12-20 10:43:14,964 epoch 27 - iter 8/83 - loss 5.85933012 - samples/sec: 177.46\n",
            "2019-12-20 10:43:16,427 epoch 27 - iter 16/83 - loss 5.47776849 - samples/sec: 178.02\n",
            "2019-12-20 10:43:17,931 epoch 27 - iter 24/83 - loss 5.34842460 - samples/sec: 172.40\n",
            "2019-12-20 10:43:19,382 epoch 27 - iter 32/83 - loss 5.24254008 - samples/sec: 179.09\n",
            "2019-12-20 10:43:21,034 epoch 27 - iter 40/83 - loss 5.28671042 - samples/sec: 156.66\n",
            "2019-12-20 10:43:22,398 epoch 27 - iter 48/83 - loss 5.28682032 - samples/sec: 190.48\n",
            "2019-12-20 10:43:23,963 epoch 27 - iter 56/83 - loss 5.38331996 - samples/sec: 165.53\n",
            "2019-12-20 10:43:25,439 epoch 27 - iter 64/83 - loss 5.40143114 - samples/sec: 175.77\n",
            "2019-12-20 10:43:27,085 epoch 27 - iter 72/83 - loss 5.35417562 - samples/sec: 157.52\n",
            "2019-12-20 10:43:28,589 epoch 27 - iter 80/83 - loss 5.32932249 - samples/sec: 172.53\n",
            "2019-12-20 10:43:28,898 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:43:28,902 EPOCH 27 done: loss 5.3337 - lr 0.1000\n",
            "2019-12-20 10:43:29,496 DEV : loss 3.8556301593780518 - score 0.9059\n",
            "2019-12-20 10:43:29,509 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:43:31,046 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:43:31,246 epoch 28 - iter 0/83 - loss 6.50395489 - samples/sec: 1297.42\n",
            "2019-12-20 10:43:32,741 epoch 28 - iter 8/83 - loss 5.16245662 - samples/sec: 173.41\n",
            "2019-12-20 10:43:34,125 epoch 28 - iter 16/83 - loss 5.06408843 - samples/sec: 187.34\n",
            "2019-12-20 10:43:35,677 epoch 28 - iter 24/83 - loss 5.04362349 - samples/sec: 167.16\n",
            "2019-12-20 10:43:37,087 epoch 28 - iter 32/83 - loss 5.08271305 - samples/sec: 184.39\n",
            "2019-12-20 10:43:38,559 epoch 28 - iter 40/83 - loss 5.10291151 - samples/sec: 175.98\n",
            "2019-12-20 10:43:40,164 epoch 28 - iter 48/83 - loss 5.10136027 - samples/sec: 161.38\n",
            "2019-12-20 10:43:41,867 epoch 28 - iter 56/83 - loss 5.20454940 - samples/sec: 152.13\n",
            "2019-12-20 10:43:43,384 epoch 28 - iter 64/83 - loss 5.21911981 - samples/sec: 170.96\n",
            "2019-12-20 10:43:44,888 epoch 28 - iter 72/83 - loss 5.21877893 - samples/sec: 172.12\n",
            "2019-12-20 10:43:46,323 epoch 28 - iter 80/83 - loss 5.16773523 - samples/sec: 180.92\n",
            "2019-12-20 10:43:46,611 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:43:46,612 EPOCH 28 done: loss 5.1752 - lr 0.1000\n",
            "2019-12-20 10:43:47,191 DEV : loss 3.9392290115356445 - score 0.8981\n",
            "2019-12-20 10:43:47,204 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:43:47,205 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:43:47,385 epoch 29 - iter 0/83 - loss 5.41724205 - samples/sec: 1428.13\n",
            "2019-12-20 10:43:48,775 epoch 29 - iter 8/83 - loss 4.62441005 - samples/sec: 186.40\n",
            "2019-12-20 10:43:50,445 epoch 29 - iter 16/83 - loss 4.89991612 - samples/sec: 155.08\n",
            "2019-12-20 10:43:51,904 epoch 29 - iter 24/83 - loss 5.04164115 - samples/sec: 177.82\n",
            "2019-12-20 10:43:53,423 epoch 29 - iter 32/83 - loss 5.04303002 - samples/sec: 170.98\n",
            "2019-12-20 10:43:54,868 epoch 29 - iter 40/83 - loss 5.04996686 - samples/sec: 179.19\n",
            "2019-12-20 10:43:56,369 epoch 29 - iter 48/83 - loss 5.01318762 - samples/sec: 172.85\n",
            "2019-12-20 10:43:57,801 epoch 29 - iter 56/83 - loss 5.05452171 - samples/sec: 181.15\n",
            "2019-12-20 10:43:59,289 epoch 29 - iter 64/83 - loss 5.07143311 - samples/sec: 174.44\n",
            "2019-12-20 10:44:00,904 epoch 29 - iter 72/83 - loss 5.11283142 - samples/sec: 160.66\n",
            "2019-12-20 10:44:02,348 epoch 29 - iter 80/83 - loss 5.12518859 - samples/sec: 179.65\n",
            "2019-12-20 10:44:02,618 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:44:02,619 EPOCH 29 done: loss 5.1237 - lr 0.1000\n",
            "2019-12-20 10:44:03,196 DEV : loss 3.7609758377075195 - score 0.905\n",
            "2019-12-20 10:44:03,210 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:44:03,211 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:44:03,387 epoch 30 - iter 0/83 - loss 5.82844067 - samples/sec: 1484.23\n",
            "2019-12-20 10:44:04,905 epoch 30 - iter 8/83 - loss 5.77953402 - samples/sec: 170.66\n",
            "2019-12-20 10:44:06,360 epoch 30 - iter 16/83 - loss 5.31793360 - samples/sec: 178.60\n",
            "2019-12-20 10:44:07,801 epoch 30 - iter 24/83 - loss 5.12370036 - samples/sec: 180.41\n",
            "2019-12-20 10:44:09,301 epoch 30 - iter 32/83 - loss 5.05334734 - samples/sec: 173.20\n",
            "2019-12-20 10:44:10,742 epoch 30 - iter 40/83 - loss 5.05448090 - samples/sec: 180.09\n",
            "2019-12-20 10:44:12,217 epoch 30 - iter 48/83 - loss 5.05438911 - samples/sec: 175.75\n",
            "2019-12-20 10:44:13,727 epoch 30 - iter 56/83 - loss 5.09866530 - samples/sec: 171.76\n",
            "2019-12-20 10:44:15,119 epoch 30 - iter 64/83 - loss 5.06704785 - samples/sec: 186.87\n",
            "2019-12-20 10:44:16,687 epoch 30 - iter 72/83 - loss 5.05993127 - samples/sec: 165.15\n",
            "2019-12-20 10:44:18,187 epoch 30 - iter 80/83 - loss 5.06286454 - samples/sec: 172.75\n",
            "2019-12-20 10:44:18,478 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:44:18,479 EPOCH 30 done: loss 5.0542 - lr 0.1000\n",
            "2019-12-20 10:44:19,068 DEV : loss 3.7198753356933594 - score 0.9047\n",
            "2019-12-20 10:44:19,080 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:44:19,081 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:44:19,269 epoch 31 - iter 0/83 - loss 5.53285980 - samples/sec: 1378.17\n",
            "2019-12-20 10:44:20,664 epoch 31 - iter 8/83 - loss 4.82006476 - samples/sec: 185.79\n",
            "2019-12-20 10:44:22,235 epoch 31 - iter 16/83 - loss 4.77464959 - samples/sec: 165.38\n",
            "2019-12-20 10:44:23,780 epoch 31 - iter 24/83 - loss 4.84773588 - samples/sec: 167.78\n",
            "2019-12-20 10:44:25,273 epoch 31 - iter 32/83 - loss 4.77670951 - samples/sec: 173.87\n",
            "2019-12-20 10:44:26,726 epoch 31 - iter 40/83 - loss 4.82100480 - samples/sec: 178.47\n",
            "2019-12-20 10:44:28,262 epoch 31 - iter 48/83 - loss 4.84848027 - samples/sec: 169.09\n",
            "2019-12-20 10:44:29,753 epoch 31 - iter 56/83 - loss 4.93060373 - samples/sec: 173.73\n",
            "2019-12-20 10:44:31,146 epoch 31 - iter 64/83 - loss 4.94109111 - samples/sec: 186.34\n",
            "2019-12-20 10:44:32,814 epoch 31 - iter 72/83 - loss 4.95559250 - samples/sec: 155.39\n",
            "2019-12-20 10:44:34,197 epoch 31 - iter 80/83 - loss 4.96684689 - samples/sec: 187.92\n",
            "2019-12-20 10:44:34,513 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:44:34,514 EPOCH 31 done: loss 4.9554 - lr 0.1000\n",
            "2019-12-20 10:44:35,117 DEV : loss 4.001920223236084 - score 0.9004\n",
            "Epoch    30: reducing learning rate of group 0 to 5.0000e-02.\n",
            "2019-12-20 10:44:35,130 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:44:35,131 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:44:35,329 epoch 32 - iter 0/83 - loss 5.96671247 - samples/sec: 1315.77\n",
            "2019-12-20 10:44:36,750 epoch 32 - iter 8/83 - loss 4.64651468 - samples/sec: 182.49\n",
            "2019-12-20 10:44:38,237 epoch 32 - iter 16/83 - loss 4.63966173 - samples/sec: 174.76\n",
            "2019-12-20 10:44:39,736 epoch 32 - iter 24/83 - loss 4.59346250 - samples/sec: 172.94\n",
            "2019-12-20 10:44:41,189 epoch 32 - iter 32/83 - loss 4.58750812 - samples/sec: 178.64\n",
            "2019-12-20 10:44:42,589 epoch 32 - iter 40/83 - loss 4.61294826 - samples/sec: 185.33\n",
            "2019-12-20 10:44:43,944 epoch 32 - iter 48/83 - loss 4.54893476 - samples/sec: 191.57\n",
            "2019-12-20 10:44:45,511 epoch 32 - iter 56/83 - loss 4.63878770 - samples/sec: 165.62\n",
            "2019-12-20 10:44:47,214 epoch 32 - iter 64/83 - loss 4.65550071 - samples/sec: 152.01\n",
            "2019-12-20 10:44:48,732 epoch 32 - iter 72/83 - loss 4.69048369 - samples/sec: 170.66\n",
            "2019-12-20 10:44:50,254 epoch 32 - iter 80/83 - loss 4.70786210 - samples/sec: 170.49\n",
            "2019-12-20 10:44:50,514 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:44:50,515 EPOCH 32 done: loss 4.7049 - lr 0.0500\n",
            "2019-12-20 10:44:51,111 DEV : loss 3.6614058017730713 - score 0.9102\n",
            "2019-12-20 10:44:51,125 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:44:52,575 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:44:52,828 epoch 33 - iter 0/83 - loss 4.46606731 - samples/sec: 1020.65\n",
            "2019-12-20 10:44:54,383 epoch 33 - iter 8/83 - loss 4.24018113 - samples/sec: 166.62\n",
            "2019-12-20 10:44:55,805 epoch 33 - iter 16/83 - loss 4.42843805 - samples/sec: 182.29\n",
            "2019-12-20 10:44:57,489 epoch 33 - iter 24/83 - loss 4.53611651 - samples/sec: 154.17\n",
            "2019-12-20 10:44:58,889 epoch 33 - iter 32/83 - loss 4.56559734 - samples/sec: 185.82\n",
            "2019-12-20 10:45:00,289 epoch 33 - iter 40/83 - loss 4.59559698 - samples/sec: 185.37\n",
            "2019-12-20 10:45:01,738 epoch 33 - iter 48/83 - loss 4.56033231 - samples/sec: 179.06\n",
            "2019-12-20 10:45:03,289 epoch 33 - iter 56/83 - loss 4.53865582 - samples/sec: 167.29\n",
            "2019-12-20 10:45:04,868 epoch 33 - iter 64/83 - loss 4.58855718 - samples/sec: 164.25\n",
            "2019-12-20 10:45:06,441 epoch 33 - iter 72/83 - loss 4.55781487 - samples/sec: 164.77\n",
            "2019-12-20 10:45:07,934 epoch 33 - iter 80/83 - loss 4.54983533 - samples/sec: 173.77\n",
            "2019-12-20 10:45:08,221 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:45:08,221 EPOCH 33 done: loss 4.5510 - lr 0.0500\n",
            "2019-12-20 10:45:08,806 DEV : loss 3.7766504287719727 - score 0.9056\n",
            "2019-12-20 10:45:08,819 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:45:08,820 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:45:08,979 epoch 34 - iter 0/83 - loss 4.76760006 - samples/sec: 1620.87\n",
            "2019-12-20 10:45:10,667 epoch 34 - iter 8/83 - loss 4.77919987 - samples/sec: 153.15\n",
            "2019-12-20 10:45:12,230 epoch 34 - iter 16/83 - loss 4.72358856 - samples/sec: 165.68\n",
            "2019-12-20 10:45:13,677 epoch 34 - iter 24/83 - loss 4.68549785 - samples/sec: 179.24\n",
            "2019-12-20 10:45:15,366 epoch 34 - iter 32/83 - loss 4.68988828 - samples/sec: 153.52\n",
            "2019-12-20 10:45:16,742 epoch 34 - iter 40/83 - loss 4.58596983 - samples/sec: 188.50\n",
            "2019-12-20 10:45:18,211 epoch 34 - iter 48/83 - loss 4.56817616 - samples/sec: 176.57\n",
            "2019-12-20 10:45:19,806 epoch 34 - iter 56/83 - loss 4.57902568 - samples/sec: 162.32\n",
            "2019-12-20 10:45:21,286 epoch 34 - iter 64/83 - loss 4.53009589 - samples/sec: 175.05\n",
            "2019-12-20 10:45:22,632 epoch 34 - iter 72/83 - loss 4.51164594 - samples/sec: 193.01\n",
            "2019-12-20 10:45:24,062 epoch 34 - iter 80/83 - loss 4.47859507 - samples/sec: 181.39\n",
            "2019-12-20 10:45:24,361 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:45:24,365 EPOCH 34 done: loss 4.4753 - lr 0.0500\n",
            "2019-12-20 10:45:24,945 DEV : loss 3.738199472427368 - score 0.9064\n",
            "2019-12-20 10:45:24,959 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:45:24,960 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:45:25,099 epoch 35 - iter 0/83 - loss 3.45929575 - samples/sec: 1864.52\n",
            "2019-12-20 10:45:26,659 epoch 35 - iter 8/83 - loss 4.34722561 - samples/sec: 165.86\n",
            "2019-12-20 10:45:28,308 epoch 35 - iter 16/83 - loss 4.34857018 - samples/sec: 157.17\n",
            "2019-12-20 10:45:29,813 epoch 35 - iter 24/83 - loss 4.35528318 - samples/sec: 172.13\n",
            "2019-12-20 10:45:31,310 epoch 35 - iter 32/83 - loss 4.42264125 - samples/sec: 173.43\n",
            "2019-12-20 10:45:32,871 epoch 35 - iter 40/83 - loss 4.46782204 - samples/sec: 166.01\n",
            "2019-12-20 10:45:34,245 epoch 35 - iter 48/83 - loss 4.45269139 - samples/sec: 189.24\n",
            "2019-12-20 10:45:35,544 epoch 35 - iter 56/83 - loss 4.35095287 - samples/sec: 200.99\n",
            "2019-12-20 10:45:36,939 epoch 35 - iter 64/83 - loss 4.33913325 - samples/sec: 186.15\n",
            "2019-12-20 10:45:38,451 epoch 35 - iter 72/83 - loss 4.38208871 - samples/sec: 171.69\n",
            "2019-12-20 10:45:39,934 epoch 35 - iter 80/83 - loss 4.39645618 - samples/sec: 174.84\n",
            "2019-12-20 10:45:40,245 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:45:40,248 EPOCH 35 done: loss 4.4493 - lr 0.0500\n",
            "2019-12-20 10:45:40,838 DEV : loss 3.6627209186553955 - score 0.9059\n",
            "2019-12-20 10:45:40,852 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:45:40,853 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:45:41,017 epoch 36 - iter 0/83 - loss 4.48793125 - samples/sec: 1591.42\n",
            "2019-12-20 10:45:42,731 epoch 36 - iter 8/83 - loss 4.39184941 - samples/sec: 151.04\n",
            "2019-12-20 10:45:44,080 epoch 36 - iter 16/83 - loss 4.23590649 - samples/sec: 192.69\n",
            "2019-12-20 10:45:45,549 epoch 36 - iter 24/83 - loss 4.26118568 - samples/sec: 176.62\n",
            "2019-12-20 10:45:47,183 epoch 36 - iter 32/83 - loss 4.40797145 - samples/sec: 158.65\n",
            "2019-12-20 10:45:48,531 epoch 36 - iter 40/83 - loss 4.48512643 - samples/sec: 192.40\n",
            "2019-12-20 10:45:49,900 epoch 36 - iter 48/83 - loss 4.45122067 - samples/sec: 190.17\n",
            "2019-12-20 10:45:51,418 epoch 36 - iter 56/83 - loss 4.46802581 - samples/sec: 170.94\n",
            "2019-12-20 10:45:52,938 epoch 36 - iter 64/83 - loss 4.46550960 - samples/sec: 170.86\n",
            "2019-12-20 10:45:54,460 epoch 36 - iter 72/83 - loss 4.48883720 - samples/sec: 170.36\n",
            "2019-12-20 10:45:55,979 epoch 36 - iter 80/83 - loss 4.47013155 - samples/sec: 170.75\n",
            "2019-12-20 10:45:56,305 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:45:56,308 EPOCH 36 done: loss 4.4691 - lr 0.0500\n",
            "2019-12-20 10:45:56,898 DEV : loss 3.6897506713867188 - score 0.9079\n",
            "Epoch    35: reducing learning rate of group 0 to 2.5000e-02.\n",
            "2019-12-20 10:45:56,911 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:45:56,912 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:45:57,090 epoch 37 - iter 0/83 - loss 3.61328745 - samples/sec: 1448.55\n",
            "2019-12-20 10:45:58,506 epoch 37 - iter 8/83 - loss 4.45982189 - samples/sec: 182.93\n",
            "2019-12-20 10:46:00,061 epoch 37 - iter 16/83 - loss 4.46008943 - samples/sec: 166.80\n",
            "2019-12-20 10:46:01,512 epoch 37 - iter 24/83 - loss 4.48442763 - samples/sec: 179.03\n",
            "2019-12-20 10:46:02,903 epoch 37 - iter 32/83 - loss 4.38178075 - samples/sec: 186.48\n",
            "2019-12-20 10:46:04,525 epoch 37 - iter 40/83 - loss 4.39957491 - samples/sec: 159.77\n",
            "2019-12-20 10:46:05,939 epoch 37 - iter 48/83 - loss 4.41095375 - samples/sec: 183.87\n",
            "2019-12-20 10:46:07,530 epoch 37 - iter 56/83 - loss 4.41373852 - samples/sec: 163.24\n",
            "2019-12-20 10:46:08,991 epoch 37 - iter 64/83 - loss 4.39923710 - samples/sec: 177.52\n",
            "2019-12-20 10:46:10,428 epoch 37 - iter 72/83 - loss 4.39542591 - samples/sec: 181.62\n",
            "2019-12-20 10:46:12,039 epoch 37 - iter 80/83 - loss 4.38952986 - samples/sec: 160.79\n",
            "2019-12-20 10:46:12,357 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:46:12,358 EPOCH 37 done: loss 4.3949 - lr 0.0250\n",
            "2019-12-20 10:46:12,943 DEV : loss 3.6472134590148926 - score 0.9105\n",
            "2019-12-20 10:46:12,955 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:46:14,416 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:46:14,679 epoch 38 - iter 0/83 - loss 3.96926975 - samples/sec: 978.56\n",
            "2019-12-20 10:46:16,230 epoch 38 - iter 8/83 - loss 4.09172975 - samples/sec: 167.14\n",
            "2019-12-20 10:46:17,664 epoch 38 - iter 16/83 - loss 4.13040966 - samples/sec: 180.98\n",
            "2019-12-20 10:46:19,018 epoch 38 - iter 24/83 - loss 4.11357802 - samples/sec: 192.56\n",
            "2019-12-20 10:46:20,581 epoch 38 - iter 32/83 - loss 4.19744781 - samples/sec: 165.89\n",
            "2019-12-20 10:46:22,201 epoch 38 - iter 40/83 - loss 4.20749050 - samples/sec: 159.88\n",
            "2019-12-20 10:46:23,724 epoch 38 - iter 48/83 - loss 4.27567960 - samples/sec: 170.16\n",
            "2019-12-20 10:46:25,497 epoch 38 - iter 56/83 - loss 4.35516635 - samples/sec: 146.02\n",
            "2019-12-20 10:46:26,809 epoch 38 - iter 64/83 - loss 4.33849134 - samples/sec: 198.29\n",
            "2019-12-20 10:46:28,302 epoch 38 - iter 72/83 - loss 4.33304736 - samples/sec: 173.83\n",
            "2019-12-20 10:46:29,755 epoch 38 - iter 80/83 - loss 4.28814705 - samples/sec: 178.46\n",
            "2019-12-20 10:46:30,045 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:46:30,046 EPOCH 38 done: loss 4.2937 - lr 0.0250\n",
            "2019-12-20 10:46:30,632 DEV : loss 3.6709959506988525 - score 0.9082\n",
            "2019-12-20 10:46:30,646 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:46:30,647 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:46:30,796 epoch 39 - iter 0/83 - loss 4.14135456 - samples/sec: 1750.76\n",
            "2019-12-20 10:46:32,294 epoch 39 - iter 8/83 - loss 4.23978596 - samples/sec: 172.86\n",
            "2019-12-20 10:46:33,701 epoch 39 - iter 16/83 - loss 4.27532155 - samples/sec: 184.30\n",
            "2019-12-20 10:46:35,096 epoch 39 - iter 24/83 - loss 4.21348653 - samples/sec: 186.13\n",
            "2019-12-20 10:46:36,430 epoch 39 - iter 32/83 - loss 4.22078854 - samples/sec: 195.09\n",
            "2019-12-20 10:46:38,103 epoch 39 - iter 40/83 - loss 4.29943705 - samples/sec: 154.90\n",
            "2019-12-20 10:46:39,608 epoch 39 - iter 48/83 - loss 4.28297766 - samples/sec: 172.35\n",
            "2019-12-20 10:46:41,139 epoch 39 - iter 56/83 - loss 4.26796992 - samples/sec: 169.23\n",
            "2019-12-20 10:46:42,709 epoch 39 - iter 64/83 - loss 4.27500236 - samples/sec: 165.60\n",
            "2019-12-20 10:46:44,121 epoch 39 - iter 72/83 - loss 4.27160730 - samples/sec: 184.30\n",
            "2019-12-20 10:46:45,782 epoch 39 - iter 80/83 - loss 4.27516831 - samples/sec: 156.03\n",
            "2019-12-20 10:46:46,121 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:46:46,122 EPOCH 39 done: loss 4.2958 - lr 0.0250\n",
            "2019-12-20 10:46:46,713 DEV : loss 3.717707395553589 - score 0.909\n",
            "2019-12-20 10:46:46,726 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:46:46,727 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:46:46,896 epoch 40 - iter 0/83 - loss 3.69340515 - samples/sec: 1560.41\n",
            "2019-12-20 10:46:48,456 epoch 40 - iter 8/83 - loss 4.17364301 - samples/sec: 166.03\n",
            "2019-12-20 10:46:49,881 epoch 40 - iter 16/83 - loss 4.08696155 - samples/sec: 182.50\n",
            "2019-12-20 10:46:51,270 epoch 40 - iter 24/83 - loss 4.10591186 - samples/sec: 187.02\n",
            "2019-12-20 10:46:52,737 epoch 40 - iter 32/83 - loss 4.11760766 - samples/sec: 176.97\n",
            "2019-12-20 10:46:54,034 epoch 40 - iter 40/83 - loss 4.06757314 - samples/sec: 199.97\n",
            "2019-12-20 10:46:55,726 epoch 40 - iter 48/83 - loss 4.16795411 - samples/sec: 153.09\n",
            "2019-12-20 10:46:57,302 epoch 40 - iter 56/83 - loss 4.19110591 - samples/sec: 164.61\n",
            "2019-12-20 10:46:58,638 epoch 40 - iter 64/83 - loss 4.15228561 - samples/sec: 194.33\n",
            "2019-12-20 10:47:00,049 epoch 40 - iter 72/83 - loss 4.17135408 - samples/sec: 183.94\n",
            "2019-12-20 10:47:01,712 epoch 40 - iter 80/83 - loss 4.20678532 - samples/sec: 155.86\n",
            "2019-12-20 10:47:02,027 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:47:02,029 EPOCH 40 done: loss 4.1765 - lr 0.0250\n",
            "2019-12-20 10:47:02,636 DEV : loss 3.729792356491089 - score 0.907\n",
            "2019-12-20 10:47:02,648 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:47:02,649 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:47:02,844 epoch 41 - iter 0/83 - loss 4.33980799 - samples/sec: 1325.43\n",
            "2019-12-20 10:47:04,451 epoch 41 - iter 8/83 - loss 4.05760834 - samples/sec: 161.34\n",
            "2019-12-20 10:47:05,817 epoch 41 - iter 16/83 - loss 3.94484115 - samples/sec: 189.86\n",
            "2019-12-20 10:47:07,332 epoch 41 - iter 24/83 - loss 3.96255342 - samples/sec: 171.30\n",
            "2019-12-20 10:47:08,673 epoch 41 - iter 32/83 - loss 3.97437087 - samples/sec: 194.04\n",
            "2019-12-20 10:47:10,194 epoch 41 - iter 40/83 - loss 4.02057534 - samples/sec: 170.81\n",
            "2019-12-20 10:47:11,635 epoch 41 - iter 48/83 - loss 4.10334211 - samples/sec: 180.09\n",
            "2019-12-20 10:47:13,264 epoch 41 - iter 56/83 - loss 4.07854710 - samples/sec: 159.04\n",
            "2019-12-20 10:47:15,047 epoch 41 - iter 64/83 - loss 4.18372215 - samples/sec: 145.24\n",
            "2019-12-20 10:47:16,605 epoch 41 - iter 72/83 - loss 4.21867148 - samples/sec: 166.19\n",
            "2019-12-20 10:47:18,172 epoch 41 - iter 80/83 - loss 4.21939025 - samples/sec: 165.48\n",
            "2019-12-20 10:47:18,477 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:47:18,478 EPOCH 41 done: loss 4.2282 - lr 0.0250\n",
            "2019-12-20 10:47:19,055 DEV : loss 3.6684658527374268 - score 0.9122\n",
            "2019-12-20 10:47:19,068 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:47:20,527 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:47:20,784 epoch 42 - iter 0/83 - loss 5.29219580 - samples/sec: 1006.83\n",
            "2019-12-20 10:47:22,328 epoch 42 - iter 8/83 - loss 4.29293185 - samples/sec: 167.86\n",
            "2019-12-20 10:47:23,872 epoch 42 - iter 16/83 - loss 4.36584187 - samples/sec: 168.02\n",
            "2019-12-20 10:47:25,464 epoch 42 - iter 24/83 - loss 4.30166352 - samples/sec: 162.87\n",
            "2019-12-20 10:47:26,834 epoch 42 - iter 32/83 - loss 4.22938263 - samples/sec: 189.19\n",
            "2019-12-20 10:47:28,435 epoch 42 - iter 40/83 - loss 4.24461547 - samples/sec: 161.89\n",
            "2019-12-20 10:47:29,862 epoch 42 - iter 48/83 - loss 4.19021623 - samples/sec: 181.74\n",
            "2019-12-20 10:47:31,190 epoch 42 - iter 56/83 - loss 4.14749072 - samples/sec: 195.51\n",
            "2019-12-20 10:47:32,655 epoch 42 - iter 64/83 - loss 4.12341504 - samples/sec: 177.17\n",
            "2019-12-20 10:47:34,210 epoch 42 - iter 72/83 - loss 4.16452433 - samples/sec: 166.84\n",
            "2019-12-20 10:47:35,656 epoch 42 - iter 80/83 - loss 4.19109606 - samples/sec: 179.37\n",
            "2019-12-20 10:47:35,994 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:47:35,995 EPOCH 42 done: loss 4.1901 - lr 0.0250\n",
            "2019-12-20 10:47:36,579 DEV : loss 3.6473121643066406 - score 0.9108\n",
            "2019-12-20 10:47:36,592 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:47:36,593 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:47:36,819 epoch 43 - iter 0/83 - loss 4.09432507 - samples/sec: 1141.90\n",
            "2019-12-20 10:47:38,389 epoch 43 - iter 8/83 - loss 3.93279412 - samples/sec: 164.90\n",
            "2019-12-20 10:47:40,144 epoch 43 - iter 16/83 - loss 4.25412954 - samples/sec: 147.65\n",
            "2019-12-20 10:47:41,519 epoch 43 - iter 24/83 - loss 4.11581423 - samples/sec: 188.87\n",
            "2019-12-20 10:47:42,921 epoch 43 - iter 32/83 - loss 4.16162830 - samples/sec: 185.12\n",
            "2019-12-20 10:47:44,555 epoch 43 - iter 40/83 - loss 4.20106940 - samples/sec: 158.54\n",
            "2019-12-20 10:47:45,995 epoch 43 - iter 48/83 - loss 4.19217763 - samples/sec: 180.16\n",
            "2019-12-20 10:47:47,436 epoch 43 - iter 56/83 - loss 4.23515075 - samples/sec: 180.30\n",
            "2019-12-20 10:47:48,733 epoch 43 - iter 64/83 - loss 4.16640822 - samples/sec: 200.10\n",
            "2019-12-20 10:47:50,334 epoch 43 - iter 72/83 - loss 4.15275228 - samples/sec: 161.94\n",
            "2019-12-20 10:47:51,728 epoch 43 - iter 80/83 - loss 4.13239466 - samples/sec: 186.28\n",
            "2019-12-20 10:47:52,038 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:47:52,039 EPOCH 43 done: loss 4.1123 - lr 0.0250\n",
            "2019-12-20 10:47:52,630 DEV : loss 3.772918224334717 - score 0.9093\n",
            "2019-12-20 10:47:52,641 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:47:52,643 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:47:52,791 epoch 44 - iter 0/83 - loss 2.33600712 - samples/sec: 1760.88\n",
            "2019-12-20 10:47:54,172 epoch 44 - iter 8/83 - loss 3.72485163 - samples/sec: 187.63\n",
            "2019-12-20 10:47:55,626 epoch 44 - iter 16/83 - loss 4.00059556 - samples/sec: 178.47\n",
            "2019-12-20 10:47:57,306 epoch 44 - iter 24/83 - loss 4.15006834 - samples/sec: 154.09\n",
            "2019-12-20 10:47:58,665 epoch 44 - iter 32/83 - loss 4.11075694 - samples/sec: 191.64\n",
            "2019-12-20 10:48:00,144 epoch 44 - iter 40/83 - loss 4.10547683 - samples/sec: 175.43\n",
            "2019-12-20 10:48:01,662 epoch 44 - iter 48/83 - loss 4.03741951 - samples/sec: 171.02\n",
            "2019-12-20 10:48:03,124 epoch 44 - iter 56/83 - loss 4.04193029 - samples/sec: 177.49\n",
            "2019-12-20 10:48:04,519 epoch 44 - iter 64/83 - loss 4.01966934 - samples/sec: 186.14\n",
            "2019-12-20 10:48:06,115 epoch 44 - iter 72/83 - loss 4.08049282 - samples/sec: 163.52\n",
            "2019-12-20 10:48:07,496 epoch 44 - iter 80/83 - loss 4.08301255 - samples/sec: 188.08\n",
            "2019-12-20 10:48:07,891 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:48:07,892 EPOCH 44 done: loss 4.0865 - lr 0.0250\n",
            "2019-12-20 10:48:08,473 DEV : loss 3.687861919403076 - score 0.9111\n",
            "2019-12-20 10:48:08,487 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:48:08,488 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:48:08,663 epoch 45 - iter 0/83 - loss 4.65882587 - samples/sec: 1470.51\n",
            "2019-12-20 10:48:10,271 epoch 45 - iter 8/83 - loss 3.85385956 - samples/sec: 161.22\n",
            "2019-12-20 10:48:11,749 epoch 45 - iter 16/83 - loss 3.98489747 - samples/sec: 175.59\n",
            "2019-12-20 10:48:13,268 epoch 45 - iter 24/83 - loss 3.94372130 - samples/sec: 170.59\n",
            "2019-12-20 10:48:14,823 epoch 45 - iter 32/83 - loss 4.00095516 - samples/sec: 166.91\n",
            "2019-12-20 10:48:16,299 epoch 45 - iter 40/83 - loss 3.99186202 - samples/sec: 175.82\n",
            "2019-12-20 10:48:17,861 epoch 45 - iter 48/83 - loss 4.00489065 - samples/sec: 165.95\n",
            "2019-12-20 10:48:19,462 epoch 45 - iter 56/83 - loss 4.06331202 - samples/sec: 161.93\n",
            "2019-12-20 10:48:20,905 epoch 45 - iter 64/83 - loss 4.09482117 - samples/sec: 179.85\n",
            "2019-12-20 10:48:22,182 epoch 45 - iter 72/83 - loss 4.07670326 - samples/sec: 203.29\n",
            "2019-12-20 10:48:23,652 epoch 45 - iter 80/83 - loss 4.12329793 - samples/sec: 176.75\n",
            "2019-12-20 10:48:23,963 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:48:23,964 EPOCH 45 done: loss 4.1121 - lr 0.0250\n",
            "2019-12-20 10:48:24,550 DEV : loss 3.6205437183380127 - score 0.9148\n",
            "2019-12-20 10:48:24,564 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 10:48:26,446 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:48:26,609 epoch 46 - iter 0/83 - loss 4.06159687 - samples/sec: 1634.20\n",
            "2019-12-20 10:48:28,174 epoch 46 - iter 8/83 - loss 3.89087584 - samples/sec: 165.43\n",
            "2019-12-20 10:48:29,774 epoch 46 - iter 16/83 - loss 4.00019249 - samples/sec: 161.80\n",
            "2019-12-20 10:48:31,070 epoch 46 - iter 24/83 - loss 3.97172933 - samples/sec: 200.73\n",
            "2019-12-20 10:48:32,596 epoch 46 - iter 32/83 - loss 3.93516755 - samples/sec: 170.22\n",
            "2019-12-20 10:48:33,964 epoch 46 - iter 40/83 - loss 3.95768633 - samples/sec: 189.96\n",
            "2019-12-20 10:48:35,512 epoch 46 - iter 48/83 - loss 3.99904808 - samples/sec: 167.44\n",
            "2019-12-20 10:48:37,045 epoch 46 - iter 56/83 - loss 4.03477972 - samples/sec: 169.18\n",
            "2019-12-20 10:48:38,549 epoch 46 - iter 64/83 - loss 4.05993865 - samples/sec: 172.63\n",
            "2019-12-20 10:48:39,949 epoch 46 - iter 72/83 - loss 4.09326221 - samples/sec: 185.16\n",
            "2019-12-20 10:48:41,562 epoch 46 - iter 80/83 - loss 4.08182998 - samples/sec: 160.71\n",
            "2019-12-20 10:48:41,866 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:48:41,867 EPOCH 46 done: loss 4.0713 - lr 0.0250\n",
            "2019-12-20 10:48:42,459 DEV : loss 3.6417245864868164 - score 0.9128\n",
            "2019-12-20 10:48:42,473 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:48:42,474 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:48:42,668 epoch 47 - iter 0/83 - loss 4.34812737 - samples/sec: 1332.40\n",
            "2019-12-20 10:48:44,160 epoch 47 - iter 8/83 - loss 3.98693127 - samples/sec: 173.69\n",
            "2019-12-20 10:48:45,901 epoch 47 - iter 16/83 - loss 4.21551580 - samples/sec: 148.78\n",
            "2019-12-20 10:48:47,362 epoch 47 - iter 24/83 - loss 4.14183300 - samples/sec: 177.33\n",
            "2019-12-20 10:48:48,866 epoch 47 - iter 32/83 - loss 4.10398094 - samples/sec: 172.21\n",
            "2019-12-20 10:48:50,271 epoch 47 - iter 40/83 - loss 4.11786942 - samples/sec: 184.53\n",
            "2019-12-20 10:48:51,670 epoch 47 - iter 48/83 - loss 4.09936878 - samples/sec: 185.50\n",
            "2019-12-20 10:48:53,144 epoch 47 - iter 56/83 - loss 4.07807616 - samples/sec: 176.13\n",
            "2019-12-20 10:48:54,560 epoch 47 - iter 64/83 - loss 4.08760670 - samples/sec: 183.24\n",
            "2019-12-20 10:48:55,916 epoch 47 - iter 72/83 - loss 4.07619158 - samples/sec: 191.73\n",
            "2019-12-20 10:48:57,590 epoch 47 - iter 80/83 - loss 4.05737944 - samples/sec: 154.69\n",
            "2019-12-20 10:48:57,860 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:48:57,861 EPOCH 47 done: loss 4.0501 - lr 0.0250\n",
            "2019-12-20 10:48:58,451 DEV : loss 3.704256772994995 - score 0.9111\n",
            "2019-12-20 10:48:58,465 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:48:58,466 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:48:58,715 epoch 48 - iter 0/83 - loss 5.45662689 - samples/sec: 1030.77\n",
            "2019-12-20 10:49:00,162 epoch 48 - iter 8/83 - loss 4.41232120 - samples/sec: 179.11\n",
            "2019-12-20 10:49:01,592 epoch 48 - iter 16/83 - loss 4.06085716 - samples/sec: 181.56\n",
            "2019-12-20 10:49:03,158 epoch 48 - iter 24/83 - loss 4.06354436 - samples/sec: 165.50\n",
            "2019-12-20 10:49:04,796 epoch 48 - iter 32/83 - loss 4.11898584 - samples/sec: 158.25\n",
            "2019-12-20 10:49:06,375 epoch 48 - iter 40/83 - loss 4.10758964 - samples/sec: 164.66\n",
            "2019-12-20 10:49:07,794 epoch 48 - iter 48/83 - loss 4.11389655 - samples/sec: 182.94\n",
            "2019-12-20 10:49:09,165 epoch 48 - iter 56/83 - loss 4.06087621 - samples/sec: 189.75\n",
            "2019-12-20 10:49:10,548 epoch 48 - iter 64/83 - loss 4.02420823 - samples/sec: 187.67\n",
            "2019-12-20 10:49:12,023 epoch 48 - iter 72/83 - loss 4.05279561 - samples/sec: 175.66\n",
            "2019-12-20 10:49:13,442 epoch 48 - iter 80/83 - loss 4.04451201 - samples/sec: 182.71\n",
            "2019-12-20 10:49:13,892 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:49:13,893 EPOCH 48 done: loss 4.0399 - lr 0.0250\n",
            "2019-12-20 10:49:14,475 DEV : loss 3.652175188064575 - score 0.9131\n",
            "2019-12-20 10:49:14,487 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:49:14,488 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:49:14,680 epoch 49 - iter 0/83 - loss 4.24225760 - samples/sec: 1344.59\n",
            "2019-12-20 10:49:16,363 epoch 49 - iter 8/83 - loss 4.08114913 - samples/sec: 153.66\n",
            "2019-12-20 10:49:18,040 epoch 49 - iter 16/83 - loss 3.88555723 - samples/sec: 154.55\n",
            "2019-12-20 10:49:19,582 epoch 49 - iter 24/83 - loss 3.94018998 - samples/sec: 168.07\n",
            "2019-12-20 10:49:20,979 epoch 49 - iter 32/83 - loss 4.00981178 - samples/sec: 185.80\n",
            "2019-12-20 10:49:22,397 epoch 49 - iter 40/83 - loss 3.96271785 - samples/sec: 182.83\n",
            "2019-12-20 10:49:23,835 epoch 49 - iter 48/83 - loss 4.04120658 - samples/sec: 180.45\n",
            "2019-12-20 10:49:25,356 epoch 49 - iter 56/83 - loss 4.04387069 - samples/sec: 170.46\n",
            "2019-12-20 10:49:26,980 epoch 49 - iter 64/83 - loss 4.02445230 - samples/sec: 159.60\n",
            "2019-12-20 10:49:28,455 epoch 49 - iter 72/83 - loss 4.02054586 - samples/sec: 175.77\n",
            "2019-12-20 10:49:29,787 epoch 49 - iter 80/83 - loss 4.01289521 - samples/sec: 195.07\n",
            "2019-12-20 10:49:30,043 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:49:30,043 EPOCH 49 done: loss 3.9904 - lr 0.0250\n",
            "2019-12-20 10:49:30,615 DEV : loss 3.65057373046875 - score 0.9111\n",
            "Epoch    48: reducing learning rate of group 0 to 1.2500e-02.\n",
            "2019-12-20 10:49:30,627 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:49:30,628 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:49:30,817 epoch 50 - iter 0/83 - loss 4.86073732 - samples/sec: 1363.24\n",
            "2019-12-20 10:49:32,220 epoch 50 - iter 8/83 - loss 4.35232923 - samples/sec: 184.70\n",
            "2019-12-20 10:49:33,603 epoch 50 - iter 16/83 - loss 4.31990266 - samples/sec: 187.61\n",
            "2019-12-20 10:49:34,958 epoch 50 - iter 24/83 - loss 4.16394303 - samples/sec: 191.55\n",
            "2019-12-20 10:49:36,491 epoch 50 - iter 32/83 - loss 4.02898054 - samples/sec: 169.19\n",
            "2019-12-20 10:49:38,047 epoch 50 - iter 40/83 - loss 3.97919995 - samples/sec: 166.80\n",
            "2019-12-20 10:49:39,654 epoch 50 - iter 48/83 - loss 3.96110170 - samples/sec: 161.24\n",
            "2019-12-20 10:49:41,260 epoch 50 - iter 56/83 - loss 3.98243258 - samples/sec: 161.49\n",
            "2019-12-20 10:49:42,631 epoch 50 - iter 64/83 - loss 3.95994372 - samples/sec: 189.56\n",
            "2019-12-20 10:49:44,250 epoch 50 - iter 72/83 - loss 3.98256953 - samples/sec: 159.76\n",
            "2019-12-20 10:49:45,719 epoch 50 - iter 80/83 - loss 3.97887008 - samples/sec: 176.66\n",
            "2019-12-20 10:49:46,056 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:49:46,057 EPOCH 50 done: loss 3.9527 - lr 0.0125\n",
            "2019-12-20 10:49:46,637 DEV : loss 3.6208527088165283 - score 0.9111\n",
            "2019-12-20 10:49:46,650 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:49:46,652 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:49:46,806 epoch 51 - iter 0/83 - loss 3.63438869 - samples/sec: 1673.56\n",
            "2019-12-20 10:49:48,246 epoch 51 - iter 8/83 - loss 4.28266189 - samples/sec: 179.92\n",
            "2019-12-20 10:49:49,679 epoch 51 - iter 16/83 - loss 4.00110748 - samples/sec: 180.78\n",
            "2019-12-20 10:49:51,080 epoch 51 - iter 24/83 - loss 3.90814564 - samples/sec: 185.22\n",
            "2019-12-20 10:49:52,394 epoch 51 - iter 32/83 - loss 3.84374093 - samples/sec: 197.93\n",
            "2019-12-20 10:49:54,080 epoch 51 - iter 40/83 - loss 3.89269876 - samples/sec: 153.66\n",
            "2019-12-20 10:49:55,591 epoch 51 - iter 48/83 - loss 3.85626348 - samples/sec: 171.80\n",
            "2019-12-20 10:49:57,004 epoch 51 - iter 56/83 - loss 3.85424500 - samples/sec: 183.61\n",
            "2019-12-20 10:49:58,852 epoch 51 - iter 64/83 - loss 3.87650279 - samples/sec: 140.07\n",
            "2019-12-20 10:50:00,265 epoch 51 - iter 72/83 - loss 3.87341879 - samples/sec: 183.74\n",
            "2019-12-20 10:50:01,810 epoch 51 - iter 80/83 - loss 3.90353821 - samples/sec: 167.97\n",
            "2019-12-20 10:50:02,116 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:50:02,117 EPOCH 51 done: loss 3.9016 - lr 0.0125\n",
            "2019-12-20 10:50:02,719 DEV : loss 3.691802978515625 - score 0.9108\n",
            "2019-12-20 10:50:02,733 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:50:02,734 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:50:02,912 epoch 52 - iter 0/83 - loss 4.10597897 - samples/sec: 1456.53\n",
            "2019-12-20 10:50:04,367 epoch 52 - iter 8/83 - loss 3.67232966 - samples/sec: 178.09\n",
            "2019-12-20 10:50:05,773 epoch 52 - iter 16/83 - loss 3.61221340 - samples/sec: 184.69\n",
            "2019-12-20 10:50:07,482 epoch 52 - iter 24/83 - loss 3.78113184 - samples/sec: 151.60\n",
            "2019-12-20 10:50:08,871 epoch 52 - iter 32/83 - loss 3.86495282 - samples/sec: 187.18\n",
            "2019-12-20 10:50:10,353 epoch 52 - iter 40/83 - loss 3.79777775 - samples/sec: 174.93\n",
            "2019-12-20 10:50:11,810 epoch 52 - iter 48/83 - loss 3.80178553 - samples/sec: 177.89\n",
            "2019-12-20 10:50:13,185 epoch 52 - iter 56/83 - loss 3.80500779 - samples/sec: 189.03\n",
            "2019-12-20 10:50:14,734 epoch 52 - iter 64/83 - loss 3.84002160 - samples/sec: 167.14\n",
            "2019-12-20 10:50:16,160 epoch 52 - iter 72/83 - loss 3.86858611 - samples/sec: 182.06\n",
            "2019-12-20 10:50:17,807 epoch 52 - iter 80/83 - loss 3.88878211 - samples/sec: 157.36\n",
            "2019-12-20 10:50:18,078 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:50:18,078 EPOCH 52 done: loss 3.8929 - lr 0.0125\n",
            "2019-12-20 10:50:18,659 DEV : loss 3.653536081314087 - score 0.9122\n",
            "2019-12-20 10:50:18,672 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:50:18,673 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:50:18,843 epoch 53 - iter 0/83 - loss 3.23122644 - samples/sec: 1520.05\n",
            "2019-12-20 10:50:20,400 epoch 53 - iter 8/83 - loss 3.67264538 - samples/sec: 166.11\n",
            "2019-12-20 10:50:21,812 epoch 53 - iter 16/83 - loss 3.90751540 - samples/sec: 184.36\n",
            "2019-12-20 10:50:23,345 epoch 53 - iter 24/83 - loss 3.81828548 - samples/sec: 169.17\n",
            "2019-12-20 10:50:24,773 epoch 53 - iter 32/83 - loss 3.79690368 - samples/sec: 181.63\n",
            "2019-12-20 10:50:26,522 epoch 53 - iter 40/83 - loss 3.80680394 - samples/sec: 147.98\n",
            "2019-12-20 10:50:27,917 epoch 53 - iter 48/83 - loss 3.81372135 - samples/sec: 186.45\n",
            "2019-12-20 10:50:29,352 epoch 53 - iter 56/83 - loss 3.76040063 - samples/sec: 180.83\n",
            "2019-12-20 10:50:30,747 epoch 53 - iter 64/83 - loss 3.79375814 - samples/sec: 185.80\n",
            "2019-12-20 10:50:32,210 epoch 53 - iter 72/83 - loss 3.78944488 - samples/sec: 177.73\n",
            "2019-12-20 10:50:33,646 epoch 53 - iter 80/83 - loss 3.85030045 - samples/sec: 180.45\n",
            "2019-12-20 10:50:33,974 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:50:33,978 EPOCH 53 done: loss 3.8514 - lr 0.0125\n",
            "2019-12-20 10:50:34,552 DEV : loss 3.6164932250976562 - score 0.9125\n",
            "Epoch    52: reducing learning rate of group 0 to 6.2500e-03.\n",
            "2019-12-20 10:50:34,565 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:50:34,566 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:50:34,824 epoch 54 - iter 0/83 - loss 4.07108831 - samples/sec: 997.90\n",
            "2019-12-20 10:50:36,212 epoch 54 - iter 8/83 - loss 3.85219211 - samples/sec: 186.86\n",
            "2019-12-20 10:50:37,897 epoch 54 - iter 16/83 - loss 3.85080391 - samples/sec: 153.68\n",
            "2019-12-20 10:50:39,377 epoch 54 - iter 24/83 - loss 3.83146889 - samples/sec: 175.01\n",
            "2019-12-20 10:50:40,993 epoch 54 - iter 32/83 - loss 3.85798363 - samples/sec: 160.38\n",
            "2019-12-20 10:50:42,397 epoch 54 - iter 40/83 - loss 3.90781263 - samples/sec: 184.84\n",
            "2019-12-20 10:50:44,008 epoch 54 - iter 48/83 - loss 3.89372171 - samples/sec: 161.08\n",
            "2019-12-20 10:50:45,396 epoch 54 - iter 56/83 - loss 3.88849362 - samples/sec: 187.06\n",
            "2019-12-20 10:50:46,813 epoch 54 - iter 64/83 - loss 3.84932817 - samples/sec: 182.86\n",
            "2019-12-20 10:50:48,526 epoch 54 - iter 72/83 - loss 3.88083988 - samples/sec: 151.00\n",
            "2019-12-20 10:50:49,910 epoch 54 - iter 80/83 - loss 3.90521666 - samples/sec: 187.50\n",
            "2019-12-20 10:50:50,262 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:50:50,263 EPOCH 54 done: loss 3.9245 - lr 0.0063\n",
            "2019-12-20 10:50:50,851 DEV : loss 3.6354153156280518 - score 0.9119\n",
            "2019-12-20 10:50:50,865 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:50:50,866 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:50:51,046 epoch 55 - iter 0/83 - loss 3.28362322 - samples/sec: 1438.03\n",
            "2019-12-20 10:50:52,494 epoch 55 - iter 8/83 - loss 3.94283697 - samples/sec: 180.13\n",
            "2019-12-20 10:50:54,091 epoch 55 - iter 16/83 - loss 3.87282285 - samples/sec: 162.25\n",
            "2019-12-20 10:50:55,468 epoch 55 - iter 24/83 - loss 3.91537705 - samples/sec: 188.15\n",
            "2019-12-20 10:50:57,083 epoch 55 - iter 32/83 - loss 3.94680914 - samples/sec: 160.34\n",
            "2019-12-20 10:50:58,734 epoch 55 - iter 40/83 - loss 3.94762290 - samples/sec: 156.68\n",
            "2019-12-20 10:51:00,164 epoch 55 - iter 48/83 - loss 3.88902700 - samples/sec: 181.17\n",
            "2019-12-20 10:51:01,549 epoch 55 - iter 56/83 - loss 3.83865881 - samples/sec: 187.50\n",
            "2019-12-20 10:51:02,917 epoch 55 - iter 64/83 - loss 3.80858773 - samples/sec: 189.80\n",
            "2019-12-20 10:51:04,302 epoch 55 - iter 72/83 - loss 3.80795878 - samples/sec: 187.41\n",
            "2019-12-20 10:51:05,947 epoch 55 - iter 80/83 - loss 3.83319686 - samples/sec: 157.67\n",
            "2019-12-20 10:51:06,279 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:51:06,280 EPOCH 55 done: loss 3.8289 - lr 0.0063\n",
            "2019-12-20 10:51:06,868 DEV : loss 3.632357358932495 - score 0.9119\n",
            "2019-12-20 10:51:06,880 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:51:06,884 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:51:07,074 epoch 56 - iter 0/83 - loss 4.19818211 - samples/sec: 1362.79\n",
            "2019-12-20 10:51:08,512 epoch 56 - iter 8/83 - loss 3.58169908 - samples/sec: 180.18\n",
            "2019-12-20 10:51:10,002 epoch 56 - iter 16/83 - loss 3.78130600 - samples/sec: 174.17\n",
            "2019-12-20 10:51:11,539 epoch 56 - iter 24/83 - loss 3.89430737 - samples/sec: 168.48\n",
            "2019-12-20 10:51:13,137 epoch 56 - iter 32/83 - loss 3.83024242 - samples/sec: 162.24\n",
            "2019-12-20 10:51:14,688 epoch 56 - iter 40/83 - loss 3.88301757 - samples/sec: 167.15\n",
            "2019-12-20 10:51:16,304 epoch 56 - iter 48/83 - loss 3.87128673 - samples/sec: 160.47\n",
            "2019-12-20 10:51:17,780 epoch 56 - iter 56/83 - loss 3.86840551 - samples/sec: 175.83\n",
            "2019-12-20 10:51:19,390 epoch 56 - iter 64/83 - loss 3.87312363 - samples/sec: 161.03\n",
            "2019-12-20 10:51:20,826 epoch 56 - iter 72/83 - loss 3.85390852 - samples/sec: 180.74\n",
            "2019-12-20 10:51:22,320 epoch 56 - iter 80/83 - loss 3.84064388 - samples/sec: 173.57\n",
            "2019-12-20 10:51:22,617 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:51:22,618 EPOCH 56 done: loss 3.8389 - lr 0.0063\n",
            "2019-12-20 10:51:23,212 DEV : loss 3.648451089859009 - score 0.9116\n",
            "2019-12-20 10:51:23,225 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:51:23,226 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:51:23,373 epoch 57 - iter 0/83 - loss 4.36879873 - samples/sec: 1767.75\n",
            "2019-12-20 10:51:24,913 epoch 57 - iter 8/83 - loss 3.75344157 - samples/sec: 168.09\n",
            "2019-12-20 10:51:26,456 epoch 57 - iter 16/83 - loss 3.69780598 - samples/sec: 168.32\n",
            "2019-12-20 10:51:27,867 epoch 57 - iter 24/83 - loss 3.75378930 - samples/sec: 183.85\n",
            "2019-12-20 10:51:29,351 epoch 57 - iter 32/83 - loss 3.77254182 - samples/sec: 174.96\n",
            "2019-12-20 10:51:30,872 epoch 57 - iter 40/83 - loss 3.77497950 - samples/sec: 170.58\n",
            "2019-12-20 10:51:32,490 epoch 57 - iter 48/83 - loss 3.83443023 - samples/sec: 160.40\n",
            "2019-12-20 10:51:33,943 epoch 57 - iter 56/83 - loss 3.85679585 - samples/sec: 178.72\n",
            "2019-12-20 10:51:35,525 epoch 57 - iter 64/83 - loss 3.84198002 - samples/sec: 164.33\n",
            "2019-12-20 10:51:36,979 epoch 57 - iter 72/83 - loss 3.82509038 - samples/sec: 178.53\n",
            "2019-12-20 10:51:38,393 epoch 57 - iter 80/83 - loss 3.79661813 - samples/sec: 183.24\n",
            "2019-12-20 10:51:38,831 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:51:38,832 EPOCH 57 done: loss 3.8401 - lr 0.0063\n",
            "2019-12-20 10:51:39,437 DEV : loss 3.64209246635437 - score 0.9108\n",
            "Epoch    56: reducing learning rate of group 0 to 3.1250e-03.\n",
            "2019-12-20 10:51:39,451 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:51:39,452 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:51:39,615 epoch 58 - iter 0/83 - loss 3.60438323 - samples/sec: 1583.67\n",
            "2019-12-20 10:51:41,399 epoch 58 - iter 8/83 - loss 3.96347393 - samples/sec: 144.78\n",
            "2019-12-20 10:51:42,807 epoch 58 - iter 16/83 - loss 3.90659752 - samples/sec: 184.39\n",
            "2019-12-20 10:51:44,364 epoch 58 - iter 24/83 - loss 4.01234066 - samples/sec: 166.35\n",
            "2019-12-20 10:51:45,746 epoch 58 - iter 32/83 - loss 3.94399563 - samples/sec: 187.78\n",
            "2019-12-20 10:51:47,050 epoch 58 - iter 40/83 - loss 3.87024426 - samples/sec: 199.28\n",
            "2019-12-20 10:51:48,723 epoch 58 - iter 48/83 - loss 3.89108829 - samples/sec: 154.85\n",
            "2019-12-20 10:51:50,180 epoch 58 - iter 56/83 - loss 3.85472134 - samples/sec: 177.77\n",
            "2019-12-20 10:51:51,771 epoch 58 - iter 64/83 - loss 3.85007000 - samples/sec: 162.73\n",
            "2019-12-20 10:51:53,316 epoch 58 - iter 72/83 - loss 3.87343817 - samples/sec: 167.94\n",
            "2019-12-20 10:51:54,700 epoch 58 - iter 80/83 - loss 3.85989965 - samples/sec: 187.55\n",
            "2019-12-20 10:51:55,019 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:51:55,020 EPOCH 58 done: loss 3.8492 - lr 0.0031\n",
            "2019-12-20 10:51:55,606 DEV : loss 3.6443073749542236 - score 0.9105\n",
            "2019-12-20 10:51:55,619 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:51:55,620 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:51:55,802 epoch 59 - iter 0/83 - loss 4.59175301 - samples/sec: 1427.52\n",
            "2019-12-20 10:51:57,186 epoch 59 - iter 8/83 - loss 4.08337379 - samples/sec: 187.28\n",
            "2019-12-20 10:51:58,662 epoch 59 - iter 16/83 - loss 3.87258803 - samples/sec: 175.57\n",
            "2019-12-20 10:52:00,182 epoch 59 - iter 24/83 - loss 3.92246152 - samples/sec: 170.59\n",
            "2019-12-20 10:52:01,874 epoch 59 - iter 32/83 - loss 3.92900594 - samples/sec: 153.30\n",
            "2019-12-20 10:52:03,211 epoch 59 - iter 40/83 - loss 3.86463592 - samples/sec: 194.21\n",
            "2019-12-20 10:52:04,555 epoch 59 - iter 48/83 - loss 3.86605739 - samples/sec: 193.44\n",
            "2019-12-20 10:52:05,798 epoch 59 - iter 56/83 - loss 3.84739289 - samples/sec: 209.46\n",
            "2019-12-20 10:52:07,359 epoch 59 - iter 64/83 - loss 3.86742222 - samples/sec: 166.11\n",
            "2019-12-20 10:52:09,008 epoch 59 - iter 72/83 - loss 3.89314738 - samples/sec: 157.12\n",
            "2019-12-20 10:52:10,525 epoch 59 - iter 80/83 - loss 3.89361735 - samples/sec: 171.00\n",
            "2019-12-20 10:52:10,887 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:52:10,888 EPOCH 59 done: loss 3.9079 - lr 0.0031\n",
            "2019-12-20 10:52:11,493 DEV : loss 3.647486925125122 - score 0.9105\n",
            "2019-12-20 10:52:11,510 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:52:11,511 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:52:11,702 epoch 60 - iter 0/83 - loss 4.75304127 - samples/sec: 1352.75\n",
            "2019-12-20 10:52:13,234 epoch 60 - iter 8/83 - loss 3.78916913 - samples/sec: 168.97\n",
            "2019-12-20 10:52:14,644 epoch 60 - iter 16/83 - loss 3.70104002 - samples/sec: 184.32\n",
            "2019-12-20 10:52:16,260 epoch 60 - iter 24/83 - loss 3.79872648 - samples/sec: 160.33\n",
            "2019-12-20 10:52:17,794 epoch 60 - iter 32/83 - loss 3.85862223 - samples/sec: 169.13\n",
            "2019-12-20 10:52:19,247 epoch 60 - iter 40/83 - loss 3.81718028 - samples/sec: 178.65\n",
            "2019-12-20 10:52:20,782 epoch 60 - iter 48/83 - loss 3.84287712 - samples/sec: 168.87\n",
            "2019-12-20 10:52:22,576 epoch 60 - iter 56/83 - loss 3.83927158 - samples/sec: 144.37\n",
            "2019-12-20 10:52:24,073 epoch 60 - iter 64/83 - loss 3.81340016 - samples/sec: 173.27\n",
            "2019-12-20 10:52:25,564 epoch 60 - iter 72/83 - loss 3.77306533 - samples/sec: 173.68\n",
            "2019-12-20 10:52:27,094 epoch 60 - iter 80/83 - loss 3.77012497 - samples/sec: 169.65\n",
            "2019-12-20 10:52:27,424 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:52:27,427 EPOCH 60 done: loss 3.7745 - lr 0.0031\n",
            "2019-12-20 10:52:28,041 DEV : loss 3.6415913105010986 - score 0.9111\n",
            "2019-12-20 10:52:28,057 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:52:28,058 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:52:28,226 epoch 61 - iter 0/83 - loss 4.34729099 - samples/sec: 1546.38\n",
            "2019-12-20 10:52:29,652 epoch 61 - iter 8/83 - loss 4.09197148 - samples/sec: 181.61\n",
            "2019-12-20 10:52:31,174 epoch 61 - iter 16/83 - loss 4.04899704 - samples/sec: 170.64\n",
            "2019-12-20 10:52:32,758 epoch 61 - iter 24/83 - loss 4.00451394 - samples/sec: 164.02\n",
            "2019-12-20 10:52:34,242 epoch 61 - iter 32/83 - loss 3.92162038 - samples/sec: 174.54\n",
            "2019-12-20 10:52:35,590 epoch 61 - iter 40/83 - loss 3.81068726 - samples/sec: 193.12\n",
            "2019-12-20 10:52:37,119 epoch 61 - iter 48/83 - loss 3.81550646 - samples/sec: 169.76\n",
            "2019-12-20 10:52:38,737 epoch 61 - iter 56/83 - loss 3.80968877 - samples/sec: 160.34\n",
            "2019-12-20 10:52:40,254 epoch 61 - iter 64/83 - loss 3.80939102 - samples/sec: 171.17\n",
            "2019-12-20 10:52:41,836 epoch 61 - iter 72/83 - loss 3.77893436 - samples/sec: 163.94\n",
            "2019-12-20 10:52:43,264 epoch 61 - iter 80/83 - loss 3.76817352 - samples/sec: 181.97\n",
            "2019-12-20 10:52:43,546 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:52:43,547 EPOCH 61 done: loss 3.7654 - lr 0.0031\n",
            "2019-12-20 10:52:44,129 DEV : loss 3.654611825942993 - score 0.9108\n",
            "Epoch    60: reducing learning rate of group 0 to 1.5625e-03.\n",
            "2019-12-20 10:52:44,140 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:52:44,144 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:52:44,380 epoch 62 - iter 0/83 - loss 3.23661542 - samples/sec: 1100.22\n",
            "2019-12-20 10:52:45,854 epoch 62 - iter 8/83 - loss 3.83390326 - samples/sec: 175.76\n",
            "2019-12-20 10:52:47,166 epoch 62 - iter 16/83 - loss 3.85117368 - samples/sec: 198.01\n",
            "2019-12-20 10:52:48,867 epoch 62 - iter 24/83 - loss 3.84182611 - samples/sec: 152.27\n",
            "2019-12-20 10:52:50,379 epoch 62 - iter 32/83 - loss 3.83812922 - samples/sec: 171.38\n",
            "2019-12-20 10:52:51,856 epoch 62 - iter 40/83 - loss 3.81246813 - samples/sec: 176.15\n",
            "2019-12-20 10:52:53,245 epoch 62 - iter 48/83 - loss 3.78573381 - samples/sec: 187.00\n",
            "2019-12-20 10:52:54,684 epoch 62 - iter 56/83 - loss 3.74324492 - samples/sec: 180.40\n",
            "2019-12-20 10:52:56,071 epoch 62 - iter 64/83 - loss 3.76036245 - samples/sec: 187.67\n",
            "2019-12-20 10:52:57,846 epoch 62 - iter 72/83 - loss 3.78993110 - samples/sec: 145.83\n",
            "2019-12-20 10:52:59,305 epoch 62 - iter 80/83 - loss 3.79097450 - samples/sec: 177.85\n",
            "2019-12-20 10:52:59,647 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:52:59,648 EPOCH 62 done: loss 3.7731 - lr 0.0016\n",
            "2019-12-20 10:53:00,236 DEV : loss 3.64003324508667 - score 0.9122\n",
            "2019-12-20 10:53:00,248 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:53:00,249 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:53:00,456 epoch 63 - iter 0/83 - loss 4.72299099 - samples/sec: 1245.12\n",
            "2019-12-20 10:53:01,884 epoch 63 - iter 8/83 - loss 3.77866568 - samples/sec: 181.55\n",
            "2019-12-20 10:53:03,424 epoch 63 - iter 16/83 - loss 3.87411606 - samples/sec: 168.03\n",
            "2019-12-20 10:53:04,837 epoch 63 - iter 24/83 - loss 3.90534235 - samples/sec: 183.74\n",
            "2019-12-20 10:53:06,372 epoch 63 - iter 32/83 - loss 3.92790933 - samples/sec: 168.62\n",
            "2019-12-20 10:53:07,846 epoch 63 - iter 40/83 - loss 3.75965049 - samples/sec: 176.06\n",
            "2019-12-20 10:53:09,239 epoch 63 - iter 48/83 - loss 3.72380990 - samples/sec: 186.22\n",
            "2019-12-20 10:53:10,732 epoch 63 - iter 56/83 - loss 3.73568232 - samples/sec: 173.73\n",
            "2019-12-20 10:53:12,221 epoch 63 - iter 64/83 - loss 3.71985655 - samples/sec: 174.18\n",
            "2019-12-20 10:53:13,699 epoch 63 - iter 72/83 - loss 3.73143757 - samples/sec: 175.59\n",
            "2019-12-20 10:53:15,232 epoch 63 - iter 80/83 - loss 3.73950254 - samples/sec: 169.25\n",
            "2019-12-20 10:53:15,547 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:53:15,548 EPOCH 63 done: loss 3.7532 - lr 0.0016\n",
            "2019-12-20 10:53:16,145 DEV : loss 3.635441780090332 - score 0.9113\n",
            "2019-12-20 10:53:16,161 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:53:16,162 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:53:16,333 epoch 64 - iter 0/83 - loss 3.58749008 - samples/sec: 1513.33\n",
            "2019-12-20 10:53:17,732 epoch 64 - iter 8/83 - loss 3.33344576 - samples/sec: 185.21\n",
            "2019-12-20 10:53:19,251 epoch 64 - iter 16/83 - loss 3.65747605 - samples/sec: 170.78\n",
            "2019-12-20 10:53:20,833 epoch 64 - iter 24/83 - loss 3.69109909 - samples/sec: 163.61\n",
            "2019-12-20 10:53:22,186 epoch 64 - iter 32/83 - loss 3.69460140 - samples/sec: 192.02\n",
            "2019-12-20 10:53:23,554 epoch 64 - iter 40/83 - loss 3.70855661 - samples/sec: 189.85\n",
            "2019-12-20 10:53:24,959 epoch 64 - iter 48/83 - loss 3.73239831 - samples/sec: 184.88\n",
            "2019-12-20 10:53:26,437 epoch 64 - iter 56/83 - loss 3.70888984 - samples/sec: 175.75\n",
            "2019-12-20 10:53:28,085 epoch 64 - iter 64/83 - loss 3.69244653 - samples/sec: 157.10\n",
            "2019-12-20 10:53:29,711 epoch 64 - iter 72/83 - loss 3.76426418 - samples/sec: 159.39\n",
            "2019-12-20 10:53:31,176 epoch 64 - iter 80/83 - loss 3.75096950 - samples/sec: 177.39\n",
            "2019-12-20 10:53:31,630 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:53:31,633 EPOCH 64 done: loss 3.7962 - lr 0.0016\n",
            "2019-12-20 10:53:32,211 DEV : loss 3.650308847427368 - score 0.9105\n",
            "2019-12-20 10:53:32,224 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:53:32,224 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:53:32,492 epoch 65 - iter 0/83 - loss 4.29611588 - samples/sec: 961.87\n",
            "2019-12-20 10:53:33,851 epoch 65 - iter 8/83 - loss 4.03599837 - samples/sec: 191.02\n",
            "2019-12-20 10:53:35,210 epoch 65 - iter 16/83 - loss 3.75197645 - samples/sec: 191.15\n",
            "2019-12-20 10:53:36,695 epoch 65 - iter 24/83 - loss 3.78943874 - samples/sec: 174.64\n",
            "2019-12-20 10:53:38,385 epoch 65 - iter 32/83 - loss 3.84389513 - samples/sec: 153.08\n",
            "2019-12-20 10:53:39,958 epoch 65 - iter 40/83 - loss 3.86710518 - samples/sec: 164.72\n",
            "2019-12-20 10:53:41,507 epoch 65 - iter 48/83 - loss 3.80924263 - samples/sec: 167.37\n",
            "2019-12-20 10:53:42,947 epoch 65 - iter 56/83 - loss 3.75867533 - samples/sec: 180.47\n",
            "2019-12-20 10:53:44,491 epoch 65 - iter 64/83 - loss 3.78077715 - samples/sec: 167.90\n",
            "2019-12-20 10:53:46,137 epoch 65 - iter 72/83 - loss 3.77440909 - samples/sec: 157.65\n",
            "2019-12-20 10:53:47,419 epoch 65 - iter 80/83 - loss 3.76795706 - samples/sec: 202.70\n",
            "2019-12-20 10:53:47,739 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:53:47,742 EPOCH 65 done: loss 3.7863 - lr 0.0016\n",
            "2019-12-20 10:53:48,328 DEV : loss 3.64406156539917 - score 0.9108\n",
            "Epoch    64: reducing learning rate of group 0 to 7.8125e-04.\n",
            "2019-12-20 10:53:48,341 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:53:48,342 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:53:48,498 epoch 66 - iter 0/83 - loss 3.62067223 - samples/sec: 1660.52\n",
            "2019-12-20 10:53:50,085 epoch 66 - iter 8/83 - loss 3.90692096 - samples/sec: 163.02\n",
            "2019-12-20 10:53:51,467 epoch 66 - iter 16/83 - loss 3.93076595 - samples/sec: 188.36\n",
            "2019-12-20 10:53:52,788 epoch 66 - iter 24/83 - loss 3.91179895 - samples/sec: 196.71\n",
            "2019-12-20 10:53:54,587 epoch 66 - iter 32/83 - loss 3.95608312 - samples/sec: 143.79\n",
            "2019-12-20 10:53:55,966 epoch 66 - iter 40/83 - loss 3.86486694 - samples/sec: 188.30\n",
            "2019-12-20 10:53:57,442 epoch 66 - iter 48/83 - loss 3.85877831 - samples/sec: 176.69\n",
            "2019-12-20 10:53:59,070 epoch 66 - iter 56/83 - loss 3.83761523 - samples/sec: 159.33\n",
            "2019-12-20 10:54:00,426 epoch 66 - iter 64/83 - loss 3.78948772 - samples/sec: 191.60\n",
            "2019-12-20 10:54:02,018 epoch 66 - iter 72/83 - loss 3.81131260 - samples/sec: 163.17\n",
            "2019-12-20 10:54:03,364 epoch 66 - iter 80/83 - loss 3.76340472 - samples/sec: 193.13\n",
            "2019-12-20 10:54:03,672 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:54:03,675 EPOCH 66 done: loss 3.7470 - lr 0.0008\n",
            "2019-12-20 10:54:04,256 DEV : loss 3.6458375453948975 - score 0.9105\n",
            "2019-12-20 10:54:04,269 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:54:04,270 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:54:04,434 epoch 67 - iter 0/83 - loss 4.31515694 - samples/sec: 1569.95\n",
            "2019-12-20 10:54:06,013 epoch 67 - iter 8/83 - loss 3.88054313 - samples/sec: 164.02\n",
            "2019-12-20 10:54:07,550 epoch 67 - iter 16/83 - loss 3.78703325 - samples/sec: 168.75\n",
            "2019-12-20 10:54:08,975 epoch 67 - iter 24/83 - loss 3.77959350 - samples/sec: 182.08\n",
            "2019-12-20 10:54:10,602 epoch 67 - iter 32/83 - loss 3.92176722 - samples/sec: 159.24\n",
            "2019-12-20 10:54:12,037 epoch 67 - iter 40/83 - loss 3.93532031 - samples/sec: 180.75\n",
            "2019-12-20 10:54:13,564 epoch 67 - iter 48/83 - loss 3.87075710 - samples/sec: 169.80\n",
            "2019-12-20 10:54:15,142 epoch 67 - iter 56/83 - loss 3.82982783 - samples/sec: 164.13\n",
            "2019-12-20 10:54:16,493 epoch 67 - iter 64/83 - loss 3.80040359 - samples/sec: 192.01\n",
            "2019-12-20 10:54:17,949 epoch 67 - iter 72/83 - loss 3.78253016 - samples/sec: 178.54\n",
            "2019-12-20 10:54:19,336 epoch 67 - iter 80/83 - loss 3.78774174 - samples/sec: 187.06\n",
            "2019-12-20 10:54:19,630 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:54:19,631 EPOCH 67 done: loss 3.7946 - lr 0.0008\n",
            "2019-12-20 10:54:20,233 DEV : loss 3.642566204071045 - score 0.9108\n",
            "2019-12-20 10:54:20,247 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:54:20,248 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:54:20,408 epoch 68 - iter 0/83 - loss 2.73265624 - samples/sec: 1614.58\n",
            "2019-12-20 10:54:21,982 epoch 68 - iter 8/83 - loss 3.63741904 - samples/sec: 164.42\n",
            "2019-12-20 10:54:23,309 epoch 68 - iter 16/83 - loss 3.61788436 - samples/sec: 195.60\n",
            "2019-12-20 10:54:24,981 epoch 68 - iter 24/83 - loss 3.64633907 - samples/sec: 155.13\n",
            "2019-12-20 10:54:26,397 epoch 68 - iter 32/83 - loss 3.63915996 - samples/sec: 183.19\n",
            "2019-12-20 10:54:27,910 epoch 68 - iter 40/83 - loss 3.65394479 - samples/sec: 171.51\n",
            "2019-12-20 10:54:29,464 epoch 68 - iter 48/83 - loss 3.70884056 - samples/sec: 166.97\n",
            "2019-12-20 10:54:30,996 epoch 68 - iter 56/83 - loss 3.78136652 - samples/sec: 169.19\n",
            "2019-12-20 10:54:32,352 epoch 68 - iter 64/83 - loss 3.78292702 - samples/sec: 191.54\n",
            "2019-12-20 10:54:33,817 epoch 68 - iter 72/83 - loss 3.79829670 - samples/sec: 177.45\n",
            "2019-12-20 10:54:35,356 epoch 68 - iter 80/83 - loss 3.79684142 - samples/sec: 168.77\n",
            "2019-12-20 10:54:35,665 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:54:35,666 EPOCH 68 done: loss 3.7939 - lr 0.0008\n",
            "2019-12-20 10:54:36,262 DEV : loss 3.641744375228882 - score 0.9111\n",
            "2019-12-20 10:54:36,276 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:54:36,277 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:54:36,430 epoch 69 - iter 0/83 - loss 3.93285966 - samples/sec: 1684.45\n",
            "2019-12-20 10:54:37,890 epoch 69 - iter 8/83 - loss 4.20532078 - samples/sec: 177.57\n",
            "2019-12-20 10:54:39,356 epoch 69 - iter 16/83 - loss 3.88504159 - samples/sec: 176.89\n",
            "2019-12-20 10:54:40,912 epoch 69 - iter 24/83 - loss 3.85422563 - samples/sec: 166.83\n",
            "2019-12-20 10:54:42,468 epoch 69 - iter 32/83 - loss 3.82172638 - samples/sec: 166.78\n",
            "2019-12-20 10:54:43,930 epoch 69 - iter 40/83 - loss 3.75908724 - samples/sec: 177.34\n",
            "2019-12-20 10:54:45,544 epoch 69 - iter 48/83 - loss 3.77676377 - samples/sec: 160.20\n",
            "2019-12-20 10:54:47,133 epoch 69 - iter 56/83 - loss 3.76050484 - samples/sec: 163.09\n",
            "2019-12-20 10:54:48,558 epoch 69 - iter 64/83 - loss 3.73295863 - samples/sec: 182.01\n",
            "2019-12-20 10:54:50,072 epoch 69 - iter 72/83 - loss 3.78542102 - samples/sec: 171.41\n",
            "2019-12-20 10:54:51,500 epoch 69 - iter 80/83 - loss 3.79672481 - samples/sec: 182.21\n",
            "2019-12-20 10:54:51,807 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:54:51,808 EPOCH 69 done: loss 3.7794 - lr 0.0008\n",
            "2019-12-20 10:54:52,399 DEV : loss 3.637953519821167 - score 0.9108\n",
            "Epoch    68: reducing learning rate of group 0 to 3.9063e-04.\n",
            "2019-12-20 10:54:52,411 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:54:52,412 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:54:52,581 epoch 70 - iter 0/83 - loss 4.36012077 - samples/sec: 1529.93\n",
            "2019-12-20 10:54:54,157 epoch 70 - iter 8/83 - loss 3.68560219 - samples/sec: 164.18\n",
            "2019-12-20 10:54:55,603 epoch 70 - iter 16/83 - loss 3.71179979 - samples/sec: 179.45\n",
            "2019-12-20 10:54:57,158 epoch 70 - iter 24/83 - loss 3.79375594 - samples/sec: 166.82\n",
            "2019-12-20 10:54:58,661 epoch 70 - iter 32/83 - loss 3.87081532 - samples/sec: 172.60\n",
            "2019-12-20 10:55:00,060 epoch 70 - iter 40/83 - loss 3.75188188 - samples/sec: 185.54\n",
            "2019-12-20 10:55:01,602 epoch 70 - iter 48/83 - loss 3.72232842 - samples/sec: 168.52\n",
            "2019-12-20 10:55:03,129 epoch 70 - iter 56/83 - loss 3.70211017 - samples/sec: 169.63\n",
            "2019-12-20 10:55:04,620 epoch 70 - iter 64/83 - loss 3.72867566 - samples/sec: 174.07\n",
            "2019-12-20 10:55:06,024 epoch 70 - iter 72/83 - loss 3.71812864 - samples/sec: 184.83\n",
            "2019-12-20 10:55:07,477 epoch 70 - iter 80/83 - loss 3.77065359 - samples/sec: 178.54\n",
            "2019-12-20 10:55:07,773 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:55:07,777 EPOCH 70 done: loss 3.7655 - lr 0.0004\n",
            "2019-12-20 10:55:08,355 DEV : loss 3.6404871940612793 - score 0.9111\n",
            "2019-12-20 10:55:08,368 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:55:08,369 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:55:08,521 epoch 71 - iter 0/83 - loss 3.36933327 - samples/sec: 1701.16\n",
            "2019-12-20 10:55:09,968 epoch 71 - iter 8/83 - loss 3.54553032 - samples/sec: 179.33\n",
            "2019-12-20 10:55:11,499 epoch 71 - iter 16/83 - loss 3.67948783 - samples/sec: 169.53\n",
            "2019-12-20 10:55:12,981 epoch 71 - iter 24/83 - loss 3.61838034 - samples/sec: 175.19\n",
            "2019-12-20 10:55:14,490 epoch 71 - iter 32/83 - loss 3.70721853 - samples/sec: 171.79\n",
            "2019-12-20 10:55:16,095 epoch 71 - iter 40/83 - loss 3.70175501 - samples/sec: 161.52\n",
            "2019-12-20 10:55:17,659 epoch 71 - iter 48/83 - loss 3.72565636 - samples/sec: 165.72\n",
            "2019-12-20 10:55:19,259 epoch 71 - iter 56/83 - loss 3.77058594 - samples/sec: 162.47\n",
            "2019-12-20 10:55:20,699 epoch 71 - iter 64/83 - loss 3.76186576 - samples/sec: 180.48\n",
            "2019-12-20 10:55:22,166 epoch 71 - iter 72/83 - loss 3.76788360 - samples/sec: 176.75\n",
            "2019-12-20 10:55:23,624 epoch 71 - iter 80/83 - loss 3.74966896 - samples/sec: 177.75\n",
            "2019-12-20 10:55:23,945 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:55:23,948 EPOCH 71 done: loss 3.7676 - lr 0.0004\n",
            "2019-12-20 10:55:24,531 DEV : loss 3.6410326957702637 - score 0.9111\n",
            "2019-12-20 10:55:24,542 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:55:24,543 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:55:24,691 epoch 72 - iter 0/83 - loss 2.88844824 - samples/sec: 1775.77\n",
            "2019-12-20 10:55:26,124 epoch 72 - iter 8/83 - loss 3.69576621 - samples/sec: 180.92\n",
            "2019-12-20 10:55:27,622 epoch 72 - iter 16/83 - loss 3.90548769 - samples/sec: 173.05\n",
            "2019-12-20 10:55:29,194 epoch 72 - iter 24/83 - loss 3.90819890 - samples/sec: 164.87\n",
            "2019-12-20 10:55:30,711 epoch 72 - iter 32/83 - loss 3.90690664 - samples/sec: 170.87\n",
            "2019-12-20 10:55:32,197 epoch 72 - iter 40/83 - loss 3.85582828 - samples/sec: 174.53\n",
            "2019-12-20 10:55:33,801 epoch 72 - iter 48/83 - loss 3.92103995 - samples/sec: 162.46\n",
            "2019-12-20 10:55:35,449 epoch 72 - iter 56/83 - loss 3.88808618 - samples/sec: 157.01\n",
            "2019-12-20 10:55:36,754 epoch 72 - iter 64/83 - loss 3.83153910 - samples/sec: 199.04\n",
            "2019-12-20 10:55:38,205 epoch 72 - iter 72/83 - loss 3.78663372 - samples/sec: 179.14\n",
            "2019-12-20 10:55:39,833 epoch 72 - iter 80/83 - loss 3.80242017 - samples/sec: 159.05\n",
            "2019-12-20 10:55:40,158 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:55:40,160 EPOCH 72 done: loss 3.8155 - lr 0.0004\n",
            "2019-12-20 10:55:40,736 DEV : loss 3.6395161151885986 - score 0.9111\n",
            "2019-12-20 10:55:40,748 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:55:40,749 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:55:40,940 epoch 73 - iter 0/83 - loss 3.71067691 - samples/sec: 1354.01\n",
            "2019-12-20 10:55:42,358 epoch 73 - iter 8/83 - loss 3.45558365 - samples/sec: 182.71\n",
            "2019-12-20 10:55:44,050 epoch 73 - iter 16/83 - loss 3.62422727 - samples/sec: 153.10\n",
            "2019-12-20 10:55:45,667 epoch 73 - iter 24/83 - loss 3.86650108 - samples/sec: 160.25\n",
            "2019-12-20 10:55:47,203 epoch 73 - iter 32/83 - loss 3.80668191 - samples/sec: 168.84\n",
            "2019-12-20 10:55:48,809 epoch 73 - iter 40/83 - loss 3.82504239 - samples/sec: 161.30\n",
            "2019-12-20 10:55:50,158 epoch 73 - iter 48/83 - loss 3.77440418 - samples/sec: 192.55\n",
            "2019-12-20 10:55:51,535 epoch 73 - iter 56/83 - loss 3.74569165 - samples/sec: 188.40\n",
            "2019-12-20 10:55:53,035 epoch 73 - iter 64/83 - loss 3.76047869 - samples/sec: 173.02\n",
            "2019-12-20 10:55:54,422 epoch 73 - iter 72/83 - loss 3.72829608 - samples/sec: 187.28\n",
            "2019-12-20 10:55:55,924 epoch 73 - iter 80/83 - loss 3.75638119 - samples/sec: 172.80\n",
            "2019-12-20 10:55:56,201 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:55:56,203 EPOCH 73 done: loss 3.7571 - lr 0.0004\n",
            "2019-12-20 10:55:56,791 DEV : loss 3.642624855041504 - score 0.9111\n",
            "Epoch    72: reducing learning rate of group 0 to 1.9531e-04.\n",
            "2019-12-20 10:55:56,804 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:55:56,804 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:55:56,968 epoch 74 - iter 0/83 - loss 3.37925434 - samples/sec: 1581.62\n",
            "2019-12-20 10:55:58,461 epoch 74 - iter 8/83 - loss 3.73823534 - samples/sec: 173.21\n",
            "2019-12-20 10:55:59,890 epoch 74 - iter 16/83 - loss 3.54780200 - samples/sec: 181.69\n",
            "2019-12-20 10:56:01,464 epoch 74 - iter 24/83 - loss 3.71239677 - samples/sec: 164.68\n",
            "2019-12-20 10:56:02,971 epoch 74 - iter 32/83 - loss 3.74829994 - samples/sec: 172.12\n",
            "2019-12-20 10:56:04,479 epoch 74 - iter 40/83 - loss 3.74261350 - samples/sec: 171.95\n",
            "2019-12-20 10:56:05,999 epoch 74 - iter 48/83 - loss 3.74931123 - samples/sec: 170.44\n",
            "2019-12-20 10:56:07,464 epoch 74 - iter 56/83 - loss 3.73703826 - samples/sec: 177.05\n",
            "2019-12-20 10:56:08,944 epoch 74 - iter 64/83 - loss 3.70880097 - samples/sec: 175.28\n",
            "2019-12-20 10:56:10,424 epoch 74 - iter 72/83 - loss 3.68311734 - samples/sec: 175.38\n",
            "2019-12-20 10:56:11,784 epoch 74 - iter 80/83 - loss 3.67828357 - samples/sec: 190.66\n",
            "2019-12-20 10:56:12,100 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:56:12,101 EPOCH 74 done: loss 3.6790 - lr 0.0002\n",
            "2019-12-20 10:56:12,675 DEV : loss 3.64237904548645 - score 0.9108\n",
            "2019-12-20 10:56:12,688 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 10:56:12,689 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:56:12,860 epoch 75 - iter 0/83 - loss 4.71500587 - samples/sec: 1513.72\n",
            "2019-12-20 10:56:14,247 epoch 75 - iter 8/83 - loss 3.70519617 - samples/sec: 187.13\n",
            "2019-12-20 10:56:15,687 epoch 75 - iter 16/83 - loss 3.96562025 - samples/sec: 180.05\n",
            "2019-12-20 10:56:17,410 epoch 75 - iter 24/83 - loss 4.06941310 - samples/sec: 150.62\n",
            "2019-12-20 10:56:18,902 epoch 75 - iter 32/83 - loss 3.86716588 - samples/sec: 173.93\n",
            "2019-12-20 10:56:20,278 epoch 75 - iter 40/83 - loss 3.83220739 - samples/sec: 188.66\n",
            "2019-12-20 10:56:21,770 epoch 75 - iter 48/83 - loss 3.74830511 - samples/sec: 173.80\n",
            "2019-12-20 10:56:23,230 epoch 75 - iter 56/83 - loss 3.76045865 - samples/sec: 177.55\n",
            "2019-12-20 10:56:24,755 epoch 75 - iter 64/83 - loss 3.77556809 - samples/sec: 170.01\n",
            "2019-12-20 10:56:26,356 epoch 75 - iter 72/83 - loss 3.74307447 - samples/sec: 162.08\n",
            "2019-12-20 10:56:27,879 epoch 75 - iter 80/83 - loss 3.73821000 - samples/sec: 170.48\n",
            "2019-12-20 10:56:28,205 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:56:28,206 EPOCH 75 done: loss 3.7550 - lr 0.0002\n",
            "2019-12-20 10:56:28,785 DEV : loss 3.6428921222686768 - score 0.9111\n",
            "2019-12-20 10:56:28,798 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 10:56:28,799 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:56:28,964 epoch 76 - iter 0/83 - loss 2.99572039 - samples/sec: 1557.32\n",
            "2019-12-20 10:56:30,432 epoch 76 - iter 8/83 - loss 3.39025807 - samples/sec: 176.44\n",
            "2019-12-20 10:56:31,869 epoch 76 - iter 16/83 - loss 3.49922869 - samples/sec: 180.57\n",
            "2019-12-20 10:56:33,506 epoch 76 - iter 24/83 - loss 3.70381613 - samples/sec: 158.34\n",
            "2019-12-20 10:56:34,796 epoch 76 - iter 32/83 - loss 3.69342438 - samples/sec: 201.59\n",
            "2019-12-20 10:56:36,440 epoch 76 - iter 40/83 - loss 3.72453312 - samples/sec: 157.79\n",
            "2019-12-20 10:56:37,941 epoch 76 - iter 48/83 - loss 3.74317856 - samples/sec: 172.75\n",
            "2019-12-20 10:56:39,407 epoch 76 - iter 56/83 - loss 3.77205095 - samples/sec: 177.17\n",
            "2019-12-20 10:56:40,882 epoch 76 - iter 64/83 - loss 3.80010295 - samples/sec: 176.01\n",
            "2019-12-20 10:56:42,384 epoch 76 - iter 72/83 - loss 3.75624888 - samples/sec: 172.83\n",
            "2019-12-20 10:56:43,837 epoch 76 - iter 80/83 - loss 3.75223074 - samples/sec: 178.69\n",
            "2019-12-20 10:56:44,288 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:56:44,289 EPOCH 76 done: loss 3.7910 - lr 0.0002\n",
            "2019-12-20 10:56:44,879 DEV : loss 3.6425845623016357 - score 0.9105\n",
            "2019-12-20 10:56:44,893 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 10:56:44,894 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:56:45,060 epoch 77 - iter 0/83 - loss 3.00887299 - samples/sec: 1548.30\n",
            "2019-12-20 10:56:46,552 epoch 77 - iter 8/83 - loss 4.05726337 - samples/sec: 173.49\n",
            "2019-12-20 10:56:48,011 epoch 77 - iter 16/83 - loss 3.86171936 - samples/sec: 177.59\n",
            "2019-12-20 10:56:49,479 epoch 77 - iter 24/83 - loss 3.85638826 - samples/sec: 176.69\n",
            "2019-12-20 10:56:51,007 epoch 77 - iter 32/83 - loss 3.81587838 - samples/sec: 169.58\n",
            "2019-12-20 10:56:52,490 epoch 77 - iter 40/83 - loss 3.82119039 - samples/sec: 174.70\n",
            "2019-12-20 10:56:53,832 epoch 77 - iter 48/83 - loss 3.85033839 - samples/sec: 193.23\n",
            "2019-12-20 10:56:55,418 epoch 77 - iter 56/83 - loss 3.84782916 - samples/sec: 163.49\n",
            "2019-12-20 10:56:56,999 epoch 77 - iter 64/83 - loss 3.88146107 - samples/sec: 163.83\n",
            "2019-12-20 10:56:58,600 epoch 77 - iter 72/83 - loss 3.84087921 - samples/sec: 161.66\n",
            "2019-12-20 10:57:00,041 epoch 77 - iter 80/83 - loss 3.86671963 - samples/sec: 180.30\n",
            "2019-12-20 10:57:00,301 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:57:00,302 EPOCH 77 done: loss 3.8336 - lr 0.0002\n",
            "2019-12-20 10:57:00,889 DEV : loss 3.643223285675049 - score 0.9105\n",
            "Epoch    76: reducing learning rate of group 0 to 9.7656e-05.\n",
            "2019-12-20 10:57:00,902 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 10:57:00,902 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:57:00,903 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:57:00,904 learning rate too small - quitting training!\n",
            "2019-12-20 10:57:00,905 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:57:02,307 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:57:02,312 Testing using best model ...\n",
            "2019-12-20 10:57:02,314 loading file resources/taggers/example-universal-pos/best-model.pt\n",
            "2019-12-20 10:57:10,938 0.9175\t0.9175\t0.9175\n",
            "2019-12-20 10:57:10,938 \n",
            "MICRO_AVG: acc 0.8475 - f1-score 0.9175\n",
            "MACRO_AVG: acc 0.8009 - f1-score 0.8807117647058823\n",
            "ADJ        tp: 188 - fp: 57 - fn: 49 - tn: 188 - precision: 0.7673 - recall: 0.7932 - accuracy: 0.6395 - f1-score: 0.7800\n",
            "ADP        tp: 359 - fp: 18 - fn: 4 - tn: 359 - precision: 0.9523 - recall: 0.9890 - accuracy: 0.9423 - f1-score: 0.9703\n",
            "ADV        tp: 169 - fp: 25 - fn: 42 - tn: 169 - precision: 0.8711 - recall: 0.8009 - accuracy: 0.7161 - f1-score: 0.8345\n",
            "AUX        tp: 203 - fp: 15 - fn: 4 - tn: 203 - precision: 0.9312 - recall: 0.9807 - accuracy: 0.9144 - f1-score: 0.9553\n",
            "CCONJ      tp: 104 - fp: 1 - fn: 2 - tn: 104 - precision: 0.9905 - recall: 0.9811 - accuracy: 0.9720 - f1-score: 0.9858\n",
            "DET        tp: 382 - fp: 21 - fn: 8 - tn: 382 - precision: 0.9479 - recall: 0.9795 - accuracy: 0.9294 - f1-score: 0.9634\n",
            "INTJ       tp: 10 - fp: 2 - fn: 7 - tn: 10 - precision: 0.8333 - recall: 0.5882 - accuracy: 0.5263 - f1-score: 0.6896\n",
            "NOUN       tp: 693 - fp: 73 - fn: 77 - tn: 693 - precision: 0.9047 - recall: 0.9000 - accuracy: 0.8221 - f1-score: 0.9023\n",
            "NUM        tp: 58 - fp: 4 - fn: 3 - tn: 58 - precision: 0.9355 - recall: 0.9508 - accuracy: 0.8923 - f1-score: 0.9431\n",
            "PART       tp: 73 - fp: 3 - fn: 4 - tn: 73 - precision: 0.9605 - recall: 0.9481 - accuracy: 0.9125 - f1-score: 0.9543\n",
            "PRON       tp: 267 - fp: 8 - fn: 25 - tn: 267 - precision: 0.9709 - recall: 0.9144 - accuracy: 0.8900 - f1-score: 0.9418\n",
            "PROPN      tp: 248 - fp: 76 - fn: 53 - tn: 248 - precision: 0.7654 - recall: 0.8239 - accuracy: 0.6578 - f1-score: 0.7936\n",
            "PUNCT      tp: 552 - fp: 3 - fn: 0 - tn: 552 - precision: 0.9946 - recall: 1.0000 - accuracy: 0.9946 - f1-score: 0.9973\n",
            "SCONJ      tp: 46 - fp: 2 - fn: 5 - tn: 46 - precision: 0.9583 - recall: 0.9020 - accuracy: 0.8679 - f1-score: 0.9293\n",
            "SYM        tp: 6 - fp: 2 - fn: 3 - tn: 6 - precision: 0.7500 - recall: 0.6667 - accuracy: 0.5455 - f1-score: 0.7059\n",
            "VERB       tp: 339 - fp: 23 - fn: 43 - tn: 339 - precision: 0.9365 - recall: 0.8874 - accuracy: 0.8370 - f1-score: 0.9113\n",
            "X          tp: 5 - fp: 0 - fn: 4 - tn: 5 - precision: 1.0000 - recall: 0.5556 - accuracy: 0.5556 - f1-score: 0.7143\n",
            "2019-12-20 10:57:10,939 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:57:10,940 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 10:57:42,366 ----------------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'test_score': 0.9175,\n",
              " 'dev_score_history': [0.7225,\n",
              "  0.7895,\n",
              "  0.8345,\n",
              "  0.8458,\n",
              "  0.842,\n",
              "  0.8703,\n",
              "  0.8631,\n",
              "  0.8776,\n",
              "  0.8758,\n",
              "  0.877,\n",
              "  0.8819,\n",
              "  0.8764,\n",
              "  0.8796,\n",
              "  0.8894,\n",
              "  0.8952,\n",
              "  0.894,\n",
              "  0.8923,\n",
              "  0.8946,\n",
              "  0.8984,\n",
              "  0.8885,\n",
              "  0.8937,\n",
              "  0.8963,\n",
              "  0.9007,\n",
              "  0.9024,\n",
              "  0.9012,\n",
              "  0.9033,\n",
              "  0.9059,\n",
              "  0.8981,\n",
              "  0.905,\n",
              "  0.9047,\n",
              "  0.9004,\n",
              "  0.9102,\n",
              "  0.9056,\n",
              "  0.9064,\n",
              "  0.9059,\n",
              "  0.9079,\n",
              "  0.9105,\n",
              "  0.9082,\n",
              "  0.909,\n",
              "  0.907,\n",
              "  0.9122,\n",
              "  0.9108,\n",
              "  0.9093,\n",
              "  0.9111,\n",
              "  0.9148,\n",
              "  0.9128,\n",
              "  0.9111,\n",
              "  0.9131,\n",
              "  0.9111,\n",
              "  0.9111,\n",
              "  0.9108,\n",
              "  0.9122,\n",
              "  0.9125,\n",
              "  0.9119,\n",
              "  0.9119,\n",
              "  0.9116,\n",
              "  0.9108,\n",
              "  0.9105,\n",
              "  0.9105,\n",
              "  0.9111,\n",
              "  0.9108,\n",
              "  0.9122,\n",
              "  0.9113,\n",
              "  0.9105,\n",
              "  0.9108,\n",
              "  0.9105,\n",
              "  0.9108,\n",
              "  0.9111,\n",
              "  0.9108,\n",
              "  0.9111,\n",
              "  0.9111,\n",
              "  0.9111,\n",
              "  0.9111,\n",
              "  0.9108,\n",
              "  0.9111,\n",
              "  0.9105,\n",
              "  0.9105],\n",
              " 'train_loss_history': [29.416899991322712,\n",
              "  14.541617221142872,\n",
              "  11.568042192114405,\n",
              "  10.18543615685888,\n",
              "  9.409052222608084,\n",
              "  8.744065669645746,\n",
              "  8.242865177522223,\n",
              "  7.902512021811612,\n",
              "  7.462084442736154,\n",
              "  7.28814155509673,\n",
              "  7.045113557792572,\n",
              "  6.909576404525573,\n",
              "  6.595078163836376,\n",
              "  6.492865272315152,\n",
              "  6.245029920555023,\n",
              "  6.2283248729016405,\n",
              "  6.03732318476022,\n",
              "  6.013873473707452,\n",
              "  5.8320087409881225,\n",
              "  5.8203843128250305,\n",
              "  5.654979177268155,\n",
              "  5.538572759513396,\n",
              "  5.5574367908110105,\n",
              "  5.461377132369812,\n",
              "  5.287198279277388,\n",
              "  5.323209903326379,\n",
              "  5.333667189241892,\n",
              "  5.175237641277083,\n",
              "  5.123664551470653,\n",
              "  5.054193126149924,\n",
              "  4.955399200140712,\n",
              "  4.704883561076888,\n",
              "  4.550996211637933,\n",
              "  4.475330668759633,\n",
              "  4.449276860938014,\n",
              "  4.469088540019759,\n",
              "  4.394944837294429,\n",
              "  4.293668330433857,\n",
              "  4.29577944939395,\n",
              "  4.1764906802809385,\n",
              "  4.228166732443384,\n",
              "  4.19014353924487,\n",
              "  4.112282422651727,\n",
              "  4.086477587021977,\n",
              "  4.112057384238185,\n",
              "  4.071311100419745,\n",
              "  4.050073500139168,\n",
              "  4.039903637874557,\n",
              "  3.990443301488118,\n",
              "  3.952728369149817,\n",
              "  3.9015593586197816,\n",
              "  3.892914312431611,\n",
              "  3.8513854153185005,\n",
              "  3.924518381256655,\n",
              "  3.828855112374547,\n",
              "  3.838907121175743,\n",
              "  3.8401307801166213,\n",
              "  3.8492027449320596,\n",
              "  3.9079157616718705,\n",
              "  3.7744618840964446,\n",
              "  3.765397278659315,\n",
              "  3.773111489881952,\n",
              "  3.7531789009829604,\n",
              "  3.796225108296038,\n",
              "  3.786288183855723,\n",
              "  3.74701688663069,\n",
              "  3.7946203702903656,\n",
              "  3.793862385922168,\n",
              "  3.779431104660034,\n",
              "  3.765487087778298,\n",
              "  3.7676229074776892,\n",
              "  3.815498308963086,\n",
              "  3.7571494292063887,\n",
              "  3.679028301353914,\n",
              "  3.755012598382421,\n",
              "  3.790986830929676,\n",
              "  3.83358985257436],\n",
              " 'dev_loss_history': [tensor(11.1819, device='cuda:0'),\n",
              "  tensor(8.4656, device='cuda:0'),\n",
              "  tensor(6.5247, device='cuda:0'),\n",
              "  tensor(5.8640, device='cuda:0'),\n",
              "  tensor(5.9723, device='cuda:0'),\n",
              "  tensor(4.9800, device='cuda:0'),\n",
              "  tensor(5.0075, device='cuda:0'),\n",
              "  tensor(4.6847, device='cuda:0'),\n",
              "  tensor(4.5715, device='cuda:0'),\n",
              "  tensor(4.6017, device='cuda:0'),\n",
              "  tensor(4.3384, device='cuda:0'),\n",
              "  tensor(4.4909, device='cuda:0'),\n",
              "  tensor(4.4226, device='cuda:0'),\n",
              "  tensor(4.0748, device='cuda:0'),\n",
              "  tensor(4.0403, device='cuda:0'),\n",
              "  tensor(4.0677, device='cuda:0'),\n",
              "  tensor(4.0204, device='cuda:0'),\n",
              "  tensor(4.0549, device='cuda:0'),\n",
              "  tensor(4.0869, device='cuda:0'),\n",
              "  tensor(4.1463, device='cuda:0'),\n",
              "  tensor(4.1827, device='cuda:0'),\n",
              "  tensor(3.8814, device='cuda:0'),\n",
              "  tensor(4.0949, device='cuda:0'),\n",
              "  tensor(3.8801, device='cuda:0'),\n",
              "  tensor(3.8793, device='cuda:0'),\n",
              "  tensor(3.9212, device='cuda:0'),\n",
              "  tensor(3.8556, device='cuda:0'),\n",
              "  tensor(3.9392, device='cuda:0'),\n",
              "  tensor(3.7610, device='cuda:0'),\n",
              "  tensor(3.7199, device='cuda:0'),\n",
              "  tensor(4.0019, device='cuda:0'),\n",
              "  tensor(3.6614, device='cuda:0'),\n",
              "  tensor(3.7767, device='cuda:0'),\n",
              "  tensor(3.7382, device='cuda:0'),\n",
              "  tensor(3.6627, device='cuda:0'),\n",
              "  tensor(3.6898, device='cuda:0'),\n",
              "  tensor(3.6472, device='cuda:0'),\n",
              "  tensor(3.6710, device='cuda:0'),\n",
              "  tensor(3.7177, device='cuda:0'),\n",
              "  tensor(3.7298, device='cuda:0'),\n",
              "  tensor(3.6685, device='cuda:0'),\n",
              "  tensor(3.6473, device='cuda:0'),\n",
              "  tensor(3.7729, device='cuda:0'),\n",
              "  tensor(3.6879, device='cuda:0'),\n",
              "  tensor(3.6205, device='cuda:0'),\n",
              "  tensor(3.6417, device='cuda:0'),\n",
              "  tensor(3.7043, device='cuda:0'),\n",
              "  tensor(3.6522, device='cuda:0'),\n",
              "  tensor(3.6506, device='cuda:0'),\n",
              "  tensor(3.6209, device='cuda:0'),\n",
              "  tensor(3.6918, device='cuda:0'),\n",
              "  tensor(3.6535, device='cuda:0'),\n",
              "  tensor(3.6165, device='cuda:0'),\n",
              "  tensor(3.6354, device='cuda:0'),\n",
              "  tensor(3.6324, device='cuda:0'),\n",
              "  tensor(3.6485, device='cuda:0'),\n",
              "  tensor(3.6421, device='cuda:0'),\n",
              "  tensor(3.6443, device='cuda:0'),\n",
              "  tensor(3.6475, device='cuda:0'),\n",
              "  tensor(3.6416, device='cuda:0'),\n",
              "  tensor(3.6546, device='cuda:0'),\n",
              "  tensor(3.6400, device='cuda:0'),\n",
              "  tensor(3.6354, device='cuda:0'),\n",
              "  tensor(3.6503, device='cuda:0'),\n",
              "  tensor(3.6441, device='cuda:0'),\n",
              "  tensor(3.6458, device='cuda:0'),\n",
              "  tensor(3.6426, device='cuda:0'),\n",
              "  tensor(3.6417, device='cuda:0'),\n",
              "  tensor(3.6380, device='cuda:0'),\n",
              "  tensor(3.6405, device='cuda:0'),\n",
              "  tensor(3.6410, device='cuda:0'),\n",
              "  tensor(3.6395, device='cuda:0'),\n",
              "  tensor(3.6426, device='cuda:0'),\n",
              "  tensor(3.6424, device='cuda:0'),\n",
              "  tensor(3.6429, device='cuda:0'),\n",
              "  tensor(3.6426, device='cuda:0'),\n",
              "  tensor(3.6432, device='cuda:0')]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6LmFmCJw9pqO",
        "colab_type": "text"
      },
      "source": [
        "### B1. Plotting Training Curves and Weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4xlf28cK4Ss3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.visual.training_curves import Plotter\n",
        "\n",
        "plotter = Plotter()\n",
        "# plotter.plot_training_curves('resources/taggers/example-universal-pos/loss.tsv')\n",
        "# plotter.plot_weights('resources/taggers/example-universal-pos/weights.txt')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k6QRHmqq-5W9",
        "colab_type": "text"
      },
      "source": [
        "## C. Resuming Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KQfGaQAr92Mj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.data import Sentence, Corpus\n",
        "from flair.datasets import WNUT_17\n",
        "from flair.embeddings import TokenEmbeddings, WordEmbeddings\n",
        "from flair.embeddings import StackedEmbeddings\n",
        "from typing import List"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PBgFFPVq_WiY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 328
        },
        "outputId": "2d01e124-db2f-401c-f007-dcce20b03b5c"
      },
      "source": [
        "# 1. get the corpus\n",
        "\n",
        "corpus: Corpus = WNUT_17().downsample(0.1)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 11:06:09,904 https://noisy-text.github.io/2017/files/wnut17train.conll not found in cache, downloading to /tmp/tmphaw5g42p\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 493781/493781 [00:00<00:00, 14362565.78B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 11:06:09,975 copying /tmp/tmphaw5g42p to cache at /root/.flair/datasets/wnut_17/wnut17train.conll\n",
            "2019-12-20 11:06:09,978 removing temp file /tmp/tmphaw5g42p\n",
            "2019-12-20 11:06:10,105 https://noisy-text.github.io/2017/files/emerging.dev.conll not found in cache, downloading to /tmp/tmpv704v7em\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "100%|██████████| 114752/114752 [00:00<00:00, 8472929.72B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 11:06:10,150 copying /tmp/tmpv704v7em to cache at /root/.flair/datasets/wnut_17/emerging.dev.conll\n",
            "2019-12-20 11:06:10,151 removing temp file /tmp/tmpv704v7em\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 11:06:10,364 https://noisy-text.github.io/2017/files/emerging.test.annotated not found in cache, downloading to /tmp/tmpsv2gz32e\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 192425/192425 [00:00<00:00, 14959943.41B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 11:06:10,408 copying /tmp/tmpsv2gz32e to cache at /root/.flair/datasets/wnut_17/emerging.test.annotated\n",
            "2019-12-20 11:06:10,409 removing temp file /tmp/tmpsv2gz32e\n",
            "2019-12-20 11:06:10,411 Reading data from /root/.flair/datasets/wnut_17\n",
            "2019-12-20 11:06:10,412 Train: /root/.flair/datasets/wnut_17/wnut17train.conll\n",
            "2019-12-20 11:06:10,414 Dev: /root/.flair/datasets/wnut_17/emerging.dev.conll\n",
            "2019-12-20 11:06:10,415 Test: /root/.flair/datasets/wnut_17/emerging.test.annotated\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qe9I2wER_bnJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 2. what tag do we want to predict?\n",
        "tag_type = 'ner'\n",
        "\n",
        "# 3. make the tag dictionary from the corpus\n",
        "tag_dictionary = corpus.make_tag_dictionary(tag_type = tag_type)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y4qtZcNk_qAY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "0e57b19a-2bf0-4b08-d332-e964e657346a"
      },
      "source": [
        "# 4. initialize embeddings\n",
        "\n",
        "embedding_types: List[TokenEmbeddings] = [\n",
        "                WordEmbeddings('glove')\n",
        "]\n",
        "\n",
        "embeddings: StackedEmbeddings = StackedEmbeddings(embeddings = embedding_types)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:402: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "koMLR93j_3sp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 5. initialize sequence tagger\n",
        "\n",
        "from flair.models import SequenceTagger\n",
        "\n",
        "tagger: SequenceTagger = SequenceTagger(hidden_size = 256,\n",
        "                                        embeddings = embeddings,\n",
        "                                        tag_dictionary = tag_dictionary,\n",
        "                                        tag_type = tag_type,\n",
        "                                        use_crf = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0QqVJUbsAKlh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 6. initialize trainer\n",
        "\n",
        "from flair.trainers import ModelTrainer\n",
        "from flair.training_utils import EvaluationMetric\n",
        "\n",
        "trainer: ModelTrainer = ModelTrainer(tagger, corpus)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mDxHnI0iAVPB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "4dda1518-b9ba-4efc-fe61-6b8c4bd1b476"
      },
      "source": [
        "# 7. start training\n",
        "\n",
        "trainer.train('resources/taggers/example-ner',\n",
        "              learning_rate = 0.1,\n",
        "              mini_batch_size = 32,\n",
        "              max_epochs = 150,\n",
        "              checkpoint = True)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 11:10:57,698 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:10:57,702 Model: \"SequenceTagger(\n",
            "  (embeddings): StackedEmbeddings(\n",
            "    (list_embedding_0): WordEmbeddings('glove')\n",
            "  )\n",
            "  (word_dropout): WordDropout(p=0.05)\n",
            "  (locked_dropout): LockedDropout(p=0.5)\n",
            "  (embedding2nn): Linear(in_features=100, out_features=100, bias=True)\n",
            "  (rnn): LSTM(100, 256, batch_first=True, bidirectional=True)\n",
            "  (linear): Linear(in_features=512, out_features=28, bias=True)\n",
            ")\"\n",
            "2019-12-20 11:10:57,703 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:10:57,705 Corpus: \"Corpus: 339 train + 101 dev + 129 test sentences\"\n",
            "2019-12-20 11:10:57,705 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:10:57,707 Parameters:\n",
            "2019-12-20 11:10:57,708  - learning_rate: \"0.1\"\n",
            "2019-12-20 11:10:57,709  - mini_batch_size: \"32\"\n",
            "2019-12-20 11:10:57,709  - patience: \"3\"\n",
            "2019-12-20 11:10:57,710  - anneal_factor: \"0.5\"\n",
            "2019-12-20 11:10:57,711  - max_epochs: \"150\"\n",
            "2019-12-20 11:10:57,712  - shuffle: \"True\"\n",
            "2019-12-20 11:10:57,714  - train_with_dev: \"False\"\n",
            "2019-12-20 11:10:57,715  - batch_growth_annealing: \"False\"\n",
            "2019-12-20 11:10:57,716 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:10:57,717 Model training base path: \"resources/taggers/example-ner\"\n",
            "2019-12-20 11:10:57,718 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:10:57,719 Device: cuda:0\n",
            "2019-12-20 11:10:57,720 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:10:57,721 Embeddings storage mode: cpu\n",
            "2019-12-20 11:10:57,723 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:10:57,875 epoch 1 - iter 0/11 - loss 60.37521744 - samples/sec: 214.07\n",
            "2019-12-20 11:10:58,012 epoch 1 - iter 1/11 - loss 53.98990250 - samples/sec: 252.94\n",
            "2019-12-20 11:10:58,145 epoch 1 - iter 2/11 - loss 44.96579615 - samples/sec: 262.56\n",
            "2019-12-20 11:10:58,273 epoch 1 - iter 3/11 - loss 36.52803993 - samples/sec: 268.38\n",
            "2019-12-20 11:10:58,408 epoch 1 - iter 4/11 - loss 31.01541939 - samples/sec: 254.93\n",
            "2019-12-20 11:10:58,529 epoch 1 - iter 5/11 - loss 27.75439199 - samples/sec: 286.95\n",
            "2019-12-20 11:10:58,661 epoch 1 - iter 6/11 - loss 25.12620899 - samples/sec: 260.97\n",
            "2019-12-20 11:10:58,783 epoch 1 - iter 7/11 - loss 22.64008832 - samples/sec: 282.47\n",
            "2019-12-20 11:10:58,914 epoch 1 - iter 8/11 - loss 21.10544788 - samples/sec: 261.20\n",
            "2019-12-20 11:10:59,028 epoch 1 - iter 9/11 - loss 19.42645402 - samples/sec: 301.60\n",
            "2019-12-20 11:10:59,133 epoch 1 - iter 10/11 - loss 18.22513658 - samples/sec: 335.40\n",
            "2019-12-20 11:10:59,142 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:10:59,143 EPOCH 1 done: loss 18.2251 - lr 0.1000\n",
            "2019-12-20 11:11:04,563 DEV : loss 6.268129825592041 - score 0.0\n",
            "2019-12-20 11:11:04,569 BAD EPOCHS (no improvement): 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type SequenceTagger. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type StackedEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type WordEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type WordDropout. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type LockedDropout. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Linear. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type LSTM. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 11:11:10,159 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:10,292 epoch 2 - iter 0/11 - loss 12.04873848 - samples/sec: 251.62\n",
            "2019-12-20 11:11:10,420 epoch 2 - iter 1/11 - loss 11.55122328 - samples/sec: 275.94\n",
            "2019-12-20 11:11:10,537 epoch 2 - iter 2/11 - loss 9.39491049 - samples/sec: 296.75\n",
            "2019-12-20 11:11:10,669 epoch 2 - iter 3/11 - loss 8.10245109 - samples/sec: 264.65\n",
            "2019-12-20 11:11:10,783 epoch 2 - iter 4/11 - loss 7.77407551 - samples/sec: 305.72\n",
            "2019-12-20 11:11:10,895 epoch 2 - iter 5/11 - loss 7.29443105 - samples/sec: 314.84\n",
            "2019-12-20 11:11:11,003 epoch 2 - iter 6/11 - loss 7.26372113 - samples/sec: 322.90\n",
            "2019-12-20 11:11:11,109 epoch 2 - iter 7/11 - loss 6.91480714 - samples/sec: 333.52\n",
            "2019-12-20 11:11:11,213 epoch 2 - iter 8/11 - loss 6.95603100 - samples/sec: 340.11\n",
            "2019-12-20 11:11:11,327 epoch 2 - iter 9/11 - loss 7.37227788 - samples/sec: 305.65\n",
            "2019-12-20 11:11:11,424 epoch 2 - iter 10/11 - loss 7.20580851 - samples/sec: 362.47\n",
            "2019-12-20 11:11:11,433 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:11,434 EPOCH 2 done: loss 7.2058 - lr 0.1000\n",
            "2019-12-20 11:11:11,576 DEV : loss 6.299071311950684 - score 0.0\n",
            "2019-12-20 11:11:11,581 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 11:11:17,104 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:17,250 epoch 3 - iter 0/11 - loss 8.81720066 - samples/sec: 226.14\n",
            "2019-12-20 11:11:17,371 epoch 3 - iter 1/11 - loss 8.70403147 - samples/sec: 291.16\n",
            "2019-12-20 11:11:17,499 epoch 3 - iter 2/11 - loss 7.96866798 - samples/sec: 273.18\n",
            "2019-12-20 11:11:17,626 epoch 3 - iter 3/11 - loss 6.96733278 - samples/sec: 272.36\n",
            "2019-12-20 11:11:17,750 epoch 3 - iter 4/11 - loss 6.55737653 - samples/sec: 280.29\n",
            "2019-12-20 11:11:17,872 epoch 3 - iter 5/11 - loss 6.68579209 - samples/sec: 284.07\n",
            "2019-12-20 11:11:17,988 epoch 3 - iter 6/11 - loss 6.60141914 - samples/sec: 299.34\n",
            "2019-12-20 11:11:18,103 epoch 3 - iter 7/11 - loss 6.62880895 - samples/sec: 304.26\n",
            "2019-12-20 11:11:18,224 epoch 3 - iter 8/11 - loss 7.13377653 - samples/sec: 285.76\n",
            "2019-12-20 11:11:18,341 epoch 3 - iter 9/11 - loss 6.88290474 - samples/sec: 297.19\n",
            "2019-12-20 11:11:18,434 epoch 3 - iter 10/11 - loss 6.76008313 - samples/sec: 382.69\n",
            "2019-12-20 11:11:18,443 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:18,444 EPOCH 3 done: loss 6.7601 - lr 0.1000\n",
            "2019-12-20 11:11:18,591 DEV : loss 5.702841281890869 - score 0.0\n",
            "2019-12-20 11:11:18,596 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 11:11:24,221 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:24,335 epoch 4 - iter 0/11 - loss 6.81490326 - samples/sec: 288.76\n",
            "2019-12-20 11:11:24,464 epoch 4 - iter 1/11 - loss 6.88209867 - samples/sec: 267.97\n",
            "2019-12-20 11:11:24,592 epoch 4 - iter 2/11 - loss 6.90967464 - samples/sec: 271.92\n",
            "2019-12-20 11:11:24,739 epoch 4 - iter 3/11 - loss 7.57851815 - samples/sec: 236.10\n",
            "2019-12-20 11:11:24,861 epoch 4 - iter 4/11 - loss 7.20996428 - samples/sec: 284.62\n",
            "2019-12-20 11:11:24,985 epoch 4 - iter 5/11 - loss 7.44907451 - samples/sec: 287.03\n",
            "2019-12-20 11:11:25,094 epoch 4 - iter 6/11 - loss 7.28464787 - samples/sec: 324.75\n",
            "2019-12-20 11:11:25,205 epoch 4 - iter 7/11 - loss 7.21342015 - samples/sec: 312.77\n",
            "2019-12-20 11:11:25,319 epoch 4 - iter 8/11 - loss 6.92129887 - samples/sec: 306.89\n",
            "2019-12-20 11:11:25,427 epoch 4 - iter 9/11 - loss 6.70083447 - samples/sec: 326.34\n",
            "2019-12-20 11:11:25,533 epoch 4 - iter 10/11 - loss 6.55698438 - samples/sec: 336.21\n",
            "2019-12-20 11:11:25,543 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:25,544 EPOCH 4 done: loss 6.5570 - lr 0.1000\n",
            "2019-12-20 11:11:25,688 DEV : loss 5.382099628448486 - score 0.0\n",
            "2019-12-20 11:11:25,695 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 11:11:31,124 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:31,256 epoch 5 - iter 0/11 - loss 4.50739384 - samples/sec: 246.87\n",
            "2019-12-20 11:11:31,382 epoch 5 - iter 1/11 - loss 7.08241272 - samples/sec: 279.26\n",
            "2019-12-20 11:11:31,505 epoch 5 - iter 2/11 - loss 6.74149418 - samples/sec: 288.52\n",
            "2019-12-20 11:11:31,623 epoch 5 - iter 3/11 - loss 6.38310516 - samples/sec: 296.27\n",
            "2019-12-20 11:11:31,742 epoch 5 - iter 4/11 - loss 6.77594366 - samples/sec: 291.95\n",
            "2019-12-20 11:11:31,854 epoch 5 - iter 5/11 - loss 6.36604857 - samples/sec: 309.07\n",
            "2019-12-20 11:11:31,965 epoch 5 - iter 6/11 - loss 6.29895401 - samples/sec: 315.82\n",
            "2019-12-20 11:11:32,077 epoch 5 - iter 7/11 - loss 6.15847445 - samples/sec: 309.01\n",
            "2019-12-20 11:11:32,191 epoch 5 - iter 8/11 - loss 6.34786028 - samples/sec: 304.58\n",
            "2019-12-20 11:11:32,310 epoch 5 - iter 9/11 - loss 6.17461219 - samples/sec: 292.33\n",
            "2019-12-20 11:11:32,399 epoch 5 - iter 10/11 - loss 6.73820023 - samples/sec: 398.46\n",
            "2019-12-20 11:11:32,408 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:32,408 EPOCH 5 done: loss 6.7382 - lr 0.1000\n",
            "2019-12-20 11:11:32,549 DEV : loss 5.153600215911865 - score 0.0\n",
            "Epoch     4: reducing learning rate of group 0 to 5.0000e-02.\n",
            "2019-12-20 11:11:32,555 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 11:11:38,158 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:38,290 epoch 6 - iter 0/11 - loss 6.29628325 - samples/sec: 256.14\n",
            "2019-12-20 11:11:38,419 epoch 6 - iter 1/11 - loss 4.50476718 - samples/sec: 272.05\n",
            "2019-12-20 11:11:38,538 epoch 6 - iter 2/11 - loss 4.94259342 - samples/sec: 295.93\n",
            "2019-12-20 11:11:38,649 epoch 6 - iter 3/11 - loss 4.98010910 - samples/sec: 317.90\n",
            "2019-12-20 11:11:38,768 epoch 6 - iter 4/11 - loss 5.53601723 - samples/sec: 290.43\n",
            "2019-12-20 11:11:38,882 epoch 6 - iter 5/11 - loss 5.72687960 - samples/sec: 307.37\n",
            "2019-12-20 11:11:38,985 epoch 6 - iter 6/11 - loss 5.95238801 - samples/sec: 343.54\n",
            "2019-12-20 11:11:39,090 epoch 6 - iter 7/11 - loss 5.74557418 - samples/sec: 334.24\n",
            "2019-12-20 11:11:39,201 epoch 6 - iter 8/11 - loss 5.72049024 - samples/sec: 311.90\n",
            "2019-12-20 11:11:39,319 epoch 6 - iter 9/11 - loss 6.27010260 - samples/sec: 295.39\n",
            "2019-12-20 11:11:39,423 epoch 6 - iter 10/11 - loss 6.18897442 - samples/sec: 339.79\n",
            "2019-12-20 11:11:39,432 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:39,433 EPOCH 6 done: loss 6.1890 - lr 0.0500\n",
            "2019-12-20 11:11:39,574 DEV : loss 5.1682329177856445 - score 0.0\n",
            "2019-12-20 11:11:39,579 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 11:11:45,031 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:45,168 epoch 7 - iter 0/11 - loss 5.70762587 - samples/sec: 235.91\n",
            "2019-12-20 11:11:45,299 epoch 7 - iter 1/11 - loss 5.97570753 - samples/sec: 268.63\n",
            "2019-12-20 11:11:45,424 epoch 7 - iter 2/11 - loss 6.16959635 - samples/sec: 279.04\n",
            "2019-12-20 11:11:45,551 epoch 7 - iter 3/11 - loss 6.45495176 - samples/sec: 276.47\n",
            "2019-12-20 11:11:45,663 epoch 7 - iter 4/11 - loss 6.52368183 - samples/sec: 327.26\n",
            "2019-12-20 11:11:45,797 epoch 7 - iter 5/11 - loss 6.19338918 - samples/sec: 256.02\n",
            "2019-12-20 11:11:45,935 epoch 7 - iter 6/11 - loss 5.93983262 - samples/sec: 255.05\n",
            "2019-12-20 11:11:46,055 epoch 7 - iter 7/11 - loss 5.80218327 - samples/sec: 291.05\n",
            "2019-12-20 11:11:46,174 epoch 7 - iter 8/11 - loss 5.69016594 - samples/sec: 292.38\n",
            "2019-12-20 11:11:46,283 epoch 7 - iter 9/11 - loss 6.11435804 - samples/sec: 321.42\n",
            "2019-12-20 11:11:46,380 epoch 7 - iter 10/11 - loss 6.02913297 - samples/sec: 370.69\n",
            "2019-12-20 11:11:46,389 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:46,390 EPOCH 7 done: loss 6.0291 - lr 0.0500\n",
            "2019-12-20 11:11:46,537 DEV : loss 5.209019660949707 - score 0.0\n",
            "2019-12-20 11:11:46,541 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 11:11:52,069 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:52,204 epoch 8 - iter 0/11 - loss 4.98759985 - samples/sec: 248.57\n",
            "2019-12-20 11:11:52,333 epoch 8 - iter 1/11 - loss 5.33788562 - samples/sec: 274.17\n",
            "2019-12-20 11:11:52,466 epoch 8 - iter 2/11 - loss 5.34203959 - samples/sec: 268.15\n",
            "2019-12-20 11:11:52,576 epoch 8 - iter 3/11 - loss 5.41793466 - samples/sec: 320.61\n",
            "2019-12-20 11:11:52,694 epoch 8 - iter 4/11 - loss 5.82299833 - samples/sec: 296.21\n",
            "2019-12-20 11:11:52,812 epoch 8 - iter 5/11 - loss 5.88631018 - samples/sec: 294.21\n",
            "2019-12-20 11:11:52,927 epoch 8 - iter 6/11 - loss 5.59487210 - samples/sec: 306.00\n",
            "2019-12-20 11:11:53,030 epoch 8 - iter 7/11 - loss 5.60407701 - samples/sec: 340.01\n",
            "2019-12-20 11:11:53,135 epoch 8 - iter 8/11 - loss 5.77084237 - samples/sec: 333.35\n",
            "2019-12-20 11:11:53,253 epoch 8 - iter 9/11 - loss 5.71250412 - samples/sec: 295.21\n",
            "2019-12-20 11:11:53,350 epoch 8 - iter 10/11 - loss 6.12697898 - samples/sec: 365.50\n",
            "2019-12-20 11:11:53,360 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:53,363 EPOCH 8 done: loss 6.1270 - lr 0.0500\n",
            "2019-12-20 11:11:53,511 DEV : loss 4.959033966064453 - score 0.0\n",
            "2019-12-20 11:11:53,517 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 11:11:59,113 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:11:59,244 epoch 9 - iter 0/11 - loss 4.75074959 - samples/sec: 253.76\n",
            "2019-12-20 11:11:59,369 epoch 9 - iter 1/11 - loss 6.28731966 - samples/sec: 281.48\n",
            "2019-12-20 11:11:59,497 epoch 9 - iter 2/11 - loss 6.34207630 - samples/sec: 272.63\n",
            "2019-12-20 11:11:59,618 epoch 9 - iter 3/11 - loss 6.60136950 - samples/sec: 289.45\n",
            "2019-12-20 11:11:59,735 epoch 9 - iter 4/11 - loss 6.68765860 - samples/sec: 297.93\n",
            "2019-12-20 11:11:59,842 epoch 9 - iter 5/11 - loss 6.07281303 - samples/sec: 330.70\n",
            "2019-12-20 11:11:59,965 epoch 9 - iter 6/11 - loss 5.97936031 - samples/sec: 282.23\n",
            "2019-12-20 11:12:00,075 epoch 9 - iter 7/11 - loss 5.80552381 - samples/sec: 321.48\n",
            "2019-12-20 11:12:00,189 epoch 9 - iter 8/11 - loss 5.92767541 - samples/sec: 304.50\n",
            "2019-12-20 11:12:00,304 epoch 9 - iter 9/11 - loss 5.96625233 - samples/sec: 300.71\n",
            "2019-12-20 11:12:00,396 epoch 9 - iter 10/11 - loss 5.87754271 - samples/sec: 390.18\n",
            "2019-12-20 11:12:00,405 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:00,406 EPOCH 9 done: loss 5.8775 - lr 0.0500\n",
            "2019-12-20 11:12:00,546 DEV : loss 4.942681312561035 - score 0.0\n",
            "Epoch     8: reducing learning rate of group 0 to 2.5000e-02.\n",
            "2019-12-20 11:12:00,549 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 11:12:06,006 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:06,141 epoch 10 - iter 0/11 - loss 4.44116449 - samples/sec: 240.36\n",
            "2019-12-20 11:12:06,274 epoch 10 - iter 1/11 - loss 5.75031209 - samples/sec: 261.30\n",
            "2019-12-20 11:12:06,390 epoch 10 - iter 2/11 - loss 6.28569078 - samples/sec: 303.44\n",
            "2019-12-20 11:12:06,508 epoch 10 - iter 3/11 - loss 5.76659667 - samples/sec: 296.20\n",
            "2019-12-20 11:12:06,630 epoch 10 - iter 4/11 - loss 5.85393896 - samples/sec: 286.23\n",
            "2019-12-20 11:12:06,739 epoch 10 - iter 5/11 - loss 5.81788770 - samples/sec: 320.91\n",
            "2019-12-20 11:12:06,860 epoch 10 - iter 6/11 - loss 5.71883440 - samples/sec: 285.16\n",
            "2019-12-20 11:12:06,967 epoch 10 - iter 7/11 - loss 5.68402356 - samples/sec: 334.14\n",
            "2019-12-20 11:12:07,089 epoch 10 - iter 8/11 - loss 5.39795393 - samples/sec: 283.56\n",
            "2019-12-20 11:12:07,211 epoch 10 - iter 9/11 - loss 5.78260503 - samples/sec: 284.37\n",
            "2019-12-20 11:12:07,307 epoch 10 - iter 10/11 - loss 5.74790616 - samples/sec: 370.14\n",
            "2019-12-20 11:12:07,317 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:07,318 EPOCH 10 done: loss 5.7479 - lr 0.0250\n",
            "2019-12-20 11:12:07,459 DEV : loss 4.893737316131592 - score 0.0\n",
            "2019-12-20 11:12:07,465 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 11:12:12,913 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:13,029 epoch 11 - iter 0/11 - loss 5.95863819 - samples/sec: 280.59\n",
            "2019-12-20 11:12:13,151 epoch 11 - iter 1/11 - loss 5.96802282 - samples/sec: 285.17\n",
            "2019-12-20 11:12:13,286 epoch 11 - iter 2/11 - loss 6.04726760 - samples/sec: 256.16\n",
            "2019-12-20 11:12:13,405 epoch 11 - iter 3/11 - loss 5.75543141 - samples/sec: 294.30\n",
            "2019-12-20 11:12:13,524 epoch 11 - iter 4/11 - loss 5.53574562 - samples/sec: 292.96\n",
            "2019-12-20 11:12:13,629 epoch 11 - iter 5/11 - loss 5.45436907 - samples/sec: 336.29\n",
            "2019-12-20 11:12:13,745 epoch 11 - iter 6/11 - loss 5.64941018 - samples/sec: 300.83\n",
            "2019-12-20 11:12:13,862 epoch 11 - iter 7/11 - loss 5.73611492 - samples/sec: 297.64\n",
            "2019-12-20 11:12:13,978 epoch 11 - iter 8/11 - loss 5.77750974 - samples/sec: 300.52\n",
            "2019-12-20 11:12:14,094 epoch 11 - iter 9/11 - loss 5.71153569 - samples/sec: 301.36\n",
            "2019-12-20 11:12:14,186 epoch 11 - iter 10/11 - loss 5.85880843 - samples/sec: 388.36\n",
            "2019-12-20 11:12:14,205 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:14,208 EPOCH 11 done: loss 5.8588 - lr 0.0250\n",
            "2019-12-20 11:12:14,547 DEV : loss 4.864536285400391 - score 0.0\n",
            "2019-12-20 11:12:14,552 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 11:12:20,063 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:20,201 epoch 12 - iter 0/11 - loss 2.78848958 - samples/sec: 235.84\n",
            "2019-12-20 11:12:20,323 epoch 12 - iter 1/11 - loss 3.84467995 - samples/sec: 289.04\n",
            "2019-12-20 11:12:20,452 epoch 12 - iter 2/11 - loss 4.68050901 - samples/sec: 270.56\n",
            "2019-12-20 11:12:20,578 epoch 12 - iter 3/11 - loss 5.07851285 - samples/sec: 279.78\n",
            "2019-12-20 11:12:20,695 epoch 12 - iter 4/11 - loss 5.43058276 - samples/sec: 298.96\n",
            "2019-12-20 11:12:20,808 epoch 12 - iter 5/11 - loss 5.54329614 - samples/sec: 307.94\n",
            "2019-12-20 11:12:20,925 epoch 12 - iter 6/11 - loss 5.51481979 - samples/sec: 298.01\n",
            "2019-12-20 11:12:21,040 epoch 12 - iter 7/11 - loss 5.18717304 - samples/sec: 306.18\n",
            "2019-12-20 11:12:21,149 epoch 12 - iter 8/11 - loss 5.57028307 - samples/sec: 323.65\n",
            "2019-12-20 11:12:21,261 epoch 12 - iter 9/11 - loss 5.61724498 - samples/sec: 312.25\n",
            "2019-12-20 11:12:21,348 epoch 12 - iter 10/11 - loss 5.87776121 - samples/sec: 416.89\n",
            "2019-12-20 11:12:21,358 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:21,359 EPOCH 12 done: loss 5.8778 - lr 0.0250\n",
            "2019-12-20 11:12:21,514 DEV : loss 4.781057357788086 - score 0.0\n",
            "2019-12-20 11:12:21,518 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 11:12:26,991 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:27,133 epoch 13 - iter 0/11 - loss 8.77454376 - samples/sec: 236.38\n",
            "2019-12-20 11:12:27,247 epoch 13 - iter 1/11 - loss 8.29852724 - samples/sec: 313.49\n",
            "2019-12-20 11:12:27,371 epoch 13 - iter 2/11 - loss 6.78616722 - samples/sec: 281.32\n",
            "2019-12-20 11:12:27,497 epoch 13 - iter 3/11 - loss 6.48881227 - samples/sec: 275.53\n",
            "2019-12-20 11:12:27,620 epoch 13 - iter 4/11 - loss 6.27636361 - samples/sec: 281.70\n",
            "2019-12-20 11:12:27,718 epoch 13 - iter 5/11 - loss 6.15851843 - samples/sec: 357.63\n",
            "2019-12-20 11:12:27,837 epoch 13 - iter 6/11 - loss 6.05537588 - samples/sec: 295.76\n",
            "2019-12-20 11:12:27,955 epoch 13 - iter 7/11 - loss 5.82230243 - samples/sec: 292.78\n",
            "2019-12-20 11:12:28,081 epoch 13 - iter 8/11 - loss 5.83929494 - samples/sec: 279.08\n",
            "2019-12-20 11:12:28,199 epoch 13 - iter 9/11 - loss 5.71624095 - samples/sec: 295.84\n",
            "2019-12-20 11:12:28,287 epoch 13 - iter 10/11 - loss 5.77815769 - samples/sec: 405.63\n",
            "2019-12-20 11:12:28,296 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:28,297 EPOCH 13 done: loss 5.7782 - lr 0.0250\n",
            "2019-12-20 11:12:28,454 DEV : loss 4.793233871459961 - score 0.0\n",
            "Epoch    12: reducing learning rate of group 0 to 1.2500e-02.\n",
            "2019-12-20 11:12:28,460 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 11:12:33,922 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:34,049 epoch 14 - iter 0/11 - loss 7.49825764 - samples/sec: 261.99\n",
            "2019-12-20 11:12:34,161 epoch 14 - iter 1/11 - loss 8.48958349 - samples/sec: 310.16\n",
            "2019-12-20 11:12:34,280 epoch 14 - iter 2/11 - loss 6.71378183 - samples/sec: 291.28\n",
            "2019-12-20 11:12:34,413 epoch 14 - iter 3/11 - loss 6.69822514 - samples/sec: 262.47\n",
            "2019-12-20 11:12:34,537 epoch 14 - iter 4/11 - loss 6.37511635 - samples/sec: 279.58\n",
            "2019-12-20 11:12:34,648 epoch 14 - iter 5/11 - loss 6.56465228 - samples/sec: 319.11\n",
            "2019-12-20 11:12:34,767 epoch 14 - iter 6/11 - loss 6.11809863 - samples/sec: 292.81\n",
            "2019-12-20 11:12:34,885 epoch 14 - iter 7/11 - loss 5.78071567 - samples/sec: 295.45\n",
            "2019-12-20 11:12:34,999 epoch 14 - iter 8/11 - loss 5.68977401 - samples/sec: 308.57\n",
            "2019-12-20 11:12:35,122 epoch 14 - iter 9/11 - loss 5.71939514 - samples/sec: 284.97\n",
            "2019-12-20 11:12:35,207 epoch 14 - iter 10/11 - loss 5.74891686 - samples/sec: 433.85\n",
            "2019-12-20 11:12:35,217 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:35,217 EPOCH 14 done: loss 5.7489 - lr 0.0125\n",
            "2019-12-20 11:12:35,372 DEV : loss 4.799848556518555 - score 0.0\n",
            "2019-12-20 11:12:35,376 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 11:12:41,227 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:41,342 epoch 15 - iter 0/11 - loss 5.59924698 - samples/sec: 297.98\n",
            "2019-12-20 11:12:41,467 epoch 15 - iter 1/11 - loss 4.97087693 - samples/sec: 280.87\n",
            "2019-12-20 11:12:41,576 epoch 15 - iter 2/11 - loss 4.64194330 - samples/sec: 326.14\n",
            "2019-12-20 11:12:41,698 epoch 15 - iter 3/11 - loss 5.06956768 - samples/sec: 284.28\n",
            "2019-12-20 11:12:41,820 epoch 15 - iter 4/11 - loss 4.61995401 - samples/sec: 283.00\n",
            "2019-12-20 11:12:41,938 epoch 15 - iter 5/11 - loss 4.87455908 - samples/sec: 296.76\n",
            "2019-12-20 11:12:42,058 epoch 15 - iter 6/11 - loss 5.29652078 - samples/sec: 293.43\n",
            "2019-12-20 11:12:42,183 epoch 15 - iter 7/11 - loss 5.30357915 - samples/sec: 277.01\n",
            "2019-12-20 11:12:42,289 epoch 15 - iter 8/11 - loss 5.20885430 - samples/sec: 332.79\n",
            "2019-12-20 11:12:42,416 epoch 15 - iter 9/11 - loss 5.50499015 - samples/sec: 273.11\n",
            "2019-12-20 11:12:42,507 epoch 15 - iter 10/11 - loss 5.71107244 - samples/sec: 394.50\n",
            "2019-12-20 11:12:42,517 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:42,518 EPOCH 15 done: loss 5.7111 - lr 0.0125\n",
            "2019-12-20 11:12:42,667 DEV : loss 4.760636329650879 - score 0.0\n",
            "2019-12-20 11:12:42,673 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 11:12:53,073 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:53,192 epoch 16 - iter 0/11 - loss 7.68400192 - samples/sec: 278.26\n",
            "2019-12-20 11:12:53,308 epoch 16 - iter 1/11 - loss 6.46769285 - samples/sec: 302.15\n",
            "2019-12-20 11:12:53,442 epoch 16 - iter 2/11 - loss 5.86793041 - samples/sec: 263.92\n",
            "2019-12-20 11:12:53,572 epoch 16 - iter 3/11 - loss 6.53362155 - samples/sec: 269.48\n",
            "2019-12-20 11:12:53,693 epoch 16 - iter 4/11 - loss 6.53520451 - samples/sec: 287.95\n",
            "2019-12-20 11:12:53,809 epoch 16 - iter 5/11 - loss 6.14612786 - samples/sec: 299.24\n",
            "2019-12-20 11:12:53,923 epoch 16 - iter 6/11 - loss 6.24340425 - samples/sec: 309.16\n",
            "2019-12-20 11:12:54,033 epoch 16 - iter 7/11 - loss 6.13599873 - samples/sec: 316.28\n",
            "2019-12-20 11:12:54,145 epoch 16 - iter 8/11 - loss 5.87926065 - samples/sec: 308.81\n",
            "2019-12-20 11:12:54,274 epoch 16 - iter 9/11 - loss 5.68195448 - samples/sec: 267.98\n",
            "2019-12-20 11:12:54,374 epoch 16 - iter 10/11 - loss 5.77711222 - samples/sec: 352.38\n",
            "2019-12-20 11:12:54,383 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:12:54,386 EPOCH 16 done: loss 5.7771 - lr 0.0125\n",
            "2019-12-20 11:12:54,528 DEV : loss 4.757718563079834 - score 0.0\n",
            "2019-12-20 11:12:54,531 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 11:13:00,119 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:00,270 epoch 17 - iter 0/11 - loss 5.59861565 - samples/sec: 214.72\n",
            "2019-12-20 11:13:00,396 epoch 17 - iter 1/11 - loss 4.79609823 - samples/sec: 277.31\n",
            "2019-12-20 11:13:00,511 epoch 17 - iter 2/11 - loss 5.05656719 - samples/sec: 306.11\n",
            "2019-12-20 11:13:00,635 epoch 17 - iter 3/11 - loss 5.38947630 - samples/sec: 286.61\n",
            "2019-12-20 11:13:00,761 epoch 17 - iter 4/11 - loss 5.75635242 - samples/sec: 277.64\n",
            "2019-12-20 11:13:00,883 epoch 17 - iter 5/11 - loss 5.64498075 - samples/sec: 285.81\n",
            "2019-12-20 11:13:01,001 epoch 17 - iter 6/11 - loss 6.11632878 - samples/sec: 296.31\n",
            "2019-12-20 11:13:01,120 epoch 17 - iter 7/11 - loss 5.92170554 - samples/sec: 296.96\n",
            "2019-12-20 11:13:01,236 epoch 17 - iter 8/11 - loss 5.67647152 - samples/sec: 298.27\n",
            "2019-12-20 11:13:01,349 epoch 17 - iter 9/11 - loss 5.74025662 - samples/sec: 313.24\n",
            "2019-12-20 11:13:01,444 epoch 17 - iter 10/11 - loss 5.55151827 - samples/sec: 378.22\n",
            "2019-12-20 11:13:01,454 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:01,455 EPOCH 17 done: loss 5.5515 - lr 0.0125\n",
            "2019-12-20 11:13:01,612 DEV : loss 4.785056114196777 - score 0.0\n",
            "Epoch    16: reducing learning rate of group 0 to 6.2500e-03.\n",
            "2019-12-20 11:13:01,618 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 11:13:08,015 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:08,132 epoch 18 - iter 0/11 - loss 6.78603745 - samples/sec: 282.45\n",
            "2019-12-20 11:13:08,254 epoch 18 - iter 1/11 - loss 5.90325069 - samples/sec: 286.75\n",
            "2019-12-20 11:13:08,374 epoch 18 - iter 2/11 - loss 6.51218287 - samples/sec: 290.92\n",
            "2019-12-20 11:13:08,489 epoch 18 - iter 3/11 - loss 5.45914257 - samples/sec: 303.93\n",
            "2019-12-20 11:13:08,608 epoch 18 - iter 4/11 - loss 5.01236057 - samples/sec: 291.73\n",
            "2019-12-20 11:13:08,717 epoch 18 - iter 5/11 - loss 5.10625243 - samples/sec: 322.89\n",
            "2019-12-20 11:13:08,827 epoch 18 - iter 6/11 - loss 5.28613329 - samples/sec: 319.64\n",
            "2019-12-20 11:13:08,951 epoch 18 - iter 7/11 - loss 5.26056665 - samples/sec: 279.31\n",
            "2019-12-20 11:13:09,063 epoch 18 - iter 8/11 - loss 5.53122033 - samples/sec: 313.87\n",
            "2019-12-20 11:13:09,183 epoch 18 - iter 9/11 - loss 5.55411153 - samples/sec: 289.06\n",
            "2019-12-20 11:13:09,287 epoch 18 - iter 10/11 - loss 5.55744366 - samples/sec: 339.80\n",
            "2019-12-20 11:13:09,297 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:09,298 EPOCH 18 done: loss 5.5574 - lr 0.0063\n",
            "2019-12-20 11:13:09,446 DEV : loss 4.767139911651611 - score 0.0\n",
            "2019-12-20 11:13:09,451 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 11:13:16,639 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:16,750 epoch 19 - iter 0/11 - loss 6.49671221 - samples/sec: 290.54\n",
            "2019-12-20 11:13:16,869 epoch 19 - iter 1/11 - loss 5.99113822 - samples/sec: 296.02\n",
            "2019-12-20 11:13:16,997 epoch 19 - iter 2/11 - loss 5.83550199 - samples/sec: 276.41\n",
            "2019-12-20 11:13:17,108 epoch 19 - iter 3/11 - loss 5.99987209 - samples/sec: 314.52\n",
            "2019-12-20 11:13:17,237 epoch 19 - iter 4/11 - loss 5.73338690 - samples/sec: 273.46\n",
            "2019-12-20 11:13:17,362 epoch 19 - iter 5/11 - loss 6.21619503 - samples/sec: 290.23\n",
            "2019-12-20 11:13:17,478 epoch 19 - iter 6/11 - loss 6.01449769 - samples/sec: 302.79\n",
            "2019-12-20 11:13:17,586 epoch 19 - iter 7/11 - loss 5.87977362 - samples/sec: 328.01\n",
            "2019-12-20 11:13:17,821 epoch 19 - iter 8/11 - loss 5.84890546 - samples/sec: 141.73\n",
            "2019-12-20 11:13:17,940 epoch 19 - iter 9/11 - loss 5.77437849 - samples/sec: 293.05\n",
            "2019-12-20 11:13:18,030 epoch 19 - iter 10/11 - loss 5.60867734 - samples/sec: 394.73\n",
            "2019-12-20 11:13:18,040 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:18,041 EPOCH 19 done: loss 5.6087 - lr 0.0063\n",
            "2019-12-20 11:13:18,190 DEV : loss 4.7482523918151855 - score 0.0\n",
            "2019-12-20 11:13:18,195 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 11:13:25,473 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:25,613 epoch 20 - iter 0/11 - loss 6.61885643 - samples/sec: 233.00\n",
            "2019-12-20 11:13:25,754 epoch 20 - iter 1/11 - loss 5.43620300 - samples/sec: 255.88\n",
            "2019-12-20 11:13:25,883 epoch 20 - iter 2/11 - loss 5.19268465 - samples/sec: 274.03\n",
            "2019-12-20 11:13:26,018 epoch 20 - iter 3/11 - loss 5.75859880 - samples/sec: 261.43\n",
            "2019-12-20 11:13:26,144 epoch 20 - iter 4/11 - loss 5.51841183 - samples/sec: 277.05\n",
            "2019-12-20 11:13:26,270 epoch 20 - iter 5/11 - loss 5.53562148 - samples/sec: 273.89\n",
            "2019-12-20 11:13:26,379 epoch 20 - iter 6/11 - loss 5.84788534 - samples/sec: 321.19\n",
            "2019-12-20 11:13:26,509 epoch 20 - iter 7/11 - loss 5.65821391 - samples/sec: 263.20\n",
            "2019-12-20 11:13:26,638 epoch 20 - iter 8/11 - loss 5.66919353 - samples/sec: 269.36\n",
            "2019-12-20 11:13:26,764 epoch 20 - iter 9/11 - loss 5.60449619 - samples/sec: 279.95\n",
            "2019-12-20 11:13:26,860 epoch 20 - iter 10/11 - loss 5.51230699 - samples/sec: 370.74\n",
            "2019-12-20 11:13:26,869 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:26,870 EPOCH 20 done: loss 5.5123 - lr 0.0063\n",
            "2019-12-20 11:13:27,021 DEV : loss 4.727533340454102 - score 0.0\n",
            "2019-12-20 11:13:27,027 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 11:13:34,276 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:34,401 epoch 21 - iter 0/11 - loss 4.54623508 - samples/sec: 263.94\n",
            "2019-12-20 11:13:34,522 epoch 21 - iter 1/11 - loss 4.98013377 - samples/sec: 299.84\n",
            "2019-12-20 11:13:34,637 epoch 21 - iter 2/11 - loss 5.49948994 - samples/sec: 309.92\n",
            "2019-12-20 11:13:35,282 epoch 21 - iter 3/11 - loss 5.58281720 - samples/sec: 266.81\n",
            "2019-12-20 11:13:35,398 epoch 21 - iter 4/11 - loss 5.71335888 - samples/sec: 299.48\n",
            "2019-12-20 11:13:35,526 epoch 21 - iter 5/11 - loss 5.53626951 - samples/sec: 278.73\n",
            "2019-12-20 11:13:35,649 epoch 21 - iter 6/11 - loss 5.46241256 - samples/sec: 285.73\n",
            "2019-12-20 11:13:35,768 epoch 21 - iter 7/11 - loss 5.50703919 - samples/sec: 291.97\n",
            "2019-12-20 11:13:35,886 epoch 21 - iter 8/11 - loss 5.61067062 - samples/sec: 295.80\n",
            "2019-12-20 11:13:36,010 epoch 21 - iter 9/11 - loss 5.68990984 - samples/sec: 283.96\n",
            "2019-12-20 11:13:36,119 epoch 21 - iter 10/11 - loss 5.42981007 - samples/sec: 322.30\n",
            "2019-12-20 11:13:36,129 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:36,130 EPOCH 21 done: loss 5.4298 - lr 0.0063\n",
            "2019-12-20 11:13:36,294 DEV : loss 4.731586456298828 - score 0.0\n",
            "Epoch    20: reducing learning rate of group 0 to 3.1250e-03.\n",
            "2019-12-20 11:13:36,300 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 11:13:43,634 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:43,757 epoch 22 - iter 0/11 - loss 4.74815989 - samples/sec: 270.43\n",
            "2019-12-20 11:13:43,884 epoch 22 - iter 1/11 - loss 4.31219089 - samples/sec: 286.69\n",
            "2019-12-20 11:13:44,007 epoch 22 - iter 2/11 - loss 5.65284848 - samples/sec: 288.42\n",
            "2019-12-20 11:13:44,149 epoch 22 - iter 3/11 - loss 5.51612133 - samples/sec: 243.95\n",
            "2019-12-20 11:13:44,254 epoch 22 - iter 4/11 - loss 5.47768760 - samples/sec: 342.12\n",
            "2019-12-20 11:13:44,373 epoch 22 - iter 5/11 - loss 5.64797986 - samples/sec: 295.25\n",
            "2019-12-20 11:13:44,480 epoch 22 - iter 6/11 - loss 5.48936227 - samples/sec: 329.31\n",
            "2019-12-20 11:13:44,599 epoch 22 - iter 7/11 - loss 5.49371418 - samples/sec: 291.09\n",
            "2019-12-20 11:13:44,714 epoch 22 - iter 8/11 - loss 5.56481910 - samples/sec: 302.06\n",
            "2019-12-20 11:13:44,843 epoch 22 - iter 9/11 - loss 5.55926220 - samples/sec: 267.59\n",
            "2019-12-20 11:13:44,937 epoch 22 - iter 10/11 - loss 5.52506653 - samples/sec: 380.61\n",
            "2019-12-20 11:13:44,946 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:44,947 EPOCH 22 done: loss 5.5251 - lr 0.0031\n",
            "2019-12-20 11:13:45,092 DEV : loss 4.7343316078186035 - score 0.0\n",
            "2019-12-20 11:13:45,097 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 11:13:52,526 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:52,667 epoch 23 - iter 0/11 - loss 5.01744413 - samples/sec: 229.42\n",
            "2019-12-20 11:13:52,804 epoch 23 - iter 1/11 - loss 5.46483541 - samples/sec: 257.68\n",
            "2019-12-20 11:13:52,928 epoch 23 - iter 2/11 - loss 7.02071969 - samples/sec: 283.69\n",
            "2019-12-20 11:13:53,042 epoch 23 - iter 3/11 - loss 6.34761274 - samples/sec: 309.66\n",
            "2019-12-20 11:13:53,171 epoch 23 - iter 4/11 - loss 6.55500898 - samples/sec: 272.06\n",
            "2019-12-20 11:13:53,302 epoch 23 - iter 5/11 - loss 6.07659547 - samples/sec: 266.63\n",
            "2019-12-20 11:13:53,423 epoch 23 - iter 6/11 - loss 6.06744637 - samples/sec: 291.30\n",
            "2019-12-20 11:13:53,554 epoch 23 - iter 7/11 - loss 5.84047413 - samples/sec: 267.31\n",
            "2019-12-20 11:13:53,675 epoch 23 - iter 8/11 - loss 5.84279288 - samples/sec: 287.91\n",
            "2019-12-20 11:13:53,800 epoch 23 - iter 9/11 - loss 5.73110290 - samples/sec: 277.40\n",
            "2019-12-20 11:13:53,904 epoch 23 - iter 10/11 - loss 5.48314428 - samples/sec: 338.44\n",
            "2019-12-20 11:13:53,914 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:13:53,914 EPOCH 23 done: loss 5.4831 - lr 0.0031\n",
            "2019-12-20 11:13:54,066 DEV : loss 4.7320332527160645 - score 0.0\n",
            "2019-12-20 11:13:54,071 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 11:14:01,033 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:14:01,140 epoch 24 - iter 0/11 - loss 3.92238307 - samples/sec: 302.15\n",
            "2019-12-20 11:14:01,268 epoch 24 - iter 1/11 - loss 5.01160586 - samples/sec: 285.54\n",
            "2019-12-20 11:14:01,403 epoch 24 - iter 2/11 - loss 6.02002279 - samples/sec: 259.59\n",
            "2019-12-20 11:14:01,535 epoch 24 - iter 3/11 - loss 5.47187281 - samples/sec: 266.46\n",
            "2019-12-20 11:14:01,681 epoch 24 - iter 4/11 - loss 5.81694775 - samples/sec: 238.26\n",
            "2019-12-20 11:14:01,808 epoch 24 - iter 5/11 - loss 5.88258298 - samples/sec: 275.72\n",
            "2019-12-20 11:14:01,954 epoch 24 - iter 6/11 - loss 5.69661563 - samples/sec: 237.35\n",
            "2019-12-20 11:14:02,075 epoch 24 - iter 7/11 - loss 5.59598827 - samples/sec: 288.54\n",
            "2019-12-20 11:14:02,204 epoch 24 - iter 8/11 - loss 5.68033886 - samples/sec: 269.05\n",
            "2019-12-20 11:14:02,323 epoch 24 - iter 9/11 - loss 5.59511008 - samples/sec: 291.43\n",
            "2019-12-20 11:14:02,430 epoch 24 - iter 10/11 - loss 5.63430201 - samples/sec: 333.33\n",
            "2019-12-20 11:14:02,440 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:14:02,441 EPOCH 24 done: loss 5.6343 - lr 0.0031\n",
            "2019-12-20 11:14:02,595 DEV : loss 4.712625503540039 - score 0.0\n",
            "2019-12-20 11:14:02,601 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 11:14:10,683 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:14:10,841 epoch 25 - iter 0/11 - loss 3.53436732 - samples/sec: 272.61\n",
            "2019-12-20 11:14:10,963 epoch 25 - iter 1/11 - loss 5.40816820 - samples/sec: 285.20\n",
            "2019-12-20 11:14:11,083 epoch 25 - iter 2/11 - loss 5.71012584 - samples/sec: 290.79\n",
            "2019-12-20 11:14:11,198 epoch 25 - iter 3/11 - loss 6.17039460 - samples/sec: 303.87\n",
            "2019-12-20 11:14:11,307 epoch 25 - iter 4/11 - loss 5.87805867 - samples/sec: 320.53\n",
            "2019-12-20 11:14:11,432 epoch 25 - iter 5/11 - loss 5.75948226 - samples/sec: 274.27\n",
            "2019-12-20 11:14:11,549 epoch 25 - iter 6/11 - loss 5.73457456 - samples/sec: 297.49\n",
            "2019-12-20 11:14:11,663 epoch 25 - iter 7/11 - loss 5.73649672 - samples/sec: 302.96\n",
            "2019-12-20 11:14:11,782 epoch 25 - iter 8/11 - loss 5.72074125 - samples/sec: 293.23\n",
            "2019-12-20 11:14:11,884 epoch 25 - iter 9/11 - loss 5.58302042 - samples/sec: 344.65\n",
            "2019-12-20 11:14:11,961 epoch 25 - iter 10/11 - loss 5.54795064 - samples/sec: 465.43\n",
            "2019-12-20 11:14:11,970 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:14:11,971 EPOCH 25 done: loss 5.5480 - lr 0.0031\n",
            "2019-12-20 11:14:12,115 DEV : loss 4.717304229736328 - score 0.0\n",
            "Epoch    24: reducing learning rate of group 0 to 1.5625e-03.\n",
            "2019-12-20 11:14:12,121 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 11:14:18,894 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:14:19,009 epoch 26 - iter 0/11 - loss 8.35386276 - samples/sec: 281.95\n",
            "2019-12-20 11:14:19,132 epoch 26 - iter 1/11 - loss 6.20010567 - samples/sec: 285.62\n",
            "2019-12-20 11:14:19,257 epoch 26 - iter 2/11 - loss 5.94851812 - samples/sec: 282.02\n",
            "2019-12-20 11:14:19,389 epoch 26 - iter 3/11 - loss 5.94628918 - samples/sec: 265.31\n",
            "2019-12-20 11:14:19,519 epoch 26 - iter 4/11 - loss 5.36258078 - samples/sec: 267.40\n",
            "2019-12-20 11:14:19,683 epoch 26 - iter 5/11 - loss 5.84049726 - samples/sec: 208.05\n",
            "2019-12-20 11:14:19,800 epoch 26 - iter 6/11 - loss 5.60067838 - samples/sec: 311.58\n",
            "2019-12-20 11:14:19,919 epoch 26 - iter 7/11 - loss 5.77328444 - samples/sec: 294.64\n",
            "2019-12-20 11:14:20,032 epoch 26 - iter 8/11 - loss 5.62370586 - samples/sec: 318.21\n",
            "2019-12-20 11:14:20,158 epoch 26 - iter 9/11 - loss 5.54744787 - samples/sec: 272.49\n",
            "2019-12-20 11:14:20,256 epoch 26 - iter 10/11 - loss 5.67406464 - samples/sec: 368.94\n",
            "2019-12-20 11:14:20,265 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:14:20,266 EPOCH 26 done: loss 5.6741 - lr 0.0016\n",
            "2019-12-20 11:14:20,410 DEV : loss 4.714450359344482 - score 0.0\n",
            "2019-12-20 11:14:20,415 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 11:14:28,449 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:14:28,602 epoch 27 - iter 0/11 - loss 4.69045925 - samples/sec: 290.61\n",
            "2019-12-20 11:14:28,732 epoch 27 - iter 1/11 - loss 5.77475643 - samples/sec: 277.24\n",
            "2019-12-20 11:14:28,861 epoch 27 - iter 2/11 - loss 6.28206507 - samples/sec: 270.02\n",
            "2019-12-20 11:14:28,977 epoch 27 - iter 3/11 - loss 6.95807981 - samples/sec: 303.64\n",
            "2019-12-20 11:14:29,088 epoch 27 - iter 4/11 - loss 6.60478497 - samples/sec: 317.74\n",
            "2019-12-20 11:14:29,206 epoch 27 - iter 5/11 - loss 6.30621028 - samples/sec: 295.56\n",
            "2019-12-20 11:14:29,329 epoch 27 - iter 6/11 - loss 5.87109358 - samples/sec: 284.61\n",
            "2019-12-20 11:14:29,448 epoch 27 - iter 7/11 - loss 5.73025414 - samples/sec: 293.92\n",
            "2019-12-20 11:14:29,570 epoch 27 - iter 8/11 - loss 5.56331033 - samples/sec: 288.84\n",
            "2019-12-20 11:14:29,688 epoch 27 - iter 9/11 - loss 5.57043650 - samples/sec: 295.30\n",
            "2019-12-20 11:14:29,781 epoch 27 - iter 10/11 - loss 5.59468601 - samples/sec: 384.43\n",
            "2019-12-20 11:14:29,790 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:14:29,791 EPOCH 27 done: loss 5.5947 - lr 0.0016\n",
            "2019-12-20 11:14:29,935 DEV : loss 4.712727069854736 - score 0.0\n",
            "2019-12-20 11:14:29,940 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 11:14:30,072 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 11:14:30,073 Exiting from training early.\n",
            "2019-12-20 11:14:30,073 Saving model ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/trainers/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, base_path, learning_rate, mini_batch_size, mini_batch_chunk_size, max_epochs, anneal_factor, patience, min_learning_rate, train_with_dev, monitor_train, monitor_test, embeddings_storage_mode, checkpoint, save_final_model, anneal_with_restarts, batch_growth_annealing, shuffle, param_selection_mode, num_workers, sampler, use_amp, amp_opt_level, eval_on_train_fraction, eval_on_train_shuffle, **kwargs)\u001b[0m\n\u001b[1;32m    507\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcheckpoint\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mparam_selection_mode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 508\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbase_path\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"checkpoint.pt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    509\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/trainers/trainer.py\u001b[0m in \u001b[0;36msave_checkpoint\u001b[0;34m(self, model_file)\u001b[0m\n\u001b[1;32m    555\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcorpus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 556\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_protocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    557\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcorpus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(obj, f, pickle_module, pickle_protocol)\u001b[0m\n\u001b[1;32m    259\u001b[0m     \"\"\"\n\u001b[0;32m--> 260\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_with_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"wb\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0m_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_protocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_with_file_like\u001b[0;34m(f, mode, body)\u001b[0m\n\u001b[1;32m    184\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f)\u001b[0m\n\u001b[1;32m    259\u001b[0m     \"\"\"\n\u001b[0;32m--> 260\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_with_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"wb\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0m_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_protocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_save\u001b[0;34m(obj, f, pickle_module, pickle_protocol)\u001b[0m\n\u001b[1;32m    331\u001b[0m     \u001b[0mpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpersistent_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 332\u001b[0;31m     \u001b[0mpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    333\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36mpersistent_id\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m    294\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'module'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m         \u001b[0;32melif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_storage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m             \u001b[0mstorage_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnormalize_storage_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/__init__.py\u001b[0m in \u001b[0;36mis_storage\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m    130\u001b[0m     \"\"\"\n\u001b[0;32m--> 131\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_storage_classes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-31-07b396305a78>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m               \u001b[0mmini_batch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m               \u001b[0mmax_epochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m150\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m               checkpoint = True)\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/trainers/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, base_path, learning_rate, mini_batch_size, mini_batch_chunk_size, max_epochs, anneal_factor, patience, min_learning_rate, train_with_dev, monitor_train, monitor_test, embeddings_storage_mode, checkpoint, save_final_model, anneal_with_restarts, batch_growth_annealing, shuffle, param_selection_mode, num_workers, sampler, use_amp, amp_opt_level, eval_on_train_fraction, eval_on_train_shuffle, **kwargs)\u001b[0m\n\u001b[1;32m    529\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mparam_selection_mode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    530\u001b[0m                 \u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Saving model ...\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 531\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbase_path\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m\"final-model.pt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    532\u001b[0m                 \u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Done.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/nn.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, model_file)\u001b[0m\n\u001b[1;32m     66\u001b[0m         \u001b[0mmodel_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_protocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(obj, f, pickle_module, pickle_protocol)\u001b[0m\n\u001b[1;32m    258\u001b[0m         \u001b[0;34m>>\u001b[0m\u001b[0;34m>\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m     \"\"\"\n\u001b[0;32m--> 260\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_with_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"wb\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0m_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_protocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_with_file_like\u001b[0;34m(f, mode, body)\u001b[0m\n\u001b[1;32m    183\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnew_fd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f)\u001b[0m\n\u001b[1;32m    258\u001b[0m         \u001b[0;34m>>\u001b[0m\u001b[0;34m>\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m     \"\"\"\n\u001b[0;32m--> 260\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_with_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"wb\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0m_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_protocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_save\u001b[0;34m(obj, f, pickle_module, pickle_protocol)\u001b[0m\n\u001b[1;32m    330\u001b[0m     \u001b[0mpickler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPickler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpickle_protocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m     \u001b[0mpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpersistent_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 332\u001b[0;31m     \u001b[0mpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    333\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    334\u001b[0m     \u001b[0mserialized_storage_keys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mserialized_storages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36mpersistent_id\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m    293\u001b[0m                               \"for correctness upon loading.\")\n\u001b[1;32m    294\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'module'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m         \u001b[0;32melif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_storage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m             \u001b[0mstorage_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnormalize_storage_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m             \u001b[0;31m# Offset is always 0, but we keep it for backwards compatibility\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/__init__.py\u001b[0m in \u001b[0;36mis_storage\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m    129\u001b[0m         \u001b[0mobj\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mObject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mObject\u001b[0m \u001b[0mto\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m     \"\"\"\n\u001b[0;32m--> 131\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_storage_classes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-7q9-LqNAh7x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 8. stop training at any point\n",
        "\n",
        "# 9. continue trainer at later point\n",
        "\n",
        "from pathlib import Path\n",
        "\n",
        "checkpoint = 'resources/taggers/example-ner/checkpoint.pt'\n",
        "# trainer = ModelTrainer.load_checkpoint(checkpoint, corpus)\n",
        "# trainer.train('resources/taggers/example-ner',\n",
        "#               learning_rate = 0.1,\n",
        "#               mini_batch_size = 32,\n",
        "#               max_epochs = 150,\n",
        "#               checkpoint = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gagQOBfeBysR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}