train_xlmroberta_earlystopping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "  Using cached transformers-4.15.0-py3-none-any.whl (3.4 MB)\n",
      "Collecting regex!=2019.12.17\n",
      "  Using cached regex-2021.11.10-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (748 kB)\n",
      "Requirement already satisfied: dataclasses in /opt/conda/lib/python3.6/site-packages (from transformers) (0.8)\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.6/site-packages (from transformers) (4.4.0)\n",
      "Collecting filelock\n",
      "  Using cached filelock-3.4.1-py3-none-any.whl (9.9 kB)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.6/site-packages (from transformers) (20.9)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.6/site-packages (from transformers) (5.4.1)\n",
      "Collecting huggingface-hub<1.0,>=0.1.0\n",
      "  Using cached huggingface_hub-0.2.1-py3-none-any.whl (61 kB)\n",
      "Collecting tokenizers<0.11,>=0.10.1\n",
      "  Using cached tokenizers-0.10.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.6/site-packages (from transformers) (1.19.1)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from transformers) (2.25.1)\n",
      "Collecting sacremoses\n",
      "  Using cached sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.6/site-packages (from transformers) (4.51.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.6/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.6/site-packages (from packaging>=20.0->transformers) (2.4.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.6/site-packages (from importlib-metadata->transformers) (3.4.1)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (2021.5.30)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (1.25.11)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (1.0.1)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (1.16.0)\n",
      "Installing collected packages: regex, filelock, tokenizers, sacremoses, huggingface-hub, transformers\n",
      "Successfully installed filelock-3.4.1 huggingface-hub-0.2.1 regex-2021.11.10 sacremoses-0.0.46 tokenizers-0.10.3 transformers-4.15.0\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Collecting sentencepiece\n",
      "  Using cached sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "Installing collected packages: sentencepiece\n",
      "Successfully installed sentencepiece-0.1.96\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Collecting nltk\n",
      "  Using cached nltk-3.6.7-py3-none-any.whl (1.5 MB)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from nltk) (1.0.1)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /opt/conda/lib/python3.6/site-packages (from nltk) (2021.11.10)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from nltk) (4.51.0)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from nltk) (7.1.2)\n",
      "Installing collected packages: nltk\n",
      "Successfully installed nltk-3.6.7\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Collecting ipywidgets\n",
      "  Using cached ipywidgets-7.6.5-py2.py3-none-any.whl (121 kB)\n",
      "Requirement already satisfied: ipython>=4.0.0 in /opt/conda/lib/python3.6/site-packages (from ipywidgets) (7.16.2)\n",
      "Collecting jupyterlab-widgets>=1.0.0\n",
      "  Using cached jupyterlab_widgets-1.0.2-py3-none-any.whl (243 kB)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.6/site-packages (from ipywidgets) (4.3.3)\n",
      "Collecting nbformat>=4.2.0\n",
      "  Using cached nbformat-5.1.3-py3-none-any.whl (178 kB)\n",
      "Requirement already satisfied: ipython-genutils~=0.2.0 in /opt/conda/lib/python3.6/site-packages (from ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: ipykernel>=4.5.1 in /opt/conda/lib/python3.6/site-packages (from ipywidgets) (5.5.6)\n",
      "Collecting widgetsnbextension~=3.5.0\n",
      "  Using cached widgetsnbextension-3.5.2-py2.py3-none-any.whl (1.6 MB)\n",
      "Requirement already satisfied: jupyter-client in /opt/conda/lib/python3.6/site-packages (from ipykernel>=4.5.1->ipywidgets) (7.1.0)\n",
      "Requirement already satisfied: tornado>=4.2 in /opt/conda/lib/python3.6/site-packages (from ipykernel>=4.5.1->ipywidgets) (6.1)\n",
      "Requirement already satisfied: pygments in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipywidgets) (2.10.0)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipywidgets) (0.7.5)\n",
      "Requirement already satisfied: pexpect in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipywidgets) (4.8.0)\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipywidgets) (5.1.0)\n",
      "Requirement already satisfied: setuptools>=18.5 in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipywidgets) (59.6.0)\n",
      "Collecting jedi<=0.17.2,>=0.10\n",
      "  Using cached jedi-0.17.2-py2.py3-none-any.whl (1.4 MB)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipywidgets) (3.0.24)\n",
      "Collecting parso<0.8.0,>=0.7.0\n",
      "  Using cached parso-0.7.1-py2.py3-none-any.whl (109 kB)\n",
      "Requirement already satisfied: jupyter-core in /opt/conda/lib/python3.6/site-packages (from nbformat>=4.2.0->ipywidgets) (4.9.1)\n",
      "Collecting jsonschema!=2.5.0,>=2.4\n",
      "  Using cached jsonschema-3.2.0-py2.py3-none-any.whl (56 kB)\n",
      "Requirement already satisfied: six>=1.11.0 in /opt/conda/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets) (1.16.0)\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets) (4.4.0)\n",
      "Collecting pyrsistent>=0.14.0\n",
      "  Using cached pyrsistent-0.18.0-cp36-cp36m-manylinux1_x86_64.whl (117 kB)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets) (21.2.0)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.6/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=4.0.0->ipywidgets) (0.2.5)\n",
      "Collecting notebook>=4.4.1\n",
      "  Using cached notebook-6.4.6-py3-none-any.whl (9.9 MB)\n",
      "Collecting argon2-cffi\n",
      "  Using cached argon2_cffi-21.3.0-py3-none-any.whl (14 kB)\n",
      "Collecting nbconvert\n",
      "  Using cached nbconvert-6.0.7-py3-none-any.whl (552 kB)\n",
      "Collecting Send2Trash>=1.8.0\n",
      "  Using cached Send2Trash-1.8.0-py3-none-any.whl (18 kB)\n",
      "Requirement already satisfied: nest-asyncio>=1.5 in /opt/conda/lib/python3.6/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (1.5.4)\n",
      "Collecting prometheus-client\n",
      "  Using cached prometheus_client-0.12.0-py2.py3-none-any.whl (57 kB)\n",
      "Requirement already satisfied: pyzmq>=17 in /opt/conda/lib/python3.6/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (22.1.0)\n",
      "Collecting terminado>=0.8.3\n",
      "  Using cached terminado-0.12.1-py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.6/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (3.0.1)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.6/site-packages (from jupyter-client->ipykernel>=4.5.1->ipywidgets) (2.8.2)\n",
      "Requirement already satisfied: entrypoints in /opt/conda/lib/python3.6/site-packages (from jupyter-client->ipykernel>=4.5.1->ipywidgets) (0.3)\n",
      "Requirement already satisfied: ptyprocess in /opt/conda/lib/python3.6/site-packages (from terminado>=0.8.3->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.7.0)\n",
      "Requirement already satisfied: dataclasses in /opt/conda/lib/python3.6/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.8)\n",
      "Collecting argon2-cffi-bindings\n",
      "  Using cached argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (86 kB)\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.6/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (3.10.0.0)\n",
      "Requirement already satisfied: cffi>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (1.14.5)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.6/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (2.20)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.6/site-packages (from importlib-metadata->jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets) (3.4.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.6/site-packages (from jinja2->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (2.0.1)\n",
      "Collecting testpath\n",
      "  Using cached testpath-0.5.0-py3-none-any.whl (84 kB)\n",
      "Collecting bleach\n",
      "  Using cached bleach-4.1.0-py2.py3-none-any.whl (157 kB)\n",
      "Collecting pandocfilters>=1.4.1\n",
      "  Using cached pandocfilters-1.5.0-py2.py3-none-any.whl (8.7 kB)\n",
      "Collecting defusedxml\n",
      "  Using cached defusedxml-0.7.1-py2.py3-none-any.whl (25 kB)\n",
      "Collecting mistune<2,>=0.8.1\n",
      "  Using cached mistune-0.8.4-py2.py3-none-any.whl (16 kB)\n",
      "Collecting jupyterlab-pygments\n",
      "  Using cached jupyterlab_pygments-0.1.2-py2.py3-none-any.whl (4.6 kB)\n",
      "Collecting nbclient<0.6.0,>=0.5.0\n",
      "  Using cached nbclient-0.5.9-py3-none-any.whl (69 kB)\n",
      "Collecting async-generator\n",
      "  Using cached async_generator-1.10-py3-none-any.whl (18 kB)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.6/site-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (20.9)\n",
      "Collecting webencodings\n",
      "  Using cached webencodings-0.5.1-py2.py3-none-any.whl (11 kB)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.6/site-packages (from packaging->bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (2.4.7)\n",
      "Installing collected packages: pyrsistent, parso, jsonschema, webencodings, nbformat, jedi, async-generator, testpath, pandocfilters, nbclient, mistune, jupyterlab-pygments, defusedxml, bleach, argon2-cffi-bindings, terminado, Send2Trash, prometheus-client, nbconvert, argon2-cffi, notebook, widgetsnbextension, jupyterlab-widgets, ipywidgets\n",
      "  Attempting uninstall: parso\n",
      "    Found existing installation: parso 0.8.0\n",
      "    Uninstalling parso-0.8.0:\n",
      "      Successfully uninstalled parso-0.8.0\n",
      "  Attempting uninstall: jedi\n",
      "    Found existing installation: jedi 0.18.0\n",
      "    Uninstalling jedi-0.18.0:\n",
      "      Successfully uninstalled jedi-0.18.0\n",
      "Successfully installed Send2Trash-1.8.0 argon2-cffi-21.3.0 argon2-cffi-bindings-21.2.0 async-generator-1.10 bleach-4.1.0 defusedxml-0.7.1 ipywidgets-7.6.5 jedi-0.17.2 jsonschema-3.2.0 jupyterlab-pygments-0.1.2 jupyterlab-widgets-1.0.2 mistune-0.8.4 nbclient-0.5.9 nbconvert-6.0.7 nbformat-5.1.3 notebook-6.4.6 pandocfilters-1.5.0 parso-0.7.1 prometheus-client-0.12.0 pyrsistent-0.18.0 terminado-0.12.1 testpath-0.5.0 webencodings-0.5.1 widgetsnbextension-3.5.2\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n",
    "!pip install sentencepiece\n",
    "!pip install nltk\n",
    "!pip install ipywidgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /opt/conda/lib/python3.6/site-packages (4.15.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.6/site-packages (from transformers) (20.9)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.6/site-packages (from transformers) (5.4.1)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /opt/conda/lib/python3.6/site-packages (from transformers) (0.10.3)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.6/site-packages (from transformers) (4.51.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /opt/conda/lib/python3.6/site-packages (from transformers) (0.2.1)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from transformers) (2.25.1)\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.6/site-packages (from transformers) (4.4.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.6/site-packages (from transformers) (3.4.1)\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.6/site-packages (from transformers) (0.0.46)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.6/site-packages (from transformers) (1.19.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.6/site-packages (from transformers) (2021.11.10)\n",
      "Requirement already satisfied: dataclasses in /opt/conda/lib/python3.6/site-packages (from transformers) (0.8)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.6/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.6/site-packages (from packaging>=20.0->transformers) (2.4.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.6/site-packages (from importlib-metadata->transformers) (3.4.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (1.25.11)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (2021.5.30)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (1.16.0)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (1.0.1)\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.6/site-packages (0.1.96)\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Collecting seqeval\n",
      "  Using cached seqeval-1.2.2-py3-none-any.whl\n",
      "Requirement already satisfied: scikit-learn>=0.21.3 in /opt/conda/lib/python3.6/site-packages (from seqeval) (0.24.2)\n",
      "Requirement already satisfied: numpy>=1.14.0 in /opt/conda/lib/python3.6/site-packages (from seqeval) (1.19.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from scikit-learn>=0.21.3->seqeval) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.6/site-packages (from scikit-learn>=0.21.3->seqeval) (1.0.1)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.6/site-packages (from scikit-learn>=0.21.3->seqeval) (1.5.4)\n",
      "Installing collected packages: seqeval\n",
      "Successfully installed seqeval-1.2.2\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Requirement already satisfied: transformers in /opt/conda/lib/python3.6/site-packages (4.15.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /opt/conda/lib/python3.6/site-packages (from transformers) (0.2.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.6/site-packages (from transformers) (1.19.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.6/site-packages (from transformers) (20.9)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.6/site-packages (from transformers) (2021.11.10)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.6/site-packages (from transformers) (4.51.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.6/site-packages (from transformers) (3.4.1)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /opt/conda/lib/python3.6/site-packages (from transformers) (0.10.3)\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.6/site-packages (from transformers) (0.0.46)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from transformers) (2.25.1)\n",
      "Requirement already satisfied: dataclasses in /opt/conda/lib/python3.6/site-packages (from transformers) (0.8)\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.6/site-packages (from transformers) (4.4.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.6/site-packages (from transformers) (5.4.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.6/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.6/site-packages (from packaging>=20.0->transformers) (2.4.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.6/site-packages (from importlib-metadata->transformers) (3.4.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (1.25.11)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (2021.5.30)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (1.16.0)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (1.0.1)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (7.1.2)\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.6/site-packages (0.1.96)\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import joblib\n",
    "import torch\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn import model_selection\n",
    "# from DataAugmentation \n",
    "# from DataAugmentation import DataAugmentation\n",
    "from transformers import AdamW\n",
    "from transformers import get_linear_schedule_with_warmup\n",
    "from importlib import reload\n",
    "import config\n",
    "from transformers import XLMRobertaForTokenClassification, XLMRobertaConfig ,BertModel, XLMRobertaTokenizer, XLMRobertaModel, BertForTokenClassification\n",
    "from data_utils import loadDatafromFile,createTokenizedDf,CompDataset,createkfoldData,createDataloaders\n",
    "from seqeval.metrics import accuracy_score, classification_report,f1_score\n",
    "from sklearn.metrics import matthews_corrcoef,classification_report\n",
    "import engine\n",
    "# from model_new import EntityModel\n",
    "import torch.nn as nn\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the data\n",
    "def process_data(filePath_src,filePath_tar, filePath_srcTags,filePath_tarTags,model_type):\n",
    "    \n",
    "    dataObj = loadDatafromFile(filePath_src,filePath_tar, filePath_srcTags,filePath_tarTags)\n",
    "    df= dataObj.createDf() # get dataframe from files\n",
    "    obj_tokenized = createTokenizedDf(df,model_type)\n",
    "    df_new= obj_tokenized.convertDf()\n",
    "#     enc_label = preprocessing.LabelEncoder()\n",
    "#     df_new['labels']= enc_label.fit_transform(df_new['labels'])\n",
    "    train_data = CompDataset(df_new,model_type)\n",
    "    return train_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if loading of the non augmented (noise-free) data is needed, then run this\n",
    "dataset_train = process_data(config.filePath_src,config.filePath_tar, config.filePath_srcTags,config.filePath_tarTags,model_type = 'xlm')\n",
    "len(dataset_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#loading dev/eval data\n",
    "dataset_eval = process_data(config.filePath_src_eval,config.filePath_tar_eval, config.filePath_srcTags_eval,config.filePath_tarTags_eval,model_type = 'xlm')\n",
    "len(dataset_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([     0,  17151,  12426,   2765,    113,    527, 110896,  36997,     71,\n",
       "          20387,   2189,    141,     99, 183124,     23,  58020,      6,      5,\n",
       "              2,      2,    656,  58020,    656, 138438,     13,    656,  17151,\n",
       "            656,  12426,   2765,    656,    113,    656,    527, 110896,    656,\n",
       "          20387,   2189,    141,    656,     23,    656, 183124,    656,      6,\n",
       "              5,    656,      2,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1]),\n",
       " tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " tensor([-100,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "            1,    1,    1,    1,    1,    1, -100, -100,    1,    1,    1,    1,\n",
       "            1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "            1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
       "         -100, -100, -100, -100]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>src_tokens</th>\n",
       "      <th>tar_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>José Ortega y Gasset visited Husserl at Freibu...</td>\n",
       "      <td>1934 besuchte José Ortega y Gasset Husserl in ...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>However , a disappointing ninth in China meant...</td>\n",
       "      <td>Eine enttäuschende Neunte in China bedeutete j...</td>\n",
       "      <td>OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...</td>\n",
       "      <td>OK BAD OK BAD OK BAD OK OK OK OK OK OK OK OK O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>In his diary , Chase wrote that the release of...</td>\n",
       "      <td>In seinem Tagebuch , Chase schrieb , dass die ...</td>\n",
       "      <td>OK OK OK OK BAD BAD OK OK OK OK OK OK OK OK BA...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK BAD OK BAD OK OK OK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Heavy arquebuses mounted on wagons were called...</td>\n",
       "      <td>Schwere Arquebuses auf Waggons montiert wurden...</td>\n",
       "      <td>OK BAD BAD OK OK OK OK OK OK OK OK</td>\n",
       "      <td>OK OK OK BAD OK OK OK OK BAD BAD OK OK OK OK O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Once North Pacific salmon die off after spawni...</td>\n",
       "      <td>Sobald der nordpazifische Lachs nach dem Laich...</td>\n",
       "      <td>OK OK OK OK BAD OK OK OK OK OK BAD OK BAD OK O...</td>\n",
       "      <td>OK OK OK BAD OK OK OK BAD OK OK OK OK OK OK OK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6995</th>\n",
       "      <td>Some may also discourage or disallow unsanitar...</td>\n",
       "      <td>Einige können auch unhygienische Praktiken wie...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6996</th>\n",
       "      <td>In the late 1860s , the crinolines disappeared...</td>\n",
       "      <td>In den späten 1860er Jahren verschwanden die K...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK BAD BAD OK OK OK...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6997</th>\n",
       "      <td>Disco was criticized as mindless , consumerist...</td>\n",
       "      <td>Disco wurde als geistlos , konsumistisch , übe...</td>\n",
       "      <td>OK OK OK OK BAD OK BAD OK OK OK OK OK</td>\n",
       "      <td>OK OK OK OK OK OK OK BAD OK OK OK BAD OK OK OK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6998</th>\n",
       "      <td>Planters would then fill large hogsheads with ...</td>\n",
       "      <td>Die Pflanzer würden dann große Heuschrecken mi...</td>\n",
       "      <td>OK OK OK BAD OK BAD OK OK BAD BAD OK OK OK OK OK</td>\n",
       "      <td>OK OK OK OK OK BAD OK OK OK OK OK BAD OK OK OK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6999</th>\n",
       "      <td>He slew Krishna 's most dangerous enemy , Jara...</td>\n",
       "      <td>Er tötete Krishnas gefährlichsten Feind Jarasa...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK BAD OK OK ...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 source  \\\n",
       "0     José Ortega y Gasset visited Husserl at Freibu...   \n",
       "1     However , a disappointing ninth in China meant...   \n",
       "2     In his diary , Chase wrote that the release of...   \n",
       "3     Heavy arquebuses mounted on wagons were called...   \n",
       "4     Once North Pacific salmon die off after spawni...   \n",
       "...                                                 ...   \n",
       "6995  Some may also discourage or disallow unsanitar...   \n",
       "6996  In the late 1860s , the crinolines disappeared...   \n",
       "6997  Disco was criticized as mindless , consumerist...   \n",
       "6998  Planters would then fill large hogsheads with ...   \n",
       "6999  He slew Krishna 's most dangerous enemy , Jara...   \n",
       "\n",
       "                                                 target  \\\n",
       "0     1934 besuchte José Ortega y Gasset Husserl in ...   \n",
       "1     Eine enttäuschende Neunte in China bedeutete j...   \n",
       "2     In seinem Tagebuch , Chase schrieb , dass die ...   \n",
       "3     Schwere Arquebuses auf Waggons montiert wurden...   \n",
       "4     Sobald der nordpazifische Lachs nach dem Laich...   \n",
       "...                                                 ...   \n",
       "6995  Einige können auch unhygienische Praktiken wie...   \n",
       "6996  In den späten 1860er Jahren verschwanden die K...   \n",
       "6997  Disco wurde als geistlos , konsumistisch , übe...   \n",
       "6998  Die Pflanzer würden dann große Heuschrecken mi...   \n",
       "6999  Er tötete Krishnas gefährlichsten Feind Jarasa...   \n",
       "\n",
       "                                             src_tokens  \\\n",
       "0                      OK OK OK OK OK OK OK OK OK OK OK   \n",
       "1     OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...   \n",
       "2     OK OK OK OK BAD BAD OK OK OK OK OK OK OK OK BA...   \n",
       "3                    OK BAD BAD OK OK OK OK OK OK OK OK   \n",
       "4     OK OK OK OK BAD OK OK OK OK OK BAD OK BAD OK O...   \n",
       "...                                                 ...   \n",
       "6995  OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...   \n",
       "6996  OK OK OK OK OK OK OK OK OK OK BAD BAD OK OK OK...   \n",
       "6997              OK OK OK OK BAD OK BAD OK OK OK OK OK   \n",
       "6998   OK OK OK BAD OK BAD OK OK BAD BAD OK OK OK OK OK   \n",
       "6999  OK OK OK OK OK OK OK OK OK OK OK OK BAD OK OK ...   \n",
       "\n",
       "                                             tar_tokens  \n",
       "0     OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...  \n",
       "1     OK BAD OK BAD OK BAD OK OK OK OK OK OK OK OK O...  \n",
       "2     OK OK OK OK OK OK OK OK OK BAD OK BAD OK OK OK...  \n",
       "3     OK OK OK BAD OK OK OK OK BAD BAD OK OK OK OK O...  \n",
       "4     OK OK OK BAD OK OK OK BAD OK OK OK OK OK OK OK...  \n",
       "...                                                 ...  \n",
       "6995  OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...  \n",
       "6996  OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...  \n",
       "6997  OK OK OK OK OK OK OK BAD OK OK OK BAD OK OK OK...  \n",
       "6998  OK OK OK OK OK BAD OK OK OK OK OK BAD OK OK OK...  \n",
       "6999  OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...  \n",
       "\n",
       "[7000 rows x 4 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataObj = loadDatafromFile(config.filePath_src,config.filePath_tar, config.filePath_srcTags,config.filePath_tarTags)\n",
    "df= dataObj.createDf() \n",
    "df\n",
    "# list(df.source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /opt/conda/lib/python3.6/site-packages (3.6.5)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from nltk) (1.0.1)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from nltk) (7.1.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /opt/conda/lib/python3.6/site-packages (from nltk) (2021.11.10)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from nltk) (4.51.0)\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "reload(DataAugmentation)\n",
    "from DataAugmentation import DataAugmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>src_tokens</th>\n",
       "      <th>tar_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>José Ortega y Gasset visited Husserl at Freibu...</td>\n",
       "      <td>1934 besuchte José Ortega y Gasset Husserl in ...</td>\n",
       "      <td>OK OK OK OK BAD OK OK OK BAD BAD OK</td>\n",
       "      <td>OK BAD OK BAD OK OK OK OK OK OK OK OK BAD OK O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>however , a disappointing ninth in China meant...</td>\n",
       "      <td>eine enttäuschende Neunte in China bedeutete j...</td>\n",
       "      <td>OK BAD BAD BAD BAD OK OK OK OK OK OK OK OK OK ...</td>\n",
       "      <td>BAD BAD OK BAD OK BAD OK OK OK OK OK OK OK OK ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>in his diary , Chase wrote that the release of...</td>\n",
       "      <td>in seinem Tagebuch , Chase schrieb , dass die ...</td>\n",
       "      <td>OK OK OK OK OK BAD OK OK BAD OK OK BAD OK BAD ...</td>\n",
       "      <td>OK OK OK OK OK OK OK BAD OK OK OK BAD OK OK OK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Heavy arquebuses mounted on wagons were called...</td>\n",
       "      <td>schwere Arquebuses auf Waggons montiert wurden...</td>\n",
       "      <td>BAD BAD OK OK BAD BAD BAD OK OK OK OK</td>\n",
       "      <td>OK OK BAD BAD OK OK OK BAD OK OK BAD OK BAD OK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>once North Pacific salmon die off after spawni...</td>\n",
       "      <td>sobald der nordpazifische Lachs nach dem Laich...</td>\n",
       "      <td>OK OK OK OK OK OK OK BAD OK BAD BAD BAD BAD OK...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK OK BAD ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76995</th>\n",
       "      <td>he mass Krishna 's most grave enemy , Jarasand...</td>\n",
       "      <td>er tötete Krishnas gefährlichsten Feind Jarasa...</td>\n",
       "      <td>OK BAD OK OK OK OK OK OK OK OK OK BAD BAD OK O...</td>\n",
       "      <td>OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76996</th>\n",
       "      <td>he great deal Krishna 's most dangerous enemy ...</td>\n",
       "      <td>er tötete Krishnas gefährlichsten Feind Jarasa...</td>\n",
       "      <td>OK BAD BAD OK OK OK OK OK OK OK OK OK BAD BAD ...</td>\n",
       "      <td>OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76997</th>\n",
       "      <td>he slew Krishna 's most dangerous enemy , Jara...</td>\n",
       "      <td>er tötete Krishnas gefährlichsten Feind Jarasa...</td>\n",
       "      <td>OK BAD OK OK OK OK OK OK OK OK OK BAD BAD OK O...</td>\n",
       "      <td>OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76998</th>\n",
       "      <td>he slue Krishna 's most dangerous enemy , Jara...</td>\n",
       "      <td>er tötete Krishnas gefährlichsten Feind Jarasa...</td>\n",
       "      <td>OK BAD OK OK OK OK OK OK OK OK OK BAD BAD OK O...</td>\n",
       "      <td>OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76999</th>\n",
       "      <td>he slew krishna 's most dangerous enemy , Jara...</td>\n",
       "      <td>er tötete Krishnas gefährlichsten Feind Jarasa...</td>\n",
       "      <td>OK BAD OK OK OK OK OK OK OK OK OK BAD BAD OK O...</td>\n",
       "      <td>OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>217000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  source  \\\n",
       "0      José Ortega y Gasset visited Husserl at Freibu...   \n",
       "1      however , a disappointing ninth in China meant...   \n",
       "2      in his diary , Chase wrote that the release of...   \n",
       "3      Heavy arquebuses mounted on wagons were called...   \n",
       "4      once North Pacific salmon die off after spawni...   \n",
       "...                                                  ...   \n",
       "76995  he mass Krishna 's most grave enemy , Jarasand...   \n",
       "76996  he great deal Krishna 's most dangerous enemy ...   \n",
       "76997  he slew Krishna 's most dangerous enemy , Jara...   \n",
       "76998  he slue Krishna 's most dangerous enemy , Jara...   \n",
       "76999  he slew krishna 's most dangerous enemy , Jara...   \n",
       "\n",
       "                                                  target  \\\n",
       "0      1934 besuchte José Ortega y Gasset Husserl in ...   \n",
       "1      eine enttäuschende Neunte in China bedeutete j...   \n",
       "2      in seinem Tagebuch , Chase schrieb , dass die ...   \n",
       "3      schwere Arquebuses auf Waggons montiert wurden...   \n",
       "4      sobald der nordpazifische Lachs nach dem Laich...   \n",
       "...                                                  ...   \n",
       "76995  er tötete Krishnas gefährlichsten Feind Jarasa...   \n",
       "76996  er tötete Krishnas gefährlichsten Feind Jarasa...   \n",
       "76997  er tötete Krishnas gefährlichsten Feind Jarasa...   \n",
       "76998  er tötete Krishnas gefährlichsten Feind Jarasa...   \n",
       "76999  er tötete Krishnas gefährlichsten Feind Jarasa...   \n",
       "\n",
       "                                              src_tokens  \\\n",
       "0                    OK OK OK OK BAD OK OK OK BAD BAD OK   \n",
       "1      OK BAD BAD BAD BAD OK OK OK OK OK OK OK OK OK ...   \n",
       "2      OK OK OK OK OK BAD OK OK BAD OK OK BAD OK BAD ...   \n",
       "3                  BAD BAD OK OK BAD BAD BAD OK OK OK OK   \n",
       "4      OK OK OK OK OK OK OK BAD OK BAD BAD BAD BAD OK...   \n",
       "...                                                  ...   \n",
       "76995  OK BAD OK OK OK OK OK OK OK OK OK BAD BAD OK O...   \n",
       "76996  OK BAD BAD OK OK OK OK OK OK OK OK OK BAD BAD ...   \n",
       "76997  OK BAD OK OK OK OK OK OK OK OK OK BAD BAD OK O...   \n",
       "76998  OK BAD OK OK OK OK OK OK OK OK OK BAD BAD OK O...   \n",
       "76999  OK BAD OK OK OK OK OK OK OK OK OK BAD BAD OK O...   \n",
       "\n",
       "                                              tar_tokens  \n",
       "0      OK BAD OK BAD OK OK OK OK OK OK OK OK BAD OK O...  \n",
       "1      BAD BAD OK BAD OK BAD OK OK OK OK OK OK OK OK ...  \n",
       "2      OK OK OK OK OK OK OK BAD OK OK OK BAD OK OK OK...  \n",
       "3      OK OK BAD BAD OK OK OK BAD OK OK BAD OK BAD OK...  \n",
       "4      OK OK OK OK OK OK OK OK OK OK OK OK OK OK BAD ...  \n",
       "...                                                  ...  \n",
       "76995  OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...  \n",
       "76996  OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...  \n",
       "76997  OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...  \n",
       "76998  OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...  \n",
       "76999  OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...  \n",
       "\n",
       "[217000 rows x 4 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataaug_obj = DataAugmentation(df,swap_words=2,syn_words=2,del_words_prob=0.2,num_sentences=10)  \n",
    "swapDataset = dataaug_obj.random_swap()\n",
    "del_augDataset = dataaug_obj.random_deletion()\n",
    "del_augDataset = del_augDataset[7000:]\n",
    "syn_dataset = dataaug_obj.synonym_replacement()\n",
    "syn_dataset = syn_dataset[7000:]\n",
    "frames = [swapDataset , del_augDataset,syn_dataset]\n",
    "aug_df  = pd.concat(frames)\n",
    "aug_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>src_tokens</th>\n",
       "      <th>tar_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Germany invaded France , , Luxembourg and the ...</td>\n",
       "      <td>Deutschland marschierte am 10. Mai 1940 in Fra...</td>\n",
       "      <td>OK OK OK OK OK OK OK BAD BAD OK OK OK</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>the enzyme chitinase grow in the stomach helps...</td>\n",
       "      <td>das im Magen produzierte Enzym Chitinase hilft...</td>\n",
       "      <td>OK OK OK OK OK BAD OK BAD BAD BAD BAD BAD BAD ...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK OK BAD ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Roger Rousseau , The French Foreign Legion in ...</td>\n",
       "      <td>Roger Rousseau , Die französische Fremdenlegio...</td>\n",
       "      <td>OK OK OK BAD OK BAD BAD OK OK OK OK OK</td>\n",
       "      <td>OK OK OK OK OK OK OK BAD OK OK BAD BAD OK OK O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>in 1927 , Lieutenant Jaime Sabater , from San ...</td>\n",
       "      <td>1927 absolvierte Lieutenant Jaime Sabater aus ...</td>\n",
       "      <td>BAD OK OK BAD OK OK OK OK OK OK OK OK OK OK OK...</td>\n",
       "      <td>BAD OK OK OK OK BAD OK OK OK OK OK OK OK OK OK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>blackmails Sharon into meeting him at a motel</td>\n",
       "      <td>Cameron erpresst Sharon , ihn in einem Motel z...</td>\n",
       "      <td>BAD BAD BAD OK OK OK OK OK</td>\n",
       "      <td>OK OK OK BAD OK OK BAD OK OK OK OK OK OK OK OK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216995</th>\n",
       "      <td>maxilla baleen exclusively the in the sit jaw ...</td>\n",
       "      <td>die Zähne oder Baleen im Oberkiefer sitzen aus...</td>\n",
       "      <td>OK OK OK OK BAD OK OK OK OK OK OK OK OK OK</td>\n",
       "      <td>OK OK OK OK OK OK OK BAD OK OK OK OK OK OK OK ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216996</th>\n",
       "      <td>the with clock with mainspring either 's motor...</td>\n",
       "      <td>die Hauptfeder der elektrischen Uhr wird entwe...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK BAD OK OK OK ...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216997</th>\n",
       "      <td>he accidentally frivol away Jane during an alt...</td>\n",
       "      <td>er schießt Jane versehentlich während einer Au...</td>\n",
       "      <td>OK OK BAD BAD OK OK OK OK OK OK OK OK OK OK OK...</td>\n",
       "      <td>OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216998</th>\n",
       "      <td>Paul Collier disability advocate , Australian ...</td>\n",
       "      <td>Paul Collier , 46 , australischer Behindertena...</td>\n",
       "      <td>OK OK BAD BAD OK OK OK OK OK OK OK OK</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK BAD BAD OK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216999</th>\n",
       "      <td>; republish with many errors by Jacques st pau...</td>\n",
       "      <td>; wiederveröffentlicht mit vielen Fehlern von ...</td>\n",
       "      <td>OK OK OK OK OK OK OK OK OK OK OK OK OK</td>\n",
       "      <td>OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>217000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   source  \\\n",
       "0       Germany invaded France , , Luxembourg and the ...   \n",
       "1       the enzyme chitinase grow in the stomach helps...   \n",
       "2       Roger Rousseau , The French Foreign Legion in ...   \n",
       "3       in 1927 , Lieutenant Jaime Sabater , from San ...   \n",
       "4           blackmails Sharon into meeting him at a motel   \n",
       "...                                                   ...   \n",
       "216995  maxilla baleen exclusively the in the sit jaw ...   \n",
       "216996  the with clock with mainspring either 's motor...   \n",
       "216997  he accidentally frivol away Jane during an alt...   \n",
       "216998  Paul Collier disability advocate , Australian ...   \n",
       "216999  ; republish with many errors by Jacques st pau...   \n",
       "\n",
       "                                                   target  \\\n",
       "0       Deutschland marschierte am 10. Mai 1940 in Fra...   \n",
       "1       das im Magen produzierte Enzym Chitinase hilft...   \n",
       "2       Roger Rousseau , Die französische Fremdenlegio...   \n",
       "3       1927 absolvierte Lieutenant Jaime Sabater aus ...   \n",
       "4       Cameron erpresst Sharon , ihn in einem Motel z...   \n",
       "...                                                   ...   \n",
       "216995  die Zähne oder Baleen im Oberkiefer sitzen aus...   \n",
       "216996  die Hauptfeder der elektrischen Uhr wird entwe...   \n",
       "216997  er schießt Jane versehentlich während einer Au...   \n",
       "216998  Paul Collier , 46 , australischer Behindertena...   \n",
       "216999  ; wiederveröffentlicht mit vielen Fehlern von ...   \n",
       "\n",
       "                                               src_tokens  \\\n",
       "0                   OK OK OK OK OK OK OK BAD BAD OK OK OK   \n",
       "1       OK OK OK OK OK BAD OK BAD BAD BAD BAD BAD BAD ...   \n",
       "2                  OK OK OK BAD OK BAD BAD OK OK OK OK OK   \n",
       "3       BAD OK OK BAD OK OK OK OK OK OK OK OK OK OK OK...   \n",
       "4                              BAD BAD BAD OK OK OK OK OK   \n",
       "...                                                   ...   \n",
       "216995         OK OK OK OK BAD OK OK OK OK OK OK OK OK OK   \n",
       "216996  OK OK OK OK OK OK OK OK OK OK OK BAD OK OK OK ...   \n",
       "216997  OK OK BAD BAD OK OK OK OK OK OK OK OK OK OK OK...   \n",
       "216998              OK OK BAD BAD OK OK OK OK OK OK OK OK   \n",
       "216999             OK OK OK OK OK OK OK OK OK OK OK OK OK   \n",
       "\n",
       "                                               tar_tokens  \n",
       "0       OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...  \n",
       "1       OK OK OK OK OK OK OK OK OK OK OK OK OK OK BAD ...  \n",
       "2       OK OK OK OK OK OK OK BAD OK OK BAD BAD OK OK O...  \n",
       "3       BAD OK OK OK OK BAD OK OK OK OK OK OK OK OK OK...  \n",
       "4       OK OK OK BAD OK OK BAD OK OK OK OK OK OK OK OK...  \n",
       "...                                                   ...  \n",
       "216995  OK OK OK OK OK OK OK BAD OK OK OK OK OK OK OK ...  \n",
       "216996  OK OK OK OK OK OK OK OK OK OK OK OK OK OK OK O...  \n",
       "216997  OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...  \n",
       "216998  OK OK OK OK OK OK OK OK OK OK OK OK BAD BAD OK...  \n",
       "216999  OK OK OK BAD OK OK OK OK OK OK OK OK OK OK OK ...  \n",
       "\n",
       "[217000 rows x 4 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aug_df =aug_df.sample(frac=1).reset_index(drop=True)\n",
    "aug_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #DataAugmentation methods\n",
    "# def random_swap(dataframe,n): # n is number of times to swap randomly 2 words\n",
    "    \n",
    "#     source_sentences  = list(dataframe.source)\n",
    "#     target_sentences = list(dataframe.target)\n",
    "#     labels_src = list(dataframe.src_tokens)\n",
    "#     labels_tar = list(dataframe.tar_tokens)\n",
    "#     source_sentences_temp =[]\n",
    "#     labels_sec_temp=[]\n",
    "#     i=0\n",
    "    \n",
    "#     for sentences, labels in zip(source_sentences,labels_src):\n",
    "        \n",
    "#         sentences = sentences.split()\n",
    "#         labels = labels.split()\n",
    "        \n",
    "#         for _ in range(5):\n",
    "#             for _ in range(n):\n",
    "#                 sentences, labels = swap_word(sentences,labels)\n",
    "#             assert(len(sentences) == len(labels))\n",
    "#             sentences_str = ' '.join(sentences)\n",
    "#             labels_str = ' '.join(labels)\n",
    "#             target_sentences.append(target_sentences[i])\n",
    "#             labels_tar.append(labels_tar[i])\n",
    "#             source_sentences_temp.append(sentences_str)\n",
    "#             labels_sec_temp.append(labels_str)\n",
    "        \n",
    "# #         break\n",
    "#         i+=1\n",
    "    \n",
    "#     source_sentences.extend(source_sentences_temp)\n",
    "#     labels_src.extend(labels_sec_temp)\n",
    "# #     print(source_sentences)\n",
    "# #     print(labels_src)\n",
    "#     column_names = [\"source\",\"target\",\"src_tokens\",\"tar_tokens\"]\n",
    "#     df = pd.DataFrame(columns=column_names,dtype=object)\n",
    "#     df = df.assign(source=source_sentences)\n",
    "#     df = df.assign(target = target_sentences)\n",
    "#     df = df.assign(src_tokens = labels_src)\n",
    "#     df = df.assign(tar_tokens = labels_tar)\n",
    "    \n",
    "#     return df\n",
    "\n",
    "# def swap_word(new_words,labels_src):\n",
    "\n",
    "#     random_idx_1 = random.randint(0, len(new_words)-1)\n",
    "#     random_idx_2 = random_idx_1\n",
    "#     counter = 0\n",
    "#     while random_idx_2 == random_idx_1:\n",
    "#         random_idx_2 = random.randint(0, len(new_words)-1)\n",
    "#         counter += 1\n",
    "#         if counter > 3:\n",
    "#             return (new_words,labels_src)\n",
    "#     new_words[random_idx_1], new_words[random_idx_2] = new_words[random_idx_2], new_words[random_idx_1]\n",
    "#     labels_src[random_idx_1], labels_src[random_idx_2] = labels_src[random_idx_2], labels_src[random_idx_1]\n",
    "# #     print(labels_src)\n",
    "# #     new_str = ''.join(new_words)\n",
    "#     return (new_words, labels_src)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_df = random_swap(df,2) # tune how much words you want to swap\n",
    "# # new_df = new_df.sample(frac=1).reset_index(drop=True) # shuffling the dafaframe with resetting the index\n",
    "# print(new_df.iloc[0].values)\n",
    "# print(new_df.iloc[7000].values)\n",
    "# print(new_df.iloc[7001].values)\n",
    "# print(new_df.iloc[7002].values)\n",
    "# print(new_df.iloc[7003].values)\n",
    "# print(new_df.iloc[7004].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def random_deletion(dataframe, p):\n",
    "    \n",
    "#     source_sentences  = list(dataframe.source)\n",
    "#     target_sentences = list(dataframe.target)\n",
    "#     labels_src = list(dataframe.src_tokens)\n",
    "#     labels_tar = list(dataframe.tar_tokens)\n",
    "#     senetences_temp=[]\n",
    "#     labels_temp= []\n",
    "#     #randomly delete words with probability p\n",
    "#     i=0\n",
    "#     for sentences, labels in zip(source_sentences,labels_src):\n",
    "            \n",
    "#         sentences = sentences.split()\n",
    "#         labels = labels.split() \n",
    "        \n",
    "        \n",
    "        \n",
    "#         if len(sentences) == 1:\n",
    "#             i+=1\n",
    "#             continue\n",
    "            \n",
    "        \n",
    "               \n",
    "#         for _ in range(5):\n",
    "            \n",
    "#             source_sentences_temp=[]\n",
    "#             labels_sec_temp=[]\n",
    "#             for word,label in zip(sentences,labels):\n",
    "#                 r = random.uniform(0, 1)\n",
    "#                 if r > p:\n",
    "#                     source_sentences_temp.append(word)\n",
    "#                     labels_sec_temp.append(label)\n",
    "#             if len(source_sentences_temp) == 0: #if you end up deleting all words, just return a random word\n",
    "#                 rand_int = random.randint(0, len(source_sentences_temp)-1)\n",
    "#                 source_sentences_temp.append(sentences[rand_int])\n",
    "#                 labels_sec_temp.append(labels[rand_int])\n",
    "#             assert(len(source_sentences_temp) == len(labels_sec_temp))\n",
    "#             sentences_str = ' '.join(source_sentences_temp)\n",
    "#             labels_str = ' '.join(labels_sec_temp)\n",
    "#             senetences_temp.append(sentences_str)\n",
    "#             labels_temp.append(labels_str)\n",
    "#             target_sentences.append(target_sentences[i])\n",
    "#             labels_tar.append(labels_tar[i])\n",
    "# #         break\n",
    "#         i+=1\n",
    "#     source_sentences.extend(senetences_temp)\n",
    "#     labels_src.extend(labels_temp)    \n",
    "#     column_names = [\"source\",\"target\",\"src_tokens\",\"tar_tokens\"]\n",
    "#     df = pd.DataFrame(columns=column_names,dtype=object)\n",
    "#     df = df.assign(source=source_sentences)\n",
    "#     df = df.assign(target = target_sentences)\n",
    "#     df = df.assign(src_tokens = labels_src)\n",
    "#     df = df.assign(tar_tokens = labels_tar)  \n",
    "\n",
    "\n",
    "#     return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data, test_data= model_selection.train_test_split(dataset, random_state = 34, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# augmented_dataset = random_deletion(df,0.2) # probabality of deleting the tokens\n",
    "# augmented_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(augmented_dataset.iloc[0].values)\n",
    "# print(augmented_dataset.iloc[0].values)\n",
    "\n",
    "# print(augmented_dataset.iloc[7003].values)\n",
    "# print(augmented_dataset.iloc[7004].values)\n",
    "# print(augmented_dataset.iloc[7002].values)\n",
    "# print(augmented_dataset.iloc[7001].values)\n",
    "# print(augmented_dataset.iloc[7000].values)\n",
    "# assert(augmented_dataset.iloc[7002].values[0] == augmented_dataset.iloc[7003].values[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: seqeval in /opt/conda/lib/python3.6/site-packages (1.2.2)\n",
      "Requirement already satisfied: scikit-learn>=0.21.3 in /opt/conda/lib/python3.6/site-packages (from seqeval) (0.24.2)\n",
      "Requirement already satisfied: numpy>=1.14.0 in /opt/conda/lib/python3.6/site-packages (from seqeval) (1.19.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from scikit-learn>=0.21.3->seqeval) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.6/site-packages (from scikit-learn>=0.21.3->seqeval) (1.0.1)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.6/site-packages (from scikit-learn>=0.21.3->seqeval) (1.5.4)\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Requirement already satisfied: transformers in /opt/conda/lib/python3.6/site-packages (4.15.0)\n",
      "Requirement already satisfied: dataclasses in /opt/conda/lib/python3.6/site-packages (from transformers) (0.8)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /opt/conda/lib/python3.6/site-packages (from transformers) (0.10.3)\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.6/site-packages (from transformers) (4.4.0)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from transformers) (2.25.1)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /opt/conda/lib/python3.6/site-packages (from transformers) (0.2.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.6/site-packages (from transformers) (20.9)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.6/site-packages (from transformers) (2021.11.10)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.6/site-packages (from transformers) (5.4.1)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.6/site-packages (from transformers) (4.51.0)\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.6/site-packages (from transformers) (0.0.46)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.6/site-packages (from transformers) (3.4.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.6/site-packages (from transformers) (1.19.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.6/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.6/site-packages (from packaging>=20.0->transformers) (2.4.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.6/site-packages (from importlib-metadata->transformers) (3.4.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (1.25.11)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->transformers) (2021.5.30)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (1.0.1)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.6/site-packages (0.1.96)\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import data_utils\n",
    "data_utils=reload(data_utils)\n",
    "from data_utils import loadDatafromFile,createTokenizedDf,CompDataset,createkfoldData,createDataloaders\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "obj_tokenized = createTokenizedDf(aug_df)\n",
    "df_new= obj_tokenized.convertDf()\n",
    "enc_label = preprocessing.LabelEncoder()\n",
    "df_new['labels']= enc_label.fit_transform(df_new['labels'])\n",
    "train_data = CompDataset(df_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "217000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([     0, 102126, 205491,     71,   9942,      6,      4,      6,      4,\n",
       "         211546,    136,     70, 231118,     98,   4347,  27712,      2,      2,\n",
       "            656,  12154,    656,  10283,    206,  32223,    656,    444,    656,\n",
       "           4068,    656,   3596,    656,  27712,    656,     23,    656, 118742,\n",
       "            656,      6,      4,    656,  41267,     33,    656,      6,      4,\n",
       "            656, 157436,    656,    165,    656,     68,    656,  82237,  30766,\n",
       "            656,    599,    656,      6,      5,    656,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "              1,      1,      1,      1]),\n",
       " tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(train_data))\n",
    "train_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch.utils.data.dataloader.DataLoader at 0x7f69c8a0ccc0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dataloaders\n",
    "loader_obj = createDataloaders(dataset_train,config.TRAIN_BATCH_SIZE)\n",
    "train_dataloader = loader_obj.createDataloaders()\n",
    "train_dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "438"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader_obj = createDataloaders(dataset_eval,config.VALID_BATCH_SIZE)\n",
    "val_dataloader = loader_obj.createDataloaders()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "125"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(val_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-01-06 20:44:14.888 pytorch-1-6-gpu-py-ml-g4dn-8xlarge-d01940f40012030c1b9f0e178db4:31 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
      "[2022-01-06 20:44:14.923 pytorch-1-6-gpu-py-ml-g4dn-8xlarge-d01940f40012030c1b9f0e178db4:31 INFO profiler_config_parser.py:102] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([     0,    581, 148639,   7432,  11764,     33,   2408,  99201,     70,\n",
       "         36770,    366,    150,  11533,   6635,  71119,    127,    111, 148639,\n",
       "          7432,      6,      5,      2,      2,    656,   1310,    656,  34288,\n",
       "         89869,    656, 148639,   7432,    656, 182211,     13,    656,    381,\n",
       "           656, 134763,    656,  73225,   4986,    656, 210479,    656,    542,\n",
       "           656, 148639,   7432,    656,      6,      5,    656,      2,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
       "             1,      1,      1,      1])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch = next(iter(val_dataloader))\n",
    "batch[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(output,target,mask):\n",
    "    lfn = nn.CrossEntropyLoss()\n",
    "    active_loss = mask.view(-1) == 1 #loss calculation for non padded tokens only (mask =1)\n",
    "    active_logits = output.view(-1,2)\n",
    "    active_labels = torch.where(\n",
    "        active_loss,\n",
    "        target.view(-1),\n",
    "        torch.tensor(lfn.ignore_index).type_as(target)    \n",
    "    )\n",
    "    loss = lfn(active_logits,active_labels)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EntityModel(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        \n",
    "        super(EntityModel, self).__init__()\n",
    "        self.bert = XLMRobertaForTokenClassification.from_pretrained(config.BASE_MODEL,output_attentions = False, output_hidden_states = False)\n",
    "#         self.bert_drop_1 = nn.Dropout(0.3)\n",
    "#         self.out_tag = nn.Linear(768, 2)\n",
    "    \n",
    "    def forward(self, ids, attention_mask, labels):\n",
    "        \n",
    "        outputs = self.bert(ids,\n",
    "                                attention_mask = attention_mask,\n",
    "                                labels = labels,return_dict=False)\n",
    "#         bo_tag = self.bert_drop_1(output_1)\n",
    "        \n",
    "#         tag = self.out_tag(bo_tag)  \n",
    "        \n",
    "#         loss_tag = loss_fn(outputs[1],labels,attention_mask)\n",
    "        \n",
    "#         return bo_tag,loss\n",
    "        return outputs[0], outputs[1]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# b_input_ids = batch[0].cuda()\n",
    "# b_input_mask = batch[1].cuda()\n",
    "# b_labels = batch[2].cuda()\n",
    "# outputs = model(b_input_ids, \n",
    "#                 b_input_mask,\n",
    "#                 labels=b_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForTokenClassification: ['lm_head.dense.weight', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
      "- This IS expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "EntityModel(\n",
       "  (bert): XLMRobertaForTokenClassification(\n",
       "    (roberta): RobertaModel(\n",
       "      (embeddings): RobertaEmbeddings(\n",
       "        (word_embeddings): Embedding(250002, 768, padding_idx=1)\n",
       "        (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
       "        (token_type_embeddings): Embedding(1, 768)\n",
       "        (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (encoder): RobertaEncoder(\n",
       "        (layer): ModuleList(\n",
       "          (0): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (1): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (2): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (3): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (4): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (5): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (6): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (7): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (8): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (9): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (10): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (11): RobertaLayer(\n",
       "            (attention): RobertaAttention(\n",
       "              (self): RobertaSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): RobertaSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): RobertaIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            )\n",
       "            (output): RobertaOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "    (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = EntityModel()\n",
    "model.cuda()\n",
    "# model = nn.DataParallel(model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.cuda.memory_summary(device=None, abbreviated=False)\n",
    "# torch.cuda.empty_cache()\n",
    "# outputs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EarlyStopping(object):\n",
    "    def __init__(self, mode='max', min_delta=0, patience=10, percentage=False):\n",
    "        self.mode = mode\n",
    "        self.min_delta = min_delta\n",
    "        self.patience = patience\n",
    "        self.best = 0\n",
    "        self.num_bad_epochs = 0\n",
    "        self.is_better = None\n",
    "        self._init_is_better(mode, min_delta, percentage)\n",
    "\n",
    "        if patience == 0:\n",
    "            self.is_better = lambda a, b: True\n",
    "            self.step = lambda a: False\n",
    "\n",
    "    def step(self, metrics):\n",
    "        if self.best is None:\n",
    "            self.best = metrics\n",
    "            return False\n",
    "\n",
    "        if np.isnan(metrics):\n",
    "            return True\n",
    "\n",
    "        if self.is_better(metrics, self.best):\n",
    "            self.num_bad_epochs = 0\n",
    "            self.best = metrics\n",
    "        else:\n",
    "            self.num_bad_epochs += 1\n",
    "\n",
    "        if self.num_bad_epochs >= self.patience:\n",
    "            print('terminating because of early stopping!')\n",
    "            return True\n",
    "\n",
    "        return False\n",
    "\n",
    "    def _init_is_better(self, mode, min_delta, percentage):\n",
    "        if mode not in {'min', 'max'}:\n",
    "            raise ValueError('mode ' + mode + ' is unknown!')\n",
    "        if not percentage:\n",
    "            if mode == 'min':\n",
    "                self.is_better = lambda a, best: a < best \n",
    "            if mode == 'max':\n",
    "                self.is_better = lambda a, best: a > best \n",
    "        else:\n",
    "            if mode == 'min':\n",
    "                self.is_better = lambda a, best: a < best - (\n",
    "                            best * min_delta / 100)\n",
    "            if mode == 'max':\n",
    "                self.is_better = lambda a, best: a > best + (\n",
    "                            best * min_delta / 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n",
    "optimizer_parameters = [\n",
    "    {\n",
    "        \"params\": [\n",
    "            p for n, p in param_optimizer if not any(nd in n for nd in no_decay)\n",
    "        ],\n",
    "        \"weight_decay\": 0.001,\n",
    "    },\n",
    "    {\n",
    "        \"params\": [\n",
    "            p for n, p in param_optimizer if any(nd in n for nd in no_decay)\n",
    "        ],\n",
    "        \"weight_decay\": 0.0,\n",
    "    },\n",
    "]\n",
    "num_train_steps = int(len(dataset_train) / config.TRAIN_BATCH_SIZE * 4) #10 is the batchsize\n",
    "optimizer = AdamW(optimizer_parameters, lr=2e-5) # used 3e-5\n",
    "scheduler = get_linear_schedule_with_warmup(\n",
    "    optimizer, num_warmup_steps=0, num_training_steps=num_train_steps\n",
    ")\n",
    "early_stopping = EarlyStopping(patience=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config.EPOCHS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:13<00:00,  9.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Validation score for class 0 : 0.0\n",
      "Average F1 Validation score for class 1 : 0.9294044211680166\n",
      "Average Accuracy Validation score  : 0.8681190540521324\n",
      "Average mcc Validation score  : 0.0\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00      9274\n",
      "           1       0.87      1.00      0.93     61047\n",
      "\n",
      "    accuracy                           0.87     70321\n",
      "   macro avg       0.43      0.50      0.46     70321\n",
      "weighted avg       0.75      0.87      0.81     70321\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "loss_test,f1_score_0_test, f1_score_1_test, accuracy_score_test, mcc_score_test,labels_test,preds_test = engine.eval_fn(val_dataloader, model)\n",
    "print(classification_report(labels_test,preds_test)) # validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 438/438 [01:40<00:00,  4.36it/s]\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Validation score for class 0 : 0.0\n",
      "Average F1 Validation score for class 1 : 0.932492299304893\n",
      "Average Accuracy Validation score  : 0.8735227846110161\n",
      "Average mcc Validation score  : 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00     61913\n",
      "           1       0.87      1.00      0.93    427606\n",
      "\n",
      "    accuracy                           0.87    489519\n",
      "   macro avg       0.44      0.50      0.47    489519\n",
      "weighted avg       0.76      0.87      0.81    489519\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "loss_test,f1_score_0_test, f1_score_1_test, accuracy_score_test, mcc_score_test,labels_test,preds_test = engine.eval_fn(train_dataloader, model)\n",
    "print(classification_report(labels_test,preds_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "engine = reload(engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name,param in model.named_parameters():\n",
    "    if 'classifier' not in name:\n",
    "        param.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/438 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 438/438 [05:45<00:00,  1.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Training score for class 0 : 0.0775124864760719\n",
      "Average F1 Training score for class 1 : 0.9317185280259772\n",
      "Average Accuracy Training score  : 0.8728486534741245\n",
      "Average mcc Training score  : 0.11088032353358976\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.47      0.04      0.08     61913\n",
      "           1       0.88      0.99      0.93    427606\n",
      "\n",
      "    accuracy                           0.87    489519\n",
      "   macro avg       0.67      0.52      0.50    489519\n",
      "weighted avg       0.83      0.87      0.82    489519\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:15<00:00,  8.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Validation score for class 0 : 0.05476314419573139\n",
      "Average F1 Validation score for class 1 : 0.9307142257530316\n",
      "Average Accuracy Validation score  : 0.8708920521608055\n",
      "Average mcc Validation score  : 0.13468518895157555\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.03      0.05      9274\n",
      "           1       0.87      1.00      0.93     61047\n",
      "\n",
      "    accuracy                           0.87     70321\n",
      "   macro avg       0.83      0.51      0.49     70321\n",
      "weighted avg       0.86      0.87      0.82     70321\n",
      "\n",
      "Train Loss = 0.3267540782349839 Valid Loss = 0.31757344281673433\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/438 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 438/438 [05:54<00:00,  1.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Training score for class 0 : 0.3017270654441372\n",
      "Average F1 Training score for class 1 : 0.9357853011311797\n",
      "Average Accuracy Training score  : 0.8823865876503262\n",
      "Average mcc Training score  : 0.30167665974313485\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.20      0.30     61913\n",
      "           1       0.89      0.98      0.94    427606\n",
      "\n",
      "    accuracy                           0.88    489519\n",
      "   macro avg       0.75      0.59      0.62    489519\n",
      "weighted avg       0.86      0.88      0.86    489519\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:15<00:00,  8.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Validation score for class 0 : 0.20455173684695474\n",
      "Average F1 Validation score for class 1 : 0.9334843476719907\n",
      "Average Accuracy Validation score  : 0.87723439655295\n",
      "Average mcc Validation score  : 0.255804477978454\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.12      0.20      9274\n",
      "           1       0.88      0.99      0.93     61047\n",
      "\n",
      "    accuracy                           0.88     70321\n",
      "   macro avg       0.79      0.56      0.57     70321\n",
      "weighted avg       0.86      0.88      0.84     70321\n",
      "\n",
      "Train Loss = 0.287871975604802 Valid Loss = 0.3175424579381943\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/438 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 438/438 [05:54<00:00,  1.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Training score for class 0 : 0.4468204233692961\n",
      "Average F1 Training score for class 1 : 0.9420242754162886\n",
      "Average Accuracy Training score  : 0.895047996094125\n",
      "Average mcc Training score  : 0.42502768013613795\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.67      0.34      0.45     61913\n",
      "           1       0.91      0.98      0.94    427606\n",
      "\n",
      "    accuracy                           0.90    489519\n",
      "   macro avg       0.79      0.66      0.69    489519\n",
      "weighted avg       0.88      0.90      0.88    489519\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:15<00:00,  8.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Validation score for class 0 : 0.24655369215910086\n",
      "Average F1 Validation score for class 1 : 0.9336108252806511\n",
      "Average Accuracy Validation score  : 0.8779738627152629\n",
      "Average mcc Validation score  : 0.2768427098848643\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.66      0.15      0.25      9274\n",
      "           1       0.88      0.99      0.93     61047\n",
      "\n",
      "    accuracy                           0.88     70321\n",
      "   macro avg       0.77      0.57      0.59     70321\n",
      "weighted avg       0.86      0.88      0.84     70321\n",
      "\n",
      "Train Loss = 0.2572144322335448 Valid Loss = 0.3323549725115299\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/438 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 438/438 [05:30<00:00,  1.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Training score for class 0 : 0.5283075991522859\n",
      "Average F1 Training score for class 1 : 0.9468832249178359\n",
      "Average Accuracy Training score  : 0.9045185171566374\n",
      "Average mcc Training score  : 0.4981704369949589\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.42      0.53     61913\n",
      "           1       0.92      0.97      0.95    427606\n",
      "\n",
      "    accuracy                           0.90    489519\n",
      "   macro avg       0.81      0.70      0.74    489519\n",
      "weighted avg       0.89      0.90      0.89    489519\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:14<00:00,  8.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Validation score for class 0 : 0.2960695076541167\n",
      "Average F1 Validation score for class 1 : 0.9338270183653944\n",
      "Average Accuracy Validation score  : 0.8790261799462465\n",
      "Average mcc Validation score  : 0.3042759630566861\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.19      0.30      9274\n",
      "           1       0.89      0.98      0.93     61047\n",
      "\n",
      "    accuracy                           0.88     70321\n",
      "   macro avg       0.76      0.59      0.61     70321\n",
      "weighted avg       0.86      0.88      0.85     70321\n",
      "\n",
      "Train Loss = 0.2342940232967405 Valid Loss = 0.3522270393371582\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/438 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 438/438 [05:54<00:00,  1.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Training score for class 0 : 0.5491326586789185\n",
      "Average F1 Training score for class 1 : 0.9487644104716568\n",
      "Average Accuracy Training score  : 0.9079851854575614\n",
      "Average mcc Training score  : 0.5197211097042358\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.44      0.55     61913\n",
      "           1       0.92      0.98      0.95    427606\n",
      "\n",
      "    accuracy                           0.91    489519\n",
      "   macro avg       0.82      0.71      0.75    489519\n",
      "weighted avg       0.90      0.91      0.90    489519\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:15<00:00,  8.19it/s]\n",
      "  0%|          | 0/438 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Validation score for class 0 : 0.2960695076541167\n",
      "Average F1 Validation score for class 1 : 0.9338270183653944\n",
      "Average Accuracy Validation score  : 0.8790261799462465\n",
      "Average mcc Validation score  : 0.3042759630566861\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.19      0.30      9274\n",
      "           1       0.89      0.98      0.93     61047\n",
      "\n",
      "    accuracy                           0.88     70321\n",
      "   macro avg       0.76      0.59      0.61     70321\n",
      "weighted avg       0.86      0.88      0.85     70321\n",
      "\n",
      "Train Loss = 0.22570576346968407 Valid Loss = 0.35185785287618637\n",
      "Epoch 6 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 438/438 [05:33<00:00,  1.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Training score for class 0 : 0.5558852621167162\n",
      "Average F1 Training score for class 1 : 0.9494270998658626\n",
      "Average Accuracy Training score  : 0.9091945358607123\n",
      "Average mcc Training score  : 0.5269368640753156\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.45      0.56     61913\n",
      "           1       0.92      0.98      0.95    427606\n",
      "\n",
      "    accuracy                           0.91    489519\n",
      "   macro avg       0.83      0.71      0.75    489519\n",
      "weighted avg       0.90      0.91      0.90    489519\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:13<00:00,  9.19it/s]\n",
      "  0%|          | 0/438 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Validation score for class 0 : 0.2960695076541167\n",
      "Average F1 Validation score for class 1 : 0.9338270183653944\n",
      "Average Accuracy Validation score  : 0.8790261799462465\n",
      "Average mcc Validation score  : 0.3042759630566861\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.19      0.30      9274\n",
      "           1       0.89      0.98      0.93     61047\n",
      "\n",
      "    accuracy                           0.88     70321\n",
      "   macro avg       0.76      0.59      0.61     70321\n",
      "weighted avg       0.86      0.88      0.85     70321\n",
      "\n",
      "Train Loss = 0.2249761511950188 Valid Loss = 0.3507982357144356\n",
      "Epoch 7 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 438/438 [05:28<00:00,  1.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Training score for class 0 : 0.5529550732762073\n",
      "Average F1 Training score for class 1 : 0.9492027454040416\n",
      "Average Accuracy Training score  : 0.9087716717839348\n",
      "Average mcc Training score  : 0.5241271868943475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.45      0.55     61913\n",
      "           1       0.92      0.98      0.95    427606\n",
      "\n",
      "    accuracy                           0.91    489519\n",
      "   macro avg       0.83      0.71      0.75    489519\n",
      "weighted avg       0.90      0.91      0.90    489519\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 125/125 [00:15<00:00,  8.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average F1 Validation score for class 0 : 0.2960695076541167\n",
      "Average F1 Validation score for class 1 : 0.9338270183653944\n",
      "Average Accuracy Validation score  : 0.8790261799462465\n",
      "Average mcc Validation score  : 0.3042759630566861\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.19      0.30      9274\n",
      "           1       0.89      0.98      0.93     61047\n",
      "\n",
      "    accuracy                           0.88     70321\n",
      "   macro avg       0.76      0.59      0.61     70321\n",
      "weighted avg       0.86      0.88      0.85     70321\n",
      "\n",
      "Train Loss = 0.2245521538605973 Valid Loss = 0.3541386137008667\n",
      "terminating because of early stopping!\n"
     ]
    }
   ],
   "source": [
    "best_accuracy = -1\n",
    "train_loss_lst = []\n",
    "val_loss_lst = []\n",
    "with open('metrics_xlmroberta__earlyStopping.txt', 'w') as f:\n",
    "    for epoch in range(config.EPOCHS):\n",
    "            print(f'Epoch {epoch+1} of {config.EPOCHS}')\n",
    "            train_metrics = engine.train_fn(train_dataloader, model, optimizer, scheduler)\n",
    "            print(classification_report(train_metrics[5],train_metrics[6]))\n",
    "            test_metrics = engine.eval_fn(val_dataloader, model)\n",
    "            print(classification_report(test_metrics[5],test_metrics[6]))\n",
    "            print(f\"Train Loss = {train_metrics[0]} Valid Loss = {test_metrics[0]}\")\n",
    "            train_loss_lst.append(train_metrics[:-2])\n",
    "            val_loss_lst.append(test_metrics[:-2])\n",
    "            f.write(f\"Train_loss {epoch+1} : {str(train_loss_lst)}\" + '\\n')\n",
    "            f.write(f\"val_loss {epoch+1} : {str(val_loss_lst)}\" + '\\n')\n",
    "            if early_stopping.step(float(\"{:.2f}\".format(test_metrics[4]))): #mcc score for early stopping \"{:.2f}\".format(a_float)\n",
    "                  break  # early stop criterion is met, we can stop now\n",
    "            if test_metrics[4] >  best_accuracy:\n",
    "                torch.save(model.state_dict(), './models/training_data/model_xlmrobertatokenclassificationmodel_TrainData_earlystopping.bin')\n",
    "                best_accuracy = test_metrics[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss_train : [0.3267540782349839, 0.287871975604802, 0.2572144322335448, 0.2342940232967405, 0.22570576346968407, 0.2249761511950188, 0.2245521538605973] \n",
      " loss_test :  [0.31757344281673433, 0.3175424579381943, 0.3323549725115299, 0.3522270393371582, 0.35185785287618637, 0.3507982357144356, 0.3541386137008667] \n",
      " f1_class0_train : [0.0775124864760719, 0.3017270654441372, 0.4468204233692961, 0.5283075991522859, 0.5491326586789185, 0.5558852621167162, 0.5529550732762073] \n",
      " f1_class0_test : [0.05476314419573139, 0.20455173684695474, 0.24655369215910086, 0.2960695076541167, 0.2960695076541167, 0.2960695076541167, 0.2960695076541167] \n",
      " f1_class1_train : [0.9317185280259772, 0.9357853011311797, 0.9420242754162886, 0.9468832249178359, 0.9487644104716568, 0.9494270998658626, 0.9492027454040416] \n",
      " f1_class1_test : [0.9307142257530316, 0.9334843476719907, 0.9336108252806511, 0.9338270183653944, 0.9338270183653944, 0.9338270183653944, 0.9338270183653944] \n",
      " accuracy_score_train : [0.8728486534741245, 0.8823865876503262, 0.895047996094125, 0.9045185171566374, 0.9079851854575614, 0.9091945358607123, 0.9087716717839348] \n",
      " accuracy_score_test : [0.8708920521608055, 0.87723439655295, 0.8779738627152629, 0.8790261799462465, 0.8790261799462465, 0.8790261799462465, 0.8790261799462465] \n",
      " mcc_score_train : [0.11088032353358976, 0.30167665974313485, 0.42502768013613795, 0.4981704369949589, 0.5197211097042358, 0.5269368640753156, 0.5241271868943475] \n",
      " mcc_score_test : [0.13468518895157555, 0.255804477978454, 0.2768427098848643, 0.3042759630566861, 0.3042759630566861, 0.3042759630566861, 0.3042759630566861]\n"
     ]
    }
   ],
   "source": [
    "train_metrices = np.array(train_loss_lst)\n",
    "test_metrices = np.array(val_loss_lst)\n",
    "loss_train = train_metrices[:,0]\n",
    "loss_test = test_metrices[:,0]\n",
    "f1_class0_train = train_metrices[:,1]\n",
    "f1_class0_test= test_metrices[:,1]\n",
    "f1_class1_train = train_metrices[:,2]\n",
    "f1_class1_test = test_metrices[:,2]\n",
    "accuracy_score_train = train_metrices[:,3]\n",
    "accuracy_score_test = test_metrices[:,3]\n",
    "mcc_score_train = train_metrices[:,4]\n",
    "mcc_score_test = test_metrices[:,4]\n",
    "print('loss_train :',list(loss_train),'\\n','loss_test : ',list(loss_test),'\\n','f1_class0_train :',list(f1_class0_train),'\\n','f1_class0_test :',list(f1_class0_test),'\\n','f1_class1_train :',list(f1_class1_train),'\\n','f1_class1_test :',list(f1_class1_test),'\\n','accuracy_score_train :',list(accuracy_score_train),'\\n','accuracy_score_test :',list(accuracy_score_test),'\\n','mcc_score_train :',list(mcc_score_train),'\\n',\n",
    "    'mcc_score_test :', list(mcc_score_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2 3 4 5 6]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA8tElEQVR4nO3dd3wUdfrA8c+zm0AINfQSEJTeSwClC4qAFAsgRQQsWECxy93589TTOwsiKiii2EVEPE5shyeCiEoJ0jsIktBrIIRAyvP7Y4a4hE0BstmU5/167Wt3+jOzu/PMfGfm+xVVxRhjjEnPE+wAjDHG5E2WIIwxxvhlCcIYY4xfliCMMcb4ZQnCGGOMX5YgjDHG+GUJIh8TkQUicnuw4wgGEekiIrEFbbkiMkVE/s+n+24R2Sci8SJSzn2/NADLXSciXXJ6voGSH3/7IvJXEXk72HGcD0sQmRCREiKyQ0SG+vQrKSI7RaS/iLwnIioi/dJN97Lbf4TbPUJEFmWwjAUikuj+8Q+KyL9FpEpAVywD5/unE5EnRSTJjf3M69ELWK7v9KkictKne2jWc8hZItJGRL4RkaMiclhElorIyNxYtqrepar/cOMIBSYA3VW1hKoect9/v5hluL/bZ9Itt5GqLriY+eZV7o55u/t7ihWRT32G5VqiUdV/qmq+SmqWIDKhqvHAncBEEang9n4BiFbVWW73ZuCWM9OISAgwENh2Hosao6olgNpACWD8xcZ+PsRxob+FT92d1pnXC+e57BDf6YGdQB+ffh9fYFwXRESuAH4AfsT5PsoBdwM9czMOVyUgDFgXhGXnSef7WxWR4cAw4Cr39xUFzAtUfAWNJYgsqOpc4GvgVfcUfCBwj88oXwIdRCTC7e4BrAb2XsCyjgL/AZqf6Sci7URkmYjEue/t0k12mXuEe0xEvhCRsj7TXi4iv7hHwqt8ixDcI6dnReRnIAH4EOgITHKPtCa5470iIjHu/JeLSMfsrIuI9HWLLY66y2rgM2yHiDwmIquBE25S9TePoiIyUUR2u6+JIlI0g3HvE5H1IhLpTjfePdPbJ06xTTF3vC7uUeRDIrJfRPakOzt4EXhfVZ9X1YPqWK6qAzNY7jgR2SYix93lX+8zrLaI/Oh+dwfPHLm6O7mX3eUfE5E1ItLYHfaeiDwjInWBTe6sjorID+5wFZHa7udiIvKSiPzhLmORz3p+JiJ73f4LRaSR238UMBR41P2ev/T5Tq7KartnY/v5lcV3EiEiX4nIARE54n6O9Jk2/W/1Up9hRcQ5y2vi06+iiCSIc1DXGpirqtsAVHWvqk51x3sW/7/5DP9zbiz/Ej//ORGp6X4/o9zttkdEHvaZ9kkR+SjduMPdbXJQRP7mM24xEXnf3R4bRORRCUKRKqpqryxeQASwBzgIjPTp/x7wDDAVuNvtNxMYDCwCRrj9RgCLMpj3AuB293M54HvgC7e7LHAE5wgoxJ3vEaCcz7S7gMZAceBz4CN3WDXgENAL50Dgare7gs+0O4FG7rxDfWPxie9mN64Q4CGcxBfmDnvyzPLSTVMXOOEuMxR4FNgKFHGH7wBWAtWBYumm3YFztAfwNLAYqAhUAH4B/uEO6wLEup+fAH7zWbeXgTnu9iuJk8T/5TNdsjvvUHf7JLjfcTiQAlyZyW8hbblu9wCgqruNb3LXu4o77BPgb+6wMKCD2/8aYDlQBhCggc807wHPuJ9rAgqE+CxPgdru58nud1YN8ALtgKLusFvddS8KTARWpv/dXsR297v9svgPZfadlANudLd/SeAz4D/p/iMZ/laB14HnfcYfC3zp8/s9DDyCc/bgzej/lwP/uTPf1yfusCbAAZ/t+qSfcd8CigHNgFNAA3f4czhnsRFAJM5BZ2xm2zgg+77cXmB+feHsuBOA0j793sNJEB2AX3H+8PvcL/x8EkQCEOf+YFYCNdxhw4Cl6cb/1We+C4DnfIY1BE7j7CweAz5MN+1cYLjPtE/7ieX2LLbDEaCZ+/lJd3lHfV5Vgf8DZvpM43H/VF3c7h3ArRnMf4fPH2ob0Mtn2DXADvdzF3eeE9xtXdrtLzg76ct8prsC2O4z3UnO3unuBy7H2dEqUD+T9e+S2R/V/f76uZ8/wDl4iEw3TlecosnLAU+6Ye+RjQThbtOTZ76LLL6zMu50pdMv4wK3u9/tl8nyM/1O/IzfHDiS7neZ4W8VaIuTQMTtjgYG+ow7FOf/ewLnIOmxjH7zXNx/7sz3Vd9n+AvANJ//S/oEEekz7lJgkPv5d+Aan2G3E4QEYUVM2SAiN+N8od8Dz6cfrqqLcI60/gZ8paonz3MR96lqaaApfx4xgLOz/SPduH/g7MjOiEk3LBQoD1wCDBCniOeoiBzFSWRVMpjWLxF52D3FjXPnUdqd/xkzVbWMz2t3+rhVNdVdVkZxZyT9+v/h9jujDDAK50g0zu1XAedIdLnPev/X7X/GIVVN9ulOwLn2cwRI5extlCkRuUVEVvosqzF/bp9HcXaOS8UpbrsVQFV/ACbhnAHsF5GpIlIqu8t0lcc5KznnWpeIeEXkOXGKvo7h7PzPTJMdWW33jLZfRjL9TkQkXETedIvKjgELgTIi4vWZR4a/F1Vd4sbQRUTq4yTQOT7DP1bVq3B+L3cB/xCRazKY3cX85zIa7rvt0vMtivbdjlXTzSc7/5ccZwkiCyJSEef0+A6cC9YDxX85/Ec4RTAfXOiyVHUNzhnJZBERYDfOjt5XDZwj5zOqpxuWhFMUFoNzBuG78y6uqs/5LjJ9CL4d7no+inPdJUJVy+Cc6UgWq3JW3O66VE8Xd/plZzkfnPXb7dN9BOgNvCsi7d1+B3GOcBv5rHdpdS5QZkpVE3COFm/MRmyIyCU4RQRjcIogygBrcbePOuXdd6hqVZzfzuviXj9Q1VdVtRXOEWhdnCKQ83EQSAQu8zNsCNAPuAonodc8E7L7ntW2z2q7n6+svpOHgHpAW1UtBXRKF292Yn4fpzhpGDBLVRPTj6CqSar6GU5xTeMM5nsx/7mMhl/IttvDnweK6eeZayxBZG0STnnofFXdg7PDfEvOvVj6Kk6Z+8IM5iMiEub7ymC893HuXukLfAPUFZEhIhIiIjfh7FC+8hn/ZhFpKCLhOOXCs1Q1BSdh9RGRa9wjyjD3AmMkGduHzwVAnPLgZJxy1BAReQLIzpHuTOBaEekmzq2aD+GUr/6SjWl9fQI8LiIVRKQ8zrWGj3xHUOfWzKHAv0WkjXu28hbwspvcEZFqmRwxpvcoMEJEHhGRcu70zURkhp9xi+PsYA64443kzx0PIjLAZ3sfccdNFZHWItLW3TYncHb0qdmM78x6pwLvABNEpKr7HV/h/i5L4mzvQzhH7v9MN3n67zm9LLf7BcSa2XdSEieBHHUv+P79AhbzEXA9TpJIO0gT5xbza8W5Pd0jIj1xrmUscUdJvy0u5j93xv+5Z0WNgJHAp5y/mcBfxLmAXw3nICTXWYLIhIhch1Msk3Z0p6pv4xwRPOE7rqoeVtV56hYY+tEO50+Q9hI/d++o6mngFeD/VPUQzhHyQzh/9keB3qrqe7TyIU6Z8l6cIof73PnE4BxF/hVnBxbjrkdm3/krQH/3zolXca5Z/BenvPwPnB1Zlqe6qroJ54/6Gs6RVR+cW1dPZzVtOs/glCevBtbgXIh+Jv1Iqvo/nIuyX4pIS5zrL1uBxW6Rxfc4R6hZUtVfcK4RdAV+F5HDONcRvvEz7nrgJZyzjn04FyV/9hmlNbBEROJxijzGqvMMQymcHeYRnO16COfuqfP1MM52WYZzIfZ5nO/3A3e+u4D1OBecfU0DGrrFPf/xM99sbffzlNl3MhHnut1BN9b/nu/M3d/7bzhJ+CefQcdw/gM7ca6RvYBzQ8mZ55LO+s1fzH/Ox4/uus4Dxqvqd+e7PjiJJxbYjrOtZuEk/VwlGe/PjDEm/xCRd4Ddqvp4AJexAOdC8zlPRItITZwdemi6azQ5sdy7cS5gd87J+WbF7/3nxhiTn7g75xuAFkEOJUeIU5vCpThnp3Vwzmgm5XYcVsRkjMkR7p1a8X5eAa0uRUT+gXNzwIuquj2Qy8pFRYA3geM4T/Z/gfO8R66yIiZjjDF+2RmEMcYYvwrMNYjy5ctrzZo1gx2GMcbkK8uXLz+oqhX8DSswCaJmzZpER0cHOwxjjMlXRCT9k+NprIjJGGOMX5YgjDHG+GUJwhhjjF8F5hqEP0lJScTGxpKYeE69XSaPCgsLIzIyktDQ0GCHYkyhV6ATRGxsLCVLlqRmzZo4FYqavExVOXToELGxsdSqVSvY4RhT6BXoIqbExETKlStnySGfEBHKlStnZ3zG5BEFOkEAlhzyGfu+jMk7CnQRkzGF2q7fYPNc57PHCyIgHhCv8+7x+nSLn34en37iZzqfac/p5wn8Mu1gIuAsQRhT0OxZBfP/BZu/DXYkgZWWMPwkoJCi4C0KIUUyeC8K3iLue6iffunf/czDWySD+flM48nfhTSWIALs6NGjTJ8+nXvuuee8puvVqxfTp0+nTJky5zXdiBEj6N27N/379z+v6UwBsHctLPgXbPwKwkpD18ehzZ1QtCSkpoCmui/3c1o/9dPvzHjq089nWGrK2dOdNa3PfC9oWs1gfqmQmtE6+LxSkiDltPNKPuXzfgqST8Op4877me6UU+7wJOdzag425eAJ8ZNcspO0/CQbb2jGSat4eajeJufidlmCCLCjR4/y+uuvn5MgkpOTCQnJePN/8805DZgZ49++9fDjc7D+CyhaCrr8BS6/20kSZ3jtr55tqSl+kku6JOP7nnI6XaLJ7rSns5G03GnPatHUj2pRcMe8HN8UheZX89SX61i/+1iOzrNh1VL8vU+jTMcZN24c27Zto3nz5oSGhhIWFkZERAQbN25k8+bNXHfddcTExJCYmMjYsWMZNWoU8GfdUvHx8fTs2ZMOHTrwyy+/UK1aNb744guKFSuWZXzz5s3j4YcfJjk5mdatW/PGG29QtGhRxo0bx5w5cwgJCaF79+6MHz+ezz77jKeeegqv10vp0qVZuDCjprVNnnFgE/z4PKz9NxQpAZ0ehSvugWIRwY4sf/N4wVMMQrP+j+Wa1JR0Seb02YnHWzQgiy00CSJYnnvuOdauXcvKlStZsGAB1157LWvXrk27z/+dd96hbNmynDx5ktatW3PjjTdSrly5s+axZcsWPvnkE9566y0GDhzI559/zs0335zpchMTExkxYgTz5s2jbt263HLLLbzxxhsMGzaM2bNns3HjRkSEo0ePAvD0008zd+5cqlWrltbP5FEHtzqJYc1nEBoOHR6AdvdCeNlgR2YCxeOFIuFAeK4uttAkiKyO9HNLmzZtznoI7NVXX2X27NkAxMTEsGXLlnMSRK1atWjevDkArVq1YseOHVkuZ9OmTdSqVYu6desCMHz4cCZPnsyYMWMICwvjtttuo3fv3vTu3RuA9u3bM2LECAYOHMgNN9yQA2tqctzh3+HHF2H1DAgJg/b3Qbv7nPJnYwKg0CSIvKJ48eJpnxcsWMD333/Pr7/+Snh4OF26dPH7kFjRon+ePnq9Xk6ePHnByw8JCWHp0qXMmzePWbNmMWnSJH744QemTJnCkiVL+Prrr2nVqhXLly8/J1GZIDnyByx8EVZOdy5UXn4PtB8LJSoGOzJTwFmCCLCSJUty/Phxv8Pi4uKIiIggPDycjRs3snjx4hxbbr169dixYwdbt26ldu3afPjhh3Tu3Jn4+HgSEhLo1asX7du359JLLwVg27ZttG3blrZt2/Ltt98SExNjCSLYjsbAT+NhxUfObZxt7nCKk0pWDnZkppCwBBFg5cqVo3379jRu3JhixYpRqVKltGE9evRgypQpNGjQgHr16nH55Zfn2HLDwsJ49913GTBgQNpF6rvuuovDhw/Tr18/EhMTUVUmTJgAwCOPPMKWLVtQVbp160azZs1yLBZznuJ2waIJsPx9p7vVCOjwIJSuFtSwTOEjqhrsGHJEVFSUpm9RbsOGDTRo0CBIEZkLVWi/t+N7YdHLEP2uc1tji2HQ8SEoUz3YkZkCTESWq2qUv2F2BmFMsMXvh0UTIXqa87BW8yHQ6RGIuCTYkZlCzhJEPjV69Gh+/vnns/qNHTuWkSNHBikic95OHISfX4Glbzn3sjcbDJ0ehrKXBjsyYwBLEPnW5MmTgx2CuVAJh+GX12DJm5CUAE0HOg+5la8d7MiMOYslCGNyy8kj8OtkWDwFTsdD4xug82NQoV6wIzPGL0sQxgRaYhwsfsNJDqeOQcProMs4qFgIL8SbfMUShDGBcuo4LJniFCclxkH93k5FepUbBzsyY7IloJWVi0gPEdkkIltFZJyf4XeJyBoRWSkii0SkYbrhNUQkXkQeDmScxuSoU/Hw0wSY2AR+eAZqtINRP8Kgjy05mHwlYAlCRLzAZKAn0BAYnD4BANNVtYmqNgdeACakGz4BKOCtnpytRIkSAOzevTvDNh26dOlC+mc+0ps4cSIJCQlp3b169crRSvhGjBjBrFmzcmx+BcLpBPj5VXilKcx7CiJbwx0/wJAZULV5sKMz5rwFsoipDbBVVX8HEJEZQD9g/ZkRVNW3/u3iQNpTeyJyHbAdOBHAGPOsqlWrXtQOeOLEidx8882Ehzu1P1r7EgGUdBKi33GeZTixHy7rCl3+CtVbBzsyYy5KIBNENSDGpzsWaJt+JBEZDTwIFAG6uv1KAI8BVwMZFi+JyChgFECNGjUyj+bbcbB3zfnEn7XKTaDnc5mOMm7cOKpXr87o0aMBePLJJwkJCWH+/PkcOXKEpKQknnnmGfr163fWdDt27KB3796sXbuWkydPMnLkSFatWkX9+vXPqqzv7rvvZtmyZZw8eZL+/fvz1FNP8eqrr7J7926uvPJKypcvz/z589PalyhfvjwTJkzgnXfeAeD222/n/vvvZ8eOHdbuxPlKSoTf3neKk+L3Qq1O0OUDuOSKYEdmTI4I+kVqVZ0MTBaRIcDjwHDgSeBlVY2XTBomV9WpwFRwqtoIfLTn76abbuL+++9PSxAzZ85k7ty53HfffZQqVYqDBw9y+eWX07dvXzJa1zfeeIPw8HA2bNjA6tWradmyZdqwZ599lrJly5KSkkK3bt1YvXo19913HxMmTGD+/PmUL392VdDLly/n3XffZcmSJagqbdu2pXPnzkRERFi7E9mVfApWfAgLX4Lju+GS9tB/GtTsEOzIjMlRgUwQuwDfSmQi3X4ZmQG84X5uC/QXkReAMkCqiCSq6qQLjiaLI/1AadGiBfv372f37t0cOHCAiIgIKleuzAMPPMDChQvxeDzs2rWLffv2Ubmy/1o6Fy5cyH333QdA06ZNadq0adqwmTNnMnXqVJKTk9mzZw/r168/a3h6ixYt4vrrr0+rdvyGG27gp59+om/fvtbuRFZSkmDlx7BwPMTFQPXL4fopzplDJgcyxuRXgUwQy4A6IlILJzEMAob4jiAidVR1i9t5LbAFQFU7+ozzJBB/UckhyAYMGMCsWbPYu3cvN910Ex9//DEHDhxg+fLlhIaGUrNmTb/tQGRl+/btjB8/nmXLlhEREcGIESMuaD5nWLsTGUhJglUzYOELcHSn0/5vn1ecaw2WGEwBFrC7mFQ1GRgDzAU2ADNVdZ2IPC0ifd3RxojIOhFZiXMdYnig4gmmm266iRkzZjBr1iwGDBhAXFwcFStWJDQ0lPnz5/PHH39kOn2nTp2YPn06AGvXrmX16tUAHDt2jOLFi1O6dGn27dvHt9/+ecNXRu1QdOzYkf/85z8kJCRw4sQJZs+eTceOHc8ZL7t8250Azmp3Ii4ujl69evHyyy+zatUq4M92J55++mkqVKhATExMZrMPrpRkWPkJTGoNc8ZAeDkYOgtu/x5qd7PkYAq8gF6DUNVvgG/S9XvC5/PYbMzjyZyPLHc1atSI48ePU61aNapUqcLQoUPp06cPTZo0ISoqivr162c6/d13383IkSNp0KABDRo0oFWrVgA0a9aMFi1aUL9+fapXr0779u3Tphk1ahQ9evSgatWqzJ8/P61/y5YtGTFiBG3atAGci9QtWrTIVnGSPwWy3YnUFFj7OSx4Dg5vc25GGDwD6vawpGAKFWsPwuQ5QfveUlNh/WwnMRzcDJUaO1Vi1O9ticEUWNYehDGZSU2FjV86iWH/eqhQHwa8Dw36gieglQ0Yk6dZgjCZKtDtTqjCxq+dxLBvDZSrAzdOg0bXg8cb7OiMCboCnyBUNcPnC0zWcrvdiVwp8lSFzXNhwT9hzyooexnc8BY0vtESgzE+CnSCCAsL49ChQ5QrV86SRD6gqhw6dIiwsLBALQC2fg/z/wm7f4OImnDdG9BkIHgL9F/BmAtSoP8VkZGRxMbGcuDAgWCHYrIpLCyMyMjInJ1p0kmnKGnJmxC7FErXgL6vOU18ekNzdlnGFCAFOkGEhoZSq1atYIdhgkEVdi13nnxe8zmcioMyNaD3y9D8ZggpEuwIjcnzCnSCMIXQ8b2w+lNY8TEc3AQhYc7dSC2GQs1OdleSMefBEoTJ/5JPw+ZvnaSw9XvQFIhs41SH0eh6CCsd7AiNyZcsQZj8a88qWDkdVs+Ek4ehZBVofx80Hwrl6wQ7OmPyPUsQJn85cRDWfOacLexbA94iUK8XtLgZLr3S7kYyJgfZv8nkfSnJsPV/sOIj5/mF1CSo0hx6jXeeXQgvG+wIjSmQLEGYvGv/BucupFWfOk15hpeHtndC8yFQqVGwozOmwLMEAcSdTKJ0MbsfPk84ecSpSXXFx87DbJ4QqHONcxdSne723IIxuajQJ4idhxK44Y2fubdrHYa3qxnscAqn1BT4fb6TFDZ+DSmnoGIjuOafzlPOJSoEO0JjCqVCnyAqlw6jZY0I/j5nHamqjGxvD9blmkPb3CKkGXBsF4SVgVbDnSKkKs2tim1jgqzQJ4giIR4mD23JvdNX8NSX60lVuK2DJYmAOXUc1s12zhZiFoN44LJucM2zzt1IIUWznocxJlcU+gQBEOr18NqQFtz3yQr+8dV6VJXbO14a7LAKjtRU+GORkxQ2zIGkBKdq7auehKaDoFSVYEdojPHDEoQr1Ovh1cEtuH/GSp75egOqcEcnSxIX5cgfsOoT52G2o39A0VLQdKDzIFtkaytCMiaPswThI9Tr4ZVBzRGBZ7/ZQKoqd3a+LNhh5S+nE5yzhBUfwY6fAIFanaDr407TnUXCgx2hMSabLEGkE+L1MPGm5ogI//p2I6kKd3exJJEpVYhZ4iSFdf+B08edthau/Bs0G+TUomqMyXcsQfgR4vXw8sBmeASe/+9GUlUZfWXtYIeV9xzb/WcR0qGtEFocGl3n3IVUo53VnGpMPhfQBCEiPYBXAC/wtqo+l274XcBoIAWIB0ap6noRuRp4DigCnAYeUdUfAhlreiFeDxMGNscjwotzN6GqjOlqFcCRlAibvnYuOP8+HzTVSQYdHoCG/aBoyWBHaIzJIQFLECLiBSYDVwOxwDIRmaOq631Gm66qU9zx+wITgB7AQaCPqu4WkcbAXKBaoGLNiNcjjB/QDAHGf7eZVIX7uhXCJKHqPNW84mNYOwsS46BUJHR8yGmVrZwVwRlTEAXyDKINsFVVfwcQkRlAPyAtQajqMZ/xiwPq9l/h038dUExEiqrqqQDG65fXI7w4oBkiwoT/bSZVlfuvqpvbYQRH/P4/G985sMFpfKd+b6fai1qdweMNdoTGmAAKZIKoBsT4dMcCbdOPJCKjgQdxipO6+pnPjcBv/pKDiIwCRgHUqBG4C6Fej/BC/6Z4BCZ+v4VUhQeuqoMUxNs0k0/DlrlOUtjyndP4TrUop6nORjdAsTLBjtAYk0uCfpFaVScDk0VkCPA4MPzMMBFpBDwPdM9g2qnAVICoqCgNZJxej/D8jU3xiPDqvC2gygNX1y04SWLvGrfxnU8h4RCUqATtxkCzIVCxfrCjM8YEQSATxC6guk93pNsvIzOAN850iEgkMBu4RVW3BSTC8+TxCP+6oQki8OoPW0lVeKh7Pk4SCYfdxnc+gr2rwRMK9Xo6je9c1s0a3zGmkAvkHmAZUEdEauEkhkHAEN8RRKSOqm5xO68Ftrj9ywBfA+NU9ecAxuhccP1idAYDz93xe0T4l8LQSnHsXJTApk0lqFe51Llj+k0afvpd1HgXMc/Eo7DtB0g5DZWbQs8XoMkAa3zHGJMmYAlCVZNFZAzOHUhe4B1VXSciTwPRqjoHGCMiVwFJwBH+LF4aA9QGnhCRJ9x+3VV1f44Hmpri1Cp67gr4GdnpJ0DjIlAxPJFjB3dxKCGUciWK+uyC/UybyfwuaLyLnac3FKJucy44V27if/7GmEJN1O9OJv+JiorS6OjoXF2mqvLEF+v4cPEf3NnpUsb1rJ9/i5uMMYWSiCxX1Sh/w6yQ+SKICE/3a4RH4M2Fv5Oqyl97NbAkYYwpECxBXCQR4cm+jRAR3vppO6kKj19rScIYk/9ZgsgBIsLf+zREBKYt2k6qKk/0bmhJwhiTr1mCyCEiwhO9G+IRYdqi7ajiJg1LEsaY/MkSRA4SER6/tgEewS1uUp5yi5+MMSa/sQSRw0SEv/ZqgEck7cL1030b4/FYkjDG5C+WIAJARNJueZ3y4zZU4R/9LEkYY/IXSxABIiI81qMeHoHXF2wjVeHZ6yxJGGPyD0sQASQiPHJNPTwiTJq/FVXln9c3sSRhjMkXLEEEmIjwUPe6eNwK/lJS1akV1pKEMSaPswSRC0SEB66uC25V4Qo8f2NTvJYkjDF5mCWIXCIiPHh1XZ9Gh5QX+zezJGGMybMsQeSy+6+qiyC8/P1mVGH8AEsSxpi8yRJEEIy9qg4egZfcNq5fGtCMEK8n2GEZY8xZLEEEyb3d6uDxCC/O3YQqTBhoScIYk7dYggii0VfWRgRe+O8mUlWZeFNzSxLGmDzDEkSQ3dOlNh4Rnvt2I6owcVBzQi1JGGPyAEsQecBdnS/DI/DPbzaiKK8MamFJwhgTdJYg8ohRnS7DI8IzX28gNXUFrw2xJGGMCS7bA+Uht3e8lP/r3ZD/rtvLmOm/cTo5NdghGWMKMUsQecxtHWrx9z4NmbtuH6MtSRhjgsgSRB40sn0tnurbiP+t38c9Hy/nVHJKsEMyxhRCAU0QItJDRDaJyFYRGedn+F0iskZEVorIIhFp6DPsL+50m0TkmkDGmRcNb1eTp/s14vsN+7nno98sSRhjcl3AEoSIeIHJQE+gITDYNwG4pqtqE1VtDrwATHCnbQgMAhoBPYDX3fkVKrdcUZN/XNeYeRv3c9eHy0lMsiRhjMk9gTyDaANsVdXfVfU0MAPo5zuCqh7z6SwOqPu5HzBDVU+p6nZgqzu/QmfY5Zfw7PWNmb/pAHd9ZEnCGJN7ApkgqgExPt2xbr+ziMhoEdmGcwZx33lOO0pEokUk+sCBAzkWeF4ztO0l/OuGJizYdIBRdiZhjMklQb9IraqTVfUy4DHg8fOcdqqqRqlqVIUKFQITYB4xuE0Nnr+xCT9tOcAdH0RbkjDGBFwgE8QuoLpPd6TbLyMzgOsucNpC4abWNXj+hqYs2nqQ29+P5uRpSxLGmMAJZIJYBtQRkVoiUgTnovMc3xFEpI5P57XAFvfzHGCQiBQVkVpAHWBpAGPNNwa2rs4LNzbl520Huf2DZZYkjDEBE7CqNlQ1WUTGAHMBL/COqq4TkaeBaFWdA4wRkauAJOAIMNyddp2IzATWA8nAaFW1PaFrQFR1PCI8PGsVt763jGkjoggvYrWmGGNylqhq1mPlA1FRURodHR3sMHLV7BWxPDRzFW1qleWdEa0tSRhjzpuILFfVKH/Dgn6R2ly461tEMmFgc5ZuP8yId5dx4lRysEMyxhQgliDyuetaVOPlm5oTveMwIy1JGGNykCWIAqBf82q8MqgFy3ceYcS7S4m3JGGMyQGWIAqIPs2q8sqg5vy28yjD31nK8cSkYIdkjMnnLEEUIL2bVuW1wS1YGWNJwhhz8SxBFDC9mlRh0uAWrI6N45Z3lnLMkoQx5gJlK0GIyFgRKSWOaSLym4h0D3Rw5sL0bFKFSUNasiY2jmHTlhJ30pKEMeb8ZfcM4la35tXuQAQwDHguYFGZi9ajcWVeH9qS9bvjuGXaEksSxpjzlt0EIe57L+BDVV3n08/kUd0bVeb1oa1Yv+cYw6YtIS7BkoQxJvuymyCWi8h3OAliroiUBKyx5Hzg6oaVmHJzKzbuOc7QaYs5mnA62CEZY/KJ7CaI24BxQGtVTQBCgZEBi8rkqG4NKjFlWEs2741n6NtLLEkYY7IluwniCmCTqh4VkZtx2m2IC1xYJqd1rV+JN29pxZb98Qx5awlHTliSMMZkLrsJ4g0gQUSaAQ8B24APAhaVCYgr61Vk6rBWbD0Qz5C3l7D/eGKwQzLG5GHZTRDJ6lT72g+YpKqTgZKBC8sESpd6FXn7lii2H4yn96uLiN5xONghGWPyqOwmiOMi8hec21u/FhEPznUIkw91qluB2fe0p1gRL4OmLubdn7dTUKp9N8bknOwmiJuAUzjPQ+zFaQL0xYBFZQKuQZVSzBnTgS71KvDUl+u5/9OVJJy2Sv6MMX/KVoJwk8LHQGkR6Q0kqqpdg8jnShcLZeqwKB7uXpc5q3Zzw+u/sOPgiWCHZYzJI7Jb1cZAnDahBwADgSUi0j+QgZnc4fEIY7rW4b2Rbdh7LJE+kxbxv/X7gh2WMSYPyG4R099wnoEYrqq3AG2A/wtcWCa3da5bgS/HdOCScuHc8UE04+duIiXVrksYU5hlN0F4VHW/T/eh85jW5BPVy4Yz6652DIyKZNL8rYx4d6k9L2FMIZbdnfx/RWSuiIwQkRHA18A3gQvLBEtYqJcX+jfjXzc0Ycnvh+n92iLWxNozkcYURtm9SP0IMBVo6r6mqupjgQzMBNfgNjX47K4rUFVunPILny7bGeyQjDG5LNvFRKr6uao+6L5mZ2caEekhIptEZKuIjPMz/EERWS8iq0Vknohc4jPsBRFZJyIbRORVEbHaY3NZs+pl+PLeDrSuGcFjn6/hL/9eTWJSSrDDMsbkkkwThIgcF5Fjfl7HReRYFtN6gclAT6AhMFhEGqYbbQUQpapNgVnAC+607YD2OGcrjYHWQOcLWD9zkcqVKMoHt7blni6X8cnSGAa++SuxRxKCHZYxJhdkmiBUtaSqlvLzKqmqpbKYdxtgq6r+rqqngRk4VXX4zn++WzsswGKcB/AAFAgDigBFcZ7atnsvg8TrER7tUZ83h7Vi+4ET9HltET9tORDssIwxARbIO5GqATE+3bFuv4zcBnwLoKq/AvOBPe5rrqpuSD+BiIwSkWgRiT5wwHZYgXZNo8p8MaY9FUoWZfg7S5k8fyupdiusMQVWnrhV1a1CPAq3+g4RqQ00wDmjqAZ0FZGO6adT1amqGqWqURUqVMjNkAutSyuUYPY97enVpAovzt3EnR8t51iitVRnTEEUyASxC6ju0x3p9juLiFyF8yBeX1U95fa+HlisqvGqGo9zZnFFAGM156F40RBeG9yCJ3o3ZP7G/fSb9DOb9h4PdljGmBwWyASxDKgjIrVEpAgwCJjjO4KItADexEkOvg/i7QQ6i0iIiITiXKA+p4jJBI+IcGuHWky/43LiTyVz3eSf+WLlOfnfGJOPBSxBqGoyMAaYi7Nzn6mq60TkaRHp6472IlAC+ExEVorImQQyC6dRojXAKmCVqn4ZqFjNhWtTqyxf39uBRlVLMXbGSp76ch1JKdZcuTEFgRSUdgCioqI0Ojo62GEUWkkpqTz79Qbe+2UHrWtGMHlISyqWCgt2WMaYLIjIclWN8jcsT1ykNvlfqNfDk30b8cqg5qzddYxrX1vEMmutzph8zRKEyVH9mldj9uh2FC/iZfDUxbyzyFqrMya/sgRhclz9yqWYc28HutSryNNfrWfsDGutzpj8yBKECYhSYaFMHdaKR66px5erd3P95F/Ybq3VGZOvWIIwAePxCKOvrM37I9uw/3gifV9bxHfr9gY7LGNMNlmCMAHXqW4Fvry3A7UqFGfUh8t5ce5Ga63OmHzAEoTJFZER4cy88woGta7O5PnbGPHuUg5ba3XG5GmWIEyuCQv18tyNTXnuhiYs2X6YPq8tYlXM0WCHZYzJgCUIk+sGtanBrLucqrUGTPmVGUuttTpj8iJLECYomkY6rdW1vbQs4/69hsdmWWt1xuQ1liBM0JQtXoT3RrZhzJW1+TQ6hgFTrLU6Y/ISSxAmqLwe4eFr6vHWLVHsOHiC3q8tYuFma/zJmLzAEoTJE65uWIk593agUskwhr+7lEk/bLHW6owJMksQJs+oVb44s0e3o0/Tqoz/bjOjPlxO3Elrrc6YYLEEYfKU8CIhvDKoOX/v05AFm/bTb9IiNu49FuywjCmULEGYPEdEGNm+Fp+MupwTp1O4fvIv1lqdMUFgCcLkWa1rOq3VNalWmrEzVvLknHWcTrbW6ozJLZYgTJ5WsVQYH9/Rllvb1+K9X3Yw5K3F7DuWGOywjCkULEGYPC/U6+GJPg15dXAL1u0+Ru/XFrF0u7VWZ0ygWYIw+UbfZlX5z+j2lCgawuC3FjPNWqszJqAsQZh8pV7lknwxpj3d6lfkH1+t595PVnDilLVWZ0wgWIIw+U6psFCm3NyKR3vU45s1e7j+9Z/5/UB8sMMypsAJaIIQkR4isklEtorIOD/DHxSR9SKyWkTmicglPsNqiMh3IrLBHadmIGM1+YvHI9zTpTYf3NqWA8dP0XfSz8y11uqMyVEBSxAi4gUmAz2BhsBgEWmYbrQVQJSqNgVmAS/4DPsAeFFVGwBtgP2BitXkXx3qlOer+zpyaYXi3Pnhcp7/r7VWZ0xOCeQZRBtgq6r+rqqngRlAP98RVHW+qp6pvnMxEAngJpIQVf2fO168z3jGnKVamWLMvPMKBrepwRsLtjH8naUcij8V7LCMyfcCmSCqATE+3bFuv4zcBnzrfq4LHBWRf4vIChF50T0jMcavsFAv/7qhCS/c2JSlO5zW6lZaa3XGXJQ8cZFaRG4GooAX3V4hQEfgYaA1cCkwws90o0QkWkSiDxywKqINDGxdnc/vaoeIMHDKr0xfstNuhTXmAgUyQewCqvt0R7r9ziIiVwF/A/qq6plygVhgpVs8lQz8B2iZflpVnaqqUaoaVaFChZyO3+RTTSJL85XbWt1fZ6/hsc+ttTpjLkQgE8QyoI6I1BKRIsAgYI7vCCLSAngTJznsTzdtGRE5s9fvCqwPYKymgIlwW6u7t2ttZkbH0n/KL8QctstYxpyPgCUI98h/DDAX2ADMVNV1IvK0iPR1R3sRKAF8JiIrRWSOO20KTvHSPBFZAwjwVqBiNQWT1yM81L0eb98SxR+HEuj92iKmL9lpdzkZk01SUMpno6KiNDo6OthhmDxqx8ETPDprNUt3HKZR1VI81bcRUTXLBjssY4JORJarapS/YXniIrUxgVazfHE+vfNyXhvcgsMnTtN/yq+MnbGCvXFWM6wxGbEEYQoNEaFPs6rMe6gz93atzbdr99L1pQVMnr/VLmIb44clCFPohBcJ4aHu9fj+gc50rFOeF+du4pqJC/nf+n12S6wxPixBmEKrRrlw3hwWxYe3tSHU6+GOD6IZ/u4ytu63iv+MAUsQxtCxTgW+HduRJ3o3ZMXOI/SYuJBnvlrPscSkYIdmTFBZgjAGp9W6WzvUYv7DXejfKpJpP2+n6/gFzIyOIdVuizWFlCUIY3yUL1GU525sypzRHahRNpxHZ63m+td/ZsXOI8EOzZhcZwnCGD+aRJbm87vb8fJNzdgTl8j1r//CQzNXsf+43RZrCg9LEMZkQES4vkUkPzzchbs6X8acVbvoOv5Hpi7cxunk1GCHZ0zAWYIwJgslioYwrmd9vnugM21rleWf32ykx8SFzN9kbViZgs0ShDHZVKt8caaNaM27I1qjwMh3l3Hbe8vYcfBEsEMzJiAsQRhznq6sX5G593fiLz3rs/j3Q3R/eSHPfbuR+FPJwQ7NmBxlCcKYC1AkxMOdnS9j/sNd6NOsKlN+3EbX8QuYvSLWnsY2BYYlCGMuQsVSYbw0sBmz72lHldJhPPDpKm584xfWxMYFOzRjLpolCGNyQIsaEcy+pz0v9G/KzsMJ9J28iHGfr+Zg/KmsJzYmj7IEYUwO8XiEgVHV+eHhLtzWvhazlsdy5fgFTFu0naQUuy3W5D+WIIzJYaXCQnm8d0P+e38nmlcvwz++Wk/PV35i0ZaDwQ7NmPNiCcKYAKldsQQf3NqGt26J4nRyKjdPW8KdH0Zb29gm37AEYUwAiQhXN6zEdw904pFr6rFw80G6TfiRl77bRMJpuy3W5G2WIIzJBWGhXkZfWZsfHu5Mz8aVee2HrXR76Ue+XLXbbos1eZYlCGNyUZXSxXhlUAs+u+sKyhYvwr2frOCmqYtZv/tYsEMz5hyWIIwJgtY1yzJnTAf+eX0Ttuw7Tu/XfuLx/6zhyInTwQ7NmDQBTRAi0kNENonIVhEZ52f4gyKyXkRWi8g8Ebkk3fBSIhIrIpMCGacxweD1CEPa1mDBw1dyyxU1+WRpDF3GL+CDX3eQbLfFmjwgYAlCRLzAZKAn0BAYLCIN0422AohS1abALOCFdMP/ASwMVIzG5AWlw0N5sm8jvrmvI42qluKJL9bR+7VF/LrtULBDM4VcIM8g2gBbVfV3VT0NzAD6+Y6gqvNV9cw9f4uByDPDRKQVUAn4LoAxGpNn1Ktcko9vb8sbQ1tyPDGZwW8tZvTHv7Hr6Mlgh2YKqUAmiGpAjE93rNsvI7cB3wKIiAd4CXg4YNEZkweJCD2bVGHeQ5154Kq6fL9hH91eWsAr328hMSkl2OGZQiZPXKQWkZuBKOBFt9c9wDeqGpvFdKNEJFpEog8cOBDoMI3JNWGhXsZeVYd5D3WmW/1KvPz9Zrq99CPfrtljt8WaXBPIBLELqO7THen2O4uIXAX8DeirqmdqNrsCGCMiO4DxwC0i8lz6aVV1qqpGqWpUhQoVcjp+Y4IuMiKcyUNb8skdl1MyLIS7P/6NoW8vYdPe48EOzRQCEqijEREJATYD3XASwzJgiKqu8xmnBc7F6R6quiWD+YzAuZA9JrPlRUVFaXR0dA5Fb0zek5ySyvSlO3npu83En0pm2OWX8MBVdSkdHhrs0Ew+JiLLVTXK37CAnUGoajIwBpgLbABmquo6EXlaRPq6o70IlAA+E5GVIjInUPEYk9+FeD3cckVN5j/chcFtqvPBrzu48qUFTF+yk5RUK3YyOS9gZxC5zc4gTGGzbnccT81Zz9Idh2lUtRRP9W1EVM2ywQ7L5DNBOYMwxgRWo6ql+fTOy3ltcAsOnzhN/ym/MnbGCvbGJQY7NFNAWIIwJh8TEfo0q8q8hzpzb9fafLt2L11fWsDk+Vvttlhz0SxBGFMAhBcJ4aHu9fj+gc50rFOeF+duovvLC/ly1W6OJlj9TubC2DUIYwqgn7Yc4Kkv17N1fzwAl1YoTovqEbSoUYaWNSKoW6kEIV47PjSZX4OwBGFMAZWUksqy7YdZEXOUFTuPsGLnUQ65tcWGF/HSNLI0LWpE0KJ6GVrUiKBCyaJBjtgEQ2YJIiS3gzHG5I5Qr4d2tcvTrnZ5AFSVmMMnWRHjJIsVO4/w1sLfSXZvkY2MKJaWMFpeEkHDKqUoEmJnGYWZJQhjCgkRoUa5cGqUC6dfc6datMSkFNbuinMSRswRoncc5stVuwEoEuKhcdVSTtKo4ZxlVC0dhogEczVMLrIiJmPMWfbEnWTlzqNpRVOrY+M4ley0T1GxZNG0ZNGyRgRNqpWmWBFvkCM2F8OKmIwx2ValdDGqNClGzyZVAOdaxoY9x9KKpVbEHGXuun2A0+hRgyol0y6At6gRQc1y4XaWUUDYGYQx5rwdij/FypijaUVTq2LiiD+VDEBEeCjNq/95ltG0emlKhVl9UXmVnUEYY3JUuRJF6dagEt0aVAIgJVXZuj8+7W6p33YeYf4mpwp+EahTscRZZxm1K5bA67GzjLzOziCMMQERdzKJ1bFHzyqaOpqQBECJoiE0q176rKRRtniRIEdcONkZhDEm15UuFkrHOhXoWMdpq0VV2XEo4ayzjDd+3JZWE23NcuF/3jFVPYL6VUoSag/zBZWdQRhjgibhdDJrYuPS7pj6bedRDhx32g0rGuJJe5ivpXuWUalUWJAjLnjsSWpjTL6gquyOS0w7y1ix8whrdx3jdIpzm23V0mE+z2WUoVHV0oSF2m22F8OKmIwx+YKIUK1MMaqVKUbvplUBOJWcwvrd7m227pnG12v2ABDqFRpW+fNhvpY1IoiMKGa32eYQO4MwxuQ7+48nnvUw36qYOE661ZsXCfFQxOshxCuEeDyEeIQQrxDqdT57Pe5nrxDq8eBNNzxtOne40y2EpJsm1Ov28/w5PNRn2rR5eAWvx0NoVvNwp0k/X69HAprw7AzCGFOgVCwZRvdGleneqDLgtNe9eV88v+08QszhBJJTleSUVJLcd6dbSU5NJSnl3H4nk5SUVCUprX8m06Rqrjfx6iQZn4Tlk1RCPR4aVi3FpCEtc3y5liCMMfleiNfZSTasWipXlpfqkyiSUlOdROKTVM70S0pJJSXVN8k4n30TT/rElNYv/XzPjHfWPJx+NcqGB2Q9LUEYY8x58niEIu6DfsUouBfJ7SZjY4wxflmCMMYY41dAE4SI9BCRTSKyVUTG+Rn+oIisF5HVIjJPRC5x+zcXkV9FZJ077KZAxmmMMeZcAUsQIuIFJgM9gYbAYBFpmG60FUCUqjYFZgEvuP0TgFtUtRHQA5goImUCFasxxphzBfIMog2wVVV/V9XTwAygn+8IqjpfVRPczsVApNt/s6pucT/vBvYDFQIYqzHGmHQCmSCqATE+3bFuv4zcBnybvqeItAGKANtyNDpjjDGZyhO3uYrIzUAU0Dld/yrAh8BwVU31M90oYBRAjRo1ciFSY4wpPAJ5BrELqO7THen2O4uIXAX8Deirqqd8+pcCvgb+pqqL/S1AVaeqapSqRlWoYCVQxhiTkwJWF5OIhACbgW44iWEZMERV1/mM0wLn4nSPM9cc3P5FcIqbvlTVidlc3gHgj4sIuTxw8CKmz48K2zoXtvUFW+fC4mLW+RJV9XuEHdDK+kSkFzAR8ALvqOqzIvI0EK2qc0Tke6AJsMedZKeq9nWLnN4F1vnMboSqrgxgrNEZVVhVUBW2dS5s6wu2zoVFoNY5oNcgVPUb4Jt0/Z7w+XxVBtN9BHwUyNiMMcZkzp6kNsYY45cliD9NDXYAQVDY1rmwrS/YOhcWAVnnAtNgkDHGmJxlZxDGGGP8sgRhjDHGr0KfIETkHRHZLyJrgx1LbhCR6iIy361Fd52IjA12TIEmImEislREVrnr/FSwY8otIuIVkRUi8lWwY8kNIrJDRNaIyEoRKRSN1ItIGRGZJSIbRWSDiFyRY/Mu7NcgRKQTEA98oKqNgx1PoLnVl1RR1d9EpCSwHLhOVdcHObSAEafF9+KqGi8iocAiYGxGT+gXJCLyIE41NqVUtXew4wk0EdmBU0N0oXlQTkTeB35S1bfdh4zDVfVoTsy70J9BqOpC4HCw48gtqrpHVX9zPx8HNpB5JYr5njri3c5Q91Xgj4xEJBK4Fng72LGYwBCR0kAnYBqAqp7OqeQAliAKNRGpCbQAlgQ5lIBzi1pW4lQd/z9VLfDrjFOLwaPAORVdFmAKfCciy93KPAu6WsAB4F23KPFtESmeUzO3BFFIiUgJ4HPgflU9Fux4Ak1VU1S1OU6lkW1EpEAXJ4pIb2C/qi4Pdiy5rIOqtsRpqGy0W4RckIUALYE3VLUFcAI4p/XOC2UJohByy+E/Bz5W1X8HO57c5J5+z8dpqbAgaw/0dcvkZwBdRaTAV1+jqrvc9/3AbJyGywqyWCDW54x4Fk7CyBGWIAoZ94LtNGCDqk4Idjy5QUQqnGmyVkSKAVcDG4MaVICp6l9UNVJVawKDgB9U9eYghxVQIlLcvfECt5ilO1Cg705U1b1AjIjUc3t1A3LshpM80WBQMInIJ0AXoLyIxAJ/V9VpwY0qoNoDw4A1bpk8wF/dihULqirA+2476R5gpqoWits+C5lKwGznGIgQYLqq/je4IeWKe4GP3TuYfgdG5tSMC/1trsYYY/yzIiZjjDF+WYIwxhjjlyUIY4wxflmCMMYY45clCGOMMX5ZgjAmiESkS2GpadXkP5YgjDHG+GUJwphsEJGb3TYlVorIm27lf/Ei8rLbxsQ8EangjttcRBaLyGoRmS0iEW7/2iLyvdsuxW8icpk7+xI+9fl/7D7tjog857bbsVpExgdp1U0hZgnCmCyISAPgJqC9W+FfCjAUKA5Eq2oj4Efg7+4kHwCPqWpTYI1P/4+ByaraDGgH7HH7twDuBxoClwLtRaQccD3QyJ3PM4FcR2P8sQRhTNa6Aa2AZW71JN1wduSpwKfuOB8BHdz6+cuo6o9u//eBTm4dQdVUdTaAqiaqaoI7zlJVjVXVVGAlUBOIAxKBaSJyA3BmXGNyjSUIY7ImwPuq2tx91VPVJ/2Md6H11pzy+ZwChKhqMk5NpLOA3kBhqFPI5DGWIIzJ2jygv4hUBBCRsiJyCc7/p787zhBgkarGAUdEpKPbfxjwo9t6X6yIXOfOo6iIhGe0QLe9jtJuJYoPAM0CsF7GZKrQ1+ZqTFZUdb2IPI7TUpkHSAJG4zTO0sYdth/nOgXAcGCKmwB8a9ccBrwpIk+78xiQyWJLAl+ISBjOGcyDObxaxmTJanM15gKJSLyqlgh2HMYEihUxGWOM8cvOIIwxxvhlZxDGGGP8sgRhjDHGL0sQxhhj/LIEYYwxxi9LEMYYY/z6fyHnfvKuhruxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "epochs = np.arange(1,epoch+1)\n",
    "print(epochs)\n",
    "plt.plot(epochs,loss_train[:-1],label='train_loss')\n",
    "plt.plot(epochs,loss_test[:-1],label='validation_loss')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('loss')\n",
    "plt.title('XMLRobertaForTokenClassification_earlyStopping')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2 3 4 5 6]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA42ElEQVR4nO3dd3gVZfbA8e8hCQRI6AkKoUOoEpXQWUVFRVFcUSygAhbUta666loWC+vaO/tTRAQERSwoIuJiQVQUDIj0XkMnlCSE9PP7YybhEhJygdxMkns+z3OfZPqZufe+Z+Z9574jqooxxpjgVcnrAIwxxnjLEoExxgQ5SwTGGBPkLBEYY0yQs0RgjDFBzhKBMcYEOUsEZZyIzBaRm72Owwsi0ltEEivadkXkLRF53Gf4dhHZKSKpIlLX/ds8ANtdJiK9S3q9gVIeP/si8oiIjPE6juMV9IlARCJEZKOIDPYZFykim0XkShEZJyIqIpcVWO4Vd/xQd3ioiPxcxDZmi0i6+wXfIyKficipAd2xIhzvl0tEnhCRLDf2vNeDJ7Bd3+VzReSQz/Dg4tdQskSki4jMEJH9IrJXROaLyLDS2Laq3qaqT7txhAEvAxeoaoSqJrl/15/MNtzP7cgC222vqrNPZr1llVsAb3A/T4ki8pHPtFJLKKr6jKqWq+QFlghQ1VTgVuBVEYlyRz8PJKjqJ+7wauCGvGVEJBS4Clh3HJu6U1UjgJZABPDiycZ+PMRxou/3R27hlPd6/ji3Heq7PLAZuNRn3KQTjOuEiEh34HvgR5z3oy5wO3BRacbhqg+EA8s82HaZdLyfVREZAlwP9HE/X/HAd4GKryIK+kQAoKrfAF8Br7uXzlcBf/OZ5Uugl4jUdof7AouBHSewrf3A58DpeeNEpIeI/C4iB9y/PQos1sI9Y00WkS9EpI7Pst1EZK57Zvun76W/eyb0bxH5BUgD3gf+Arzpnjm96c73mohscde/QET+4s++iEh/t7phv7uttj7TNorIQyKyGDjoJs/C1lFFRF4VkW3u61URqVLEvHeLyHIRiXGXe9G9ctspTnVLVXe+3u5Z4f0isktEthc4238BGK+qz6nqHnUsUNWritjuwyKyTkRS3O1f7jOtpYj86L53e/LORN3C7BV3+8kiskREOrjTxonISBGJBVa5q9ovIt+701VEWrr/VxWRl0Rkk7uNn33282MR2eGOnyMi7d3xw4HBwIPu+/ylz3vSp7jj7sfxK1Qx70ltEZkuIrtFZJ/7f4zPsgU/q819plUW56rtNJ9x0SKSJs7JW2fgG1VdB6CqO1R1tDvfvyn8M1/kd86N5T9SyHdORJq6789w97htF5EHfJZ9QkQmFph3iHtM9ojIoz7zVhWR8e7xWCEiD4oHVaEAqKq9nG42agPbgT3AMJ/x44CRwGjgdnfcFOBa4GdgqDtuKPBzEeueDdzs/l8X+Bb4wh2uA+zDOaMJdde7D6jrs+xWoANQHfgUmOhOawgkARfjJPXz3eEon2U3A+3ddYf5xuIT33VuXKHA/TgJLtyd9kTe9gosEwscdLcZBjwIrAUqu9M3AouARkDVAstuxDl7A3gK+A2IBqKAucDT7rTeQKL7/7+AhT779gowzT1+kTjJ+j8+y2W76w5zj0+a+x5XA3KAc47xWcjfrjs8EGjgHuOr3f0+1Z32IfCoOy0c6OWOvxBYANQCBGjrs8w4YKT7f1NAgVCf7SnQ0v1/lPueNQRCgB5AFXfaje6+VwFeBRYV/NyexHEv9PgV8x061ntSF7jCPf6RwMfA5wW+I0V+VoH/As/5zH8P8KXP53cv8A+cq4GQor5/JfCdy3u/PnSnnQbs9jmuTxQy7ztAVSAOyADautOfxbkqrQ3E4JxcJh7rGAes/PNio2X1hVNApwE1C36hgF7Arzhf7J3uG3s8iSANOOB+MBYBjd1p1wPzC8z/q896ZwPP+kxrB2TiFAoPAe8XWPYbYIjPsk8d60tRRLz7gDg9/MHOBPb7vBoAjwNTfJap5H55ervDG4Ebi1j/Rp8vzjrgYp9pFwIb3f97u+t82T3WNd3xglMYt/BZrjuwwWe5QxxZuO4CuuEUqAq0Ocb+9z7WF9J9/y5z/5+Ac5IQU2Cec3GqFLsBlQpMG4cficA9pofy3oti3rNa7nI1C27jBI97ocfvGNs/5ntSyPynA/sKfC6L/KwCXXEShbjDCcBVPvMOxvn+HsQ5GXqoqM88J/edy3u/2vhMfx541+f7UjARxPjMOx+4xv1/PXChz7Sb8SgRWNWQS0Suw3njvgWeKzhdVX/GOXN6FJiuqoeOcxN3q2pNoCOHzwDAKVQ3FZh3E06BlWdLgWlhQD2gCTBQnKqZ/SKyHydhnVrEsoUSkQfcS9MD7jpquuvPM0VVa/m8thWMW1Vz3W0VFXdRCu7/JndcnlrAcJwzywPuuCicM8sFPvs90x2fJ0lVs32G03DaZvYBuRx5jI5JRG4QkUU+2+rA4ePzIE4hOF+carIbAVT1e+BNnDP6XSIyWkRq+LtNVz2cq4yj2qJEJEREnhWnyioZp5DPW8YfxR33oo5fUY75nohINRF5263iSgbmALVEJMRnHUV+XlR1nhtDbxFpg5Mop/lMn6SqfXA+L7cBT4vIhUWs7mS+c0VN9z12BflWIfsexwYF1uPP9yUgLBHg1DfiXNbegtNwfJUUXk8+EafqZMKJbktVl+BcYYwSEQG24RTovhrjnAnnaVRgWhZOFdYWnCsC30K6uqo+67vJgiH4Drj7+SBOu0htVa2Fc+UixezKEXG7+9KoQNwFt13senD2b5vP8D7gEuA9EenpjtuDc8ba3me/a6rTUHhMqpqGc/Z3hR+xISJNcC7t78SpOqgFLMU9PurUR9+iqg1wPjv/Fbd+X1VfV9VOOGeUsThVF8djD5AOtChk2iDgMqAPTuJumhey+7e4Y1/ccT9exb0n9wOtga6qWgM4q0C8/sQ8Hqca6HrgE1VNLziDqmap6sc41SwdiljvyXznipp+IsduO4dPCAuus1RZInC8iVNf+YOqbscpGN+RoxstX8epE59TxHpERMJ9X0XMNx7nbpH+wAwgVkQGiUioiFyNU3BM95n/OhFpJyLVcOptP1HVHJzEdKmIXOieIYa7DX0xFG0nPg1xOPW12Tj1nKEi8i/AnzPXKUA/ETlPnFsg78ep/5zrx7K+PgQeE5EoEamH0xYw0XcGdW55HAx8JiJd3KuPd4BX3CSOiDQ8xhlgQQ8CQ0XkHyJS110+TkQmFzJvdZyCZLc73zAOFzCIyECf473PnTdXRDqLSFf32BzEKdBz/Ywvb79zgbHAyyLSwH2Pu7ufy0ic452Ecyb+TIHFC77PBRV73E8g1mO9J5E4iWK/2/A64gQ2MxG4HCcZ5J+MiXPrdj9xbvuuJCIX4bQ1zHNnKXgsTuY7l+dx9yqnPTAM+IjjNwX4pzgN6Q1xTjY8EfSJQET+ilOdkn+2pqpjcDL8v3znVdW9qvqduhV6heiB82HPf0khd8uoaibwGvC4qibhnPHej/OlfhC4RFV9zz7ex6nz3YFTVXC3u54tOGeFj+AUVFvc/TjW+/oacKV7p8LrOG0KM3HqszfhFFjFXqKq6iqcL+QbOGdKl+LcEppZ3LIFjMSp710MLMFpEB5ZcCZVnYXTOPqliJyJ0z6yFvjNrWr4FueMs1iqOhenDv9cYL2I7MWp559RyLzLgZdwriJ24jQO/uIzS2dgnoik4lRV3KPObwBq4BSM+3COaxLO3UrH6wGc4/I7ToPoczjv7wR3vVuB5TgNv77eBdq51TSfF7Jev477cTrWe/IqTrvaHjfWmce7cvfzvhAn2f7kMykZ5zuwGacN63mcGzvyftdzxGf+ZL5zPn509/U74EVV/d/x7g9OgkkENuAcq09wknupk6LLNGOMKVtEZCywTVUfC+A2ZuM0+B71C2ERaYpTcIcVaEMpie3ejtOQfHZJrtcfhd7bbYwxZY1bCA8AzvA4lBIhTu8CzXGuNlvhXKG86UUsQV81ZIzxn3tnVGohr4B2EyIiT+M00r+gqhsCua1SVBl4G0jB+aX7Fzi/lyh1VjVkjDFBzq4IjDEmyJW7NoJ69epp06ZNvQ7DGGPKlQULFuxR1ajCppW7RNC0aVMSEhK8DsMYY8oVESn4a+p8VjVkjDFBzhKBMcYEOUsExhgT5MpdG0FhsrKySExMJD39qD6ozAkIDw8nJiaGsLAwr0MxxpSCCpEIEhMTiYyMpGnTpjidYJoTpaokJSWRmJhIs2bNvA7HGFMKKkTVUHp6OnXr1rUkUAJEhLp169rVlTFBpEIkAsCSQAmyY2lMcKkQVUPGGFOe5eYqaVk5pKZnk5KeRUpGNinp2aSmZ5OakUVKujN8bpto4hrVKvHtWyIwxpgTpKqkZ+WSkpHlFuLZpGa4hXn+/0eOyxv2LfRTM7Lxp9u3qMgqlgiC3ezZs6lcuTI9evQ4ruUSEhKYMGECr7/+eoAiM6b8yczOJSU9K7+w9i2wjxyXlX92npKe7Rbch8dl5xZfgoeHVSIyPIzIKqFEhocSER5KVER1IsJDiagSSg13XGR4GBHuPJHhoURUCcufP6JyKJUqBaba1hJBOTJ79mwiIiIKTQTZ2dmEhhb+dsbHxxMfHx/o8IwpNWmZ2SSlZjpn1oWcbaeku2foPlUseWftqRnZJKdnk5ld/JNDw0LEKcDdAjsyPJSGtaoSGR7pMy6MiHC3MK9yeFxeYV69SihhIWW7ObbCJYInv1zG8m3JJbrOdg1qMOLS9secZ+PGjfTt25du3boxd+5cOnfuzLBhwxgxYgS7du1i0qRJtGvXjrvuuouEhAREhBEjRnDFFVcwc+ZMHnnkEXJycqhXrx7fffddoet/6623CAkJYeLEibzxxhu8++67hIeH88cff9CzZ0+uueYa7rnnHtLT06latSrvvfcerVu3Zvbs2bz44otMnz6dJ554gs2bN7N+/Xo2b97Mvffey913F3wKnzFlQ1pmNmt3pbJ6ZyprdqawemcKq3emsnX/oWMuV0k46uw6KqIKzetFuGfeoe7ZuTPP4XGHz8Ajw0OpEhpSSnvqrQqXCLy0du1aPv74Y8aOHUvnzp354IMP+Pnnn5k2bRrPPPMMrVu3pmbNmixZsgSAffv2sXv3bm655RbmzJlDs2bN2Lt3b6Hrbtq0KbfddhsRERE88MADALz77rskJiYyd+5cQkJCSE5O5qeffiI0NJRvv/2WRx55hE8//fSoda1cuZIffviBlJQUWrduze23324/HjOeSs/KcQv8lMOF/q4UEvcdyq87rxxSieZR1enUpDbXdmlEdGS4T6EddkQVS9WwELv77ThUuERQ3Jl7IDVr1ozTTjsNgPbt23PeeechIpx22mls3LiRLVu2MHny5Pz5a9euzZdffslZZ52V/+OtOnXqHNc2Bw4cSEiIc9Zy4MABhgwZwpo1axARsrKyCl2mX79+VKlShSpVqhAdHc3OnTuJiYk5kV025rikZ+WwfvdB1uxKOaLQ37Q3Lb/ADwsRmteLIC6mFgM7NSK2fgSt6kfSpE41Qst4FUt5VeESgZeqVKmS/3+lSpXyhytVqkR2dnZ+gV2Sqlevnv//448/zjnnnMPUqVPZuHEjvXv3LjbOkJAQsrNL9BncxpCZncv6PUdW6azZmcrGpIPkta2GVhKa1qtOuwY1+OsZDYmtH0ls/Qia1K1e5uvUKxpLBKXo/PPPZ9SoUbz66quAUzXUrVs3/va3v7Fhw4b8qqGirgoiIyNJTi66/ePAgQM0bNgQgHHjxpV0+MYcJSsnlw17Dh5ZpbMzhY1JaeS4JX5IJaFJ3WrE1o/kko6n0qp+JLH1I2lWrzqVQ63ALwssEZSixx57jDvuuIMOHToQEhLCiBEjGDBgAKNHj2bAgAHk5uYSHR3NrFmzCl3+0ksv5corr+SLL77gjTfeOGr6gw8+yJAhQxg5ciT9+vUL9O6YIJKdk8vGpDS3oE9l9a4U1uxMYcOeg2TlOAW+CDStW51W0RFc1OFUWtWPILZ+JM2jqgdNo2t5Ve4eXh8fH68Fn1C2YsUK2rZt61FEFZMd0+CUk6tsSjro02Dr/F2/+yCZOc7tliLQqHa1/Lr72PoRtIqOpGV0BOFhVuCXVSKyQFULvY88oFcEItIXeA0IAcao6rMFpg8FXgC2uqPeVNUxgYzJGOMU+Fv2pjl19z5366zbnXrE/fUxtasSWz+Ss1tHERvtVOm0jI6gamUr8CuSgCUCEQkBRgHnA4nA7yIyTVWXF5j1I1W9M1BxlEfvvfcer7322hHjevbsyahRozyKyJRXubnK1v2HjqjDX7UzhXW7U0nPOlzgN6gZTuwpkfylVT1aRUfkF/jVq1jtcTAI5LvcBVirqusBRGQycBlQMBGYAoYNG8awYcO8DsOUI6pOgb9mp8+9+LtSWLsrlbTMnPz5TqkRTqv6EXRr3iS/aqdVdASR4fY7kmAWyETQENjiM5wIdC1kvitE5CxgNfB3Vd1ScAYRGQ4MB2jcuHEAQjWmfFq9M4XnZ67k13VJHPQp8KMjqxBbP5KrOzfKvy2zZXQkNatagW+O5vV135fAh6qaISK3AuOBcwvOpKqjgdHgNBaXbojGlD17D2byyqzVfDB/M9UrhzDgzBhanxKZX+jXqlbZ6xBNORLIRLAVaOQzHMPhRmEAVDXJZ3AM8HwA4zGm3MvMzmXCrxt57bs1pGXmMLhrY+7tE0ud6lbwmxMXyETwO9BKRJrhJIBrgEG+M4jIqaq63R3sD6wIYDzl3ol2Qw1Op3Vz585l0KBBxc9syhxV5fuVu/j3VytYv+cgf2lVj8cvaUds/UivQzMVQMASgapmi8idwDc4t4+OVdVlIvIUkKCq04C7RaQ/kA3sBYYGKp6K4FjdUBdn48aNfPDBB5YIyqFVO1IY+dVyflqzh+b1qjN2aDzntI62TtVMyVHVcvXq1KmTFrR8+fKjxpW2DRs2aOvWrXXIkCHaqlUrHTRokM6aNUt79OihLVu21Hnz5mlKSooOHTpUO3TooKeddpp+8sknqqr69ddf6xlnnKEdO3bUc889t8j1169fXxs0aKBxcXE6Z84c3bVrlw4YMEDj4+M1Pj5ef/75Z1VVnT17tsbFxWlcXJyefvrpmpycrF27dtUaNWpoXFycvvzyy8XuT1k4psEuKTVDH526WJs9PF1PGzFT3/1pvWZm53gdlimncE7ACy1XvW4sLnlfPww7lpTsOk85DS56ttjZSrsb6kGDBvH3v/+dXr16sXnzZi688EJWrFjBiy++yKhRo+jZsyepqamEh4fz7LPP5j+TwJRtBdsBru/WhHv7xFLb2gFMgFS8ROCh0u6G+ttvv2X58sM/y0hOTiY1NZWePXty3333MXjwYAYMGGBdTJcTqsp3K3bx7xkr2LDnIGfFRvF4v7a0snYAE2AVLxH4ceYeKKXdDXVubi6//fYb4eHhR4x/+OGH6devHzNmzKBnz5588803JbpdU/JW7khm5PQV/Lx2Dy2iqvPesM6c0zra67BMkLA+YEtRXjfUefK6oZ4zZw4bNmwAKLJqCJxuqFNSUvKHL7jggiN6IV20aBEA69at47TTTuOhhx6ic+fOrFy58qhlTdmQlJrBY58v4eLXfmLJ1gOMuLQdM+89y5KAKVWWCErRY489xr59++jQoQNxcXH88MMPREVF5XdDHRcXx9VXX13k8pdeeilTp07l9NNP56effuL1118nISGBjh070q5dO9566y0AXn31VTp06EDHjh0JCwvjoosuomPHjoSEhBAXF8crr7xSWrtsipCZncuYn9bT+8XZfDh/Czd0b8rsB3ozrGczeyiLKXXWDbUplB3TwFBVvl2xi39/tZyNSWmcHRvF45e0pWW0tQOYwPKsG2pjzGErdyTz9PTl/LI2ydoBTJliiaAMsm6oK5ak1AxenrWaD+dvJjI8jCcubcfgbk2sCsiUGZYIyiDrhrpiyMzOZfzcjbz+3RrSsnK4oXtT7u3TyjqEM2VOhUkEqmo/uS8h5a3dqKxRVWYt38kzM1awMSmNc1pH8Wg/awcwZVeFSATh4eEkJSVRt25dSwYnSVVJSko66rcJxj8rticz8iunHaBldATjhnWmt7UDmDKuQiSCmJgYEhMT2b17t9ehVAjh4eH2a+TjtMdtB5g8fzM1qobxZP/2DOra2NoBTLlQIRJBWFhYfhcNxpSmjOwcxs/dyBvfreVQVg5DejTlnvOsHcCULxUiERhT2lSV/7ntAJuS0ji3TTSPXNyWltERXodmzHGzRGDMcVqxPZmnvlzOr+uTaBUdwfgbu3B2bJTXYRlzwiwRGOOnPakZvPS/1Xz0u9MO8NRl7RnUpTGh1g5gyjlLBMYUIyM7h3G/bOTN7512gKE9mnHPea2oWS3M69CMKRGWCIwpgqryzTKnHWDz3jTOaxPNI/3a0iLK2gFMxWKJwJhCLN/m9Av06/okYutHMOHGLpxl7QCmgrJEYIyP3SkZvDxrFZN/30KtqmE8fVl7rrV2AFPBWSIwBqcd4D23HSA9K4cbezbj7nOtHcAEB0sEJqgVbAfo09b5PUBzawcwQcQSgQlay7Yd4Onpy/lt/V5i60fw/k1d+EsrawcwwccSgQk6u1MyeOl/q/goYQu1q1Vm5F87cE3nRtYOYIKWJQITNNKznHaAUT847QA39WzGXee1omZVawcwwc0SganwnHaAHfx7xgq27D1En7b1eeTiNtYOYIzLEoGp0JZuddoB5m3YS+v6kUy8qSu9WtXzOixjyhRLBKZC2pWSzkvfrGbKAmsHMKY4lghMheLbDpCRncPNvZpx57nWDmDMsVgiMBXG3oOZ3DB2Hku3JnN+u/o8cnFbmtWr7nVYxpR5lghMhbArJZ3rx8xnY9JBRl/fiQvan+J1SMaUG5YITLm3/cAhBr8zjx3J6bw3tDM9WlpjsDHHwxKBKde27E1j0Jjf2H8wiwk3diG+aR2vQzKm3LFEYMqt9btTGTxmHmmZOUy6pSsdY2p5HZIx5ZIlAlMurdqRwuAx81BVJg/vRttTa3gdkjHllt1UbcqdpVsPcM3oX6kk8NGtlgSMOVmWCEy5snDzPq595zeqVQ5lyq3daRkd6XVIxpR7VjVkyo1565O4cdzv1IuswqSbuxJTu5rXIRlTIQT0ikBE+orIKhFZKyIPH2O+K0RERSQ+kPGY8uunNbsZ8t58TqkZzpRbu1sSMKYEBSwRiEgIMAq4CGgHXCsi7QqZLxK4B5gXqFhM+fbdip3cND6BpnWr89Gt3alfI9zrkIypUAJ5RdAFWKuq61U1E5gMXFbIfE8DzwHpAYzFlFMzlmzn1vcX0OaUSCYP70a9iCpeh2RMhRPIRNAQ2OIznOiOyyciZwKNVPWrAMZhyqmpfyRy5wcLOb1RLSbe3JVa1Sp7HZIxFZJnjcUiUgl4GRjqx7zDgeEAjRs3DmxgpkyYPH8z/5y6hO7N6/LODfFUr2L3NRgTKIG8ItgKNPIZjnHH5YkEOgCzRWQj0A2YVliDsaqOVtV4VY2PirKHi1d0437ZwMOfLeHs2CjGDu1sScCYAAvkN+x3oJWINMNJANcAg/ImquoBIL93MBGZDTygqgkBjMmUcW/9uI5nv17Jhe3r8/q1Z1AlNMTrkIyp8AJ2RaCq2cCdwDfACmCKqi4TkadEpH+gtmvKJ1XllVmrefbrlfSPa8Cbg860JGBMKQnoNbeqzgBmFBj3ryLm7R3IWEzZpao8O3Mlb/+4noGdYnj2io6EVBKvwzImaFjlq/FUbq7y5JfLGP/rJq7r1pin+negkiUBY0qVJQLjmZxc5dGpS5j8+xZu7tWMR/u1RcSSgDGlzRKB8UR2Ti4PfPwnny/axl3ntuS+82MtCRjjEUsEptRlZudyz+Q/+HrpDv5xYWvuOKel1yEZE9QsEZhSlZ6Vwx2TFvLdyl08fkk7burVzOuQjAl6lghMqUnLzGb4hAX8vHYP/768A4O7NvE6JGMMlghMKUnNyObG934nYdNeXhwYx5WdYrwOyRjjskRgAu5AWhZD3pvP0q0HeP3aM7ikYwOvQzLG+LBEYAIqKTWD69+dz9pdqfx38Jlc0P4Ur0MyxhRgicAEzK7kdAaPmcfmvWm8MySes2Otw0BjyqJi+xoSkfoi8q6IfO0OtxORmwIfminPtu0/xNWjf2Pr/kOMG9bFkoAxZZg/nc6Nw+k4Lq9idzVwb4DiMRXA5qQ0rnr7V/akZPD+TV3o3qKu1yEZY47Bn0RQT1WnALmQ36toTkCjMuXWut2pXPX2r6RmZPPBLd3o1KSO1yEZY4rhTxvBQRGpCyiAiHQDDgQ0KlMurdyRzHVj5gHw4S3daHtqDY8jMsb4w59EcB8wDWghIr8AUcCVAY3KlDtLtx7gunfnUSW0EpNu7kbL6AivQzLG+OmYiUBEQoCz3VdrQIBVqppVCrGZcmLh5n0MGTufGuFhfHBLV5rUre51SMaY43DMNgJVzQGuVdVsVV2mqkstCRhfv61P4vox86hbvTJTbutuScCYcsifqqFfRORN4CPgYN5IVV0YsKhMuTBn9W6Gv59Ao9rVmHRzV6JrhHsdkjHmBPiTCE53/z7lM06Bc0s8GlNufLt8J3+btJAW0RFMvKkLdSOqeB2SMeYEFZsIVPWc0gjElB9fLd7OPZP/oH3DmkwY1oWa1cK8DskYcxL8+WVxTRF5WUQS3NdLIlKzNIIzZc9nCxO568OFnNG4FhNvsiRgTEXgzw/KxgIpwFXuKxl4L5BBmbLpg3mbuf/jP+neoi7jb+xCZLglAWMqAn/aCFqo6hU+w0+KyKIAxWPKqLE/b+Cp6cs5t000/x18JuFhIV6HZIwpIf5cERwSkV55AyLSEzgUuJBMWfPf2Wt5avpy+rY/hbeu62RJwJgKxp8rgtuB8T7tAvuAoQGLyJQZqsors1bz+vdruez0Brw0MI7QEH/OHYwx5Yk/dw0tAuJEpIY7nBzooIz3VJX/fL2S0XPWc1V8DP8Z0JGQSuJ1WMaYAPDnrqFnRKSWqiararKI1BaRkaURnPFGbq4yYtoyRs9Zzw3dm/CsJQFjKjR/rvMvUtX9eQOqug+4OGARGU/l5Cr//GwJE37dxPCzmvNk//ZUsiRgTIXmTxtBiIhUUdUMABGpCtjPSCug7Jxc7v/4T75YtI27z2vF3/u0QsSSgDEVnT+JYBLwnYjk/XZgGDA+cCEZL2Rm53L3h38wc9kOHuzbmr/1bul1SMaYUuJPY/FzIvIn0Mcd9bSqfhPYsExpSs/K4faJC/hh1W5GXNqOYT2beR2SMaYUFZsIRKQ68D9VnSkirYHWIhJm3VFXDGmZ2dwyIYG565J45vLTGNS1sdchGWNKmT+NxXOAcBFpCMwErsd5oL0p51LSsxgydj6/rkvipYFxlgSMCVL+JAJR1TRgAPB/qjoQaB/YsEyg7U/L5Lox8/hj837euPZMBpwZ43VIxhiP+JUIRKQ7MBj4yh1nfQyUY0mpGVz7zjxWbE/hres60a/jqV6HZIzxkD93Dd0D/BOYqqrLRKQ58ENgwzKBsis5nUFj5pG4L40xQ+I5KzbK65CMMR7z566hOTjtBHnD64G784ZF5A1VvSsw4ZmStHX/IQa/8xu7UjIYN6wL3ZrX9TokY0wZ4M8VQXF6lsA6TIBtSjrIoHfmkZyexfs3daVTk9peh2SMKSNKIhGYMm7trlQGj/mNzOxcPrylGx0a2gPmjDGHBTQRiEhf4DWcxuUxqvpsgem3AXcAOUAqMFxVlwcypmCzckcy142ZBwiTh3en9SmRXodkSkJWOuxYDIkJsHUBHNrndUSmNHS9FWIvLPHVlkQiKLQzGhEJAUYB5wOJwO8iMq1AQf+Bqr7lzt8feBnoWwIxGWDp1gNc9+48wkNDmHRLV1pERXgdkjkRubmwd51b6Cc4f3cuhdxsZ3qNhhBpd34FheyMgKy2JBLBa0WM7wKsdRuXEZHJwGVAfiIo8GyD6oCWQDwG5xfDd3ywkGphIUwe3p3Gdat5HZLx18E9Rxb62xZC+gFnWuUIaHAG9LgLGsZDw05Qw5KAOTn+dDExCxiY1xW1iNQGJqvqhQCqOq6IRRsCW3yGE4Guhaz/DuA+oDJwbhExDAeGAzRubL9+9cfzM1exKSmNycO7WRIoy7IOwfbFhwv9rQtg/yZnmlSC6PbQ/nKnwG8YD1GtoZL9jMeULH+uCOoVfB6BiESXVACqOgoYJSKDgMeAIYXMMxoYDRAfH29XDcX4bX0S4+ZuZGiPpnaLaFmSmwtJa48s9I+o4omBhmdC55ucQr/B6VC5uqchm+DgTyLIFZHGqroZQESa4F8Vzlagkc9wjDuuKJOB//NjveYYDmZk849P/qRp3Wo82Le11+EEt9TdTqG/dYFb8C+EjCKqeGLiIfIUb+M1QcufRPAo8LOI/IjTMPwX3GqaYvwOtBKRZjgJ4BpgkO8MItJKVde4g/2ANZiT8uzXK0ncd4gpt3anWmW7O7jUZB2C7X/6FPoJsH+zMy2viqfD5YcL/XqxVsVjygx/flk8U0TOBLq5o+5V1T1+LJctIncC3+DcPjrW7aLiKSBBVacBd4pIHyAL2Ech1ULGf7+s3cP7v23i5l7N6Ny0jtfhVFxHVfEkwM5lR1bxxHSCzrc4hf6pcVbFY8o0UT12LY+IXA58r6oH3OFaQG9V/Tzg0RUiPj5eExISvNh0mZaSnkXfV3+iSmglZtzzF8LD7GyzxORV8eQV+lv/8KniiYSGZxy+g8eqeEwZJSILVDW+sGn+1B2MUNWpeQOqul9ERgCfl1B8pgQ8M2Ml2w8c4pPbe1gSOBl5VTz5t28ugAN5VTwhUL8ddBhwuNC3Kh5TAfiTCArrqtoqn8uQOat38+H8zdx6dnPObGx9CPktNxeS1hx5z/6u5UdX8XSxKh5TsflToCeIyMs4vxIGuBNYELiQzPFITs/ioU8X0zI6gr/3ifU6nLItdZdP9c6Cwqt4etztFPoNO1kVjwka/iSCu4DHcW7vBJiFcyeRKQNGTl/OrpQMPruuk1UJ+cpMO3wXz7GqeGLinfr9erFQyZ/nNBlT8fiTCNoCrd15Q4FLgUuAjgGMy/jhh5W7mJKQyN96tyCuUS2vw/GGKhxIhN2rYPdK57X9T+cuHs1x5qnZyDnD7zrcKfRPjYPK9mtrY/L4kwgmAQ8AS4HcwIZj/HUgLYuHP1tMbP0I7unTyutwAi831+l6Ib/Ad//uWQ2ZqYfnq1YP6reHXvce7pYhsr5nYRtTHviTCHar6pcBj8Qcl6emL2dPaiZjbuhMldAKVCWUmwP7Nh4+u88v+FdD9qHD80Wc4vS7c/pg529UG+dv9XqehW5MeeXX7aMiMgb4DsjvA1VVPwtYVOaYvl2+k08XJnL3uS05LaacPmQmJwv2ri9Q2K+CPWsgx6er3RoxTgEf38unwI+FqnZ3lDElxZ9EMAxoA4RxuGpIAUsEHtiflsk/py6hzSmR3HluOagSys5wfoVbsMBPWnv4Nk2AWo2dQr7FOW5h38ZpwA2v4V3sxgQJfxJBZ1W13svKiCemLWPfwUzGDetM5dAydJdLZppzT37BOvy960Hzzh8E6jRzCvnWFx2uzqkXa/fnG+MhfxLBXBFpZ4+Q9N7MpTv4fNE2/t4nlvYNPKoSykiFPauOLvD3bSK/U1oJgbotILqt05d+XoFftyWEVfUmbmNMkfxJBN2ARSKyAaeNQABVVbt9tBTtPZjJY58voX2DGvztnBaB32D6AaeB9ohG21WH78UHqBQG9Vo53SnHXXu4Dr9OCwitHPgYjTElwp9EYM8QLgP+9cVSDhzKYuLNXQkLKcEqobS9R5/d714FKdsOzxMa7hT4jbtC1A2H6/BrN4WQsJKLxRjjCX+6od5UGoGYon21eDvTF2/nHxe2ps0pJ9B4quo8B/eoWzJXwcFdh+cLq+bU1zc76/DZfXQbqNXEOlYzpgKzzuPKuD2pGTz+xVI6xtTk1rOaH9/Cy7+A395yCv1Dew+PrxzpFPStLjjyHvyajaybBWOCkCWCMkxVefzzpaSmZ/PiwDhCj6dKKGEsTL/PqdJpe+nhwj6qDdRoACKBC9wYU65YIijDpi/eztdLd/BQ3zbE1o/0f8FfXodZj0OrC+Gq8XanjjHmmCwRlFG7UtJ5/IulnN6oFrf8pZl/C6nCD8/AnOed2zYvH2137xhjimWJoAxSVR6dupS0zBz/q4Ryc+GbR2De/8EZ18Glr1sDrzHGL9YyWAZ9sWgbs5bv5B8XtKZldETxC+TmwJd3OUmg6+1w6RuWBIwxfrMrgjJmZ3I6I6Yto1OT2tzYy48qoexM+OwWWP45nP0Q9P6nNQQbY46LJYIyRFV55LMlZGTn8MKVHQmpVEyBnnUIptwAa/4H5z8NPe8unUCNMRWKJYIy5NOFW/lu5S7+dUk7mkcVUyWUkQIfXAObfoFLXoX4YaUSozGm4rFEUEZsP3CIJ79cRpemdRjao+mxZ07bC5OuhG2L4IoxcNqVpRGiMaaCskRQBqgqD3+6hOwc5YWBHal0rCqhlJ3w/l+d/vyvnghtLi61OI0xFZMlgjJgSsIWfly9myf7t6dJ3WP0y79/M0y4DFJ2wKApzkNcjDHmJFki8NjW/YcYOX0F3ZrX4fpuTYqecc9aJwlkpMD1nzs9gRpjTAmwROAhp0poMTmqvHBlXNFVQjuWOtVBqjB0Opxqj4IwxpQc+0GZhz6cv4Wf1uzhkYvb0qhOtcJnSkyAcRdDSGUY9rUlAWNMibNE4JEte9P491fL6dWyHoO7Ni58pg1zYHx/qFrbSQJRsaUbpDEmKFgi8EBurvLQp4sREZ694jSksF8Cr/4GJl4JtRrBsJlQ+xjtB8YYcxIsEXhg0rxNzF2XxGP92hJTu5AqoaWfwuRBzsPfh86AGqeWfpDGmKBhiaCUbU5K45kZKzkrNoqrOzc6eoaFE+CTmyCmCwz5EqrXLf0gjTFBxRJBKcrNVR745E9CQ4TnCqsS+vW/MO0uaHkeXPcphJ/A84mNMeY42e2jpWj8rxuZv2EvL1zZkVNr+jw1TBV+fB5mPwNt+8MV79oDZYwxpcYSQSnZsOcgz81cybltormyU8zhCarwv8fg1zchbhD0fwNC7G0xxpQeK3FKQU6u8o+P/6RySCX+M8CnSig3B766DxaMgy7Doe9zUMlq64wxpcsSQSl475cNJGzax8tXxVG/RrgzMicLpt4GSz+Bv9wP5z5uD5QxxngioKefItJXRFaJyFoRebiQ6feJyHIRWSwi34lIhbtZft3uVF74ZhV92tbn8jMaOiOz0uGj650k0OcJOO9flgSMMZ4JWCIQkRBgFHAR0A64VkTaFZjtDyBeVTsCnwDPByoeL+TkKg98/CdVK4fwzIAOTpVQRip8MBBWfw0Xvwi9/u51mMaYIBfIK4IuwFpVXa+qmcBk4DLfGVT1B1VNcwd/A2KoQMb8tJ4/Nu/nyf7tiY4Mh0P7nM7jNv4Ml78NXW7xOkRjjAloImgIbPEZTnTHFeUm4OsAxlOq1uxM4aVZq+nb/hT6xzWA1N0w7lLnqWIDx0PcNV6HaIwxQBlpLBaR64B44Owipg8HhgM0blxEB21lSHZOLg98/CcRVUIZeXkHJHkrTPgrHEiEQZOhZR+vQzTGmHyBvCLYCvj2oRDjjjuCiPQBHgX6q2pGYStS1dGqGq+q8VFRUQEJtiS9PWc9fyYe4OnLOlAvIxHGXgSpO+H6qZYEjDFlTiATwe9AKxFpJiKVgWuAab4ziMgZwNs4SWBXAGMpNSt3JPPqt6vp1/FU+tXfB+9dBJmpTr9BTbp7HZ4xxhwlYIlAVbOBO4FvgBXAFFVdJiJPiUh/d7YXgAjgYxFZJCLTilhduZDlVgnVrBrGM12znQfKSCXnWQINTvc6PGOMKVRA2whUdQYwo8C4f/n8X6HqSd6avY6lW5P56MIcan40DKrVhhumQZ1mXodmjDFFsv4MSsjybcm8/v0aHmq5ha6/3OI8Q+DGbywJGGPKvDJx11B5l5ntVAkNqLKA27a95jxQ5vqpUL2e16EZY0yxLBGUgFE/rKXtzuk8W3k00qgLDPoIqtbyOixjjPGLJYKTtHTrAZJ/HMVLlcdB83PgmklQubrXYRljjN+sjeAkZGTn8Pv7jzEidBxZrS52rgQsCRhjyhlLBCdKlUVj72VY+gR2NOlP2DUTILSK11EZY8xxs0RwInJz2fPRnXTdNoFfa1/GKUPGQ0iY11EZY8wJsURwvHKyyZl6G/VWTuT9kL/S7pYx9lQxY0y5ZiXY8cjOgI+HELLkI57PuorGV71AzWr2kHljTPlmdw35K/MgfHQdrPueJ7KGkNHpZs5uHe11VMYYc9IsEfgj/QBMugpNnM8L4XcxK/xcZl7c1uuojDGmRFgiKM7BPTBxAOxcztQWT/Pfpc2YeFNHIsOtcdgYUzFYIjiW5O0w4TLYv4k1543m/ulVGNy1Mb1aWdcRxpiKwxJBUfZtdJLAwSQyrv6Y4V/k0rBWLv+0KiFjTAVjdw0VZvcqGNvXaRsY8gXPrazHhj0Hef7KjkRUsdxpjKlYLBEUtG2R81Sx3BwYOoN5GU15b+4GhnRvQo8WViVkjKl4LBH42vwbjL8UwqrBjTNJqx3LPz5ZTKPa1XjoojZeR2eMMQFhiSDPuu/h/cshIhpunAl1W/Dc1yvZsi+NFwfGUa2yVQkZYyomSwQAK6bDB1dDnRbO84VrxjB33R7G/7qJYT2a0aVZHa8jNMaYgLFE8OdHMOUGODUOhn4JEdEczMjmwU8W06xedf5xYWuvIzTGmIAK7kTw+7sw9VZo2hOu/xyq1gbgP1+vYOv+Q7xwZUeqVg7xNkZjjAmw4E0EP78KX90HsX1h0MdQJcIZvWYPE3/bzM29mhHf1KqEjDEVX/C1gKrC9yPhpxehwxVw+dv5zxJISc/ioU8X0zyqOvdfYFVCxpjgEFyJIDcXvvknzHsLzrwBLnkVKh2u+nlmxgq2HzjEJ7f3IDzMqoSMMcEheBJBbg5MuxsWTYTud8IFI0Ekf/KPq3fz4fwt3HZ2C85sXNvDQI0xpnQFTyKY/ayTBHo/Amc/eEQSOHAoi4c+WUyr6Aju7dPKwyCNMab0BU8i6HY71G4KZww+atLI6cvZnZrB29d3siohY0zQCZ67hqrVKTQJfL9yJx8vSOT2s1sQ16hW6cdljDEeC55EUIgDaVk8/OkS2pwSyV3ntfQ6HGOM8UTwVA0V4snpy9h7MJOxQztTJdSqhIwxwSlorwhmLd/JZwu3csc5LenQsKbX4RhjjGeCMhHsO5jJI1OX0PbUGtxxjlUJGWOCW1BWDT3x5TL2Hcxk/LAuVA4NylxojDH5gq4UnLl0O18s2sbd57WiXYMaXodjjDGeC6pEkJSawaNTl9KhYQ1u793C63CMMaZMCKqqoX9NW0ZyehYfDOxGWEhQ5UBjjClS0JSG0xdv46vF27m3TyytT4n0OhxjjCkzgiYR1AgP4/x29bn1rOZeh2KMMWVK0FQNnRUbxVmxUV6HYYwxZU5ArwhEpK+IrBKRtSLycCHTzxKRhSKSLSJXBjIWY4wxhQtYIhCREGAUcBHQDrhWRNoVmG0zMBT4IFBxGGOMObZAVg11Adaq6noAEZkMXAYsz5tBVTe603IDGIcxxphjCGTVUENgi89wojvuuInIcBFJEJGE3bt3l0hwxhhjHOXiriFVHa2q8aoaHxVlDb7GGFOSApkItgKNfIZj3HHGGGPKkEAmgt+BViLSTEQqA9cA0wK4PWOMMScgYIlAVbOBO4FvgBXAFFVdJiJPiUh/ABHpLCKJwEDgbRFZFqh4jDHGFE5U1esYjouI7AY2neDi9YA9JRhOeWD7HBxsn4PDyexzE1UttJG13CWCkyEiCaoa73Ucpcn2OTjYPgeHQO1zubhryBhjTOBYIjDGmCAXbIlgtNcBeMD2OTjYPgeHgOxzULURGGOMOVqwXREYY4wpwBKBMcYEuaBIBCIyVkR2ichSr2MpLSLSSER+EJHlIrJMRO7xOqZAE5FwEZkvIn+6+/yk1zGVBhEJEZE/RGS617GUBhHZKCJLRGSRiCR4HU9pEJFaIvKJiKwUkRUi0r1E1x8MbQQichaQCkxQ1Q5ex1MaRORU4FRVXSgikcAC4K+quryYRcstERGguqqmikgY8DNwj6r+5nFoASUi9wHxQA1VvcTreAJNRDYC8aoaND8mE5HxwE+qOsbtsqeaqu4vqfUHxRWBqs4B9nodR2lS1e2qutD9PwWnm48T6ga8vFBHqjsY5r4q9JmOiMQA/YAxXsdiAkNEagJnAe8CqGpmSSYBCJJEEOxEpClwBjDP41ACzq0mWQTsAmapakXf51eBB4FgeriTAv8TkQUiMtzrYEpBM2A38J5bBThGRKqX5AYsEVRwIhIBfArcq6rJXscTaKqao6qn43R73kVEKmxVoIhcAuxS1QVex1LKeqnqmTiPwb3DrfqtyEKBM4H/U9UzgIPAUc+APxmWCCowt578U2CSqn7mdTylyb10/gHo63EogdQT6O/WmU8GzhWRid6GFHiqutX9uwuYivNY3IosEUj0ubr9BCcxlBhLBBWU23D6LrBCVV/2Op7SICJRIlLL/b8qcD6w0tOgAkhV/6mqMaraFOd5H9+r6nUehxVQIlLdvfkBt3rkAqBC3w2oqjuALSLS2h11Hj7Pfi8JgXx4fZkhIh8CvYF67vMPRqjqu95GFXA9geuBJW6dOcAjqjrDu5AC7lRgvIiE4JzkTFHVoLilMojUB6Y65zmEAh+o6kxvQyoVdwGT3DuG1gPDSnLlQXH7qDHGmKJZ1ZAxxgQ5SwTGGBPkLBEYY0yQs0RgjDFBzhKBMcYEOUsExgSYiPQOlp5BTflkicAYY4KcJQJjXCJynfs8g0Ui8rbbgV2qiLziPt/gOxGJcuc9XUR+E5HFIjJVRGq741uKyLfuMxEWikgLd/URPv3JT3J/+Y2IPOs+M2KxiLzo0a6bIGeJwBhARNoCVwM93U7rcoDBQHUgQVXbAz8CI9xFJgAPqWpHYInP+EnAKFWNA3oA293xZwD3Au2A5kBPEakLXA60d9czMpD7aExRLBEY4zgP6AT87nbJcR5OgZ0LfOTOMxHo5fYPX0tVf3THjwfOcvvAaaiqUwFUNV1V09x55qtqoqrmAouApsABIB14V0QGAHnzGlOqLBEY4xBgvKqe7r5aq+oThcx3on2yZPj8nwOEqmo2Ts+ZnwCXAMHQZ44pgywRGOP4DrhSRKIBRKSOiDTB+Y5c6c4zCPhZVQ8A+0TkL+7464Ef3SfBJYrIX911VBGRakVt0H1WRE23I8C/A3EB2C9jihUUvY8aUxxVXS4ij+E8+aoSkAXcgfMQkC7utF047QgAQ4C33ILetzfI64G3ReQpdx0Dj7HZSOALEQnHuSK5r4R3yxi/WO+jxhyDiKSqaoTXcRgTSFY1ZIwxQc6uCIwxJsjZFYExxgQ5SwTGGBPkLBEYY0yQs0RgjDFBzhKBMcYEuf8HjVrpK2bJLfIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "epochs = np.arange(1,epoch+1)\n",
    "plt.plot(epochs,mcc_score_train[:-1],label='mcc_train')\n",
    "plt.plot(epochs,mcc_score_test[:-1],label='mcc_test')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('mcc_score')\n",
    "plt.title('XMLRobertaForTokenClassification_earlyStopping')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoConfig\n",
    "config = AutoConfig.from_pretrained('xlm-roberta-base')\n",
    "config.num_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "boo = None\n",
    "\n",
    "if 1<2:\n",
    "    boo = True\n",
    "print(boo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "instance_type": "ml.g4dn.8xlarge",
  "kernelspec": {
   "display_name": "Python 3 (PyTorch 1.6 Python 3.6 GPU Optimized)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/pytorch-1.6-gpu-py36-cu110-ubuntu18.04-v3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}