Two_Berts.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Two_Berts.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "2onnw-J1NCx1",
        "colab_type": "code",
        "outputId": "c02a6793-f329-440c-abf8-6a607d32b395",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xTk7kzw7NPqk",
        "colab_type": "code",
        "outputId": "1dde6e76-58cd-47c9-bb3c-8367f48ddeef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 410
        }
      },
      "source": [
        "!pip install pytorch-pretrained-bert"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pytorch-pretrained-bert\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/e0/c08d5553b89973d9a240605b9c12404bcf8227590de62bae27acbcfe076b/pytorch_pretrained_bert-0.6.2-py3-none-any.whl (123kB)\n",
            "\r\u001b[K     |██▋                             | 10kB 29.6MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 20kB 6.4MB/s eta 0:00:01\r\u001b[K     |████████                        | 30kB 7.2MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 40kB 5.9MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 51kB 6.3MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 61kB 7.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 71kB 8.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 81kB 7.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 92kB 8.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 102kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 112kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 122kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 133kB 8.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.4.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.11.15)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (2.21.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.17.5)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (4.28.1)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (2019.12.20)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-pretrained-bert) (0.3.3)\n",
            "Requirement already satisfied: botocore<1.15.0,>=1.14.15 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-pretrained-bert) (1.14.15)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-pretrained-bert) (0.9.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (2019.11.28)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (1.24.3)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (3.0.4)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (2.8)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->pytorch-pretrained-bert) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->pytorch-pretrained-bert) (2.6.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.15.0,>=1.14.15->boto3->pytorch-pretrained-bert) (1.12.0)\n",
            "Installing collected packages: pytorch-pretrained-bert\n",
            "Successfully installed pytorch-pretrained-bert-0.6.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bo9gfjpgNWed",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp /content/drive/'My Drive'/test_task/bert.py .\n",
        "!cp /content/drive/'My Drive'/test_task/utils.py ."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X_OJIst-Nt8H",
        "colab_type": "code",
        "outputId": "2dbccb39-97d3-4a60-806d-4fc7e2ca910a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from bert import *\n",
        "from utils import *"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 231508/231508 [00:00<00:00, 899889.47B/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rulcOAamN8b0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pytorch_pretrained_bert import BertConfig\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.model_selection import StratifiedShuffleSplit\n",
        "import pickle\n",
        "import torch.optim as optim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pD-tz5eOfMML",
        "colab_type": "text"
      },
      "source": [
        "Let's define the configuration of our model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "feyPD50mOEMW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "config = BertConfig(vocab_size_or_config_json_file=32000, \n",
        "                    hidden_size=768,\n",
        "                    num_hidden_layers=12,\n",
        "                    num_attention_heads=12,\n",
        "                    intermediate_size=3072)\n",
        "\n",
        "num_labels = 5"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3uev9wwnOGsf",
        "colab_type": "code",
        "outputId": "b240cc48-bb6a-4cce-ced6-71e123e15a47",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model = BertForSequenceClassification(config, num_labels)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 407873900/407873900 [00:15<00:00, 26522987.62B/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kysBSASLfUBa",
        "colab_type": "text"
      },
      "source": [
        "First of all, let's have a look at the target distribution."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "05kH-BgjOI1M",
        "colab_type": "code",
        "outputId": "c912b0cd-fd6a-4a4c-93a1-7f7bf60eb4f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 247
        }
      },
      "source": [
        "import pandas as pd\n",
        "df = pd.read_json(\"/content/drive/My Drive/test_task/reviews.json\", lines=True)\n",
        "x = df['overall'].value_counts()\n",
        "plt.pie(x, labels = x.index, shadow=True)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXgb1d0v8O+ZRctYkrc4tuNFchKF\n7AsJERQoLYWyuPCylNJCgRZouS30fWnpYui9t31pe2soXW7pe1vK0re0ZaeAwUCgpRBWh4RAdsch\n2PEm75ZkyZJmOfcPKWmaxbEdac5IPp/n8WM/yljnl8Rfz8yZsxBKKTiOsx6BdQEcxx0ZDyfHWRQP\nJ8dZFA8nx1kUDyfHWRQPJ8dZFA8nx1kUDyfHWRQPJ8dZFA8nx1kUDyfHWRQPJ8dZFA8nx1kUDyfH\nWRQPJ8dZFA8nx1kUDyfHWRQPJ8dZFA8nx1kUDyfHWRQPJ8dZFA8nx1kUDyfHWRQPJ8dZFA8nx1kU\nDyfHWRQPJ8dZlMS6AO7ofA3NEgAvgLr05yoAJQCKABSnPxcB8AAgAIxDPnQAowAGD/kYAPARgN3t\njfW95v2NuKkgfCMja/A1NFcCWJv+WAFgAaW0jhCS7V+gYQBtAHYDaAXwHoC32xvrB7PcLncMPJwM\n+BqaBQAnAfgEgACldC0hpIptVYdpA/B2+uMtAFvbG+v5D4uJeDhN4mtoLgNwLqX0XICeQ4hQyrqm\nKQoCeB7AswBebm+sjzKuJ+/xcGaRr6G5CsAVlBqXA+REQghhXVOGJAD8A0ATgCfaG+sHGNeTl3g4\nM8zX0OwCcAnVtWshiKcTQvK9R1wF8ByAPwB4ob2xXmNcT97g4cwQX0PzSdTQvgkiXEyI4GBdDyN9\nAP4E4IH2xvqdrIvJdTycx8HX0CxQXbuEGvptgmxfxboei3kZwM/bG+vXsS4kV/FwToOvobmAasmv\nAuQWIslW62W1mq0AGgE82t5Yr7MuJpfwcE6Br6HZZiSi3ySS/VYiSoWs68kxHwL4MYAH2xvrDdbF\n5IK8CCchpB1ABKkRMRqldE0m39/X0CzosdBXiOz4kSDbyzL53jPQFgDfbm+sf5l1IVaXT+FcQynN\n+KiWmn9/+FIiyT8XbE5vpt97hlsH4DvtjfVbWRdiVTycR1F1w73zBHvBn0Sl8JRMvSd3GAPAAwC+\n195YP8y6GKvJl3B+BGAEAAVwD6X099N9r4ov/kwQC4p+Knlm30xEyZaxIrmJ9AH4Rntj/eOsC7GS\nfAlnFaW0mxAyG6ku/G9QStdP9X3mXPdfnxBdpX8Une7azFfJTcJTAG7kM2VS8iKcByOE/BDAGKX0\nrsl+T/EnviwrC065Ryqu/BIhQr4MsctVowBuaW+sf4B1IazlfDgJIQUABEppJP31ywBup5S+OJnv\nL7vo1lX2qkVPSu7SuqwWyk3V4wCub2+sD7MuhJV8COdcpC6HgNTk8YcopT851vcp/gDxnHTRt22V\nJ9wuyPaZOtzO6vYAuKy9sf591oWwkPPhnI6iM64uVOaf/KStzPsp1rVwxxQHcHN7Y/09rAsx24wL\nZ8nZ/2O1suCUpyX3rGrWtXBT8hCA69ob6+OsCzHLjAmn4g8Qp//kawoWnHq34Chwsa6Hm5a3AVw4\nU5ZQmRHhVPwBqWDRGf+pLDjlO0Syyazr4Y7LHgDntTfW72FdSLblfTgVf6DAteKc+5xzT/ocEYR8\nn/g8UwwidQZ9m3Uh2ZTX4VT8gVLXyvP+7Jy75tz8WSGES4sD+Hx7Y/0zrAvJlrwNp+IPVLpWnvew\nMu+kM1jXwmWNCuDy9sb6p455ZA7Ky3Aq/kCVe1X9o865q09lXQuXdSpSz0Lz7gyad+FU/IFq94kX\nPOasW8Vnk8wcKoDPtjfWN7EuJJPyqoNE8QdqXSvPe4QHc8aRATzua2j+DOtCMilvwqn4A7XKgo/d\n55y7hl/Kzkw2AE/4Gprz5v8/L8Kp+ANljtplvyxY8skzea/sjGYH8IyvoXk+60IyIefDqfgDblv5\nvJ+4V9XXE0EUWdfDMVcK4HlfQ3OubXdxmJwOp+IP2KWiils9J118BZFsdtb1cJbhB/C0r6E5p38m\ncjacij8gQpSu96y95DrBrhSwroeznNMA3M+6iOORs+EEcJFnzb99XXLPms26EM6yrvQ1NH+ddRHT\nlZPhVPyB5c55J93kqF6ymHUtnOX9wtfQvJJ1EdORc+FU/IHZUknV91zLzvoY61q4nGAH8JivodnN\nupCpyqlwKv6AnUi2mwoDl55DRJkvW8lNlh9Azq2kkDPhVPwBAuBz7lXnXyQqRTnfTc6Z7gu+hubr\nWBcxFTkTTgArbeXzPmevWbqEdSFczvqlr6G5hnURk5UT4VT8AQ9E+Xr36gtOJoRPmOamzQ1g2rsB\nmM3yP+jpy9nL3SvPO0V0emaxrofLeef6GpqvYl3EZFg+nACWyWW+Cxze5StYF8LljZ/7GppLWBdx\nLJYOp+IPuABc7151/ip+OctlUBmAO1kXcSxW/4G/2Dlv7WK+xiyXBdf6GppXsS5iIpYNp+IP1EAQ\nzypYdLql/wG5nEUA3MG6iIlYMpz7O4Fcy86eL9gLLH9vwOWss30NzWezLuJoLBlOAEsEh3u107fq\nRNaFcHnvDl9DsyVn6FsunIo/IAG40rXinPlEkvnuX1y2rQJwBesijsRy4QRwiuBwee2VC5azLoSb\nMW73NTRbbhUNS4VT8QdkAJcWLP1UNRElPrCdM8tcAJeyLuJQlgongNVEspU6qhatZl0IN+N8h3UB\nh7JMOBV/QARwccGST84hkk1hXQ8346zxNTRbausOy4QTwFIIYoWjdjk/a3KsfJt1AQezRDjTzzUv\nds5dXSzYnIWs6+FmrHpfQ/NC1kXsZ4lwIjVT3evwrlrAuhBuRiMAbmBdxH5WCecZoqtUkgrL/KwL\n4Wa8K3wNzRLrIgALhDM98ySgLDxtDp95wlnAbADnsS4CsEA4AawEINor5ufk8oVcXrqGdQEA43Cm\nO4LOsdcsdfIB7pyFXGCFydisz5w1AGoc3hV1jOvguIPZAFzOugjW4VwLQJNLqizTfc1xaRexLoBZ\nOBV/QABwmr1qkSzIDg+rOjjuKM7wNTQz3SCL5ZmzBoDHXr3Ey7AGjjsaO4CzWBbAMpzLAFC5pIoP\nPOCsqp5l4yzDebLoKkkITk8lwxo4biLns2ycSTgVf6AEQJW9ekkpIZZcIYLjAKDK19DMbL1kVmfO\nuQCoXFpdy6h9jpusU1k1zCqcCwGokqcsZzaV4Wask1k1zCqcy4hsj/L7TS4HBFg1bHo4FX/ADaDM\nXrW4mA9053KAn9VQPhbhqAVAbbO8/JKWywUEqZFspmMRzrkAILpKyhi0zXHTMWPCuRhARHC6eTi5\nXLGIRaOmhjM9RawaQFSwF5Sa2TbHHQcmK3SYvRyDAqBAKqpQ+aLRXA5hEk6zL2tnATDk0hq+fTyX\nSzy+hubZZjfKIpxEdJfxS1ou15h+9jQ7nJUAIDpcbpPb5bjjNd/sBs0OZw2AcWJzMp3EynHTUGF2\ng2aHsxhAksgOHk4u15h+K2Z2OIsAqIJs5+Hkco3pQ/jMDmchgCSRbDycXK7J33Aq/oANqSUHdb7F\nH5eD8jecSA1AMFKtCrKJ7XJcJuT1PacCgKa+JHyqGJdrTL/aMzMkIgCAEMLXDeJykOknFDMbFACA\nCBI/a3K5SDS7QTMHvqdOl4LIT5tZ4lU/Gj0l9Ghvcdd8UZZqpbirRpKUMrcs2h2sa8tVFFQQiKiJ\nghg2u20zw5k6YwoiP3NmyaLYB7b/Xbll4dnzhqP+DzfSz7YY4jzVUAbkQnWfqzY+6Ko1xt01RHBV\n2hRHsV3gvygnyw7A9Md/5odTS+omtjmjLNRaZUUEaY71FJzrn6PeerpgW7aBjF/2RsRYPbLVjZGt\nB45NQjK6C6oiQbcvEXJ7qeaaI8sFswtkyc6n8h2ZZnaDpl/WUl3VqaFrRBAtsbV3Plno7E4CkItE\ngzwd6RYvsFVGt66VCrauJfBvQfyzr1CsiFGHQAAbNKEu2uGpi3YAwX++x4iteKzX7QsNeerUmLtW\npAXlLtnmKiJEmOln2bwOp7r/C2roCR7OzFvuDCWRvvyqkKj4yECv/TKhMqEWSfa25YLjp8sB7x4t\n/PkXdHVVRCgRjtBtXpwccRUPjbgwtPnAawkia0FXzUi/u2485PFR1TXHITpLiyRRtpv3t2MuanaD\nZgYkceArXU2AD+HLKEI1Otuu/stUvHk2Kj04EFSvlCo0wyVJANAxX/Lc8Q0JFd1a+KrntNETh8Qq\nkZAJeyLtVJW8kb1l3sheoOefrw/ZS8NBd11kyONTx121Ei2Y7ZJtrsI8fVY2OJmDCCE1AB4EUI7U\nc/3fU0r/73QaZBJOqmtxE9udEebq7WFJQOGhry+VDOe9ff0j18kVCuzCgTNdsEry/OwGyVMyqEev\neFYdPKVXrJQJmdL9ZmliyFOaGPJgcOOB1+LEpva6vCMDnrrxsMcLtWCOQ3KWFIu5vyzN0CSP0wDc\nQil9jxDiBrCJEPIypXTHVBs0M5xxHLjv1BLHOJabomXajjHg8HACwFpBK/5Vd9/gf9SWg0jCv1yK\nDs8SC37zZbHgv0P6+OXNavsn2sUKOyHTfvTioEm5LtI2uy7SBnSnXjMoMOQsCwXddZFhT5027qqR\nUVDmkuWCwhw6yU4qnJTSXgC96a8jhJCdAKoAWDqcCaQf5FI1bvr1e75bjj0Tdlh8iqizftQ50Pe/\nameXEpEc9v8+Vig6779C9P0lZiQvfl7tOKeNlCoQXJmoTSBAWXygsCw+UIiBDQdejwmORNDlHR3w\n1MXDHi/RCiqdkrO4SBQkK469Dh77kH9FCPEBWAWgZToNmhbOWFuLrvgDSQCCEY+MmtXuTLFQ6Dzm\n8+OLkSgP7RvouctbVkGEI49vjiuC7eHP2r2PJwz9My+pHRdsI0VuCEc8Ix8vxYjb54Zby+eGWw+8\nplNCB5XZo33uuWPDHp8Wd9XIRCnzyDaF9dI23VM5mBDiAvAkgJsppdMawGB2j+kIALseDY2Y3G7e\n88kjzskc9yXE54x0DnXeX1taPVHHjWYXxKcvsHubzjPoWa9qnZdspAUlVMz6tCmRUFI+3ldUPt5X\nhP63D7weE53xHpdvdMBTl4h4vEQvqHTKjqJiwbxe/0mHkxAiIxXMv1BK/zrdBs0OZxCATx8b4mfO\nTKIGymzJSZ/dvmnEaka6xI6naoq9xzrWkATy0lm2mpfONHDam1r35W9TuVwXTV8mUtHHHfNDOyvm\nh3YeeE2nhA4oFSNBT93YqKdOj7uqbUQp88iyMyOX44eYVDjTv/DuB7CTUvqL42nQ7HD2AFikhQd4\nODPIp3VEZCemdNl3uxbxjnQL7a9WFfom9Q2CgDdOt1W9cTqweqPae8VrBqlJiqYvenUwkVBSMd5b\nXDHeW4y+tw68PiYVjPe4fKFBT118zO0VdFeFItsLiwVBnNbgdUqpTgjZO8nDTwVwFYCthJD306/d\nRil9fqrtmh3OXgCyNtLTRymlefo8zHTLjB1hYGrhBIC7kyHf1b1C++ZKt28q37dpjVy5aQ2waLs2\n8MWX9eT8mFBlpf9KlxZ1Lhjd7lwwuv3AaxoEo79gznCfuy466vbpcXe1TVBmFcrSsRebI4R8eOPv\nzpzUEwZK6RvYP8njOLG45zSoruo0OT5K7Eqxye3npeV0t3rso47swfiI76I+oePD8oJjXuIeaucS\nqez7SyR492rDV7+gRZeExOojjTqyAgmGMCfaVTIn2lWC4OsHXg9L7livuy406PElo+5aohdUFMiO\nwmLhX/eO3X74O2af2eEcRXo1BD022iPwcGbEQqHruGb6PBEdrD1/UOjsneWc1p6pHXOlkh/dKJVU\ndGuhq57TQpMZdWQVHi2ieEa2KCeMbDnwmgpR7yuoGupx+7RksX/Apsx6a4K3yBqzwzmI1OwUooUH\neuXiOUtMbj8v1cnDk+qpPRqJEPJsuL/qbLGie6TYXjXd9wlWSYU/u0EqPJ5RR1YgQxero/tKq6P7\ngOD6cgAfAV8yvQ5T51bG2lrGAfQBcGrDXT3HOp6bnNm2xHE/h7QTIjw/EixXwskpP2w/VGrUkcN7\nw42Cvq5W70hQmuvDNd9j0SiLic+tANyJ3t09lFIGzeeXam1fxCYiI2cnFyHSCwPBEtuYOpCJ9xsr\nFJ33X2n3fuVmUXjar3eMUyPnRoZRSocX7dr5EYu2WYSzDYDdGI8kaHJ8mEH7eWWZtiOSyfcrEWB7\nti/oEmNaxgaKxBXB9tBn7d7rbpEcDy/V90VgmL7kx3QRQl4/9lHZwSKcPUivX6uPDXUxaD+vrMD0\ne2qPZo5AnU8EgzKJ6xkNkWYXxKcusNd+5TuS+741eucI0XPhl/M/WDXMIpy96XaJOtTJ5HIhnywS\nOrPy6GI+MVx/6glSJDN/KWpIAnnpbHvNDd+VS+4+zejuE/WMXEZnyausGjY9nLG2ljiALgCueOe2\nyY664I6iThrK2sp6K4he+NuuYIJqRnY6dAQBr59uq/rGd+1ld5xNeztt+nF3RmUSpXQYwJZjHpgl\nrFbC2wigUBsNho14dFIzzLkjK7fHszJjZL/TiFZyR2d/mOo045fPB9u0Rq685RZ7xQ8uxECbU++2\nSGfha4t27WRWCKtwtiI9xEkd6WljVEPOq9S6o3YRWV/Hpx7J2bftGxikBs36yok7l0hl37/ZXvW9\ny8nQNrfWaTBMKSFkHau2AXbhbEdqpJCYDLbxcE7Tcn2Hab2eVyBe+fV9Q73UpLC0z5NKb7/JUXPz\nNSS8sUTbp9Ps/2I4WPrv2XSs4wghDxBC+gkh2zJdA5NwxtpaEgC2ASga73i/g2pqrj+kZmIZ3Z00\ns72v01j1FftGOs08mQWrpMI7b3DU3vQVYfz1cq1Do9m9vN7PADYt2rWzdxKH/jeAc7NRA8vV1zcA\ncEHXDHWke8rrq3DAYqHD9EHmtxljted3hzrMbneoTHTdfa3D+9UbBe2lWr09meVRRyIhj07mOErp\negBZeSTEMpy70p9JfN9WZj1iuWyuNMRk3dg71bD35J5wO4u2xwpF531X2n3Xp0YdtWdx1NFjWXrf\nSWMWzlhbyzBSAS2Ot2/uMJLxnBk1YhUVtvGs9tRO5N7EqG9RcMz0M+h+6VFHvmyMOtIofXfRrp37\nMvV+08V6U6FXkZ4krA52bJ34UO5gs/VgzCGB6e5hj8SGvDUDMaY/xAeNOnLdn6FRRxIh92SituPF\nOpzbAOgAxPH2zTycU7BM2878SkMgBM9EBmpmDceZD8M0JEFYd9Coo/5pjjrSKY0BeCTD5U0L03DG\n2lqiADYDKE327u7TYyE+jWySltPdlliYWyaEPD/SV+keTU6mZzP70qOObvquvezOs2mwU57aqCMd\n9JFFu3ZO+j6WEPIwgLcBnEAI6SKEXDfVko+G9ZkTAF4H4ASA+L6t7zCuJWcsFpjd7h3GKRDx+eHg\nLEdE7Wddy8E2rpErbvm2veKHF9D+PU5tUqOObET4zVTaoJR+gVJaSSmVKaXVlNL7p13wIawQzp0A\nxgA4o7vWbzfUeEanQOWredKgpXarLiKQnxsIeqSoNtk9RUyzY6k8+7abHVUNnyPDE406SlK6bdGu\nnZuP9GcsMA9nrK1FBdAMoAy6ZiR7295lXVMuqLDFWK+AfphyQh1P9/U6hXE9xLqWI/lovlRy+02O\nmm9efeRRRyJwJ6vajoR5ONPeQmqOpxTd+domauimb1SaS0q1gZgiQWFdx5F4CVUe7uklSBhjrGs5\nmt7qf446eqNc61Ap1ZLU6BNT94+WYYlwxtpawgDWA5itjw3H1KFOPihhAstMHFM7HYsFw3Nfd1CF\naoyzrmUiQ2Wi69fXOrw3fE3QX1hBfrBo105LnRQsEc60VwDYAJDotldep9QwWBdkVcvpLkv01E4k\nQLTiX3b1RalGLV/rWAEZ/0u9/CDrOg5lmXDG2lq6kHruWaYOd42q/e2bWNdkVYuJdXpqJ3IW1Fm3\nd/aPUJ1a6ox0mLjxi63XbLXcWd4y4Ux7CoACgES2rFtPDd2UGQi5Zp40wGRM7XRcgkTFtzoH+6lB\nLXklZGhGCB7pLtZ1HInVwrkXqUEJs/XwwFgyuGdam47muzkW7KmdyLV0fM61ncPdZs0FnQqaoD+2\n4lkTsFg4Y20tFMBfATgACJEPXnyT6nyu58GKtOF4gUyPufmO1XzLiNZc3DnKfDD5wfSY3i4WiMe1\nTV82WSqcABBra+lE6tFKhRELxeOd29azrslKluk7LPkMcTJ+pEe8Z3SH2lnXAQCUUuhj+le3XrPV\nkpfbgAXDmdaE1D4uYmTz8y36eLiPdUFWkQs9tRP5TTLkW9EbaWddhxbS1rV+p/Vl1nVMxJLhjLW1\n9AF4CcAcGLoxtvXvz1nwdoWJJaQ95/8h/hwf8c3tjzLrcjZUI041ei2r9ifLkuFMa0JqzK0r0bm1\nSx3gj1YAYJ7Un3O7dh3Jk2ODtRVD450s2tZC2l2tt7RafgaUZcMZa2uJIbV4UhkAEt74zN8MLZlz\nG+FkWpUtmlM9tUcjEUKeDfVXFY0kTA2JHtO7jLjxQzPbnC7LhjPtfaQerZQb4+H4eNvbL7AuiCWP\nPppwydTFuo5McRAiPD/SN1sJJ03pU6AGpVpI+3Lb99tMXWZzuiwdzvSjlYeQ6hyyRXe8tn0mj7td\nquVuT+3RuAmk5sFgkW1MzfrK/4lg4o+7G3b/LdvtZIqlwwkAsbaWfqRWQqsCgNA7jzcbiVjGtqfL\nJcvprrx85juLwP5Mf1DJ5LaDh1JH1X2hDaEbsvX+2WD5cKb9HakNZSqN+Fgy8v4LT8zEgfH50FN7\nNNWEKo8Fg1Kmtx0EAEM1kvGO+GX9T/Wbugj38cqJcMbaWnQADwDQALgSXdt74vu2vsK4LNPNl/pk\n1jVk0wJiuB/sCRpIGrFMvu/4h+O3t/+ifcNExxBCHISQDYSQDwgh2wkh/5nJGqYjJ8IJALG2lhEA\n9wCYDUCMbHzmTS088CHjskxVJedHT+1EVhK96L+6+sapZmRksEW8M/6PwRcHfzqJQxMAzqSUrgCw\nEsC5hJCTM1HDdOVMOAEg1tayBcCLAKoBYPSNvzwxU7YQLNAjSbes5304AeDjRC29Y19/6Hi3HVRH\n1Z7QxtCl4c3hY94C0ZT9qzfI6Q+mtxE5Fc60JwHsQ/rxSqjliYepplpyVkEmLdV3hAgxfWsUZupJ\ncvatx7HtoBbVwuFN4Qv7n+6fdCcTIUQkhLwPoB/Ay5RSprOici6c6R3Kfg1ABVCkDnYMR7aseyzf\nO4iWGfnZUzuRKxGv/No0th00EkZ89M3Rr/U82DOlUWWUUp1SuhKpK7O1hJClU/n+TMu5cAJArK1l\nCMCvkNrKwRn/6L328T3vNjMuK6uWko/y+pfP0dxIY9Wf75z8toNUo9roW6M/iu6ITnuxLkrpKIB/\nIEtb+01WToYTAGJtLXsB/A5AJQBpbMu69xLdu15nXFbW+MX87qmdyP/Ux2rPmcRUM2pQGno39PvI\nB5E7wpvDUzrbEkLKCCFF6a+dAM7GP3fCYyJnwwkAsbaWd5G6B60FQELvPPZKItiWl6vGV9kieTNs\nbzp+roZ9a4+x7WBkS+SpUEvoW+HN4encp1YC+AchZAuAd5G653xuGu+TMSTXp2Ip/oAA4MsAPo70\ndvZFp19Vb5tdt4ZpYRnkMKLqTuf18kzqEDqay5TSjl3lBd5DX4/uir4x+OLg+eHN4bzZMSCnz5wA\nEGtrMQD8EanNZLwAyOjrf2pODu57n21lmbNU2zmjemon8mh00Ft9yLaDYzvHNgy+OHhJPgUTyINw\nAkCsrUUDcD+ATUhd4mJ0/R+b1KGuvNhWcLmxK+8fFU2WQAiaIgM1pcPxbgCIbIlsGFo3dFl4c3ha\nW/5ZWV6EEziw58o9SI3BrQWldOS1PzyV7P9oI+PSjtsS7M2JKU5mkQkhzSN9FfLbw68NvzL8hfDm\nsKUWDsuUvAknAMTaWpIAfgtgBwAvKMXo639qjnfteI1xacfFLwVnbE/tkagGpT9vxSt7WiLXhDeH\n97KuJ1vyKpwAEGtriQO4G6lLXB8AIdzyxKuxPS3PWXHd1MmokcMzuqf2YAmNqrdu0DfctRvXhjeH\nc2Pp+2nKu3ACB0YR/Q6pB8k+ANLYB+s2jW19+WGqazk1bchuxLRCm+5hXYcVxFQ6ftdbyYd2f5S8\nMLw5zHyr+2zLy3ACBzqJHkRqkepaAPbxtnfaQm8/el8uDZZfpLWGBN5Vi30hI3jb3+O/aenW/6Op\n1Vo7aGdL3oYTOPCY5RkA9yL1kNmT7PtwYPjv99yrjvTsYFvd5Cw3dmV0bmOuMSilf9urbbn5xfid\ne0foD5pa1bxbquVocn4QwmQp/sAJAG4EYAfQCwDu1Rec4vCuOIsQwbK/pO6M/7jjc0U7DnvoPhNE\nkzT6u43JN1/r0O8B0NTUqlp7t7IMmzHhBADFHygBcAOAEwB0AtAd3pW1rhWfvlSQHZa8r3tau7F7\npWukinUdZts7YnTd8UZiXe8Y/UVTq5oTVzmZNqPCCQCKPyADuBTA+QCCAGKC0233rL3kHNss7yq2\n1R1uE7lqtNSuF7GuwywGpcYLbdr7v9+kPkyB+5ta1Rm5mBswA8MJAIo/QACsBvCV9Eu9AOCcH5hf\nsPiMCwXZYYkVB2Qjobc6v0QEQix72Z1JkQSN3L0h+fo7Xfr/A/BiU6s6owdfzMhw7qf4A7MAXANg\nBYAeAHHB6XGkzqK1K9lWByxTtww/624sYV1HtmkG1dZ36Fvv3ZR8K6ri102t6m7WNVnBjA4ncGBW\ny6kArgJgIHWpC+fc1XXKwo+fIzrd5axquzL+aOdPip6pYdW+GXYN6q2/2ZDcvi9E1wP4Q1OrmvGl\nMXPVjA/nfoo/UAbgagDLkVpDJgpCiGv5p090+ladSSSbYnZNP43/n/YvFG3zmd2uGQaiRvCBzerm\nNzv1VgB/BrC5qVWdkas9HA0P50HSZ9G1AK4A4ELqXlQVHG67+8T6j9vK5weIIIhm1fNX9RtdJ7qH\nqs1qzwzjKo02tWqbHt6mtvwglQAAAAT5SURBVBkUTwP4W1Mr3738SHg4j0DxB5wAPg3gAqQudXsB\nUKmkusi15JOnyrO8q8wI6bvk6pEyu1ac7XbMoBlU29Ctb/3tu8mdoQTWA3iyqTX7+6PkMh7OCaQv\ndS8B8DEAUQADAKhUWO4uWHLmKbbyujVEkLIyY0QyEkar40sQhdzuqY0maeSdLv39h7aq3QMx2grg\nL02tahvrunIBD+ckKP7APAD/htT9aBxAHwAqFhQ7C5addbK9fP5JRJKdmWxzcXL7yPOen+TsWbNv\nzOh6ea+25a87tQHNQASp3eI2zPTHI1PBwzkFij/gBfAZACchtXx/HwCDSDZROeHUxfaqxSdK7lJf\nJtr6fPyJrsaiv+bU/aZuUGPPsLH9iR3a7pZuPYLUv08TgPf4feXU8XBOg+IPVCG1punHABAAgwBi\nACCX+UoV/8kn2sp8K4+nh/fH8Ts6vlj0QU6MqY2pNLK5V//gz1vUzu4IjSO1GsWLAHbzHtjp4+E8\nDumxumuR6jwqRuqSdwCAAVESnL4T6+xzTlgoFc9ZKMj2KU2Yflz9966T3IOWPXOOJWlo16Cx49V2\n7aPXO/QkTa3A/3cArzW1qkHW9eUDHs4MUPwBEcACAJ8EsAaps2kEwAjSm+E4apdV26uXLJJLqk4Q\n7AWlx3rPFlw9XO7QLDM6yKCUDsZoz+4hY8/6Dr3jnS5dRervOQLgOQDvNrWqUbZV5hcezgxT/IEi\nAEuRuuRdmH55HMAwAB0ApKIKj33Oolq5tNorembVCnbX7IPnUwtUo232qwxRIKY9Uz2UblAjlMBg\ncMzo2dJnfPjyh1rfQIw68M9AvgHgfQAd/NI1O3g4s0jxB9xITU87Ganxu/sfi0QAhJF6hgpBKXI6\nqhfX2DylPo+dLPG69eEXSu6qNWvAe1yjscEYDQbHaF/HqNHXOmT0be7VRxM6CpGa/wqkpti1ANgG\noLOpVeU/OFnGw2kSxR9wILXo9VykgjoPqbASpDqTooUYq/CTrhOKSOxFWYBwwiyhaF6xUFrtEUrK\nXaRUkeG0icRmE2GXBdhsIrHJImySALskQD54ORPdoEZSx3hCRzyh0fGEjnhco/GYivFxlcYjScQ6\nRo2BD/r0vn0hSpEaEWVH6hcGQeps/z6AzQD2zKQVCKyCh5MRxR+wAagCUIfU/ercYkQWLiIdikKS\nHUgFJIFUSJIANKQui4/4H0YAuO2QPXZiCydoMpzA/o1nBQBS+kNOf3amX9//Xr0A9gLYg9TjjyCA\n8FTOjoQQEcBGAN2U0s9M9vu4o+PhtJBy/1LnGrK7RCS0FEApUguTVSPVE+xIf4hIhWr/x5HsP4MK\nSIU6htSldBSpy+lupDYg7gPQ39SqHtcO0gBACPkWUp1hHh7OzODhzDEXniBLAGwHfdgP+tpA6nHO\n+EGftWzfHxJCqpHar+YnAL7Fw5kZEusCuKlJL3K1/2xoFb8C8F2kNjPmMiSnB1Vz7BFCPgOgn1I6\npS3euWPj4eSO16kALiSEtAN4BMCZhJA/sy0pP/B7Ti5jCCGfAPBtfs+ZGfzMyXEWxc+cHGdR/MzJ\ncRbFw8lxFsXDyXEWxcPJcRbFw8lxFsXDyXEWxcPJcRbFw8lxFsXDyXEWxcPJcRbFw8lxFsXDyXEW\nxcPJcRbFw8lxFsXDyXEWxcPJcRbFw8lxFsXDyXEWxcPJcRbFw8lxFvX/AfyoT+Q4i/E7AAAAAElF\nTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ls95QMePfkT1",
        "colab_type": "text"
      },
      "source": [
        "As we can see, there are much more good marks, so we deal with the unbalanced dataset. Let's still just text fields and target and transform target to valid format for our model, as it has to looks like [0, 1, 2, 3, 4]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d8-kFgE9OP5n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = df[['reviewText', 'summary', 'overall']]\n",
        "df.loc[:, 'overall'] = df.overall.apply(lambda target: target -1)\n",
        "\n",
        "X = df.drop(columns=['overall'])\n",
        "y = df['overall']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hs01hnJQgl_I",
        "colab_type": "text"
      },
      "source": [
        "Also, we will need to know lengths of tests, so let's visualize it:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cXpVArHAOitR",
        "colab_type": "code",
        "outputId": "2b4e66c3-27c4-4dc8-a7e9-677c3fbd0698",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "source": [
        "df['reviewText'].astype(str).str.split().apply(len).value_counts().sort_index().hist(bins = 100, range = (0, 100), density = True, figsize = (12, 6));"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAFmCAYAAACflwdLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAYsUlEQVR4nO3df8xd910f8Pdn9lJGorUpqazhhDoV\nhhGWraUPCRNbcdu0uAtK+CNd0xWWoFYWWyO6tWgyY0q3oEoBpm1MRKwWzfixDVMCYhYxi0rbZz8E\nLXbariXpsrrBNLYKBVLC3EKL28/+eG7g5pvnia/j+/zwc18vyfI953zPPZ+bT4799nm+95zq7gAA\nAH/hL212AQAAsNUIyQAAMBCSAQBgICQDAMBASAYAgIGQDAAAg5lCclXtr6pHqupEVR1cZfv3VtXH\nquojVfW/quqaqW0/MNnvkar69nkWDwAA66HOdZ/kqtqR5P8meVWSU0mOJXl9dz88NeavdvcfT17f\nlOQfd/f+SVj+uSTXJfnqJL+W5Ou6+0trHe+KK67oPXv2XNCHerY+97nP5dJLL92UY7Nx9Hn70+PF\noM+LQZ8Xw2b1+cEHH/yD7n7Batt2zrD/dUlOdPejSVJVh5PcnOTPQ/KTAXni0iRPJu+bkxzu7i8k\n+e2qOjF5v99Y62B79uzJ8ePHZyhr/paXl7Nv375NOTYbR5+3Pz1eDPq8GPR5MWxWn6vqd9baNktI\n3p3ksanlU0muX+Ugb07y1iSXJHnF1L4fGPbdPcMxAQBg08wSkmfS3fckuaeq/kGSf5Hktln3raoD\nSQ4kya5du7K8vDyvss7LmTNnNu3YbBx93v70eDHo82LQ58WwFfs8S0g+neSqqeUrJ+vWcjjJT5zP\nvt19KMmhJFlaWurN+rGKH+ksBn3e/vR4MejzYtDnxbAV+zzL3S2OJdlbVVdX1SVJbk1yZHpAVe2d\nWrwxyScmr48kubWqnlNVVyfZm+Q3L7xsAABYP+e8ktzdZ6vqjiQPJNmR5N7ufqiq7kpyvLuPJLmj\nqm5I8mdJPpvJVIvJuHdn5Ut+Z5O8+ZnubAEAAFvBTHOSu/tokqPDujunXr/lGfZ9R5J3PNsCAQBg\no3niHgAADIRkAAAYCMkAADAQkgEAYCAkAwDAQEgGAICBkAwAAIOZ7pO86PYcvH/V9SfvvnGDKwEA\nYCO4kgwAAAMhGQAABkIyAAAMhGQAABgIyQAAMBCSAQBgICQDAMBASAYAgIGQDAAAAyEZAAAGQjIA\nAAyEZAAAGAjJAAAwEJIBAGAgJAMAwEBIBgCAgZAMAAADIRkAAAZCMgAADIRkAAAYCMkAADAQkgEA\nYCAkAwDAQEgGAICBkAwAAAMhGQAABkIyAAAMhGQAABgIyQAAMBCSAQBgICQDAMBASAYAgIGQDAAA\nAyEZAAAGQjIAAAyEZAAAGAjJAAAwmCkkV9X+qnqkqk5U1cFVtr+1qh6uqo9W1Xur6oVT275UVR+Z\n/Doyz+IBAGA97DzXgKrakeSeJK9KcirJsao60t0PTw37cJKl7v58Vf2jJD+S5HWTbX/S3S+ec90A\nALBuZrmSfF2SE939aHd/McnhJDdPD+ju93f35yeLH0hy5XzLBACAjTNLSN6d5LGp5VOTdWt5Y5Jf\nnVr+iqo6XlUfqKrvfBY1AgDAhqrufuYBVbck2d/db5osf3eS67v7jlXGfleSO5J8W3d/YbJud3ef\nrqoXJXlfkld29yeH/Q4kOZAku3bteunhw4cv/JM9C2fOnMlll132tPUfO/3EquOv3f3c9S6JdbBW\nn9k+9Hgx6PNi0OfFsFl9fvnLX/5gdy+ttu2cc5KTnE5y1dTylZN1T1FVNyT5wUwF5CTp7tOT3x+t\nquUkL0nylJDc3YeSHEqSpaWl3rdv3wxlzd/y8nJWO/btB+9fdfzJNzx9LFvfWn1m+9DjxaDPi0Gf\nF8NW7PMs0y2OJdlbVVdX1SVJbk3ylLtUVNVLkrwzyU3d/Zmp9ZdX1XMmr69I8q1Jpr/wBwAAW845\nryR399mquiPJA0l2JLm3ux+qqruSHO/uI0l+NMllSX6hqpLkU919U5JvSPLOqvpyVgL53cNdMQAA\nYMuZZbpFuvtokqPDujunXt+wxn6/nuTaCykQAAA2mifuAQDAQEgGAICBkAwAAAMhGQAABkIyAAAM\nhGQAABgIyQAAMBCSAQBgICQDAMBASAYAgIGQDAAAAyEZAAAGQjIAAAyEZAAAGAjJAAAwEJIBAGAg\nJAMAwEBIBgCAgZAMAAADIRkAAAZCMgAADIRkAAAYCMkAADAQkgEAYCAkAwDAQEgGAICBkAwAAAMh\nGQAABkIyAAAMhGQAABgIyQAAMBCSAQBgICQDAMBASAYAgIGQDAAAAyEZAAAGQjIAAAyEZAAAGAjJ\nAAAwEJIBAGAgJAMAwEBIBgCAgZAMAAADIRkAAAZCMgAADGYKyVW1v6oeqaoTVXVwle1vraqHq+qj\nVfXeqnrh1LbbquoTk1+3zbN4AABYD+cMyVW1I8k9SV6T5Jokr6+qa4ZhH06y1N1/M8l9SX5ksu/z\nk7w9yfVJrkvy9qq6fH7lAwDA/M1yJfm6JCe6+9Hu/mKSw0lunh7Q3e/v7s9PFj+Q5MrJ629P8p7u\nfry7P5vkPUn2z6d0AABYHztnGLM7yWNTy6eycmV4LW9M8qvPsO/ucYeqOpDkQJLs2rUry8vLM5Q1\nf2fOnFn12G+79uyq4zerTi7MWn1m+9DjxaDPi0GfF8NW7PMsIXlmVfVdSZaSfNv57Nfdh5IcSpKl\npaXet2/fPMua2fLyclY79u0H7191/Mk3PH0sW99afWb70OPFoM+LQZ8Xw1bs8yzTLU4nuWpq+crJ\nuqeoqhuS/GCSm7r7C+ezLwAAbCWzhORjSfZW1dVVdUmSW5McmR5QVS9J8s6sBOTPTG16IMmrq+ry\nyRf2Xj1ZBwAAW9Y5p1t099mquiMr4XZHknu7+6GquivJ8e4+kuRHk1yW5BeqKkk+1d03dffjVfVD\nWQnaSXJXdz++Lp8EAADmZKY5yd19NMnRYd2dU69veIZ9701y77MtEAAANpon7gEAwEBIBgCAgZAM\nAAADIRkAAAZCMgAADIRkAAAYCMkAADAQkgEAYCAkAwDAQEgGAICBkAwAAAMhGQAABkIyAAAMhGQA\nABgIyQAAMBCSAQBgICQDAMBASAYAgIGQDAAAAyEZAAAGQjIAAAyEZAAAGAjJAAAwEJIBAGAgJAMA\nwEBIBgCAgZAMAAADIRkAAAZCMgAADIRkAAAYCMkAADAQkgEAYCAkAwDAQEgGAICBkAwAAAMhGQAA\nBkIyAAAMhGQAABgIyQAAMBCSAQBgICQDAMBASAYAgIGQDAAAAyEZAAAGQjIAAAxmCslVtb+qHqmq\nE1V1cJXtL6uqD1XV2aq6Zdj2par6yOTXkXkVDgAA62XnuQZU1Y4k9yR5VZJTSY5V1ZHufnhq2KeS\n3J7k+1d5iz/p7hfPoVYAANgQ5wzJSa5LcqK7H02Sqjqc5OYkfx6Su/vkZNuX16FGAADYUNXdzzxg\nZfrE/u5+02T5u5Nc3913rDL2p5L8SnffN7XubJKPJDmb5O7u/uVV9juQ5ECS7Nq166WHDx9+1h/o\nQpw5cyaXXXbZ09Z/7PQTq46/dvdz17sk1sFafWb70OPFoM+LQZ8Xw2b1+eUvf/mD3b202rZZriRf\nqBd29+mqelGS91XVx7r7k9MDuvtQkkNJsrS01Pv27duAsp5ueXk5qx379oP3rzr+5BuePpatb60+\ns33o8WLQ58Wgz4thK/Z5li/unU5y1dTylZN1M+nu05PfH02ynOQl51EfAABsuFlC8rEke6vq6qq6\nJMmtSWa6S0VVXV5Vz5m8viLJt2ZqLjMAAGxF5wzJ3X02yR1JHkjy8STv7u6HququqropSarqm6vq\nVJLXJnlnVT002f0bkhyvqv+d5P1ZmZMsJAMAsKXNNCe5u48mOTqsu3Pq9bGsTMMY9/v1JNdeYI0A\nALChPHEPAAAGQjIAAAyEZAAAGAjJAAAwEJIBAGAgJAMAwEBIBgCAgZAMAAADIRkAAAZCMgAADIRk\nAAAYCMkAADAQkgEAYCAkAwDAQEgGAICBkAwAAAMhGQAABkIyAAAMhGQAABgIyQAAMBCSAQBgICQD\nAMBASAYAgIGQDAAAAyEZAAAGQjIAAAyEZAAAGAjJAAAwEJIBAGAgJAMAwEBIBgCAgZAMAAADIRkA\nAAY7N7uAi9meg/evuv7k3TducCUAAMyTK8kAADAQkgEAYCAkAwDAQEgGAICBkAwAAAMhGQAABkIy\nAAAMhGQAABgIyQAAMBCSAQBgICQDAMBgppBcVfur6pGqOlFVB1fZ/rKq+lBVna2qW4Ztt1XVJya/\nbptX4QAAsF7OGZKrakeSe5K8Jsk1SV5fVdcMwz6V5PYk/2XY9/lJ3p7k+iTXJXl7VV1+4WUDAMD6\nmeVK8nVJTnT3o939xSSHk9w8PaC7T3b3R5N8edj325O8p7sf7+7PJnlPkv1zqBsAANbNzhnG7E7y\n2NTyqaxcGZ7FavvuHgdV1YEkB5Jk165dWV5envHt5+vMmTOrHvtt1549r/fZrPqZzVp9ZvvQ48Wg\nz4tBnxfDVuzzLCF53XX3oSSHkmRpaan37du3KXUsLy9ntWPffvD+83qfk294+nuwdazVZ7YPPV4M\n+rwY9HkxbMU+zzLd4nSSq6aWr5ysm8WF7AsAAJtilpB8LMneqrq6qi5JcmuSIzO+/wNJXl1Vl0++\nsPfqyToAANiyzhmSu/tskjuyEm4/nuTd3f1QVd1VVTclSVV9c1WdSvLaJO+sqocm+z6e5IeyErSP\nJblrsg4AALasmeYkd/fRJEeHdXdOvT6WlakUq+17b5J7L6BGAADYUJ64BwAAAyEZAAAGQjIAAAyE\nZAAAGAjJAAAwEJIBAGAgJAMAwEBIBgCAgZAMAAADIRkAAAZCMgAADHZudgHb0Z6D96+6/uTdN25w\nJQAAPBuuJAMAwEBIBgCAgZAMAAADIRkAAAZCMgAADIRkAAAYCMkAADAQkgEAYCAkAwDAQEgGAICB\nkAwAAAMhGQAABkIyAAAMhGQAABgIyQAAMBCSAQBgICQDAMBASAYAgIGQDAAAAyEZAAAGQjIAAAyE\nZAAAGAjJAAAwEJIBAGAgJAMAwEBIBgCAgZAMAAADIRkAAAZCMgAADIRkAAAYCMkAADAQkgEAYDBT\nSK6q/VX1SFWdqKqDq2x/TlX9/GT7B6tqz2T9nqr6k6r6yOTXf5hv+QAAMH87zzWgqnYkuSfJq5Kc\nSnKsqo5098NTw96Y5LPd/bVVdWuSH07yusm2T3b3i+dcNwAArJtZriRfl+REdz/a3V9McjjJzcOY\nm5P89OT1fUleWVU1vzIBAGDjzBKSdyd5bGr51GTdqmO6+2ySJ5J81WTb1VX14ar671X1dy+wXgAA\nWHfnnG5xgT6d5Gu6+w+r6qVJfrmqvrG7/3h6UFUdSHIgSXbt2pXl5eV1Lmt1Z86cWfXYb7v27Fze\nf7M+F0+1Vp/ZPvR4MejzYtDnxbAV+zxLSD6d5Kqp5Ssn61Ybc6qqdiZ5bpI/7O5O8oUk6e4Hq+qT\nSb4uyfHpnbv7UJJDSbK0tNT79u07/08yB8vLy1nt2LcfvH8u73/yDU9/bzbeWn1m+9DjxaDPi0Gf\nF8NW7PMs0y2OJdlbVVdX1SVJbk1yZBhzJMltk9e3JHlfd3dVvWDyxb9U1YuS7E3y6HxKBwCA9XHO\nK8ndfbaq7kjyQJIdSe7t7oeq6q4kx7v7SJJ3JfnZqjqR5PGsBOkkeVmSu6rqz5J8Ocn3dvfj6/FB\nAABgXmaak9zdR5McHdbdOfX6T5O8dpX9fjHJL15gjQAAsKE8cQ8AAAZCMgAADNb7FnBM2bPGXTJO\n3n3jBlcCAMAzcSUZAAAGQjIAAAyEZAAAGAjJAAAwEJIBAGAgJAMAwEBIBgCAgZAMAAADIRkAAAZC\nMgAADIRkAAAYCMkAADAQkgEAYCAkAwDAQEgGAICBkAwAAIOdm10AyZ6D95/X+JN337jux57nMQAA\nLjZC8jYj9AIAXDjTLQAAYCAkAwDAwHSLi9D5zmEGAOD8CMkLQrAGAJid6RYAADAQkgEAYGC6BXPh\n1nMAwHbiSjIAAAyEZAAAGAjJAAAwEJIBAGAgJAMAwEBIBgCAgVvAsa7mdWu4jbjFnNvYAQBPEpLZ\nFAIpALCVCcmsSogFABaZOckAADAQkgEAYGC6BedlrWkY29n5Tj3ZDlNVtsNnAIAL4UoyAAAMXEkG\nNpwr1QBsdUIycNFaK2z/1P5LN7gSALYbIZmF87HTT+T2OcytPt/52ec7fl4PXHk278X24ar9YtBn\nmD9zkgEAYDDTleSq2p/kx5LsSPKT3X33sP05SX4myUuT/GGS13X3ycm2H0jyxiRfSvJ93f3A3Kpn\n4W3nq6fzvJPIel9l2mp3PVnrpwXP9Hnn9Rm24iPXz+e4z+Ri+Wzb2VY717azed3Z6Jn2WUQX0zS5\nc4bkqtqR5J4kr0pyKsmxqjrS3Q9PDXtjks9299dW1a1JfjjJ66rqmiS3JvnGJF+d5Neq6uu6+0vz\n/iAwWutEfNu1G1zIFrbeU0bmZV7H3Yj617vWzfzLdl41XSz/aNuK/63X+/23c5jbin+OLGIfLiaz\nXEm+LsmJ7n40SarqcJKbk0yH5JuT/MvJ6/uS/HhV1WT94e7+QpLfrqoTk/f7jfmUz3bjKsli0/9z\n24r/jTYrfLzt2rNz+X7BWp7NlcH1/u7B+dqI3szz+xOrWavPguS5bdb/j1vxz6lnY5Y5ybuTPDa1\nfGqybtUx3X02yRNJvmrGfQEAYEup7n7mAVW3JNnf3W+aLH93kuu7+46pMb81GXNqsvzJJNdn5ery\nB7r7P03WvyvJr3b3fcMxDiQ5MFn8+iSPXPhHe1auSPIHm3RsNo4+b396vBj0eTHo82LYrD6/sLtf\nsNqGWaZbnE5y1dTylZN1q405VVU7kzw3K1/gm2XfdPehJIdmqGVdVdXx7l7a7DpYX/q8/enxYtDn\nxaDPi2Er9nmW6RbHkuytqqur6pKsfBHvyDDmSJLbJq9vSfK+XrlEfSTJrVX1nKq6OsneJL85n9IB\nAGB9nPNKcnefrao7kjyQlVvA3dvdD1XVXUmOd/eRJO9K8rOTL+Y9npUgncm4d2flS35nk7zZnS0A\nANjqZrpPcncfTXJ0WHfn1Os/TfLaNfZ9R5J3XECNG2nTp3ywIfR5+9PjxaDPi0GfF8OW6/M5v7gH\nAACLxmOpAQBgICRn5bHbVfVIVZ2oqoObXQ/zUVVXVdX7q+rhqnqoqt4yWf/8qnpPVX1i8vvlm10r\nF66qdlTVh6vqVybLV1fVByfn9c9PvnjMRayqnldV91XV/6mqj1fV33Y+by9V9U8nf17/VlX9XFV9\nhXP54ldV91bVZya3DH5y3arnbq3495N+f7Sqvmmz6l74kDz12O3XJLkmyesnj9Pm4nc2ydu6+5ok\n35LkzZPeHkzy3u7em+S9k2Uufm9J8vGp5R9O8m+7+2uTfDbJGzelKubpx5L8t+7+60n+Vlb67Xze\nJqpqd5LvS7LU3X8jKzcLuDXO5e3gp5LsH9atde6+Jit3Q9ublWdo/MQG1fg0Cx+SM/XY7e7+YpIn\nH7vNRa67P93dH5q8/n9Z+Qt1d1b6+9OTYT+d5Ds3p0LmpaquTHJjkp+cLFeSVyR58sFF+nyRq6rn\nJnlZVu6mlO7+Ynf/UZzP283OJH9l8syFr0zy6TiXL3rd/T+ycvezaWuduzcn+Zle8YEkz6uqv7Yx\nlT6VkOzR2QuhqvYkeUmSDybZ1d2fnmz63SS7Nqks5uffJflnSb48Wf6qJH/U3Wcny87ri9/VSX4/\nyX+cTKv5yaq6NM7nbaO7Tyf510k+lZVw/ESSB+Nc3q7WOne3TC4Tktn2quqyJL+Y5J909x9Pb5s8\n9MYtXi5iVfUdST7T3Q9udi2sq51JvinJT3T3S5J8LsPUCufzxW0yJ/XmrPyD6KuTXJqn/4iebWir\nnrtC8oyPzubiVFV/OSsB+T939y9NVv/ekz+6mfz+mc2qj7n41iQ3VdXJrEyXekVW5q4+b/Ij28R5\nvR2cSnKquz84Wb4vK6HZ+bx93JDkt7v797v7z5L8UlbOb+fy9rTWubtlcpmQPNtjt7kITealvivJ\nx7v730xtmn6M+m1J/utG18b8dPcPdPeV3b0nK+fv+7r7DUnen+SWyTB9vsh19+8meayqvn6y6pVZ\neZqr83n7+FSSb6mqr5z8+f1kj53L29Na5+6RJP9wcpeLb0nyxNS0jA3lYSJJqurvZWVO45OP3b5Y\nnhDIM6iqv5Pkfyb5WP5iruo/z8q85Hcn+Zokv5Pk73f3+IUCLkJVtS/J93f3d1TVi7JyZfn5ST6c\n5Lu6+wubWR8XpqpenJUvZ16S5NEk35OViz3O522iqv5Vktdl5e5EH07ypqzMR3UuX8Sq6ueS7Ety\nRZLfS/L2JL+cVc7dyT+QfjwrU20+n+R7uvv4ptQtJAMAwFOZbgEAAAMhGQAABkIyAAAMhGQAABgI\nyQAAMBCSAQBgICQDAMBASAYAgMH/B4pnPKig8R7AAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dfvhq2CFg_eM",
        "colab_type": "text"
      },
      "source": [
        "There are reviews even longer than 100 tokens, but most pars of them are shorter and I decided to consider that part. I would like to choose the length of 60 (shorter will be padded, which longer will be truncated) to save time on training. It possible to put longer sequences into the transformer, but we will not use this benefit now."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-O2fNQEPB-v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_seq_length = 60"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dnGLQAnfiR26",
        "colab_type": "text"
      },
      "source": [
        "As we have really huge dataset, training on whole data will take probably several days, so I would like to use a subsample from the dataset."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7bgsWddEjE7h",
        "colab_type": "text"
      },
      "source": [
        "As data is unbalanced, we need to do a representative subsample. To do this, we will use StratifiedShuffleSplit."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iJ0SKpY5Ol0T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sss = StratifiedShuffleSplit(n_splits=2, test_size=0.1, random_state=0)\n",
        "_, sample_index = next(sss.split(X, y))\n",
        "\n",
        "sub_sample = df.iloc[sample_index]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tY_9zc5gPuLL",
        "colab_type": "code",
        "outputId": "8437f955-84fb-4e35-a0c6-306c8f256490",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 247
        }
      },
      "source": [
        "x = sub_sample['overall'].value_counts()\n",
        "plt.pie(x, labels = x.index, shadow=True)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxcVf038M+5yyw3M5OtaZJmm7Sd\n0n2h0AEBQQVZIigggiCLIOJP8CeKaNDneXwelYfgD/T3cnkUWVRUEKgsgQAFRSgVSKEEmm5pSps0\nSbNvM5nJzNzlPH/caamlTbPM3HNnct6vV16t43TOtyWfnHvPPQuhlILjOPsRWBfAcdzR8XBynE3x\ncHKcTfFwcpxN8XBynE3xcHKcTfFwcpxN8XBynE3xcHKcTfFwcpxN8XBynE3xcHKcTfFwcpxN8XBy\nnE3xcHKcTfFwcpxN8XBynE3xcHKcTfFwcpxN8XBynE3xcHKcTfFwcpxN8XBynE3xcHKcTfFwcpxN\n8XBynE3xcHKcTUmsC+COzV/bIAGoAlCd/LUMQAGAPAD5yV/zAPgAEADGEV86gBEAA0d89QPYB2B3\nW11Nt3V/I24qCD/IyB78tQ2lANYlv1YBWEQprSaEpPsHaAhAK4DdAFoAvAvgzba6moE0t8sdBw8n\nA/7aBgHAyQDOAhCklK4jhJSxreojWgG8mfx6A0BzW10N/2axEA+nRfy1DUUAzqOUngfQcwkRClnX\nNEU9AJ4H8CyAl9vqaiKM68l6PJxp5K9tKANwJaXG5QA5kRBCWNeUInEA/wRQD2B9W11NP+N6shIP\nZ4r5axs8AC6hunY9BPEMQki2j4irAJ4D8HsAL7TV1WiM68kaPJwp4q9tOJka2rdAhIsJEVys62Gk\nF8CfADzUVlezk3UxmY6Hcwb8tQ0C1bVLqKF/X5Cda1jXYzMvA7i3ra5mA+tCMhUP5zT4axtyqJb4\nKkBuI5Jst1FWu2kGUAfgsba6Gp11MZmEh3MK/LUNDiMe+RaRnHcQUcplXU+G+QDATwA83FZXY7Au\nJhNkTTgJISKAdwB0UUo/k8rP9tc2CHp09EYiu34syM6iVH72LLQVwHfa6mpeZl2I3WVTOL8N4CQA\nvlSGs+I/H72USPK9gsNdlarP5AAAGwDc3lZX08y6ELvKinASQsoB/BHAnQC+nYpwlt10/wLBmfMn\nUck9dcYFcsdiAHgIwPfa6mqGWBdjN9kSzvUA7gLgBfCdmYSz5Ev/JYg5eXdJvrm3ElFypKxIbiK9\nAL7RVlfzBOtC7CTjH5ATQj4DoI9SumWmnzXvhl+fJRdW7JPz532XB9NSxQAe99c2PJlcAMAhC3pO\nQshdAK4GoAFwwVw+9SSl9EuT/Yz8s74sK4tOvU/KL72OECFbpthlqhEAt7XV1TzEuhDWMj6chyOE\nnIUpXtYWfe6ONc6yJX+TvIXV6auMm4YnAHylra4mxLoQVjL+sna6lECQlFx51+3uBSe/wYNpS5cB\n2OKvbVjNuhBWsqrnnKy8M6/JVRae8jdHUdWnWNfCHVcMwK1tdTX3sS7EarMunAXnfG2tsujUpyXv\nnHLWtXBT8giAG9rqamKsC7HKrAmnEggSd+CUa3MWnfZLwZXjYV0PNy1vArhotmyhMivCqQSCUs6S\nM/+PsujU24nkkFnXw83IHgDnt9XV7GFdSLplfTiVQDDHs+rcB9zzT/4CEYRZOwCWZQZg9qBvsi4k\nnbI6nEogWOhZff6f3fNPOi97dgjhkmIArmirq3mGdSHpkrXhVALBUs/q8x9VFpx8JutauLRRAVze\nVlfzFOtC0iErw6kEgmXeNTWPueevPY11LVzaqQAuy8YeNOvCqQSC5d4TL3zcXb2GryaZPVQAn2+r\nq6lnXUgqZdUAiRIIVnpWn/9XHsxZRwbwhL+2IaWL7FnLmnAqgWClsuhjD7jnn8QvZWcnB4D1/tqG\nrPnvnxXhVALBIlflip/nLPvEJ/mo7KzmBPC0v7ZhIetCUiHjw6kEgl5H8YI7vWtqaoggiqzr4Zib\nA6DBX9tQwLqQmcrocCqBoFPKK7nDd/LFVxLJ4WRdD2cbi2D2oBm9YD5jw6kEgiJE6Su+dZfcIDiV\nHNb1cLZzBoAHWRcxExkbTgCf85302a9L3jlzWRfC2daX/LUN/8G6iOnKyHAqgeBK94KTb3GVL1vK\nuhbO9n6eqQu2My6cSiA4Vyoo+55nxdkfY10LlxGcMDcP87IuZKoyKpxKIOgkkuMbucFLzyWinNE3\n+5ylAgAybieFjAmnEggSAF/wrrngs6KSl2mnQnPsfdFf23AD6yKmImPCCWC1o3jB5c6K5ctYF8Jl\nrJ/7axsqWBcxWRkRTiUQ9EGUv+Jde2GQEL5gmps2L4DfsS5ismz/jZ68nL3cu/r8U0W3bw7reriM\nd56/tuFq1kVMhu3DCWCFXOS/0FW1chXrQriscW8mTO+zdTiVQNAD4CveNRes4ZezXAoVAfgp6yKO\nx+7f8Be7F6xbyveY5dLgen9twxrWRUzEtuFUAsEKCOLZOUvOsPU/IJexCIC7WRcxEVuG8+AgkGfF\nOQsFZ47t7w24jHWOv7bhHNZFHIstwwlgmeDyrnX715zIuhAu693tr22w5Qp924VTCQQlAFd5Vp27\nkEiyi3U9XNZbA+BK1kUcje3CCeBUweWpcpYuWsm6EG7W+JG/tsF2u2jYKpxKICgDuDRn+afK+bHv\nnIXmA7iUdRFHslU4AawlkqPQVbZkLetCuFnndtYFHMk24VQCQRHAxTnLPjGPSA6FdT3crHOSv7bB\nVkd32CacAJZDEEtclSt5r8mx8h3WBRzOFuFMPte82D1/bb7gcOeyroebtWr8tQ2LWRdxkC3CCXOl\nepWras0i1oVwsxoBcBPrIg6ySzjPFD2FkpRbFGBdCDfrXemvbZBYFwHYIJzJlSdBZfHp8/jKE84G\n5gI4n3URgA3CCWA1ANFZsjAjty/kstK1rAsAGIczORB0rrNiuZtPcOds5EI7LMZm3XNWAKhwVa2q\nZlwHxx3OAeBy1kWwDuc6AJpcUGab4WuOS/oc6wKYhVMJBAUApzvLlsiC7PKxqoPjjuFMf20D0wOy\nWPacFQB8zvJlVQxr4LhjcQI4m2UBLMO5AgCVC8r4xAPOrmpYNs4ynKeInoK44PaVMqyB4yZyAcvG\nmYRTCQQLAJQ5y5cVEmLLHSI4DgDK/LUNzPZLZtVzzgdA5cLySkbtc9xkncaqYVbhXAxAlXxFGXOo\nDDdrncKqYVbhXEFkZ4Tfb3IZIMiqYcvDqQSCXgBFzrKl+XyiO5cBAqym8rEIRyUA6phTxS9puUxA\nYM5ksxyLcM4HANFTUMSgbY6bjlkTzqUAwoLby8PJZYolLBq1NJzJJWLlACKCM6fQyrY5bgaY7NBh\n9XYMCoAcKa9E5ZtGcxmESTitvqydA8CQCyv48fFcJvH5axvmWt0oi3AS0VvEL2m5TGN572l1OEsB\nQHR5vBa3y3EztdDqBq0OZwWAceJwM13EynHTUGJ1g1aHMx9AgsguHk4u01h+K2Z1OPMAqILs5OHk\nMo3lU/isDmcugASRHDycXKbJ3nAqgaAD5paDOj/ij8tA2RtOmBMQDLNVQbawXY5Lhay+51QAUPO3\nhC8V4zKN5Vd7VoZEBAAQQvi+QVwGsrxDsbJBAQCIIPFek8tEotUNWjnx3ewuBZF3m2lSpe4bOXX0\nse78zoWiLFVKMU+5JCpzPQ7R6WZdW6aioIJARE0UxJDVbVsZTrPHFETec6bJkuj7jv9VunXxOQuG\nIoEP3qGfbzTEBaqhDMi5arunMjbgqTTGvRVE8JQ6FFe+U+A/KCfLCcDyx3/Wh1NL6Ba2Oass1lpk\nRQRpiB7IOS8wT73jDMGxYjMZv2xT2Fg73OzFcPOh9yYgGV05ZeEerz8+6q2immeeLOfMzZElJ1/K\nd3Sa1Q1afllLdVWnhq4RQbTF0d7ZZLG7KwFAzhMN8nS4S7zQURppXiflNK8jCGxF7POvUKyKUpdA\nAAc0oTrS7quOtAM9H37GsCNvrNtbPTroq1aj3kqR5hR7ZIcnjxBhtveyWR1O9eBvqKHHeThTb6V7\nNIHk5VeJRMW/9nc7LxNK42qe5GxdKbjuWglU7dFCV7yoq2tCQoFwlGHz/MSIJ3+wyYPBpkOvxYms\n9XjKh/u888dHfX6qeua5RHdhniTKTuv+dsxFrG7QyoDED/1OV+PgU/hSilCNznWq/7YUb4GDSg/3\n96hXSSWa4ZEkAGhfKPnuvkVCSZcW+lKDNrx2QCwXCZlwJNJJVakqvK+oKrwPOPDh64POwlCPtzo8\n6POr455KiebM9cgOT26WPisbmMybCCEPAfgMgD5K6fKZNMgknFTXYha2OyvM19tCkoDcI19fLhnu\n+3v7hm+QSxQ4hUM9XU+Z5Lvnq5KvYECPXPmsOnBqt1gqEzKl+83C+KCvMD7ow8A7h16LEYfa460c\n6fNWR0M+P9SceS7JXZAvZv62NIOTfN8fAPwKwMMzbdDKcMZw6L5Tix/nvdwUrdB2jAEfDScArBO0\n/P/u6h34ZmUxiCT826Xo0Bwx51dfFnP+MKqPX96gtp3VJpY4CXFNtw4XTcj+0J4if2gP0GW+ZlBg\n0FUU6vVVhwZ9fm3cUyEhZ65XlnNyM6iTnVQ4KaUbCSH+VDRodc8pAgBVY5Zfv2e7ldgz4YDFp4g6\n58cd/b3/s3JuIRHJR/67j+WK7gevFP1/iRqJi59X289tJYUKBE8qahMIUBTv9xX19/vQv/nQ6+OC\nM9Ht8Q/3+/yxkNcPzVPqltz5+aIg2XHudc/x35JaloUz2tqoK4FgAoBgxMIjVrU7WywWOo77/Phi\nxItH9/cfuKeqqIQIR5/fHFMEx6Ofd1Y9ETf0z7yktl+4jeR5IRy1R54ptxF3zA+1FM8PtRx6TaeE\nDihzR3q91WNDXr8W81bIUIq8DkeOLx01TEGX1Q1aPWI6DMCpR0aHLW436/nl4UnNAroOsXnDHYMd\nD1YWlk80cKM5BfHpC51V9ecb9OxXtY5L3qE5BVRM+7IpkVBSPN6bVzzem4e+tw69HhXdsQMe/+iA\nzx8Le6uI5il1y678fMG6Uf+sD2c3gGp9bJD3nKlEDRQ5EpPu3b5lRCuGO8X2pyryq473XkMSyEtn\nOype+qSB09/Qui5/g8rFumj5NpGKPu5aOLrTtXB056HXdEpov1Iy3OurHhv2+vWYt0ImSlGuLLtT\ncjl+hM40fOaEWIRzqRbq5+FMIb/WHpbdmNKOhj/SwlXDXULbq2W5/kn9AUHAptMdZZtOB9ZsUXuu\netVAZUK0fNOrw4mEkpLx7vyS8e589L5x6PUxURnv9laPDvj88bC3iug5pYrsys0XBHFak9cppToh\nZN9k3ksIeRTAWQDmEEI6AfyQUvrgdNplEU5ZGz7QSymlWfo8zHIrjB0hYGrhBIBfJkb913QLbU2l\nXv9U/lzTWrmkaS2weIfWd/VLurowKpTZ6T+lR4+6AyPb3YGR7Yde0yAYfUrpUK+vOjLi9esxT7lD\nUIpy5UlsNkcI2XPzbz85qScMlNIvzqD0f8PintOguqrTxPgIcSr5FreflVbS3erx33V0D8eG/Z/r\nFdo/KM457iXukXYtleb+YKmEqr3a0DUvaJFlo2L50WYd2YEEQ5gX7SqYF+0qQM+mQ6+HJU/0gLc6\nNODzxyLeKkHPKcmRnbl5giAc3svusL5i68M5guRuCHp05IDAw5kSi4XOGa30WR8ZqLxgQOjonuOe\n1pmp7fOlgh/fLBWUdGmjVz+njZ44KJYdb9aRXXi1MeWE4WblhMMWBagQjN6cssFuT7UWzw/0O3Lm\nvDHBR6SN1eEcgLk6hWih/m45f94yi9vPStXy0IzWa0qEkGdDfWXniCVdw/nOsul+Tk+ZlPtfN0m5\nM5l1ZAcyDKE80lFYHukAejcWA9gHXGd5HZaurYy2No4D6AXg1oY6Dxzv/dzkzHXEZ/wc0kmI8Pxw\nT7ESSsz4Ybs568hVddPNgr6hUmuLU5rp0zXfZdEoi4XPLQC88e7dByilDJrPLuXa/rBDREp6Jw8h\n0gv9PQWOMbU/FZ83liu6H7zK5b/xVlF4KqC3j1Mj42aGUUqHluzaOamR2lRjEc5WAE5jPBynifEh\nBu1nlRXajnAqP69AgOPZ3h6PGNVSNlHk4KyjG26TXI8s19vDMEZT9dnpRgh5nVXbLMJ5AMn9a/Wx\nQcsf7GabVZj+SO2xzBOoe31Pj0xiekr3zTk46+jG2yXfAycbHUNEz4Qfzv9k1TCLcHYn2yXqYAeT\ny4VsskToSMuji4XE8PzpQA9FIvWXogdnHX3tu3LBL043unpFvS/VbaTQq6watjyc0dbGGIAOAJ5Y\nx7a9VrefbaqlwWkv7zqeVUTP/U1nT5xqRnoGdAQBm85wlH3ju865d59DuzscuuUrPyZCKR0CsJVV\n+6x2wnsHQK420hMyYpFJrTDnjq7YGUvLipGDTidawd0dfSGq05RfPh9uy0ly6W23OUt+eBH6W916\nl00GC19bsmsns0JYhXM3kguv1eEDrYxqyHilWlfEKSLt+/jUIDH3+/v7B6hB075z4s5lUtEPbnWW\nffcKMtTs0zoMhiklhGxg1TbALpxtMGcKiYmeVh7OaVqp77Bso+MrESv9+v7BbmpRWMxZR66KW68l\nobcLtP06Tf8PhsMl/571k3kvIeQ8QkgLIWQPIaQ2VTUwCWe0tTEOYBuAvPH299qppmb6Q2omVtDd\nCSvb+zqNll+5f7jDys7MnHXkqrz5q0JsY4nWrlJqyd/ZALYs2bWz+3jvI+Y0xV8DOB/AUgBfJIQs\nTUUNLHdf3wzAA10z1OEuJhOLM91Sod3ySebfN8YqL+gabbe63X+fdaS3p3vWkUjIY5N86zoAeyil\ne6n5g+OvAD6bihpYhnNX8lcS29/MbEQsk82XBpnsG/tTNVR1yoFQG4u2zVlHzqobbxWFp9M76+jx\nSb6vDObTh4M6k6/NGLNwRlsbh2AGND/W1tRuJGKWHxST6Uoc42kdqZ3I/fER/5KeMct70INiiuB4\nJDnr6NHl+v4wjJR9/2iUvr1k1879qfq86WJ9qNCrSC4SVgfamyd+K3e4uXpP1CUhbc84J+Ov0cGq\niv4o029izSmIT13orLzxdsn7wEl6x3AKZh1JhNw3hbd3ATh8qV05UrTfEOtwbgegAxDH25p4OKdg\nhbad+ZWGQAieCfdXzBmKMZ+GaUgCeekcZ8VNH846mtbkfZ3SCMz7xsl6G0CAEFJNzOVxV2CSo7zH\nwzSc0dbGMQBNAAoT3bt79egoX0Y2SSvpbltszC0TQp4f7i31jiSOO7JpiQ9nHRVNZ9aRDvx1ya6d\nk76PpZRqAG4BsAHATgCPU0q3T/ynJod1zwkAmwC4ASC2v/mt47yXS1oqMLvd+wi3QMTnh3rmuMKq\nrebITmfWkYOQX0+1HUrp85TSRZTSBZTSO6dV7FHYIZw7AIwBcEd2bdxuqLGULoHKVgukAab3m0fK\nI5Cf6+/xSRFtsmeKWObgrKPvXU4Gt3mPPesoQem2Jbt2Nh3t/2OBeTijrY0qgAYARdA1I9Hd+jbr\nmjJBiSM65d320q2YUNfTvd1uYVy35XrNtgVS4Y9uMWcdvXOUWUcicDer2o6GeTiT3oC5xlOK7Hxt\nCzV0yw8qzSSFWn9UkaCwruNoqghVHj3QTRA3xljXciw9ZVLuT29yVd5yozD+erHWrlKqJajRKxIy\nlYGgtLNFOKOtjSEAGwHM1ceGoupgB5+UMIEVFs6pnY6lguF7oKtHhWqMs65lIoNFoueX17uqbvq6\noL+wivxwya6dtuoUbBHOpFcAOACQyLZXXqfUMFgXZFcr6S5bjNROJEi0/J939kaoRm1f65hCon+p\nkf/Iuo4j2Sac0dbGTpiT4YvUoc4Rta9tC+ua7Gopsc9I7UTOhjrnRx19w1SntuqRPiJm/Kz52mbb\nLb6wTTiTngKQA4CEt27YSA09rQt8M9UCqZ/JnNrpuATxkm93DPRRg9rySshQjRH4pHtZ13E0dgvn\nXph7hBbrof6xRM+eRtYF2dE8G47UTuR6Oj7v+o6hLqvWgk4FTdA7m69ttuW9sa3CGW1tpACeBOAE\nIITff/FfVOdrPQ+Xpw3FcmR63MN37ObbRqTi4o4R5pPJD6dH9TYxR/wZ6zqOxVbhBIBoa2MHzEcr\nJUZ0NBbr2LaRdU12skLfYctniJPxYz1cdWbXaBvrOgCAUgotrN3UfG2zLS+3ARuGM6ke5jkuYrjp\n+UZ9PNTLuiC7yISR2on8KjHqX9UdbmNdhzaqbdj93d0vsa5jIrYMZ7S1sRfASwDKYOjGWPM/nrPh\n7QoTy0hbxv9D/Dk27J/fF2E25GyoRoxq9HpW7U+WLcOZVA8gDMAT72juVPv5oxUAWCD1ZdypXUfz\nt7GBypLB8Y7jvzP1tFHtnpbbWmy/Asq24Yy2NkYB/AFAEQASeueZvxtaIuMOwkm1Mkcko0Zqj0Ui\nhDw72leWNxy3NCR6VO8yYsb/trLN6bJtOJPeg7nes9gYD8XGW998gXVBLPn0kbhHph7WdaSKixDh\n+eHeuUooYcmYAjUo1Ua161p/0GrpNpvTZetwJh+tPAJzcMgR2fHa9tk873a5lrkjtcfiJZAaBnry\nHGNq2nf+j/fEH95du/vv6W4nVWwdTgCItjb2wdwJrQwARt96osGIR1N2PF0mWUl3ZeUz3zkEzmf6\nepRUHjt4JHVE3T+6efSr6fr8dLB9OJP+AfNAmVIjNpYIv/fC+tk4MT4bRmqPpZxQ5fGeHinVxw4C\ngKEaidj+2Of7nuqzdBPumcqIcEZbG3UADwHQAHjindsPxPY3v8K4LMstlHpl1jWk0yJieB8+0GMg\nYURT+bnjH4z/qO3etgkX8RNCKggh/ySE7CCEbCeEfDOVNUxHRoQTAKKtjcMA7gMwF4AYfueZf2mh\n/g8Yl2WpMjk7Rmonsproeb/u7B2nmpGSyRaxjtg/B14cuGsSb9UA3EYpXQrgFAA3p+pYhenKmHAC\nQLS1cSuAF2HuDYqRTX9ZP1uOEMzRwwmvrGd9OAHg40QtvHt/3+hMjx1UR9QDoXdDl4SaQse9BaKU\ndlNK303+PgxzJ72U7Nw+XRkVzqS/AdiP5OOV0cb1j1JNteWqglRaru8YJcTyo1GYqSGJuXfM4NhB\nLaKFQltCF/U+2Tsy1T9LCPEDWAOA6aqojAtn8oSyXwBQAeSpA+1D4a0bHs/2AaIVRnaO1E7kKsRK\n/2Maxw4acSM2smnk6wcePjDlWWWEEA/MDuBWSinT7WAyLpwAEG1tHATw3zCPcnDH9r3bNr7n7QbG\nZaXVcrIvq3/4HMvNNFp+Rcfkjx2kGtVG3hj5cWRn5JGptkUIkWEG8y+U0ien+udTLSPDCQDR1sa9\nAH4LoBSANLZ1w7vxrl2vMy4rbQJido/UTuR/6GOV505iqRk1KB19e/R34ffDd4eaQlPqbYl5z/Ag\ngJ2UUlus8czYcAJAtLXxbZg/6SoBkNG3Hn8l3tOalbvGlznCWTNtbzruVUP+dcc5djC8NfzUaOPo\nt0NNoencp54G4GoAnySEvJf8umA6taYKyfSlWEogKAD4MoCPI3mcfd4ZV9c45lafxLSwFHIZEXWn\n+yvybBoQOpbLlML2XcU5VUe+HtkV2TTw4sAFoaZQ1pwYkNE9JwBEWxsNAH8E8CaAKgBk5PU/NSQG\n9r/HtrLUWa7tnFUjtRN5LDJQVX7EsYNjO8c2D7w4cEk2BRPIgnACQLS1UYN5v7AF5iUuRjb+sV4d\n7MyKYwVXGruy/lHRZAmEoD7cX1E4FOsCgPDW8ObBDYOXhZpC0zryz86yIpzAoTNX7oM5B7cSlNLh\n137/VKJv3zuMS5uxZdibEUucrCITQhqGe0vkN4deG3pl6IuhppCtNg5LlawJJwBEWxsTAH4D8+Sy\nKlCKkdf/1BDr3PEa49JmJCD1zNqR2qNRDUrvbcErexrD14aaQntZ15MuWRVOAIi2NsYA/BLmJa4f\nAAk1rn81uqfxOTvumzoZFXJoVo/UHi6uUfWOzfrme3bj+lBTKDO2vp+mrAsncGgW0W8B/BNANQBp\n7P0NW8aaX36U6lpGLRtyGlEt16H7WNdhB1GVjt/zRuKR3fsSF4WaQsyPuk+3rAwncGiQ6GGYm1RX\nAnCOt77VOvrmYw9k0mT5JVrLqMCHarF/1Oj5/j9iv2rs0r9Z32KvE7TTJWvDCRx6zPIMgPthziTy\nJXo/6B/6x333q8MHdrCtbnJWGrtSurYx0xiU0r/v1bbe+mLsp3uH6Q/rW9Ss26rlWDJ+EsJkKYHg\nIgC3wDzqoRsAvGsvPNVVtepsQgTb/pD6aewn7V/I2/GRh+6zQSRBI799J/Gv19r1+wDU17eo9j6t\nLMVmTTgBQAkE8wF8DcAJADoA6K6q1ZWeVZ++VJBdtryve1q7uWu1Z5jpukIW9g4bnXdvim/oHqM/\nq29RM+IqJ9VmVTgBQAkEZQCXArgAQA+AqOD2On3rLjnXMadqDdvqPmoLuXqk0Knnsa7DKgalxgut\n2nu/26I+SoEH61vUWbmZGzALwwkASiBIAKwFcGPypW4AcC8MLsxZeuZFguyyxY4DshHXW9zXEYEQ\n2152p1I4TsO/3Jx4/a1O/f8BeLG+RZ3Vky9mZTgPUgLBOQCuBbAKwAEAMcHtc5m9aOVqttUBK9St\nQ8966wpY15FumkG1je168/1bEm9EVPyivkXdzbomO5jV4QQOrWr5GIBrABgwL3Xhnr+2Wln88XNF\nt7eYVW1XxR7ruDPvmQpW7Vth14De8qvNie37R+lGAL+vb1GZ7j5gJ7M+nAcd0Yv2AoiAEOJZ+ekT\nXf41nxAkh+UH1t4V+79tX8zb5re6XSv0R4yeh5rUpn916C0A/gygqb5FnZW7PRwLD+dhkr3oOgBX\nAvDAvBdVBZfX6T2x5uOO4oVBIgiiVfU8qX6j80TvYLlV7VlhXKWR+hZty6Pb1FaD4mkAf69v4aeX\nHw0P51EogaAbwKcBXAjzUrcbAJUKyvM8yz5xmjynao0VIX2bXDNc5NTy092OFTSDapu79ObfvJ3Y\nORrHRgB/q29J//komYyHc+huaKkAAATOSURBVAJKIFgE4BKY96QRAP0AqJRb7M1Z9slTHcXVJxFB\nSsuKEcmIGy2u6yAKmT1SG0nQ8Fud+nuPNKtd/VHaAuAv9S1qK+u6MgEP5yQogeACAJ8FsBJADOY9\nKRVz8t05K84+xVm88GQiye5Utrk0sX34ed+dGdtr9o4ZnS/v1bY+uVPr1wyEYZ4Wt3m2Px6ZCh7O\nKVACwSoAnwFwMoA4zJAaRHKIygmnLXWWLT1R8hb6U9HWFbH1nXV5T2bU/aZuUGPPkLF9/Q5td2OX\nHob571MP4F1+Xzl1PJzToASCZQDOg3m5SwAMAIgCgDynqkBZdOpauci/aiYjvD+J3d3+pbz3M2JO\nbVSl4aZu/f0/b1U7usI0BnM3ihcB7OYjsNPHwzkDSiBYAHN099MA8mFe8vYDMCBKgtt/YrVz3gmL\npfx5iwXZOaUF00+o/9l5snfAtj3nWIKO7howdrzapu17vV1PUHMH/n8AeK2+Re1hXV824OFMASUQ\nFAEsAvAJACfB7E3DAIYBUABwVa4od5YvWyIXlJ0gOHMKj/eZjbhmqNil2WZ2kEEpHYjSA7sHjT0b\n2/X2tzp1FebfcxjAcwDerm9RI2yrzC48nCmmBIJ5AJbDvORdnHx5HMAQAB0ApLwSn3Pekkq5sLxK\n9M2pFJyeuYevpxaoRludVxuiQCx7pnok3aDGaBwDPWPGga29xgcvf6D19kepCx8GchOA9wC080vX\n9ODhTCMlEPTCXJ52CsyZRwcfi4QBhGA+Q4Wg5Lld5UsrHL5Cv89JllV59aEXCu6ptGrCe0yj0YEo\n7ekZo73tI0Zvy6DR29Stj8R15MJc/wqYS+waAWwD0FHfovJvnDTj4bSIEgi6YG56PR9mUBfADCuB\nOZgUycVYSYB0npBHoi/KAoQT5gh5C/KFwnKfUFDsIYWKDLdDJA6HCKcswOEQiUMW4ZAEOCUB8uHb\nmegGNRI6xuM6YnGNjsd1xGIajUVVjI+rNBZOINo+YvS/36v37h+lFOaMKCfMHxgEZm//HoAmAHtm\n0w4EdsHDyYgSCDpgHs5aDfN+dX4+wouXkHZFIYl2mAGJwwxJAubJyzqS97BHIgC8Tsg+J3GE4jQR\niuPgwbMCACn5JSd/dSdfP/hZ3QD2AtgD8/FHD4DQZHpHQogLwEaYwZYArKeU/nAq/xbc0fFw2khx\nYLn7JLK7QCS0EEAhzI3JymGOBLuSXyLMUB38OpqDPagAM9RRmJfSEZiX010wDyDuBdBX36JO+wTp\n5OlcOZTSseQRepsAfJNSmpUHSllJYl0A96He1m3jMIPTdaz3XHSCLAFwHPblPOz3BszHOeOH/aql\n8/4wuRfwWPJ/yskv/hM/BXjPyc0YIUSEuYn3QgC/ppR+j3FJWSGjJ1Vz9kAp1Smlq2Fegq8jhCxn\nXVM24OHkUoZSOgJzl/3zWNeSDXg4uRkhhBQRQvKSv3cDOAfALrZVZQc+IMTNVCmAPybvOwUAj1NK\nn2NcU1bgA0IcZ1P8spbjbIqHk+NsioeT42yKh5PjbIqHk+NsioeT42yKh5PjbIqHk+NsioeT42yK\nh5PjbIqHk+NsioeT42yKh5PjbIqHk+NsioeT42yKh5PjbIqHk+NsioeT42yKh5PjbIqHk+Ns6v8D\nGC4wGcBEbtYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7GqaFXkAjdXx",
        "colab_type": "text"
      },
      "source": [
        "Looks like that we needed."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oscWu2wakGsl",
        "colab_type": "text"
      },
      "source": [
        "Getting the feature column and target:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SVUdI69xOyym",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = sub_sample['reviewText'].astype(str)\n",
        "y = sub_sample['overall']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A2996VWijwCL",
        "colab_type": "text"
      },
      "source": [
        "At this step, we will split our data to train and validation dataset (60% for training and another for validation) "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiVUf8v0O7YU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sss = StratifiedShuffleSplit(n_splits=2, test_size=0.4, random_state=0)\n",
        "train_idxs, validation_idxs = next(sss.split(X, y))\n",
        "\n",
        "X_train = X.iloc[train_idxs].to_list()\n",
        "y_train = y.iloc[train_idxs].to_list()\n",
        "\n",
        "X_val = X.iloc[validation_idxs].to_list()\n",
        "y_val = y.iloc[validation_idxs].to_list()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wsUppz-NkUOo",
        "colab_type": "text"
      },
      "source": [
        "As I plan to use several models, I found useful to save indexes of train and val. data to use them in the future:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_5BQsN6fdDG-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "indexes = (train_idxs, validation_idxs)\n",
        "with open('/content/drive/My Drive/test_task/indexes.pickle', 'wb') as f:\n",
        "    pickle.dump(indexes, f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lmP2pFGrkvcs",
        "colab_type": "text"
      },
      "source": [
        "Let's put our data to dataloader using our class 'text_dataset'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_2u4i3e2O-0Q",
        "colab_type": "code",
        "outputId": "b124ea9e-15a8-428b-d9a1-e993fd747c15",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "batch_size = 128\n",
        "\n",
        "train_lists = [X_train, y_train]\n",
        "val_lists = [X_val, y_val]\n",
        "\n",
        "training_dataset = text_dataset(x_y_list = train_lists, max_seq_length = max_seq_length)\n",
        "test_dataset = text_dataset(x_y_list = val_lists, max_seq_length = max_seq_length)\n",
        "\n",
        "dataloaders_dict = {'train': torch.utils.data.DataLoader(training_dataset, \n",
        "                                                         batch_size=batch_size, \n",
        "                                                         shuffle=True, \n",
        "                                                         num_workers=0),\n",
        "                    'val':torch.utils.data.DataLoader(test_dataset, \n",
        "                                                      batch_size=batch_size, \n",
        "                                                      shuffle=False, \n",
        "                                                      num_workers=0)}\n",
        "\n",
        "dataset_sizes = {'train':len(train_lists[0]),\n",
        "                 'val':len(val_lists[0])}\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(device)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0kgaIQhhPWCR",
        "colab_type": "code",
        "outputId": "40f30c83-0717-4b06-ebb2-a1af6f4d2cb7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model.to(device)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): BertLayerNorm()\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=5, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PUm8BXNOlI88",
        "colab_type": "text"
      },
      "source": [
        "This is an important step where we will define two learning rates: the first one for the last layer and the second one for pre-trained Bert. As we don't want to change weights in a pre-trained part the second lr will be much lower than first.\n",
        "\n",
        "Also, we will use a step scheduler to decrease lr at every 4 steps (but probably, we will even do these steps as it takes much time."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VsvY2304Si0x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lrlast = .001\n",
        "lrmain = .00001\n",
        "optimizer = optim.Adam([\n",
        "                        {\"params\":model.bert.parameters(),\"lr\": lrmain},\n",
        "                        {\"params\":model.classifier.parameters(), \"lr\": lrlast},\n",
        "                      ])\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "# Decay LR by a factor of 0.1 every 4 epochs\n",
        "exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.1)\n",
        "\n",
        "model_name = 'review_preds'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YitVZQyHmj8c",
        "colab_type": "text"
      },
      "source": [
        "Let's train it!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t7uOiejtJzqt",
        "colab_type": "text"
      },
      "source": [
        "(By unknown reason using scheduler before optimizer.steps() usually gives better performance even if I would use smaller learning rate on the very beginning, so let's ignore this warning ; )"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ofCwAXBxSncT",
        "colab_type": "code",
        "outputId": "929bd15f-9a5a-4256-91fc-303bd1f1e449",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 552
        }
      },
      "source": [
        "model, review_preds = train_model(model, model_name, criterion, optimizer, exp_lr_scheduler, dataloaders_dict, dataset_sizes, device, num_epochs=3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "starting\n",
            "Epoch 1/3\n",
            "----------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:122: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
            "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "train total loss: 0.7513 \n",
            "train F-1 score : 0.7391 \n",
            "val total loss: 0.6631 \n",
            "val F-1 score : 0.7609 \n",
            "saving with loss of 0.6630992859840393 improved over previous 100\n",
            "\n",
            "Epoch 2/3\n",
            "----------\n",
            "train total loss: 0.6303 \n",
            "train F-1 score : 0.7687 \n",
            "val total loss: 0.6313 \n",
            "val F-1 score : 0.7694 \n",
            "saving with loss of 0.6312508131980896 improved over previous 0.6630992859840393\n",
            "\n",
            "Epoch 3/3\n",
            "----------\n",
            "train total loss: 0.5861 \n",
            "train F-1 score : 0.7818 \n",
            "val total loss: 0.6277 \n",
            "val F-1 score : 0.7728 \n",
            "saving with loss of 0.6277210783958435 improved over previous 0.6312508131980896\n",
            "\n",
            "Training complete in 24m 12s\n",
            "Best val loss: 0.627721\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5gEPyFZzmpKc",
        "colab_type": "text"
      },
      "source": [
        "Looks not bad, so let's save predictions on validation part, as I plan to da Blend then"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vi_XB-34d7pq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/drive/My Drive/test_task/review_preds.pickle', 'wb') as f:\n",
        "    pickle.dump(review_preds, f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zLn4hbipm_fA",
        "colab_type": "text"
      },
      "source": [
        "Summary column's texts look pretty same to reviews, so I will take the same length"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "upOBYlkgeytR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_seq_length = 60"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EZjBIkVpnmeI",
        "colab_type": "text"
      },
      "source": [
        "The same steps for next column"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TBtCB0zNe1gV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = sub_sample['summary'].astype(str)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y_wZb3Oqe9F1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train = X.iloc[train_idxs].to_list()\n",
        "X_val = X.iloc[validation_idxs].to_list()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JWvQiR5EfEnj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 128\n",
        "\n",
        "train_lists = [X_train, y_train]\n",
        "val_lists = [X_val, y_val]\n",
        "\n",
        "training_dataset = text_dataset(x_y_list = train_lists, max_seq_length = max_seq_length)\n",
        "test_dataset = text_dataset(x_y_list = val_lists, max_seq_length = max_seq_length)\n",
        "\n",
        "dataloaders_dict = {'train': torch.utils.data.DataLoader(training_dataset, \n",
        "                                                         batch_size=batch_size, \n",
        "                                                         shuffle=True, \n",
        "                                                         num_workers=0),\n",
        "                    'val':torch.utils.data.DataLoader(test_dataset, \n",
        "                                                      batch_size=batch_size, \n",
        "                                                      shuffle=False, \n",
        "                                                      num_workers=0)}\n",
        "\n",
        "dataset_sizes = {'train':len(train_lists[0]),\n",
        "                 'val':len(val_lists[0])}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7cKXvtBNfOLZ",
        "colab_type": "code",
        "outputId": "317c15db-5b58-4fb1-d4ed-12b5bdfee680",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model = BertForSequenceClassification(config, num_labels)\n",
        "model_name = 'summury_preds'\n",
        "model.to(device)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): BertLayerNorm()\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=5, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2q6MEUJKfenW",
        "colab_type": "code",
        "outputId": "a4e8d735-9f50-4a59-d04a-fa68f096eab0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 710
        }
      },
      "source": [
        "model, summury_preds = train_model(model, model_name, criterion, optimizer, exp_lr_scheduler, dataloaders_dict, dataset_sizes, device, num_epochs=5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "starting\n",
            "Epoch 1/5\n",
            "----------\n",
            "train total loss: 1.0865 \n",
            "train F-1 score : 0.6418 \n",
            "val total loss: 1.0565 \n",
            "val F-1 score : 0.7041 \n",
            "saving with loss of 1.0564682235717773 improved over previous 100\n",
            "\n",
            "Epoch 2/5\n",
            "----------\n",
            "train total loss: 1.0850 \n",
            "train F-1 score : 0.6445 \n",
            "val total loss: 1.0565 \n",
            "val F-1 score : 0.7041 \n",
            "\n",
            "Epoch 3/5\n",
            "----------\n",
            "train total loss: 1.0844 \n",
            "train F-1 score : 0.6425 \n",
            "val total loss: 1.0565 \n",
            "val F-1 score : 0.7041 \n",
            "\n",
            "Epoch 4/5\n",
            "----------\n",
            "train total loss: 1.0868 \n",
            "train F-1 score : 0.6422 \n",
            "val total loss: 1.0565 \n",
            "val F-1 score : 0.7041 \n",
            "\n",
            "Epoch 5/5\n",
            "----------\n",
            "train total loss: 1.0855 \n",
            "train F-1 score : 0.6412 \n",
            "val total loss: 1.0565 \n",
            "val F-1 score : 0.7041 \n",
            "\n",
            "Training complete in 36m 42s\n",
            "Best val loss: 1.056468\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C8FAniCnnweg",
        "colab_type": "text"
      },
      "source": [
        "I thought that this column will be easier for Bert than previous, anyway let's save predictions and target (it will be useful too)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ydC68G8ofjEy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/drive/My Drive/test_task/summury_preds.pickle', 'wb') as f:\n",
        "    pickle.dump(summury_preds, f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NaW3mDK6abrg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/drive/My Drive/test_task/target.pickle', 'wb') as f:\n",
        "    pickle.dump(y_val, f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8fS0-pLYA0hO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}