klue_pytorch_baseline.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "import sys\n",
        "sys.path.append('/content/drive/MyDrive/깃허브에서받아옴')  # eda_nlp 경로를 추가\n",
        "%cd /content/drive/MyDrive/깃허브에서받아옴\n",
        "%pwd"
      ],
      "metadata": {
        "id": "I6DidjB6TKEb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "7afb85ec-597d-4e87-ffb7-6c61f351a1d9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "/content/drive/MyDrive/project_klue\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/drive/MyDrive/project_klue'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qDUc9SIgn0UC"
      },
      "outputs": [],
      "source": [
        "# https://dacon.io/en/competitions/official/236037/overview/description\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "import os\n",
        "import random\n",
        "from sklearn.model_selection import train_test_split\n",
        "from transformers import AutoModel, AutoTokenizer\n",
        "from torch.utils.data import Dataset\n",
        "from torch.utils.data import DataLoader\n",
        "from torch import nn\n",
        "from tqdm import tqdm\n",
        "# from eda import *\n",
        "\n",
        "# for graphing\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PTvC_fMpn0UD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9437d27f-9d62-4e50-c493-df61c5c7ac53"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((16541, 6), (7090, 1))"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "train_original = pd.read_csv('/content/train.csv')\n",
        "train_original.drop(columns=['ID'], inplace=True)\n",
        "test = pd.read_csv('/content/test.csv')\n",
        "test.drop(columns=['ID'], inplace=True)\n",
        "# submission = pd.read_csv('/content/sample_submission.csv')\n",
        "train_original.shape, test.shape"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "import pickle\n",
        "import re\n",
        "\n",
        "wordnet = {}\n",
        "with open(\"wordnet.pickle\", \"rb\") as f:\n",
        "\twordnet = pickle.load(f)\n",
        "\n",
        "\n",
        "# 한글만 남기고 나머지는 삭제\n",
        "def get_only_hangul(line):\n",
        "\tparseText= re.compile('/ ^[ㄱ-ㅎㅏ-ㅣ가-힣]*$/').sub('',line)\n",
        "\n",
        "\treturn parseText\n",
        "\n",
        "\n",
        "\n",
        "########################################################################\n",
        "# Synonym replacement\n",
        "# Replace n words in the sentence with synonyms from wordnet\n",
        "########################################################################\n",
        "def synonym_replacement(words, n):\n",
        "\tnew_words = words.copy()\n",
        "\trandom_word_list = list(set([word for word in words]))\n",
        "\trandom.shuffle(random_word_list)\n",
        "\tnum_replaced = 0\n",
        "\tfor random_word in random_word_list:\n",
        "\t\tsynonyms = get_synonyms(random_word)\n",
        "\t\tif len(synonyms) >= 1:\n",
        "\t\t\tsynonym = random.choice(list(synonyms))\n",
        "\t\t\tnew_words = [synonym if word == random_word else word for word in new_words]\n",
        "\t\t\tnum_replaced += 1\n",
        "\t\tif num_replaced >= n:\n",
        "\t\t\tbreak\n",
        "\n",
        "\tif len(new_words) != 0:\n",
        "\t\tsentence = ' '.join(new_words)\n",
        "\t\tnew_words = sentence.split(\" \")\n",
        "\n",
        "\telse:\n",
        "\t\tnew_words = \"\"\n",
        "\n",
        "\treturn new_words\n",
        "\n",
        "\n",
        "def get_synonyms(word):\n",
        "\tsynomyms = []\n",
        "\n",
        "\ttry:\n",
        "\t\tfor syn in wordnet[word]:\n",
        "\t\t\tfor s in syn:\n",
        "\t\t\t\tsynomyms.append(s)\n",
        "\texcept:\n",
        "\t\tpass\n",
        "\n",
        "\treturn synomyms\n",
        "\n",
        "########################################################################\n",
        "# Random deletion\n",
        "# Randomly delete words from the sentence with probability p\n",
        "########################################################################\n",
        "def random_deletion(words, p):\n",
        "\tif len(words) == 1:\n",
        "\t\treturn words\n",
        "\n",
        "\tnew_words = []\n",
        "\tfor word in words:\n",
        "\t\tr = random.uniform(0, 1)\n",
        "\t\tif r > p:\n",
        "\t\t\tnew_words.append(word)\n",
        "\n",
        "\tif len(new_words) == 0:\n",
        "\t\trand_int = random.randint(0, len(words)-1)\n",
        "\t\treturn [words[rand_int]]\n",
        "\n",
        "\treturn new_words\n",
        "\n",
        "########################################################################\n",
        "# Random swap\n",
        "# Randomly swap two words in the sentence n times\n",
        "########################################################################\n",
        "def random_swap(words, n):\n",
        "\tnew_words = words.copy()\n",
        "\tfor _ in range(n):\n",
        "\t\tnew_words = swap_word(new_words)\n",
        "\n",
        "\treturn new_words\n",
        "\n",
        "def swap_word(new_words):\n",
        "\trandom_idx_1 = random.randint(0, len(new_words)-1)\n",
        "\trandom_idx_2 = random_idx_1\n",
        "\tcounter = 0\n",
        "\n",
        "\twhile random_idx_2 == random_idx_1:\n",
        "\t\trandom_idx_2 = random.randint(0, len(new_words)-1)\n",
        "\t\tcounter += 1\n",
        "\t\tif counter > 3:\n",
        "\t\t\treturn new_words\n",
        "\n",
        "\tnew_words[random_idx_1], new_words[random_idx_2] = new_words[random_idx_2], new_words[random_idx_1]\n",
        "\treturn new_words\n",
        "\n",
        "########################################################################\n",
        "# Random insertion\n",
        "# Randomly insert n words into the sentence\n",
        "########################################################################\n",
        "def random_insertion(words, n):\n",
        "\tnew_words = words.copy()\n",
        "\tfor _ in range(n):\n",
        "\t\tadd_word(new_words)\n",
        "\n",
        "\treturn new_words\n",
        "\n",
        "\n",
        "def add_word(new_words):\n",
        "\tsynonyms = []\n",
        "\tcounter = 0\n",
        "\twhile len(synonyms) < 1:\n",
        "\t\tif len(new_words) >= 1:\n",
        "\t\t\trandom_word = new_words[random.randint(0, len(new_words)-1)]\n",
        "\t\t\tsynonyms = get_synonyms(random_word)\n",
        "\t\t\tcounter += 1\n",
        "\t\telse:\n",
        "\t\t\trandom_word = \"\"\n",
        "\n",
        "\t\tif counter >= 10:\n",
        "\t\t\treturn\n",
        "\n",
        "\trandom_synonym = synonyms[0]\n",
        "\trandom_idx = random.randint(0, len(new_words)-1)\n",
        "\tnew_words.insert(random_idx, random_synonym)\n",
        "\n",
        "\n",
        "\n",
        "def EDA(sentence, alpha_sr=0.1, alpha_ri=0.1, alpha_rs=0.1, p_rd=0.1, num_aug=9):\n",
        "\tsentence = get_only_hangul(sentence)\n",
        "\twords = sentence.split(' ')\n",
        "\twords = [word for word in words if word != \"\"]\n",
        "\tnum_words = len(words)\n",
        "\n",
        "\taugmented_sentences = []\n",
        "\tnum_new_per_technique = int(num_aug/4) + 1\n",
        "\n",
        "\tn_sr = max(1, int(alpha_sr*num_words))\n",
        "\tn_ri = max(1, int(alpha_ri*num_words))\n",
        "\tn_rs = max(1, int(alpha_rs*num_words))\n",
        "\n",
        "\t# sr\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = synonym_replacement(words, n_sr)\n",
        "\t\taugmented_sentences.append(' '.join(a_words))\n",
        "\n",
        "\t# ri\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = random_insertion(words, n_ri)\n",
        "\t\taugmented_sentences.append(' '.join(a_words))\n",
        "\n",
        "\t# rs\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = random_swap(words, n_rs)\n",
        "\t\taugmented_sentences.append(\" \".join(a_words))\n",
        "\n",
        "\t# rd\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = random_deletion(words, p_rd)\n",
        "\t\taugmented_sentences.append(\" \".join(a_words))\n",
        "\n",
        "\taugmented_sentences = [get_only_hangul(sentence) for sentence in augmented_sentences]\n",
        "\trandom.shuffle(augmented_sentences)\n",
        "\n",
        "\tif num_aug >= 1:\n",
        "\t\taugmented_sentences = augmented_sentences[:num_aug]\n",
        "\telse:\n",
        "\t\tkeep_prob = num_aug / len(augmented_sentences)\n",
        "\t\taugmented_sentences = [s for s in augmented_sentences if random.uniform(0, 1) < keep_prob]\n",
        "\n",
        "\taugmented_sentences.append(sentence)\n",
        "\n",
        "\treturn augmented_sentences"
      ],
      "metadata": {
        "id": "-Lvb2RGYfve8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MxvLRcKxn0UE"
      },
      "outputs": [],
      "source": [
        "def seed_everything(seed):\n",
        "    random.seed(seed)\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = True\n",
        "\n",
        "CFG = {\n",
        "    'EPOCHS':20,\n",
        "    'LEARNING_RATE':1e-5,\n",
        "    'BATCH_SIZE':32,\n",
        "    'SEED':41\n",
        "}\n",
        "\n",
        "seed_everything(CFG['SEED']) # Seed 고정\n",
        "device = torch.device('cuda')\n",
        "\n",
        "\n",
        "def augment_data(df, text_column, label_column, num_aug=5):\n",
        "    augmented_data = []\n",
        "    for _, row in df.iterrows():\n",
        "        augmented_sentences = EDA(row[text_column], num_aug=num_aug)\n",
        "        # print('증강',augmented_sentences)\n",
        "        for sentence in augmented_sentences:\n",
        "            # print(sentence)\n",
        "            new_row = {text_column: sentence}  # Initialize new_row with the text_column and sentence\n",
        "            for label in label_column:  # Iterate through label_columns\n",
        "                new_row[label] = row[label]  # Add each label column to new_row\n",
        "\n",
        "            augmented_data.append(new_row)         # Append the new row to the list\n",
        "    return pd.DataFrame(augmented_data)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text_column = '문장'\n",
        "label_column = ['유형','극성', '시제', '확실성','label']\n",
        "augmented_data = augment_data(train_original, text_column, label_column)\n",
        "augmented_data.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HtvlclliS5_W",
        "outputId": "28e3379a-5c16-453f-d0ca-451b28980eed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(99246, 6)"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "augmented_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "DTEl1CzanC1M",
        "outputId": "ff260785-44cb-4a8d-dce2-81513c76ceee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                      문장   유형  극성  시제 확실성         label\n",
              "0  0.75%포인트 금리 인상은 1994년 28년 이후 만에 처음이다.  사실형  긍정  현재  확실  사실형-긍정-현재-확실\n",
              "1  0.75%포인트 금리 인상은 1994년 이후 28년 만에 처음이다.  사실형  긍정  현재  확실  사실형-긍정-현재-확실\n",
              "2                     인상은 1994년 이후 처음이다.  사실형  긍정  현재  확실  사실형-긍정-현재-확실\n",
              "3  0.75%포인트 금리 인상은 1994년 이후 28년 만에 처음이다.  사실형  긍정  현재  확실  사실형-긍정-현재-확실\n",
              "4  0.75%포인트 금리 인상은 1994년 이후 28년 만에 처음이다.  사실형  긍정  현재  확실  사실형-긍정-현재-확실"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1f83dc77-271b-4218-8515-dffb9675b68a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>문장</th>\n",
              "      <th>유형</th>\n",
              "      <th>극성</th>\n",
              "      <th>시제</th>\n",
              "      <th>확실성</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.75%포인트 금리 인상은 1994년 28년 이후 만에 처음이다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>현재</td>\n",
              "      <td>확실</td>\n",
              "      <td>사실형-긍정-현재-확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.75%포인트 금리 인상은 1994년 이후 28년 만에 처음이다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>현재</td>\n",
              "      <td>확실</td>\n",
              "      <td>사실형-긍정-현재-확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>인상은 1994년 이후 처음이다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>현재</td>\n",
              "      <td>확실</td>\n",
              "      <td>사실형-긍정-현재-확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.75%포인트 금리 인상은 1994년 이후 28년 만에 처음이다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>현재</td>\n",
              "      <td>확실</td>\n",
              "      <td>사실형-긍정-현재-확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.75%포인트 금리 인상은 1994년 이후 28년 만에 처음이다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>현재</td>\n",
              "      <td>확실</td>\n",
              "      <td>사실형-긍정-현재-확실</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1f83dc77-271b-4218-8515-dffb9675b68a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-1f83dc77-271b-4218-8515-dffb9675b68a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-1f83dc77-271b-4218-8515-dffb9675b68a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1e2e7897-7734-4cbd-8bea-502d69498a27\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1e2e7897-7734-4cbd-8bea-502d69498a27')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1e2e7897-7734-4cbd-8bea-502d69498a27 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "augmented_data",
              "summary": "{\n  \"name\": \"augmented_data\",\n  \"rows\": 99246,\n  \"fields\": [\n    {\n      \"column\": \"\\ubb38\\uc7a5\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 69542,\n        \"samples\": [\n          \"\\ub514\\uc9c0\\ud138 \\uc138(\\u7a05)\\uad6c\\uae00, \\uc544\\ub9c8\\uc874 \\ub4f1 \\uc628\\ub77c\\uc778\\u00b7\\ubaa8\\ubc14\\uc77c \\ud50c\\ub7ab\\ud3fc \\uae30\\uc5c5\\uc758 \\uc790\\uad6d \\ub0b4 \\uacfc\\uc138\\uc774\\uba70, \\ub9e4\\ucd9c\\uc5d0 \\ub300\\ud55c \\ub514\\uc9c0\\ud138 \\uc774\\uc5d0 OECD\\ub97c \\uc911\\uc2ec\\uc73c\\ub85c \\ud604\\uc7ac \\ub300\\ud55c \\ub300\\ucc45 \\uae30\\ubcf8\\uad6c\\uc131\\uc774 \\ub17c\\uc758 \\uc911\\uc5d0 \\uc788\\ub2e4.\",\n          \"\\uc608\\uc804\\uc5d0\\ub294 \\uc2a4\\ud29c\\ub514\\uc624\\uc5d0\\uc11c \\ucd2c\\uc601\\uc744 \\ud558\\uace0, \\ud559\\uc0dd\\ub4e4\\uc744 \\uc2a4\\ud06c\\ub9b0\\uc73c\\ub85c \\ub9cc\\ub098\\ub294 \\uc0ac\\uc774\\ubc84\\ub300\\ud559 \\uc790\\uccb4\\uac00 \\ub0af\\uc124\\uc5c8\\uc9c0\\ub9cc, \\uc9c0\\uae08\\uc740 \\ub300\\ud55c\\ubbfc\\uad6d\\uc758 \\ubaa8\\ub4e0 \\ub300\\ud559\\uc774 \\uc0ac\\uc2e4\\uc0c1 \\uac19\\uc740 \\ud658\\uacbd\\uc5d0 \\ub193\\uc600\\ub2e4.\",\n          \"\\ubc15\\ub3d9\\ub7c9\\uc740 \\ud669\\ud574\\ub3c4 \\ubd09\\uc0b0\\uad70\\uc758 \\uad70\\uc218\\ub85c \\uc788\\uc5c8\\ub358 \\ubc31\\ubd80 \\ubc15\\uc751\\ucc9c\\uc5d0\\uac8c\\uc11c \\uc774 \\uc77c\\ub300\\ub97c \\ubb34\\ub300\\ub85c \\ud65c\\uc57d\\ud588\\ub358 \\uc784\\uaebd\\uc815\\uc5d0 \\ub300\\ud574 \\ub4e4\\uc5b4\\uc11c \\uc798 \\uc54c\\uace0 \\uc788\\uc5c8\\ub358 \\uac83\\uc73c\\ub85c\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"\\uc720\\ud615\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"\\ucd94\\ub860\\ud615\",\n          \"\\ub300\\ud654\\ud615\",\n          \"\\uc0ac\\uc2e4\\ud615\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"\\uadf9\\uc131\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"\\uae0d\\uc815\",\n          \"\\ubd80\\uc815\",\n          \"\\ubbf8\\uc815\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"\\uc2dc\\uc81c\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"\\ud604\\uc7ac\",\n          \"\\uacfc\\uac70\",\n          \"\\ubbf8\\ub798\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"\\ud655\\uc2e4\\uc131\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"\\ubd88\\ud655\\uc2e4\",\n          \"\\ud655\\uc2e4\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 64,\n        \"samples\": [\n          \"\\uc0ac\\uc2e4\\ud615-\\ubd80\\uc815-\\ud604\\uc7ac-\\ubd88\\ud655\\uc2e4\",\n          \"\\ub300\\ud654\\ud615-\\ubbf8\\uc815-\\uacfc\\uac70-\\ud655\\uc2e4\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4mIq3sWLn0UE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "928dc608-b91c-470b-fcd7-ea11f46d3e85"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((79396, 5), (19850, 5), (79396,), (19850,))"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "train, val, train_target, val_target = train_test_split(augmented_data.drop('label', axis=1), augmented_data['label'], test_size=0.2, random_state=CFG['SEED'])\n",
        "train = train.reset_index(drop=True)\n",
        "val = val.reset_index(drop=True)\n",
        "train_target = train_target.reset_index(drop=True)\n",
        "val_target = val_target.reset_index(drop=True)\n",
        "train.shape, val.shape, train_target.shape, val_target.shape"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "bnyWhRppjdA1",
        "outputId": "baceefab-38d8-429c-b47f-c2d8c15ae08b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                      문장   유형  극성  시제 확실성\n",
              "0      당시 청소년의 바 보장하기 위해 게임 정하는 심야시간에 청소년에 대한 대통령령이 제...  사실형  긍정  과거  확실\n",
              "1                         낙화생은 옹크려져 누에 같으면서 몸뚱이가 형체가 있다.  사실형  긍정  현재  확실\n",
              "2               휴일이면 자동차를 자가 정비하는 모습을 동네 곳곳에서 쉽게 볼 수 있다.  추론형  긍정  현재  확실\n",
              "3          백병전 대신 지형에 매복해 활을 쏘는 게릴라전에서는 승전 소식이 들리기 시작했다.  사실형  긍정  과거  확실\n",
              "4      횡령 추정액 중 2017∼2019년 사이 사라진 3억4000만원은 지난달 숨진 경리...  사실형  긍정  과거  확실\n",
              "...                                                  ...  ...  ..  ..  ..\n",
              "79391                     한국 정부도 노인 일자리를 노 만들기 위해 노력해왔다.  사실형  긍정  과거  확실\n",
              "79392                  그런데 이렇게 맛 좋은 우곡주를 맛 제가 잘 팔지 못했어요.  대화형  부정  과거  확실\n",
              "79393        셀트리온이 밝힌 공식적인 블록딜 사유는 ＇투자 기간별 포트폴리오 조정＇이었다.  사실형  긍정  과거  확실\n",
              "79394  화장실은 단지 배 속 평안을 간구하는 생리학적 배 공간이 아니라 인류의 평안을 추구...  추론형  긍정  현재  확실\n",
              "79395             이스포츠 사업도 함께 있으며 이 분야는 글로벌을 모두 담당하고 있다.  사실형  긍정  과거  확실\n",
              "\n",
              "[79396 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-85fefb9e-bc2c-4ca7-a3e1-410969457e79\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>문장</th>\n",
              "      <th>유형</th>\n",
              "      <th>극성</th>\n",
              "      <th>시제</th>\n",
              "      <th>확실성</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>당시 청소년의 바 보장하기 위해 게임 정하는 심야시간에 청소년에 대한 대통령령이 제...</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>과거</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>낙화생은 옹크려져 누에 같으면서 몸뚱이가 형체가 있다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>현재</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>휴일이면 자동차를 자가 정비하는 모습을 동네 곳곳에서 쉽게 볼 수 있다.</td>\n",
              "      <td>추론형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>현재</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>백병전 대신 지형에 매복해 활을 쏘는 게릴라전에서는 승전 소식이 들리기 시작했다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>과거</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>횡령 추정액 중 2017∼2019년 사이 사라진 3억4000만원은 지난달 숨진 경리...</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>과거</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79391</th>\n",
              "      <td>한국 정부도 노인 일자리를 노 만들기 위해 노력해왔다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>과거</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79392</th>\n",
              "      <td>그런데 이렇게 맛 좋은 우곡주를 맛 제가 잘 팔지 못했어요.</td>\n",
              "      <td>대화형</td>\n",
              "      <td>부정</td>\n",
              "      <td>과거</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79393</th>\n",
              "      <td>셀트리온이 밝힌 공식적인 블록딜 사유는 ＇투자 기간별 포트폴리오 조정＇이었다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>과거</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79394</th>\n",
              "      <td>화장실은 단지 배 속 평안을 간구하는 생리학적 배 공간이 아니라 인류의 평안을 추구...</td>\n",
              "      <td>추론형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>현재</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79395</th>\n",
              "      <td>이스포츠 사업도 함께 있으며 이 분야는 글로벌을 모두 담당하고 있다.</td>\n",
              "      <td>사실형</td>\n",
              "      <td>긍정</td>\n",
              "      <td>과거</td>\n",
              "      <td>확실</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>79396 rows × 5 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-85fefb9e-bc2c-4ca7-a3e1-410969457e79')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-85fefb9e-bc2c-4ca7-a3e1-410969457e79 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-85fefb9e-bc2c-4ca7-a3e1-410969457e79');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-26a9edb2-bc32-46cd-beed-b257cf31ff30\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-26a9edb2-bc32-46cd-beed-b257cf31ff30')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-26a9edb2-bc32-46cd-beed-b257cf31ff30 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_22fa6c2e-64f7-4886-b234-c216eeb90f4c\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('train')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_22fa6c2e-64f7-4886-b234-c216eeb90f4c button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('train');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "train",
              "summary": "{\n  \"name\": \"train\",\n  \"rows\": 79396,\n  \"fields\": [\n    {\n      \"column\": \"\\ubb38\\uc7a5\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 58052,\n        \"samples\": [\n          \"\\uc758\\uc6d0\\uc774 \\uace0\\ud658\\uc744 \\ucc9c\\ucc9c\\ud788 \\uacc4 \\ub123\\uace0 \\ub0a9\\uac70\\ubbf8(\\uac70\\ubbf8\\uc758 \\uc77c\\uc885)\\ub97c \\uc7a1\\uc544 \\uc9d3\\uc774\\uaca8 \\uc0c1\\ucc98\\uc5d0 \\uacc4\\uc18d \\ubd99\\uc5ec\\uc8fc\\uc790 \\ub2e4\\uce58\\uae30 \\uc804\\uacfc \\uac19\\uc544\\uc84c\\ub2e4.\",\n          \"\\uc774 \\ub9db\\uc758 \\ube44\\uacb0\\uc740 \\uc8fc\\uc778\\uc7a5\\uc758 \\uc190\\ub9db\\uc5d0 \\ub354\\ud574\\uc9c4 \\uad6d\\ub0b4\\uc0b0 \\uc7ac\\ub8cc\\uc640 \\ub9e4\\uc77c \\uc900\\ube44\\ud558\\ub294 \\uc2e0\\uc120\\ud55c \\ub9cc\\ub450 \\uc18c\\uac00 \\uc544\\ub2d0\\uae4c \\uc2f6\\ub2e4.\",\n          \"\\ud575\\uc2ec\\uc801\\uc778 \\ucd94\\uc9c4\\uacfc\\uc81c\\ub85c \\ubbfc\\uac04\\uc0ac\\uc5c5 \\ubd80\\ubb38 \\uacbd\\uc720\\ucc28 \\ud1f4\\ucd9c \\uc720\\ub3c4, \\uc2dc\\ubbfc\\uacfc \\ud568\\uaed8 \\uff07\\uacbd\\uc720\\ucc28 \\uc548 \\uace0 \\uc548\\ud0c0\\uae30\\uff07 \\ub3d9 \\uc804\\uac1c, \\uacbd\\uc720\\ucc28 \\uac10\\ucd95\\uc744 \\uc704\\ud55c \\uc81c\\ub3c4\\uac1c\\uc120 \\uc815\\ucc45\\uc744 \\uacfc\\uac10\\ud558\\uac8c \\ucd94\\uc9c4\\ud55c\\ub2e4\\ub294 \\uacc4\\ud68d\\uc774\\ub2e4.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"\\uc720\\ud615\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"\\ucd94\\ub860\\ud615\",\n          \"\\uc608\\uce21\\ud615\",\n          \"\\uc0ac\\uc2e4\\ud615\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"\\uadf9\\uc131\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"\\uae0d\\uc815\",\n          \"\\ubd80\\uc815\",\n          \"\\ubbf8\\uc815\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"\\uc2dc\\uc81c\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"\\uacfc\\uac70\",\n          \"\\ud604\\uc7ac\",\n          \"\\ubbf8\\ub798\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"\\ud655\\uc2e4\\uc131\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"\\ubd88\\ud655\\uc2e4\",\n          \"\\ud655\\uc2e4\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fM3gqk-ln0UE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 358,
          "referenced_widgets": [
            "917bde512ea44912bd502521697fea2b",
            "e223659f110642f8b60b9e7e3d2ee505",
            "8c258cd9f69748ca9dcc52a42cc34d71",
            "3c616fc3b5384506850ee5e4696f4ef9",
            "d412cdd33ebd46319945a90db1132854",
            "6c7bbc00074d4dde96fc3f0d120da9fa",
            "a402990e6f4049c2af8f29b0430bed45",
            "9d51058108504dd99e42409cc297d330",
            "b64522848b694087808077d410f331e1",
            "105338fdc55647e2a587322d209b540a",
            "696c13e6f41d4ab98973c5f660d69117",
            "85efc5e3c1a24bdb9a4fc47cf55d87f7",
            "681bcd37c7cb4b0286eef70355f2984c",
            "9b333f4b75564871a253bfc70f5d5c97",
            "bf18a288c68b406da6e880adad033273",
            "5c63dd6717da48378b800cb8cff2d917",
            "bd44b35965f042329008e82f0a0f664f",
            "3207598613d34f229376c1932e1e5e43",
            "259a63b07f6b406faec160d1f08bfde4",
            "01fddb42b16e4e30a6b6cfacea25d9e9",
            "9f0f662ec72f4c27a34e3eedd80eb526",
            "a5825d1b77ba404d97a9f7ac066fba95",
            "badc8137c2f04361a8913e4cad041ff6",
            "6aadae7669894aeeb5772dbfb38adf1c",
            "195f43f2976d42be8f7bbaa901b91a4c",
            "179ed6e211104c15b5af0144e3e56a74",
            "3fed7027f600453d99a50527389c3f8b",
            "284a28dca5754a9d9b32b5c402e1e75c",
            "76b7aca84b074e909337b9f360c2f698",
            "9e5a623911584e67a6da195cdecdf464",
            "d4f1dec4538942f7ab1d9e7af1faa9a6",
            "6b9d04ebe1ce4db59fb4e78264853912",
            "2e2fe5376cac418785c618656aa010bb",
            "c6f834ae5f59434e877e21e1c9868d0d",
            "a2766d9fc60c47159682c5af2c39cab6",
            "7d7c0e5db59a4bb98b99d8b49d684a21",
            "8f2c7a510de34481a439f08b7093d1fb",
            "f553d72c17ed4d7e9dc8ddfc1f73e7bd",
            "9e6ee617ede7441484361cc30fa17b18",
            "a92a3b16c97f4ff3bc6f334dfef3600e",
            "64781b40de6e4eb08ca3fefc15319dae",
            "1cf8305a1a394ec58bbee60471093b98",
            "2e1b12b0dee345b08b1a33ca847d7b06",
            "858f135828684db4837a86c918c00847",
            "0b5c6b2877ba4924a3ead77c4e56583b",
            "8b8986ce16d146c1bffa652ef61f4f92",
            "9ccd2565eeaf4db0af4983d3904f3d9b",
            "e76ab28c0d9948fea3954a431c45c46c",
            "4b02d4241f2142d39f03de8afaf20be7",
            "218cfe08f0a14d1f889032e2f39f6a7a",
            "9d868a274ea642a58a41e24b18ab11f9",
            "50fb4e979ed1499a9acc47cab1a79ecc",
            "0fc198b6f5b9400682d90fb968960a02",
            "34f566f492c7499b9e0cb292a0c39da5",
            "2f6dda289f5845d3ac498456cdf40b33",
            "dce670de3a544dd08db2c2ee3c569189",
            "0c45c801b79a4c4ba76cfa96770570b5",
            "79d7a148cbc043108027265f5d40ce3a",
            "62d4708974e543b391c66c3e07484883",
            "d84236a6c117471f9108ba1cbfde4558",
            "61589b104d7a44638844ebd606a1d66e",
            "1403624143284e55886a8b2bcaa849b4",
            "059af35461e845b988c2febbb8d8f471",
            "c4c0cb7aace2483884b0e0c6d1e26f03",
            "ffbead7d1ba64d77907563302faa0069",
            "cdb05c643c8a4027a52e93e634c598eb"
          ]
        },
        "outputId": "6bf5b01e-dced-4015-d29b-36f8c45efed7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/545 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "917bde512ea44912bd502521697fea2b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/273M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "85efc5e3c1a24bdb9a4fc47cf55d87f7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of RobertaModel were not initialized from the model checkpoint at klue/roberta-small and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/375 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "badc8137c2f04361a8913e4cad041ff6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/248k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c6f834ae5f59434e877e21e1c9868d0d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/752k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0b5c6b2877ba4924a3ead77c4e56583b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/173 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dce670de3a544dd08db2c2ee3c569189"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "model_nm = 'klue/roberta-small'\n",
        "base_model = AutoModel.from_pretrained(model_nm)\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_nm)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KEQLLKFwn0UF"
      },
      "outputs": [],
      "source": [
        "# tokenizer_len = [len(tokenizer(s)['input_ids']) for s in train['문장']]\n",
        "# sns.histplot(tokenizer_len)\n",
        "# plt.show()\n",
        "\n",
        "# print(f'log value : {np.mean(tokenizer_len)+3*np.std(tokenizer_len)}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H8zpSwjKn0UF"
      },
      "outputs": [],
      "source": [
        "# tokenizer_log = np.log(tokenizer_len)\n",
        "# sns.histplot(tokenizer_log)\n",
        "# plt.show()\n",
        "\n",
        "# print(f'log value : {np.mean(tokenizer_log)+3*np.std(tokenizer_log)}')\n",
        "# print(f'original value : {np.exp(np.mean(tokenizer_log)+3*np.std(tokenizer_log))}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Dm6gWhsAn0UF"
      },
      "outputs": [],
      "source": [
        "class SentenceTypeDataset(Dataset):\n",
        "    def __init__(self, dataframe, tokenizer, labels=None):\n",
        "        texts = dataframe['문장'].values.tolist()\n",
        "        # print('texts', texts)\n",
        "        self.texts = [tokenizer(text, padding='max_length', max_length=90, truncation=True, return_tensors='pt') for text in texts]\n",
        "        self.labels = labels\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.texts)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        text = self.texts[idx]\n",
        "        # print('text********************\\n', text)\n",
        "        if self.labels is not None:\n",
        "            type_tmp = self.labels['type'][idx]\n",
        "            polarity_tmp = self.labels['polarity'][idx]\n",
        "            tense_tmp = self.labels['tense'][idx]\n",
        "            certainty_tmp = self.labels['certainty'][idx]\n",
        "            return text, torch.Tensor(type_tmp), torch.Tensor(polarity_tmp), torch.Tensor(tense_tmp), torch.Tensor(certainty_tmp)\n",
        "        else:\n",
        "            return text, torch.Tensor([-1,-1,-1,-1]), torch.Tensor([-1,-1,-1]), torch.Tensor([-1,-1,-1]), torch.Tensor([-1,-1])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FjU_XG1Mn0UG"
      },
      "outputs": [],
      "source": [
        "class SentenceClassifier(nn.Module):\n",
        "    def __init__(self, base_model):\n",
        "        super().__init__()\n",
        "        self.klue = base_model # from transformers package\n",
        "\n",
        "        self.fc1 = nn.Linear(self.klue.config.hidden_size, 32)\n",
        "        self.layernorm = nn.LayerNorm(32)\n",
        "        self.dropout = nn.Dropout(p = 0.1)\n",
        "        self.relu = nn.ReLU()\n",
        "        self.type_clf = nn.Linear(32,4)\n",
        "        self.polarity_clf = nn.Linear(32,3)\n",
        "        self.tense_clf = nn.Linear(32,3)\n",
        "        self.certainty_clf = nn.Linear(32,2)\n",
        "        self.softmax = nn.Softmax(dim=1)\n",
        "\n",
        "    def forward(self, input_ids, attention_mask, token_type_ids=None):\n",
        "        # input_ids : token's id / attention_mask : make a model to focus on which token\n",
        "        klue_out = self.klue(input_ids= input_ids, attention_mask = attention_mask)[0][:,0]\n",
        "\n",
        "        x = self.fc1(klue_out)\n",
        "        # print('x*************************', x)\n",
        "        x = self.layernorm(x)\n",
        "        x = self.dropout(x)\n",
        "        x = self.relu(x)\n",
        "\n",
        "        type_output = self.type_clf(x)\n",
        "        type_output = self.softmax(type_output)\n",
        "        polarity_output = self.polarity_clf(x)\n",
        "        polarity_output = self.softmax(polarity_output)\n",
        "        tense_output = self.tense_clf(x)\n",
        "        tense_output = self.softmax(tense_output)\n",
        "        certainty_output = self.certainty_clf(x)\n",
        "        certainty_output = self.softmax(certainty_output)\n",
        "\n",
        "        return type_output, polarity_output, tense_output, certainty_output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uChaWMcLn0UG"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import f1_score\n",
        "import numpy as np\n",
        "import torch\n",
        "\n",
        "def sentence_train(model, train_dataloader, val_dataloader, learning_rate, epochs, model_nm):\n",
        "    best_val_loss = 99999999999999  # setting max (act as infinity)\n",
        "    early_stopping_threshold_count = 0\n",
        "\n",
        "    criterion = {\n",
        "        'type': nn.CrossEntropyLoss().to(device),\n",
        "        'polarity': nn.CrossEntropyLoss().to(device),\n",
        "        'tense': nn.CrossEntropyLoss().to(device),\n",
        "        'certainty': nn.CrossEntropyLoss().to(device)\n",
        "    }\n",
        "\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "    model = model.to(device)\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        total_f1_train = 0  # F1 score for training set\n",
        "        total_loss_train = 0\n",
        "\n",
        "        model.train()  # sets into the training mode\n",
        "\n",
        "        all_true_labels = []\n",
        "        all_pred_labels = []\n",
        "\n",
        "        for train_input, type_label, polarity_label, tense_label, certainty_label in tqdm(train_dataloader):\n",
        "            attention_mask = train_input['attention_mask'].to(device)\n",
        "            input_ids = train_input['input_ids'].squeeze(1).to(device)\n",
        "            type_label = type_label.to(device)\n",
        "            polarity_label = polarity_label.to(device)\n",
        "            tense_label = tense_label.to(device)\n",
        "            certainty_label = certainty_label.to(device)\n",
        "            optimizer.zero_grad()  # 가중치를 업데이트할 때 사용할 기울기를 초기화\n",
        "\n",
        "            type_output, polarity_output, tense_output, certainty_output = model(input_ids, attention_mask)  # from the forward function\n",
        "            # print('output***********************************************', type_output, polarity_output, tense_output, certainty_output)\n",
        "            loss = 0.25 * criterion['type'](type_output, type_label) + \\\n",
        "                   0.25 * criterion['polarity'](polarity_output, polarity_label) + \\\n",
        "                   0.25 * criterion['tense'](tense_output, tense_label) + \\\n",
        "                   0.25 * criterion['certainty'](certainty_output, certainty_label)\n",
        "            total_loss_train += loss.item()\n",
        "\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            # Collect true labels and predicted labels for F1 score calculation\n",
        "            all_true_labels.append(type_label.cpu().numpy())  # True labels\n",
        "            all_pred_labels.append(torch.argmax(type_output, dim=1).cpu().numpy())  # Predicted labels\n",
        "\n",
        "        # Flatten the list of true and predicted labels\n",
        "\n",
        "        all_true_labels = np.argmax(np.concatenate(all_true_labels), axis=1) if all_true_labels[0].ndim > 1 else np.concatenate(all_true_labels)\n",
        "        all_pred_labels = np.argmax(np.concatenate(all_pred_labels), axis=1) if all_pred_labels[0].ndim > 1 else np.concatenate(all_pred_labels)\n",
        "\n",
        "        print(all_true_labels)\n",
        "        print(all_pred_labels)\n",
        "        # Compute F1 score for training\n",
        "        total_f1_train = f1_score(all_true_labels, all_pred_labels, average='weighted')  # 'weighted' for multiclass\n",
        "\n",
        "        with torch.no_grad():  # since we should not change gradient for validation\n",
        "            total_f1_val = 0  # F1 score for validation set\n",
        "            total_loss_val = 0\n",
        "\n",
        "            model.eval()  # deactivate training\n",
        "\n",
        "            all_true_val_labels = []\n",
        "            all_pred_val_labels = []\n",
        "\n",
        "            for val_input, vtype_label, vpolarity_label, vtense_label, vcertainty_label in tqdm(val_dataloader):\n",
        "                attention_mask = val_input['attention_mask'].to(device)\n",
        "                input_ids = val_input['input_ids'].squeeze(1).to(device)\n",
        "\n",
        "                vtype_label = vtype_label.to(device)\n",
        "                vpolarity_label = vpolarity_label.to(device)\n",
        "                vtense_label = vtense_label.to(device)\n",
        "                vcertainty_label = vcertainty_label.to(device)\n",
        "\n",
        "                vtype_output, vpolarity_output, vtense_output, vcertainty_output = model(input_ids, attention_mask)  # from the forward function\n",
        "\n",
        "                loss = 0.25 * criterion['type'](vtype_output, vtype_label) + \\\n",
        "                        0.25 * criterion['polarity'](vpolarity_output, vpolarity_label) + \\\n",
        "                        0.25 * criterion['tense'](vtense_output, vtense_label) + \\\n",
        "                        0.25 * criterion['certainty'](vcertainty_output, vcertainty_label)\n",
        "\n",
        "                total_loss_val += loss.item()\n",
        "\n",
        "                # Collect true labels and predicted labels for F1 score calculation\n",
        "                all_true_val_labels.append(vtype_label.cpu().numpy())  # True labels\n",
        "                all_pred_val_labels.append(torch.argmax(vtype_output, dim=1).cpu().numpy())  # Predicted labels\n",
        "\n",
        "            # Flatten the list of true and predicted labels\n",
        "            all_true_val_labels = np.argmax(np.concatenate(all_true_val_labels), axis=1) if all_true_val_labels[0].ndim > 1 else np.concatenate(all_true_val_labels)\n",
        "            all_pred_val_labels = np.argmax(np.concatenate(all_pred_val_labels), axis=1) if all_pred_val_labels[0].ndim > 1 else np.concatenate(all_pred_val_labels)\n",
        "\n",
        "            print(all_true_val_labels)\n",
        "            print(all_pred_val_labels)\n",
        "            # Compute F1 score for validation\n",
        "            total_f1_val = f1_score(all_true_val_labels, all_pred_val_labels, average='weighted')  # 'weighted' for multiclass\n",
        "\n",
        "            os.makedirs(\"model1\", exist_ok=True)\n",
        "\n",
        "            print(f'Epochs: {epoch + 1} '\n",
        "                  f'| Train Loss: {total_loss_train / len(train_dataloader): .3f} '\n",
        "                  f'| Train F1 Score: {total_f1_train: .3f} '\n",
        "                  f'| Val Loss: {total_loss_val / len(val_dataloader): .3f} '\n",
        "                  f'| Val F1 Score: {total_f1_val: .3f}')\n",
        "\n",
        "            if best_val_loss > total_loss_val:\n",
        "                best_val_loss = total_loss_val  # saving only the best one\n",
        "                torch.save(model, f\"model1/{model_nm}.pt\")\n",
        "                print(\"Saved model\")\n",
        "                early_stopping_threshold_count = 0\n",
        "            else:\n",
        "                early_stopping_threshold_count += 1  # checking how many epochs have passed that val_loss didn't increase\n",
        "\n",
        "            if early_stopping_threshold_count >= 3:  # ==> patience=1\n",
        "                print(\"Early stopping\")\n",
        "                break\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2cp8CqOPn0UG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "825ddd50-54ca-4d5b-dd52-6a3beb2e9533"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "before shape (79396, 5)\n",
            "train_tmp                                                   문장  유형_대화형  유형_사실형  유형_예측형  \\\n",
            "0  당시 청소년의 바 보장하기 위해 게임 정하는 심야시간에 청소년에 대한 대통령령이 제...   False    True   False   \n",
            "1                     낙화생은 옹크려져 누에 같으면서 몸뚱이가 형체가 있다.   False    True   False   \n",
            "2           휴일이면 자동차를 자가 정비하는 모습을 동네 곳곳에서 쉽게 볼 수 있다.   False   False   False   \n",
            "3      백병전 대신 지형에 매복해 활을 쏘는 게릴라전에서는 승전 소식이 들리기 시작했다.   False    True   False   \n",
            "4  횡령 추정액 중 2017∼2019년 사이 사라진 3억4000만원은 지난달 숨진 경리...   False    True   False   \n",
            "\n",
            "   유형_추론형  극성_긍정  극성_미정  극성_부정  시제_과거  시제_미래  시제_현재  확실성_불확실  확실성_확실  \n",
            "0   False   True  False  False   True  False  False    False    True  \n",
            "1   False   True  False  False  False  False   True    False    True  \n",
            "2    True   True  False  False  False  False   True    False    True  \n",
            "3   False   True  False  False   True  False  False    False    True  \n",
            "4   False   True  False  False   True  False  False    False    True  \n",
            "after shape (79396, 13)\n"
          ]
        }
      ],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    print('before shape', train.shape)\n",
        "    train_tmp = train[['문장', '유형', '극성', '시제', '확실성']]\n",
        "    train_tmp = pd.get_dummies(train_tmp, columns=['유형', '극성', '시제', '확실성'])\n",
        "    print('train_tmp', train_tmp.head())\n",
        "\n",
        "    print('after shape', train_tmp.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nePqb56pn0UG"
      },
      "outputs": [],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    train_type = train_tmp.iloc[:,1:5].values.tolist()\n",
        "    train_polarity = train_tmp.iloc[:,5:8].values.tolist()\n",
        "    train_tense = train_tmp.iloc[:,8:11].values.tolist()\n",
        "    train_certainty = train_tmp.iloc[:,11:13].values.tolist()\n",
        "\n",
        "    train_labels = {\n",
        "        'type': train_type,\n",
        "        'polarity': train_polarity,\n",
        "        'tense': train_tense,\n",
        "        'certainty': train_certainty\n",
        "    }\n",
        "    pd.DataFrame(train_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ii91MruAn0UH"
      },
      "outputs": [],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    val_tmp = val[['문장', '유형', '극성', '시제', '확실성']]\n",
        "    val_tmp = pd.get_dummies(val_tmp, columns=['유형', '극성', '시제', '확실성'])\n",
        "    # print('val_tmp', val_tmp.head())\n",
        "    val_type = val_tmp.iloc[:,1:5].values.tolist()\n",
        "    val_polarity = val_tmp.iloc[:,5:8].values.tolist()\n",
        "    val_tense = val_tmp.iloc[:,8:11].values.tolist()\n",
        "    val_certainty = val_tmp.iloc[:,11:13].values.tolist()\n",
        "    val_labels = {\n",
        "        'type': val_type,\n",
        "        'polarity': val_polarity,\n",
        "        'tense': val_tense,\n",
        "        'certainty': val_certainty\n",
        "    }\n",
        "    pd.DataFrame(val_labels)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    pd.DataFrame(train_labels).iloc[:, 1]"
      ],
      "metadata": {
        "id": "PY7R5epjELUA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LAOt7rVwn0UH"
      },
      "outputs": [],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    train_dataloader = DataLoader(SentenceTypeDataset(train_tmp, tokenizer, train_labels), batch_size=CFG['BATCH_SIZE'], shuffle=True, num_workers=0) # num_workers: how many subprocesses to use for data loading\n",
        "    val_dataloader = DataLoader(SentenceTypeDataset(val_tmp, tokenizer, val_labels), batch_size=CFG['BATCH_SIZE'], num_workers=0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mXHw1x46n0UH"
      },
      "outputs": [],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    model = SentenceClassifier(base_model)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    print(len(train_dataloader.dataset))  # 데이터셋 크기 확인\n",
        "\n",
        "    print(len(val_dataloader.dataset))    # 데이터셋 크기 확인"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N20RYezTwgWr",
        "outputId": "ff36c4bb-a727-4470-bdbc-cba80d22ffbf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "79396\n",
            "19850\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    for batch in train_dataloader:\n",
        "        print('batch', batch)  # 데이터가 올바르게 로드되는지 확인\n",
        "        break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-jFqx7qwwl9X",
        "outputId": "a2eead7e-7fcb-4dd4-ad10-6cff2f268eae"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "batch [{'input_ids': tensor([[[    0,  3625,    23,  ...,     1,     1,     1]],\n",
            "\n",
            "        [[    0,     3, 26947,  ...,     1,     1,     1]],\n",
            "\n",
            "        [[    0,  6910,  2259,  ...,     1,     1,     1]],\n",
            "\n",
            "        ...,\n",
            "\n",
            "        [[    0, 11821,  6724,  ...,     1,     1,     1]],\n",
            "\n",
            "        [[    0, 25668,  2850,  ...,     1,     1,     1]],\n",
            "\n",
            "        [[    0,   170,  1583,  ...,     1,     1,     1]]]), 'token_type_ids': tensor([[[0, 0, 0,  ..., 0, 0, 0]],\n",
            "\n",
            "        [[0, 0, 0,  ..., 0, 0, 0]],\n",
            "\n",
            "        [[0, 0, 0,  ..., 0, 0, 0]],\n",
            "\n",
            "        ...,\n",
            "\n",
            "        [[0, 0, 0,  ..., 0, 0, 0]],\n",
            "\n",
            "        [[0, 0, 0,  ..., 0, 0, 0]],\n",
            "\n",
            "        [[0, 0, 0,  ..., 0, 0, 0]]]), 'attention_mask': tensor([[[1, 1, 1,  ..., 0, 0, 0]],\n",
            "\n",
            "        [[1, 1, 1,  ..., 0, 0, 0]],\n",
            "\n",
            "        [[1, 1, 1,  ..., 0, 0, 0]],\n",
            "\n",
            "        ...,\n",
            "\n",
            "        [[1, 1, 1,  ..., 0, 0, 0]],\n",
            "\n",
            "        [[1, 1, 1,  ..., 0, 0, 0]],\n",
            "\n",
            "        [[1, 1, 1,  ..., 0, 0, 0]]])}, tensor([[0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 0., 0., 1.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 0., 0., 1.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 0., 0., 1.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 0., 0., 1.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 0., 0., 1.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [0., 0., 0., 1.],\n",
            "        [0., 1., 0., 0.],\n",
            "        [1., 0., 0., 0.],\n",
            "        [0., 1., 0., 0.]]), tensor([[1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [0., 0., 1.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.]]), tensor([[1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [0., 1., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 1., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [0., 0., 1.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [0., 0., 1.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [1., 0., 0.],\n",
            "        [0., 0., 1.],\n",
            "        [1., 0., 0.]]), tensor([[0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [0., 1.],\n",
            "        [1., 0.],\n",
            "        [0., 1.]])]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IBmBdZSdn0UH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5ee940bc-6dbd-488d-cc2e-7c51f3854516"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:45<00:00,  4.24it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 3 1 3]\n",
            "[2 2 2 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.39it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 1]\n",
            "Epochs: 1 | Train Loss:  0.832 | Train F1 Score:  0.787 | Val Loss:  0.771 | Val F1 Score:  0.820\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.25it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 0 1 1]\n",
            "[1 1 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.41it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 1]\n",
            "Epochs: 2 | Train Loss:  0.740 | Train F1 Score:  0.824 | Val Loss:  0.704 | Val F1 Score:  0.842\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.25it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 0 1 ... 1 3 1]\n",
            "[1 1 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.41it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 1]\n",
            "Epochs: 3 | Train Loss:  0.685 | Train F1 Score:  0.842 | Val Loss:  0.661 | Val F1 Score:  0.847\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:42<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 1]\n",
            "[1 1 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.43it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 1]\n",
            "Epochs: 4 | Train Loss:  0.649 | Train F1 Score:  0.883 | Val Loss:  0.631 | Val F1 Score:  0.919\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:42<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 1]\n",
            "[1 1 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.43it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 1]\n",
            "Epochs: 5 | Train Loss:  0.624 | Train F1 Score:  0.919 | Val Loss:  0.612 | Val F1 Score:  0.927\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 0 0 ... 1 1 3]\n",
            "[1 0 0 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.40it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 1]\n",
            "Epochs: 6 | Train Loss:  0.607 | Train F1 Score:  0.936 | Val Loss:  0.598 | Val F1 Score:  0.945\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 0 1 1]\n",
            "[1 1 1 ... 0 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.42it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 1]\n",
            "Epochs: 7 | Train Loss:  0.595 | Train F1 Score:  0.946 | Val Loss:  0.588 | Val F1 Score:  0.949\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 1]\n",
            "[1 1 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.39it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 8 | Train Loss:  0.586 | Train F1 Score:  0.954 | Val Loss:  0.581 | Val F1 Score:  0.959\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:42<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 3 1 1]\n",
            "[1 1 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.42it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 9 | Train Loss:  0.580 | Train F1 Score:  0.960 | Val Loss:  0.576 | Val F1 Score:  0.963\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.25it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 0 ... 1 0 1]\n",
            "[1 1 0 ... 1 0 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.31it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 10 | Train Loss:  0.575 | Train F1 Score:  0.964 | Val Loss:  0.573 | Val F1 Score:  0.963\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.25it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 0 1 ... 0 1 1]\n",
            "[1 0 1 ... 0 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.44it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 11 | Train Loss:  0.572 | Train F1 Score:  0.967 | Val Loss:  0.569 | Val F1 Score:  0.970\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:42<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[3 1 1 ... 1 0 1]\n",
            "[3 1 1 ... 1 0 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.41it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 12 | Train Loss:  0.569 | Train F1 Score:  0.970 | Val Loss:  0.567 | Val F1 Score:  0.971\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:42<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[3 0 1 ... 1 1 1]\n",
            "[3 0 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.35it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 13 | Train Loss:  0.566 | Train F1 Score:  0.972 | Val Loss:  0.565 | Val F1 Score:  0.973\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 3]\n",
            "[1 1 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.42it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 14 | Train Loss:  0.564 | Train F1 Score:  0.974 | Val Loss:  0.563 | Val F1 Score:  0.973\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[3 1 1 ... 3 1 1]\n",
            "[3 1 1 ... 3 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.38it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 15 | Train Loss:  0.563 | Train F1 Score:  0.975 | Val Loss:  0.562 | Val F1 Score:  0.975\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[3 1 1 ... 1 1 1]\n",
            "[3 1 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.40it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 16 | Train Loss:  0.562 | Train F1 Score:  0.976 | Val Loss:  0.561 | Val F1 Score:  0.976\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.25it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 0 3 ... 1 1 1]\n",
            "[1 0 3 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.40it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 17 | Train Loss:  0.560 | Train F1 Score:  0.977 | Val Loss:  0.560 | Val F1 Score:  0.976\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:43<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 1]\n",
            "[1 1 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.41it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 18 | Train Loss:  0.559 | Train F1 Score:  0.978 | Val Loss:  0.559 | Val F1 Score:  0.977\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:42<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 0 1 ... 1 1 1]\n",
            "[1 0 1 ... 1 1 1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.43it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 19 | Train Loss:  0.559 | Train F1 Score:  0.979 | Val Loss:  0.559 | Val F1 Score:  0.977\n",
            "Saved model\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2482/2482 [09:42<00:00,  4.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 3]\n",
            "[1 1 1 ... 1 1 3]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 621/621 [00:46<00:00, 13.42it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 1 ... 1 1 2]\n",
            "[1 1 1 ... 1 1 2]\n",
            "Epochs: 20 | Train Loss:  0.558 | Train F1 Score:  0.978 | Val Loss:  0.559 | Val F1 Score:  0.977\n",
            "Saved model\n"
          ]
        }
      ],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    sentence_train(model, train_dataloader, val_dataloader, CFG['LEARNING_RATE'], CFG['EPOCHS'], 'kclue')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "giX1utUYn0UI"
      },
      "outputs": [],
      "source": [
        "def get_type_predictions(model, loader):\n",
        "\n",
        "    device = torch.device('cuda')\n",
        "    model = model.to(device)\n",
        "\n",
        "    type_probs, polarity_probs, tense_probs, clarity_probs = [], [], [], []\n",
        "    with torch.no_grad():\n",
        "        model.eval()\n",
        "        for data_input, _, _, _, _ in tqdm(loader):\n",
        "            attention_mask = data_input['attention_mask'].to(device)\n",
        "            input_ids = data_input['input_ids'].squeeze(1).to(device)\n",
        "\n",
        "\n",
        "            type_output, polarity_output, tense_output, clarity_output = model(input_ids, attention_mask)\n",
        "            type_probs.append(type_output)\n",
        "            polarity_probs.append(polarity_output)\n",
        "            tense_probs.append(tense_output)\n",
        "            clarity_probs.append(clarity_output)\n",
        "\n",
        "    return torch.cat(type_probs).cpu().detach().numpy(), \\\n",
        "            torch.cat(polarity_probs).cpu().detach().numpy(), \\\n",
        "            torch.cat(tense_probs).cpu().detach().numpy(), \\\n",
        "            torch.cat(clarity_probs).cpu().detach().numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fkMqTaDHn0UI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "366711d6-a43c-4d89-b522-18eb0381f302"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-26-c0667a2f3bc6>:3: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  model = torch.load(\"model/kclue.pt\")\n"
          ]
        }
      ],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    model = torch.load(\"model1/kclue.pt\")\n",
        "    test_dataloader = DataLoader(SentenceTypeDataset(test, tokenizer), batch_size=CFG['BATCH_SIZE'], shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z04H4Qpgn0UI"
      },
      "outputs": [],
      "source": [
        "#val_pred_type, val_pred_polarity, val_pred_tense, val_pred_certainty = get_type_predictions(model, val_dataloader)\n",
        "\n",
        "#val_type = ['대화형' if i==0 else '사실형' if i==1 else '예측형' if i==2 else '추론형' for i in [np.argmax(p) for p in val_pred_type]]\n",
        "#val_polarity = ['긍정' if i==0 else '미정' if i==1 else '부정' for i in [np.argmax(p) for p in val_pred_polarity]]\n",
        "#val_type = ['과거' if i==0 else '미래' if i==1 else '현재' for i in [np.argmax(p) for p in val_pred_tense]]\n",
        "#val_type = ['불확실' if i==0 else '확실' for i in [np.argmax(p) for p in val_pred_certainty]]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gseX3IdOn0UI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "242c386a-83d6-46d9-f87b-16606c076c82"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 222/222 [00:16<00:00, 13.49it/s]\n"
          ]
        }
      ],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    test_pred_type, test_pred_polarity, test_pred_tense, test_pred_certainty = get_type_predictions(model, test_dataloader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lDXSTVJLn0UM"
      },
      "outputs": [],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    test_pred_tense"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z-XjnLZ6n0UM"
      },
      "outputs": [],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    sum(test_pred_tense[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uyym_mHNn0UM"
      },
      "outputs": [],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    test_type = ['대화형' if i==0 else '사실형' if i==1 else '예측형' if i==2 else '추론형' for i in [np.argmax(p) for p in test_pred_type]]\n",
        "    test_polarity = ['긍정' if i==0 else '미정' if i==1 else '부정' for i in [np.argmax(p) for p in test_pred_polarity]]\n",
        "    test_tense = ['과거' if i==0 else '미래' if i==1 else '현재' for i in [np.argmax(p) for p in test_pred_tense]]\n",
        "    test_certainty = ['불확실' if i==0 else '확실' for i in [np.argmax(p) for p in test_pred_certainty]]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "otzbsMVqn0UM"
      },
      "outputs": [],
      "source": [
        "if __name__ == '__main__':\n",
        "\n",
        "    label_sum = []\n",
        "    for i in range(len(test_type)):\n",
        "        label_sum.append(f'{test_type[i]}-{test_polarity[i]}-{test_tense[i]}-{test_certainty[i]}')\n",
        "    # os.makedirs('submission', exist_ok=True)\n",
        "\n",
        "    # submission['label'] = label_sum\n",
        "    # submission.to_csv('submission/klue1.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kiU6ya37n0UN"
      },
      "outputs": [],
      "source": [
        "# submission"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ET-0nZTCVzjk"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.13"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "1201aed7278f566d08684214e947d9aa97ba318061e22672851b23b6bee3a7a3"
      }
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "917bde512ea44912bd502521697fea2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e223659f110642f8b60b9e7e3d2ee505",
              "IPY_MODEL_8c258cd9f69748ca9dcc52a42cc34d71",
              "IPY_MODEL_3c616fc3b5384506850ee5e4696f4ef9"
            ],
            "layout": "IPY_MODEL_d412cdd33ebd46319945a90db1132854"
          }
        },
        "e223659f110642f8b60b9e7e3d2ee505": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c7bbc00074d4dde96fc3f0d120da9fa",
            "placeholder": "​",
            "style": "IPY_MODEL_a402990e6f4049c2af8f29b0430bed45",
            "value": "config.json: 100%"
          }
        },
        "8c258cd9f69748ca9dcc52a42cc34d71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d51058108504dd99e42409cc297d330",
            "max": 545,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b64522848b694087808077d410f331e1",
            "value": 545
          }
        },
        "3c616fc3b5384506850ee5e4696f4ef9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_105338fdc55647e2a587322d209b540a",
            "placeholder": "​",
            "style": "IPY_MODEL_696c13e6f41d4ab98973c5f660d69117",
            "value": " 545/545 [00:00&lt;00:00, 32.8kB/s]"
          }
        },
        "d412cdd33ebd46319945a90db1132854": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c7bbc00074d4dde96fc3f0d120da9fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a402990e6f4049c2af8f29b0430bed45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9d51058108504dd99e42409cc297d330": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b64522848b694087808077d410f331e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "105338fdc55647e2a587322d209b540a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "696c13e6f41d4ab98973c5f660d69117": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "85efc5e3c1a24bdb9a4fc47cf55d87f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_681bcd37c7cb4b0286eef70355f2984c",
              "IPY_MODEL_9b333f4b75564871a253bfc70f5d5c97",
              "IPY_MODEL_bf18a288c68b406da6e880adad033273"
            ],
            "layout": "IPY_MODEL_5c63dd6717da48378b800cb8cff2d917"
          }
        },
        "681bcd37c7cb4b0286eef70355f2984c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bd44b35965f042329008e82f0a0f664f",
            "placeholder": "​",
            "style": "IPY_MODEL_3207598613d34f229376c1932e1e5e43",
            "value": "model.safetensors: 100%"
          }
        },
        "9b333f4b75564871a253bfc70f5d5c97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_259a63b07f6b406faec160d1f08bfde4",
            "max": 272514464,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_01fddb42b16e4e30a6b6cfacea25d9e9",
            "value": 272514464
          }
        },
        "bf18a288c68b406da6e880adad033273": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9f0f662ec72f4c27a34e3eedd80eb526",
            "placeholder": "​",
            "style": "IPY_MODEL_a5825d1b77ba404d97a9f7ac066fba95",
            "value": " 273M/273M [00:02&lt;00:00, 138MB/s]"
          }
        },
        "5c63dd6717da48378b800cb8cff2d917": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd44b35965f042329008e82f0a0f664f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3207598613d34f229376c1932e1e5e43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "259a63b07f6b406faec160d1f08bfde4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01fddb42b16e4e30a6b6cfacea25d9e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9f0f662ec72f4c27a34e3eedd80eb526": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5825d1b77ba404d97a9f7ac066fba95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "badc8137c2f04361a8913e4cad041ff6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6aadae7669894aeeb5772dbfb38adf1c",
              "IPY_MODEL_195f43f2976d42be8f7bbaa901b91a4c",
              "IPY_MODEL_179ed6e211104c15b5af0144e3e56a74"
            ],
            "layout": "IPY_MODEL_3fed7027f600453d99a50527389c3f8b"
          }
        },
        "6aadae7669894aeeb5772dbfb38adf1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_284a28dca5754a9d9b32b5c402e1e75c",
            "placeholder": "​",
            "style": "IPY_MODEL_76b7aca84b074e909337b9f360c2f698",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "195f43f2976d42be8f7bbaa901b91a4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e5a623911584e67a6da195cdecdf464",
            "max": 375,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d4f1dec4538942f7ab1d9e7af1faa9a6",
            "value": 375
          }
        },
        "179ed6e211104c15b5af0144e3e56a74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6b9d04ebe1ce4db59fb4e78264853912",
            "placeholder": "​",
            "style": "IPY_MODEL_2e2fe5376cac418785c618656aa010bb",
            "value": " 375/375 [00:00&lt;00:00, 25.2kB/s]"
          }
        },
        "3fed7027f600453d99a50527389c3f8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "284a28dca5754a9d9b32b5c402e1e75c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "76b7aca84b074e909337b9f360c2f698": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9e5a623911584e67a6da195cdecdf464": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4f1dec4538942f7ab1d9e7af1faa9a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6b9d04ebe1ce4db59fb4e78264853912": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e2fe5376cac418785c618656aa010bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c6f834ae5f59434e877e21e1c9868d0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a2766d9fc60c47159682c5af2c39cab6",
              "IPY_MODEL_7d7c0e5db59a4bb98b99d8b49d684a21",
              "IPY_MODEL_8f2c7a510de34481a439f08b7093d1fb"
            ],
            "layout": "IPY_MODEL_f553d72c17ed4d7e9dc8ddfc1f73e7bd"
          }
        },
        "a2766d9fc60c47159682c5af2c39cab6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e6ee617ede7441484361cc30fa17b18",
            "placeholder": "​",
            "style": "IPY_MODEL_a92a3b16c97f4ff3bc6f334dfef3600e",
            "value": "vocab.txt: 100%"
          }
        },
        "7d7c0e5db59a4bb98b99d8b49d684a21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64781b40de6e4eb08ca3fefc15319dae",
            "max": 248477,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1cf8305a1a394ec58bbee60471093b98",
            "value": 248477
          }
        },
        "8f2c7a510de34481a439f08b7093d1fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e1b12b0dee345b08b1a33ca847d7b06",
            "placeholder": "​",
            "style": "IPY_MODEL_858f135828684db4837a86c918c00847",
            "value": " 248k/248k [00:00&lt;00:00, 597kB/s]"
          }
        },
        "f553d72c17ed4d7e9dc8ddfc1f73e7bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e6ee617ede7441484361cc30fa17b18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a92a3b16c97f4ff3bc6f334dfef3600e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "64781b40de6e4eb08ca3fefc15319dae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1cf8305a1a394ec58bbee60471093b98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2e1b12b0dee345b08b1a33ca847d7b06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "858f135828684db4837a86c918c00847": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0b5c6b2877ba4924a3ead77c4e56583b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8b8986ce16d146c1bffa652ef61f4f92",
              "IPY_MODEL_9ccd2565eeaf4db0af4983d3904f3d9b",
              "IPY_MODEL_e76ab28c0d9948fea3954a431c45c46c"
            ],
            "layout": "IPY_MODEL_4b02d4241f2142d39f03de8afaf20be7"
          }
        },
        "8b8986ce16d146c1bffa652ef61f4f92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_218cfe08f0a14d1f889032e2f39f6a7a",
            "placeholder": "​",
            "style": "IPY_MODEL_9d868a274ea642a58a41e24b18ab11f9",
            "value": "tokenizer.json: 100%"
          }
        },
        "9ccd2565eeaf4db0af4983d3904f3d9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50fb4e979ed1499a9acc47cab1a79ecc",
            "max": 751504,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0fc198b6f5b9400682d90fb968960a02",
            "value": 751504
          }
        },
        "e76ab28c0d9948fea3954a431c45c46c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_34f566f492c7499b9e0cb292a0c39da5",
            "placeholder": "​",
            "style": "IPY_MODEL_2f6dda289f5845d3ac498456cdf40b33",
            "value": " 752k/752k [00:00&lt;00:00, 1.73MB/s]"
          }
        },
        "4b02d4241f2142d39f03de8afaf20be7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "218cfe08f0a14d1f889032e2f39f6a7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9d868a274ea642a58a41e24b18ab11f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "50fb4e979ed1499a9acc47cab1a79ecc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0fc198b6f5b9400682d90fb968960a02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "34f566f492c7499b9e0cb292a0c39da5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2f6dda289f5845d3ac498456cdf40b33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dce670de3a544dd08db2c2ee3c569189": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0c45c801b79a4c4ba76cfa96770570b5",
              "IPY_MODEL_79d7a148cbc043108027265f5d40ce3a",
              "IPY_MODEL_62d4708974e543b391c66c3e07484883"
            ],
            "layout": "IPY_MODEL_d84236a6c117471f9108ba1cbfde4558"
          }
        },
        "0c45c801b79a4c4ba76cfa96770570b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_61589b104d7a44638844ebd606a1d66e",
            "placeholder": "​",
            "style": "IPY_MODEL_1403624143284e55886a8b2bcaa849b4",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "79d7a148cbc043108027265f5d40ce3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_059af35461e845b988c2febbb8d8f471",
            "max": 173,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c4c0cb7aace2483884b0e0c6d1e26f03",
            "value": 173
          }
        },
        "62d4708974e543b391c66c3e07484883": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ffbead7d1ba64d77907563302faa0069",
            "placeholder": "​",
            "style": "IPY_MODEL_cdb05c643c8a4027a52e93e634c598eb",
            "value": " 173/173 [00:00&lt;00:00, 6.98kB/s]"
          }
        },
        "d84236a6c117471f9108ba1cbfde4558": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61589b104d7a44638844ebd606a1d66e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1403624143284e55886a8b2bcaa849b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "059af35461e845b988c2febbb8d8f471": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c4c0cb7aace2483884b0e0c6d1e26f03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ffbead7d1ba64d77907563302faa0069": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cdb05c643c8a4027a52e93e634c598eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}