bert_discord.ipynb

{
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.13",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kaggle": {
      "accelerator": "nvidiaTeslaT4",
      "dataSources": [
        {
          "sourceId": 7837629,
          "sourceType": "datasetVersion",
          "datasetId": 4594308
        },
        {
          "sourceId": 7839317,
          "sourceType": "datasetVersion",
          "datasetId": 4595521
        }
      ],
      "dockerImageVersionId": 30665,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": true
    },
    "colab": {
      "name": "bert discord",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Windshield-Viper/Puffin-Discord-Bot/blob/master/bert_discord.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "source": [
        "\n",
        "# IMPORTANT: RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES\n",
        "# TO THE CORRECT LOCATION (/kaggle/input) IN YOUR NOTEBOOK,\n",
        "# THEN FEEL FREE TO DELETE THIS CELL.\n",
        "# NOTE: THIS NOTEBOOK ENVIRONMENT DIFFERS FROM KAGGLE'S PYTHON\n",
        "# ENVIRONMENT SO THERE MAY BE MISSING LIBRARIES USED BY YOUR\n",
        "# NOTEBOOK.\n",
        "\n",
        "import os\n",
        "import sys\n",
        "from tempfile import NamedTemporaryFile\n",
        "from urllib.request import urlopen\n",
        "from urllib.parse import unquote, urlparse\n",
        "from urllib.error import HTTPError\n",
        "from zipfile import ZipFile\n",
        "import tarfile\n",
        "import shutil\n",
        "\n",
        "CHUNK_SIZE = 40960\n",
        "DATA_SOURCE_MAPPING = 'disc-data:https%3A%2F%2Fstorage.googleapis.com%2Fkaggle-data-sets%2F4594308%2F7837629%2Fbundle%2Farchive.zip%3FX-Goog-Algorithm%3DGOOG4-RSA-SHA256%26X-Goog-Credential%3Dgcp-kaggle-com%2540kaggle-161607.iam.gserviceaccount.com%252F20240315%252Fauto%252Fstorage%252Fgoog4_request%26X-Goog-Date%3D20240315T014941Z%26X-Goog-Expires%3D259200%26X-Goog-SignedHeaders%3Dhost%26X-Goog-Signature%3D5d4f46c137b152ef1c79c43c01802c9edb0951c58127b8d6d8ba51c869adcf060bc8001ee2e359edc935b13e2d0d8b8302a29d19c15e5c041ec287b5a3979e6ce988c1e08162bcca5451a043fe0ee9a09d01edf02bd99eb008ffcf6a6fab0ab782f9d04c9f3977598ad4b9a51d38a5e916cfb28ecc9e4392d91b861d85d8c433179744f289397ae7c7645ce32aeee70810cce55ef4feed78efd0e82d81502c9c8801a9c24a7bb4c5af8b390f8ea4ae70c492d63436ed4f4062241b8f55ec213183be84d5e5d60a4c235ba99e9243c429e78e07560e9fc2f477f08a4645a5ef39802c69b8aeb71197c28ab3216b33bdbc79a367be6d41ad21eb1f09be11101451,all-discord-data:https%3A%2F%2Fstorage.googleapis.com%2Fkaggle-data-sets%2F4595521%2F7839317%2Fbundle%2Farchive.zip%3FX-Goog-Algorithm%3DGOOG4-RSA-SHA256%26X-Goog-Credential%3Dgcp-kaggle-com%2540kaggle-161607.iam.gserviceaccount.com%252F20240315%252Fauto%252Fstorage%252Fgoog4_request%26X-Goog-Date%3D20240315T014941Z%26X-Goog-Expires%3D259200%26X-Goog-SignedHeaders%3Dhost%26X-Goog-Signature%3D166d99ce2ef43be08b25b698531db915fc362e5fe17f05e94d32fe7728af232df4be495fba6c9942ebdfee8894c8b2543eeabc3daa314bf9a27dcec0d0b57768cffd9f625efbcb3976b41ed36a6dab64d34921e0839983117c7f9c52daabdb1a67c595e7553665a10e130d657afcd70f6cdf4800290a94ca73db8f2a957348c1be1ef07b74be3648f0ae2430161304abdfd6d7207a341d5d2c3f6daefa1291641baec78fa9a00b09fd429f6b755be53ea7f73dfdc7d95f37304363da776adf7924029618c674472911c679eaa3933e43ab7664aa28f254f2aec842022cc43308e67404f74346d5519852dff2f3e3519688025381b5eca7a296f59fcfee24a548'\n",
        "\n",
        "KAGGLE_INPUT_PATH='/kaggle/input'\n",
        "KAGGLE_WORKING_PATH='/kaggle/working'\n",
        "KAGGLE_SYMLINK='kaggle'\n",
        "\n",
        "!umount /kaggle/input/ 2> /dev/null\n",
        "shutil.rmtree('/kaggle/input', ignore_errors=True)\n",
        "os.makedirs(KAGGLE_INPUT_PATH, 0o777, exist_ok=True)\n",
        "os.makedirs(KAGGLE_WORKING_PATH, 0o777, exist_ok=True)\n",
        "\n",
        "try:\n",
        "  os.symlink(KAGGLE_INPUT_PATH, os.path.join(\"..\", 'input'), target_is_directory=True)\n",
        "except FileExistsError:\n",
        "  pass\n",
        "try:\n",
        "  os.symlink(KAGGLE_WORKING_PATH, os.path.join(\"..\", 'working'), target_is_directory=True)\n",
        "except FileExistsError:\n",
        "  pass\n",
        "\n",
        "for data_source_mapping in DATA_SOURCE_MAPPING.split(','):\n",
        "    directory, download_url_encoded = data_source_mapping.split(':')\n",
        "    download_url = unquote(download_url_encoded)\n",
        "    filename = urlparse(download_url).path\n",
        "    destination_path = os.path.join(KAGGLE_INPUT_PATH, directory)\n",
        "    try:\n",
        "        with urlopen(download_url) as fileres, NamedTemporaryFile() as tfile:\n",
        "            total_length = fileres.headers['content-length']\n",
        "            print(f'Downloading {directory}, {total_length} bytes compressed')\n",
        "            dl = 0\n",
        "            data = fileres.read(CHUNK_SIZE)\n",
        "            while len(data) > 0:\n",
        "                dl += len(data)\n",
        "                tfile.write(data)\n",
        "                done = int(50 * dl / int(total_length))\n",
        "                sys.stdout.write(f\"\\r[{'=' * done}{' ' * (50-done)}] {dl} bytes downloaded\")\n",
        "                sys.stdout.flush()\n",
        "                data = fileres.read(CHUNK_SIZE)\n",
        "            if filename.endswith('.zip'):\n",
        "              with ZipFile(tfile) as zfile:\n",
        "                zfile.extractall(destination_path)\n",
        "            else:\n",
        "              with tarfile.open(tfile.name) as tarfile:\n",
        "                tarfile.extractall(destination_path)\n",
        "            print(f'\\nDownloaded and uncompressed: {directory}')\n",
        "    except HTTPError as e:\n",
        "        print(f'Failed to load (likely expired) {download_url} to path {destination_path}')\n",
        "        continue\n",
        "    except OSError as e:\n",
        "        print(f'Failed to load {download_url} to path {destination_path}')\n",
        "        continue\n",
        "\n",
        "print('Data source import complete.')\n"
      ],
      "metadata": {
        "id": "rDLaHAAZfkTS"
      },
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load\n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the read-only \"../input/\" directory\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
        "\n",
        "import os\n",
        "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
        "    for filename in filenames:\n",
        "        print(os.path.join(dirname, filename))\n",
        "\n",
        "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n",
        "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
      ],
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "execution": {
          "iopub.status.busy": "2024-03-15T01:23:32.799915Z",
          "iopub.execute_input": "2024-03-15T01:23:32.800237Z",
          "iopub.status.idle": "2024-03-15T01:23:33.910083Z",
          "shell.execute_reply.started": "2024-03-15T01:23:32.80021Z",
          "shell.execute_reply": "2024-03-15T01:23:33.909047Z"
        },
        "trusted": true,
        "id": "BPpcBJklfkTT",
        "outputId": "70a56ff5-1c32-4970-99d4-6f31d2e9d6db"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "/kaggle/input/disc-data/disc_6.csv\n/kaggle/input/all-discord-data/combined_df.csv\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install -U accelerate\n",
        "! pip install datasets transformers\n",
        "!apt install git-lfs"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:23:33.911745Z",
          "iopub.execute_input": "2024-03-15T01:23:33.912224Z",
          "iopub.status.idle": "2024-03-15T01:24:04.215466Z",
          "shell.execute_reply.started": "2024-03-15T01:23:33.912182Z",
          "shell.execute_reply": "2024-03-15T01:24:04.214261Z"
        },
        "trusted": true,
        "id": "iX3E6gAkfkTU",
        "outputId": "f3b6f749-6727-4057-f351-8bc10ff03782"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Requirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (0.27.2)\nCollecting accelerate\n  Downloading accelerate-0.28.0-py3-none-any.whl.metadata (18 kB)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate) (1.26.4)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (21.3)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.9.3)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate) (6.0.1)\nRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (2.1.2)\nRequirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.20.3)\nRequirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.4.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->accelerate) (3.1.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.13.1)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (4.9.0)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.2.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (2024.2.0)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (2.31.0)\nRequirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (4.66.1)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (2024.2.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\nDownloading accelerate-0.28.0-py3-none-any.whl (290 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.1/290.1 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0mm\n\u001b[?25hInstalling collected packages: accelerate\n  Attempting uninstall: accelerate\n    Found existing installation: accelerate 0.27.2\n    Uninstalling accelerate-0.27.2:\n      Successfully uninstalled accelerate-0.27.2\nSuccessfully installed accelerate-0.28.0\nRequirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (2.1.0)\nRequirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.38.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from datasets) (1.26.4)\nRequirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (11.0.0)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.8)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (2.1.4)\nRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.31.0)\nRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.66.1)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.16)\nRequirement already satisfied: fsspec>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from fsspec[http]>=2021.05.0->datasets) (2024.2.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.9.1)\nRequirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.20.3)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (21.3)\nRequirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.18.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.13.1)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0.1)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.12.25)\nRequirement already satisfied: tokenizers<0.19,>=0.14 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.15.2)\nRequirement already satisfied: safetensors>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.4.2)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (23.2.0)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.3)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.9.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->datasets) (3.1.1)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2024.2.2)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2023.3.post1)\nRequirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2023.4)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\nReading package lists... Done\nBuilding dependency tree       \nReading state information... Done\ngit-lfs is already the newest version (2.9.2-1).\n0 upgraded, 0 newly installed, 0 to remove and 38 not upgraded.\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import notebook_login, login\n",
        "\n",
        "login(token=\"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\")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:04.217102Z",
          "iopub.execute_input": "2024-03-15T01:24:04.217407Z",
          "iopub.status.idle": "2024-03-15T01:24:04.944095Z",
          "shell.execute_reply.started": "2024-03-15T01:24:04.21738Z",
          "shell.execute_reply": "2024-03-15T01:24:04.943145Z"
        },
        "trusted": true,
        "id": "GEjhMNwbfkTU",
        "outputId": "65aa69bd-a9a7-4ad1-b0ff-b6712d8b7d5d"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\nToken is valid (permission: write).\nYour token has been saved to /root/.cache/huggingface/token\nLogin successful\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import transformers\n",
        "\n",
        "print(transformers.__version__)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:04.946388Z",
          "iopub.execute_input": "2024-03-15T01:24:04.946771Z",
          "iopub.status.idle": "2024-03-15T01:24:10.865687Z",
          "shell.execute_reply.started": "2024-03-15T01:24:04.946746Z",
          "shell.execute_reply": "2024-03-15T01:24:10.864621Z"
        },
        "trusted": true,
        "id": "0YkqBDmGfkTV",
        "outputId": "2f2e3125-0f74-49e0-9471-7f57e732649d"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "4.38.1\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install wandb"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:10.86701Z",
          "iopub.execute_input": "2024-03-15T01:24:10.867427Z",
          "iopub.status.idle": "2024-03-15T01:24:23.106806Z",
          "shell.execute_reply.started": "2024-03-15T01:24:10.867402Z",
          "shell.execute_reply": "2024-03-15T01:24:23.105567Z"
        },
        "trusted": true,
        "id": "OZZza02VfkTV",
        "outputId": "6b56d7e8-5dd2-4e1b-fe39-37ad3090b000"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Requirement already satisfied: wandb in /opt/conda/lib/python3.10/site-packages (0.16.3)\nRequirement already satisfied: Click!=8.0.0,>=7.1 in /opt/conda/lib/python3.10/site-packages (from wandb) (8.1.7)\nRequirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (3.1.41)\nRequirement already satisfied: requests<3,>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (2.31.0)\nRequirement already satisfied: psutil>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (5.9.3)\nRequirement already satisfied: sentry-sdk>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (1.40.5)\nRequirement already satisfied: docker-pycreds>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (0.4.0)\nRequirement already satisfied: PyYAML in /opt/conda/lib/python3.10/site-packages (from wandb) (6.0.1)\nRequirement already satisfied: setproctitle in /opt/conda/lib/python3.10/site-packages (from wandb) (1.3.3)\nRequirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from wandb) (69.0.3)\nRequirement already satisfied: appdirs>=1.4.3 in /opt/conda/lib/python3.10/site-packages (from wandb) (1.4.4)\nRequirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (3.20.3)\nRequirement already satisfied: six>=1.4.0 in /opt/conda/lib/python3.10/site-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\nRequirement already satisfied: gitdb<5,>=4.0.1 in /opt/conda/lib/python3.10/site-packages (from GitPython!=3.1.29,>=1.0.0->wandb) (4.0.11)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (2024.2.2)\nRequirement already satisfied: smmap<6,>=3.0.1 in /opt/conda/lib/python3.10/site-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb) (5.0.1)\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import wandb"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:23.108354Z",
          "iopub.execute_input": "2024-03-15T01:24:23.108645Z",
          "iopub.status.idle": "2024-03-15T01:24:24.173069Z",
          "shell.execute_reply.started": "2024-03-15T01:24:23.10862Z",
          "shell.execute_reply": "2024-03-15T01:24:24.172013Z"
        },
        "trusted": true,
        "id": "-X66XkbAfkTV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "wandb.login(key=\"8ff81dcac301a265e86bec8669d939dcd3b76bdc\")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:24.174783Z",
          "iopub.execute_input": "2024-03-15T01:24:24.175111Z",
          "iopub.status.idle": "2024-03-15T01:24:26.402709Z",
          "shell.execute_reply.started": "2024-03-15T01:24:24.175084Z",
          "shell.execute_reply": "2024-03-15T01:24:26.401781Z"
        },
        "trusted": true,
        "id": "iiLC3uO2fkTV",
        "outputId": "aad1981a-2aab-4eda-8354-f76e2c801898"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n",
          "output_type": "stream"
        },
        {
          "execution_count": 7,
          "output_type": "execute_result",
          "data": {
            "text/plain": "True"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset, Dataset, DatasetDict\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:26.404016Z",
          "iopub.execute_input": "2024-03-15T01:24:26.405015Z",
          "iopub.status.idle": "2024-03-15T01:24:28.381818Z",
          "shell.execute_reply.started": "2024-03-15T01:24:26.404974Z",
          "shell.execute_reply": "2024-03-15T01:24:28.381051Z"
        },
        "trusted": true,
        "id": "5AdRYoszfkTW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"/kaggle/input/all-discord-data/combined_df.csv\").dropna().drop(\"Author_ID\", axis=1)\n",
        "# df_2 = pd.read_csv(\"/content/drive/MyDrive/Data/disc_data/disc_7.csv\")\n",
        "\n",
        "df = df.sample(frac=0.05, replace=True, random_state=1)\n",
        "\n",
        "train, test = train_test_split(df, test_size=0.2)\n",
        "test, val = train_test_split(test, test_size=0.5)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:28.383102Z",
          "iopub.execute_input": "2024-03-15T01:24:28.383758Z",
          "iopub.status.idle": "2024-03-15T01:24:48.478663Z",
          "shell.execute_reply.started": "2024-03-15T01:24:28.383721Z",
          "shell.execute_reply": "2024-03-15T01:24:48.477731Z"
        },
        "trusted": true,
        "id": "NgqDqwjlfkTW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train.head()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:48.482488Z",
          "iopub.execute_input": "2024-03-15T01:24:48.48277Z",
          "iopub.status.idle": "2024-03-15T01:24:48.495534Z",
          "shell.execute_reply.started": "2024-03-15T01:24:48.482746Z",
          "shell.execute_reply": "2024-03-15T01:24:48.494614Z"
        },
        "trusted": true,
        "id": "K5ebhURCfkTW",
        "outputId": "bb5f1e27-3051-45fb-d0cf-3ae21ff7b066"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 10,
          "output_type": "execute_result",
          "data": {
            "text/plain": "                    Message\n11936715   Tohka\\nSleep\\n..\n11165328  @Khan Mileff fake\n1575371         hi\\nwelcome\n834803                 .pik\n14173488                  L",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Message</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>11936715</th>\n      <td>Tohka\\nSleep\\n..</td>\n    </tr>\n    <tr>\n      <th>11165328</th>\n      <td>@Khan Mileff fake</td>\n    </tr>\n    <tr>\n      <th>1575371</th>\n      <td>hi\\nwelcome</td>\n    </tr>\n    <tr>\n      <th>834803</th>\n      <td>.pik</td>\n    </tr>\n    <tr>\n      <th>14173488</th>\n      <td>L</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train.shape"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:48.496651Z",
          "iopub.execute_input": "2024-03-15T01:24:48.497024Z",
          "iopub.status.idle": "2024-03-15T01:24:48.503038Z",
          "shell.execute_reply.started": "2024-03-15T01:24:48.49699Z",
          "shell.execute_reply": "2024-03-15T01:24:48.502158Z"
        },
        "trusted": true,
        "id": "8wlj32GBfkTX",
        "outputId": "cca63965-776d-4994-8a6b-ffb2d9a42958"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 11,
          "output_type": "execute_result",
          "data": {
            "text/plain": "(678328, 1)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test.head()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:48.504251Z",
          "iopub.execute_input": "2024-03-15T01:24:48.504792Z",
          "iopub.status.idle": "2024-03-15T01:24:48.516892Z",
          "shell.execute_reply.started": "2024-03-15T01:24:48.504759Z",
          "shell.execute_reply": "2024-03-15T01:24:48.515967Z"
        },
        "trusted": true,
        "id": "wCpHSssifkTX",
        "outputId": "23c3f0e4-3404-4900-dfae-14da7de6d1fe"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 12,
          "output_type": "execute_result",
          "data": {
            "text/plain": "                                                   Message\n10364349                    @vic also there is no date wth\n589910    I'm screebshotting everything nora\\n@LunaAmherst\n7184787                                        toast is 13\n4075592                                                Idk\n8037680                                         hirp\\nshut",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Message</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>10364349</th>\n      <td>@vic also there is no date wth</td>\n    </tr>\n    <tr>\n      <th>589910</th>\n      <td>I'm screebshotting everything nora\\n@LunaAmherst</td>\n    </tr>\n    <tr>\n      <th>7184787</th>\n      <td>toast is 13</td>\n    </tr>\n    <tr>\n      <th>4075592</th>\n      <td>Idk</td>\n    </tr>\n    <tr>\n      <th>8037680</th>\n      <td>hirp\\nshut</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_ds = Dataset.from_pandas(train)\n",
        "test_ds = Dataset.from_pandas(test)\n",
        "validation_ds = Dataset.from_pandas(val)\n",
        "\n",
        "datasets = DatasetDict({\n",
        "    'train': train_ds,\n",
        "    'test': test_ds,\n",
        "    'valid': validation_ds})"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:48.518195Z",
          "iopub.execute_input": "2024-03-15T01:24:48.518484Z",
          "iopub.status.idle": "2024-03-15T01:24:48.722376Z",
          "shell.execute_reply.started": "2024-03-15T01:24:48.518461Z",
          "shell.execute_reply": "2024-03-15T01:24:48.721377Z"
        },
        "trusted": true,
        "id": "Guvxu1EmfkTX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "datasets[\"train\"][11]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:48.723512Z",
          "iopub.execute_input": "2024-03-15T01:24:48.723782Z",
          "iopub.status.idle": "2024-03-15T01:24:48.732387Z",
          "shell.execute_reply.started": "2024-03-15T01:24:48.723759Z",
          "shell.execute_reply": "2024-03-15T01:24:48.731574Z"
        },
        "trusted": true,
        "id": "dB7AU8qefkTX",
        "outputId": "f8022a7c-55a4-4985-e610-27a7d4b6a1f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 14,
          "output_type": "execute_result",
          "data": {
            "text/plain": "{'Message': 'HELLO SOMEONE EXPLAIN PLS', '__index_level_0__': 12770734}"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_checkpoint = \"distilroberta-base\""
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:48.733693Z",
          "iopub.execute_input": "2024-03-15T01:24:48.734048Z",
          "iopub.status.idle": "2024-03-15T01:24:48.742518Z",
          "shell.execute_reply.started": "2024-03-15T01:24:48.734019Z",
          "shell.execute_reply": "2024-03-15T01:24:48.741581Z"
        },
        "trusted": true,
        "id": "vlUkP9WafkTY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:48.743459Z",
          "iopub.execute_input": "2024-03-15T01:24:48.743752Z",
          "iopub.status.idle": "2024-03-15T01:24:50.77152Z",
          "shell.execute_reply.started": "2024-03-15T01:24:48.743729Z",
          "shell.execute_reply": "2024-03-15T01:24:50.770739Z"
        },
        "trusted": true,
        "id": "6BYD0SqVfkTY",
        "outputId": "d75b62a3-fb1d-4824-c809-8816f35c98b6",
        "colab": {
          "referenced_widgets": [
            "4fdbf10ddb494ccdab4b9f5cc0ff4872",
            "b76a7928fabe494b93c9cd466da7e4b1",
            "3b97a4e8ab0e4f29b1cc305e684f95be",
            "8eff91b779974e18b748d1051c17abb8",
            "2e1ebd07550045dbbc95ae534b435b13"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4fdbf10ddb494ccdab4b9f5cc0ff4872"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "config.json:   0%|          | 0.00/480 [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b76a7928fabe494b93c9cd466da7e4b1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3b97a4e8ab0e4f29b1cc305e684f95be"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8eff91b779974e18b748d1051c17abb8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2e1ebd07550045dbbc95ae534b435b13"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_function(examples):\n",
        "    return tokenizer(examples[\"Message\"])"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:50.772585Z",
          "iopub.execute_input": "2024-03-15T01:24:50.77288Z",
          "iopub.status.idle": "2024-03-15T01:24:50.777445Z",
          "shell.execute_reply.started": "2024-03-15T01:24:50.772855Z",
          "shell.execute_reply": "2024-03-15T01:24:50.776329Z"
        },
        "trusted": true,
        "id": "z9PZvq7rfkTY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_datasets = datasets.map(tokenize_function, batched=True, num_proc=4, remove_columns=[\"Message\",\"__index_level_0__\"])"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:24:50.77855Z",
          "iopub.execute_input": "2024-03-15T01:24:50.778838Z",
          "iopub.status.idle": "2024-03-15T01:25:13.02625Z",
          "shell.execute_reply.started": "2024-03-15T01:24:50.778816Z",
          "shell.execute_reply": "2024-03-15T01:25:13.025028Z"
        },
        "trusted": true,
        "id": "Qss8-IgnfkTY",
        "outputId": "0a658902-8f64-456c-d88a-c5b13db4c6f2",
        "colab": {
          "referenced_widgets": [
            "f9197c52d642417eb4f5eced436403e3",
            "dbe6096173784c38bc6872f323f9de1a",
            "cccac512934c4f74a136e181cd509d5d",
            "aee1d870fd5642b0b531de9059606d44",
            "207b49d605e74833a09b01f2b30719e6",
            "817d0c714d23486cab2be57648090745",
            "6ef1ef6c82674ffcaac04c301505aa7d",
            "b2b325026a4945ce884ffe4a66539ccf",
            "d9c5de95b7b745818f645d354cb904fd",
            "0f6b5f8b72a64e97bf91946c0809a2d8",
            "d5d2a3b854bf4562a2dbe24a1bd41517",
            "d9fda486c3da42458ae7545a1d18fd48"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "     ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#0:   0%|          | 0/170 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f9197c52d642417eb4f5eced436403e3"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#1:   0%|          | 0/170 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dbe6096173784c38bc6872f323f9de1a"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#2:   0%|          | 0/170 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cccac512934c4f74a136e181cd509d5d"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#3:   0%|          | 0/170 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "aee1d870fd5642b0b531de9059606d44"
            }
          },
          "metadata": {}
        },
        {
          "name": "stderr",
          "text": "Token indices sequence length is longer than the specified maximum sequence length for this model (1122 > 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (560 > 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (2673 > 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (568 > 512). Running this sequence through the model will result in indexing errors\n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "     ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#0:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "207b49d605e74833a09b01f2b30719e6"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#1:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "817d0c714d23486cab2be57648090745"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": "  ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#2:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6ef1ef6c82674ffcaac04c301505aa7d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#3:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b2b325026a4945ce884ffe4a66539ccf"
            }
          },
          "metadata": {}
        },
        {
          "name": "stderr",
          "text": "Token indices sequence length is longer than the specified maximum sequence length for this model (525 > 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (788 > 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (633 > 512). Running this sequence through the model will result in indexing errors\n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "     ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#0:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d9c5de95b7b745818f645d354cb904fd"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#1:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0f6b5f8b72a64e97bf91946c0809a2d8"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#2:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d5d2a3b854bf4562a2dbe24a1bd41517"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#3:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d9fda486c3da42458ae7545a1d18fd48"
            }
          },
          "metadata": {}
        },
        {
          "name": "stderr",
          "text": "Token indices sequence length is longer than the specified maximum sequence length for this model (514 > 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (606 > 512). Running this sequence through the model will result in indexing errors\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_datasets"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:25:13.027851Z",
          "iopub.execute_input": "2024-03-15T01:25:13.028139Z",
          "iopub.status.idle": "2024-03-15T01:25:13.034925Z",
          "shell.execute_reply.started": "2024-03-15T01:25:13.028113Z",
          "shell.execute_reply": "2024-03-15T01:25:13.034044Z"
        },
        "trusted": true,
        "id": "MJ9tooWWfkTZ",
        "outputId": "7933df51-cf73-4d6e-9c09-e6aa962959fa"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 19,
          "output_type": "execute_result",
          "data": {
            "text/plain": "DatasetDict({\n    train: Dataset({\n        features: ['input_ids', 'attention_mask'],\n        num_rows: 678328\n    })\n    test: Dataset({\n        features: ['input_ids', 'attention_mask'],\n        num_rows: 84791\n    })\n    valid: Dataset({\n        features: ['input_ids', 'attention_mask'],\n        num_rows: 84791\n    })\n})"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "block_size = 32\n",
        "def group_texts(examples):\n",
        "    # Concatenate all texts.\n",
        "    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
        "    total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
        "    # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n",
        "        # customize this part to your needs.\n",
        "    total_length = (total_length // block_size) * block_size\n",
        "    # Split by chunks of max_len.\n",
        "    result = {\n",
        "        k: [t[i : i + block_size] for i in range(0, total_length, block_size)]\n",
        "        for k, t in concatenated_examples.items()\n",
        "    }\n",
        "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
        "    return result"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:25:13.036135Z",
          "iopub.execute_input": "2024-03-15T01:25:13.036397Z",
          "iopub.status.idle": "2024-03-15T01:25:13.047837Z",
          "shell.execute_reply.started": "2024-03-15T01:25:13.036374Z",
          "shell.execute_reply": "2024-03-15T01:25:13.046951Z"
        },
        "trusted": true,
        "id": "UhTzeELVfkTZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lm_datasets = tokenized_datasets.map(\n",
        "    group_texts,\n",
        "    batched=True,\n",
        "    batch_size=1000,\n",
        "    num_proc=4,\n",
        ")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:25:13.049026Z",
          "iopub.execute_input": "2024-03-15T01:25:13.049607Z",
          "iopub.status.idle": "2024-03-15T01:25:44.845138Z",
          "shell.execute_reply.started": "2024-03-15T01:25:13.049576Z",
          "shell.execute_reply": "2024-03-15T01:25:44.844096Z"
        },
        "trusted": true,
        "id": "RJDSVHcCfkTZ",
        "outputId": "b01bc4a4-ba45-4a8f-ece8-5e7dc3167d03",
        "colab": {
          "referenced_widgets": [
            "0b11196457b54151ba3edf31218ab4bc",
            "4043379796894b1588531a6825d655ac",
            "7dbe8053ac6c43d7b38ffb5ceb19d6fc",
            "0b607b24407343e0a1bc855316ebdbf3",
            "62412c9b5a984e239417ddf1235266bc",
            "56ed57da4e6c49c3ac1d0b27573cfe3d",
            "8532342477c941cb9ddb8f36069a445b",
            "f266afcbef094e03bc14697ff8564abc",
            "1e416624903341e3bdf4fe099712cce0",
            "2552f69eef794392872f9d1ed3c43aab",
            "57bf7329df91440c959c4cbcb8facfe6",
            "0b12fd94b84a41b5bd59ce8f9e62c690"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "     ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#0:   0%|          | 0/170 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0b11196457b54151ba3edf31218ab4bc"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#1:   0%|          | 0/170 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4043379796894b1588531a6825d655ac"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#2:   0%|          | 0/170 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7dbe8053ac6c43d7b38ffb5ceb19d6fc"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#3:   0%|          | 0/170 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0b607b24407343e0a1bc855316ebdbf3"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": "     ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#0:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "62412c9b5a984e239417ddf1235266bc"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#1:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "56ed57da4e6c49c3ac1d0b27573cfe3d"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#2:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8532342477c941cb9ddb8f36069a445b"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#3:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f266afcbef094e03bc14697ff8564abc"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": "     ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#0:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1e416624903341e3bdf4fe099712cce0"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#1:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2552f69eef794392872f9d1ed3c43aab"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#2:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "57bf7329df91440c959c4cbcb8facfe6"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": " ",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "#3:   0%|          | 0/22 [00:00<?, ?ba/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0b12fd94b84a41b5bd59ce8f9e62c690"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.decode(lm_datasets[\"train\"][1][\"input_ids\"])"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:25:44.846708Z",
          "iopub.execute_input": "2024-03-15T01:25:44.847043Z",
          "iopub.status.idle": "2024-03-15T01:25:55.37692Z",
          "shell.execute_reply.started": "2024-03-15T01:25:44.847011Z",
          "shell.execute_reply": "2024-03-15T01:25:55.375866Z"
        },
        "trusted": true,
        "id": "3PQC6qMHfkTZ",
        "outputId": "e9aeafa4-1c51-4bc4-b39c-4419b503422c"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "2024-03-15 01:25:46.869332: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n2024-03-15 01:25:46.869437: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n2024-03-15 01:25:47.036991: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
          "output_type": "stream"
        },
        {
          "execution_count": 22,
          "output_type": "execute_result",
          "data": {
            "text/plain": "\"</s><s>God vic</s><s>@Tornadotracker what</s><s>But you can easily text or call people you know irl. That's what\""
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoModelForCausalLM\n",
        "model = AutoModelForCausalLM.from_pretrained(model_checkpoint)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:25:55.378071Z",
          "iopub.execute_input": "2024-03-15T01:25:55.378673Z",
          "iopub.status.idle": "2024-03-15T01:25:59.490146Z",
          "shell.execute_reply.started": "2024-03-15T01:25:55.378645Z",
          "shell.execute_reply": "2024-03-15T01:25:59.489013Z"
        },
        "trusted": true,
        "id": "-W1UZomrfkTZ",
        "outputId": "7f20de95-114d-4935-e4e4-cf353b77865f",
        "colab": {
          "referenced_widgets": [
            "3d979e7a408241a7956f2fd36b52bbbb"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "model.safetensors:   0%|          | 0.00/331M [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3d979e7a408241a7956f2fd36b52bbbb"
            }
          },
          "metadata": {}
        },
        {
          "name": "stderr",
          "text": "If you want to use `RobertaLMHeadModel` as a standalone, add `is_decoder=True.`\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import Trainer, TrainingArguments"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:25:59.491693Z",
          "iopub.execute_input": "2024-03-15T01:25:59.492502Z",
          "iopub.status.idle": "2024-03-15T01:25:59.554934Z",
          "shell.execute_reply.started": "2024-03-15T01:25:59.492463Z",
          "shell.execute_reply": "2024-03-15T01:25:59.554171Z"
        },
        "trusted": true,
        "id": "3JOyqBrpfkTZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_name = model_checkpoint.split(\"/\")[-1]\n",
        "training_args = TrainingArguments(\n",
        "    f\"RoBERTa_for_Discord\",\n",
        "    evaluation_strategy = \"epoch\",\n",
        "    learning_rate=2e-5,\n",
        "    weight_decay=0.01,\n",
        "    push_to_hub=True,\n",
        "    #report_to=None\n",
        ")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:25:59.555951Z",
          "iopub.execute_input": "2024-03-15T01:25:59.556227Z",
          "iopub.status.idle": "2024-03-15T01:25:59.643435Z",
          "shell.execute_reply.started": "2024-03-15T01:25:59.556197Z",
          "shell.execute_reply": "2024-03-15T01:25:59.642599Z"
        },
        "trusted": true,
        "id": "2422HoX0fkTa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=lm_datasets[\"train\"],\n",
        "    eval_dataset=lm_datasets[\"valid\"],\n",
        ")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:25:59.64645Z",
          "iopub.execute_input": "2024-03-15T01:25:59.646962Z",
          "iopub.status.idle": "2024-03-15T01:26:00.016103Z",
          "shell.execute_reply.started": "2024-03-15T01:25:59.646933Z",
          "shell.execute_reply": "2024-03-15T01:26:00.015262Z"
        },
        "trusted": true,
        "id": "haHUXA6WfkTa",
        "outputId": "65b75468-626a-430e-cf6d-608415fd7eb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead: \ndataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)\n  warnings.warn(\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.train()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-03-15T01:26:00.017247Z",
          "iopub.execute_input": "2024-03-15T01:26:00.017541Z"
        },
        "trusted": true,
        "id": "yWtBAqA_fkTa",
        "outputId": "258760c8-7570-4dca-92b5-d7ab23240033"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33memailsiddha\u001b[0m (\u001b[33mwindshields\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "wandb version 0.16.4 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Tracking run with wandb version 0.16.3"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Run data is saved locally in <code>/kaggle/working/wandb/run-20240315_012600-wluf4l0n</code>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Syncing run <strong><a href='https://wandb.ai/windshields/huggingface/runs/wluf4l0n' target=\"_blank\">snowy-valley-10</a></strong> to <a href='https://wandb.ai/windshields/huggingface' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View project at <a href='https://wandb.ai/windshields/huggingface' target=\"_blank\">https://wandb.ai/windshields/huggingface</a>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run at <a href='https://wandb.ai/windshields/huggingface/runs/wluf4l0n' target=\"_blank\">https://wandb.ai/windshields/huggingface/runs/wluf4l0n</a>"
          },
          "metadata": {}
        },
        {
          "name": "stderr",
          "text": "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "\n    <div>\n      \n      <progress value='9586' max='37173' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [ 9586/37173 23:00 < 1:06:14, 6.94 it/s, Epoch 0.77/3]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Epoch</th>\n      <th>Training Loss</th>\n      <th>Validation Loss</th>\n    </tr>\n  </thead>\n  <tbody>\n  </tbody>\n</table><p>"
          },
          "metadata": {}
        },
        {
          "name": "stderr",
          "text": "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import math"
      ],
      "metadata": {
        "trusted": true,
        "id": "H0FyuKjXfkTa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate()\n",
        "print(f\"Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")"
      ],
      "metadata": {
        "trusted": true,
        "id": "3nxAeZGffkTa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.push_to_hub()"
      ],
      "metadata": {
        "trusted": true,
        "id": "BuR_A5-nfkTa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.push_to_hub(\"RoBERTa_for_Discord\")"
      ],
      "metadata": {
        "jupyter": {
          "source_hidden": true
        },
        "trusted": true,
        "id": "ZQDXNqIgfkTb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vPcccVmYfkTc"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}