finetuning_BERT_for_NER.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m-Trazo_m3oI",
        "outputId": "b876a859-2e7d-4989-a295-2dcb9fbe1b65"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n",
            "Token is valid (permission: write).\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "!pip install -q accelerate -U\n",
        "!pip install -q transformers[torch]\n",
        "!pip install -q evaluate\n",
        "\n",
        "# Mount Google Drive\n",
        "drive.mount('/content/drive')\n",
        "!huggingface-cli login --token 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"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_IaoW-XNbj5l"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "\n",
        "def merge_consecutive_entities(data):\n",
        "    entities = data\n",
        "    merged_entities = []\n",
        "    current_entity = entities[0]\n",
        "    for next_entity in entities[1:]:\n",
        "        if current_entity[\"label\"] == next_entity[\"label\"] :\n",
        "            if current_entity[\"end_offset\"] == next_entity[\"start_offset\"] or current_entity[\"start_offset\"] == next_entity[\"end_offset\"] or current_entity[\"end_offset\"]+1 == next_entity[\"start_offset\"] or current_entity[\"start_offset\"]+1 == next_entity[\"end_offset\"]:\n",
        "                current_entity[\"start_offset\"] = min(current_entity[\"start_offset\"], next_entity[\"start_offset\"])\n",
        "                current_entity[\"end_offset\"] = max(current_entity[\"end_offset\"], next_entity[\"end_offset\"])\n",
        "            else:\n",
        "                merged_entities.append(current_entity)\n",
        "                current_entity = next_entity\n",
        "        else:\n",
        "\n",
        "            merged_entities.append(current_entity)\n",
        "            current_entity = next_entity\n",
        "\n",
        "    merged_entities.append(current_entity)\n",
        "    return merged_entities\n",
        "\n",
        "\n",
        "input_file_path = \"/content/drive/MyDrive/a.jsonl\"\n",
        "\n",
        "merged_data = []\n",
        "\n",
        "with open(input_file_path, \"r\") as file:\n",
        "    for line in file:\n",
        "        data = json.loads(line)\n",
        "        #data.pop(\"relations\", None)  # Remove \"relations\" key if present\n",
        "        #data.pop(\"Comments\", None)\n",
        "        data[\"entities\"].sort(key=lambda x: x[\"id\"])\n",
        "        if len(data[\"entities\"]) == 0:\n",
        "            continue\n",
        "        #data[\"entities\"]= merge_consecutive_entities(data[\"entities\"])\n",
        "        merged_data.append(data)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RXCLoujpbj5m"
      },
      "outputs": [],
      "source": [
        "unique_labels = set(entity[\"label\"] for sample in merged_data for entity in sample[\"entities\"])\n",
        "\n",
        "label2id = {label: idx for idx, label in enumerate(unique_labels)}\n",
        "\n",
        "for sample in merged_data:\n",
        "    for entity in sample[\"entities\"]:\n",
        "        entity[\"label_id\"] = label2id.get(entity[\"label\"], -1)  # -1 for unknown labels\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2fGc3RcKbj5m",
        "outputId": "37f5b9f5-1817-4f8d-c5af-1d30871d3765"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Input IDs: [101, 8327, 2184, 1012, 2410, 4101, 6957, 3820, 8145, 7028, 8474, 2177, 1010, 4297, 1012, 1006, 1000, 10507, 5856, 1000, 1007, 1998, 20369, 2969, 2326, 6627, 1010, 4297, 1012, 1006, 1000, 26189, 4757, 1000, 1007, 1010, 1006, 1996, 1000, 4243, 1000, 2030, 1000, 4101, 6957, 2869, 1000, 2065, 3615, 2000, 13643, 1010, 2030, 1996, 1000, 2283, 1000, 2030, 4101, 6957, 2099, 1000, 2065, 3615, 2000, 13048, 2135, 1007, 1010, 2011, 2023, 3820, 5482, 3209, 2004, 2449, 9228, 1010, 1998, 2025, 2004, 5826, 1010, 1999, 1996, 4195, 1997, 1037, 4101, 6957, 1006, 1996, 1000, 4101, 6957, 1000, 1007, 1010, 2005, 1996, 3800, 1997, 11973, 3227, 1999, 1996, 2449, 3024, 2005, 2011, 3408, 1998, 8910, 1997, 2023, 3820, 1012, 1015, 1012, 1050, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "Labels: [0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "Attention Mask: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "def tokenize_and_add_labels(data):\n",
        "    tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "    unique_labels = set(entity[\"label\"] for sample in data for entity in sample.get(\"entities\", []))\n",
        "\n",
        "    label2id = {label: idx + 1 for idx, label in enumerate(unique_labels)}\n",
        "    label2id[\"[PAD]\"] = 0\n",
        "    tokenized_data = []\n",
        "\n",
        "    for sample in data:\n",
        "        text = sample.get(\"text\", \"\")\n",
        "        entities = sample.get(\"entities\", [])\n",
        "        text = text[:510]\n",
        "        tokenized_inputs = tokenizer(\n",
        "            text,\n",
        "            return_tensors=\"pt\",\n",
        "            padding=\"max_length\",\n",
        "            truncation=True,\n",
        "            max_length=512,\n",
        "        )\n",
        "\n",
        "        input_ids = tokenized_inputs[\"input_ids\"].squeeze().tolist()\n",
        "        # tokens = tokenizer.convert_ids_to_tokens(input_ids)\n",
        "        labels = [0] * len(input_ids)\n",
        "        attention_mask = tokenized_inputs[\"attention_mask\"].squeeze().tolist()\n",
        "\n",
        "        for entity in entities:\n",
        "            start_offset = entity.get(\"start_offset\", 0)\n",
        "            end_offset = entity.get(\"end_offset\", 0)\n",
        "            label = entity.get(\"label\", \"\")\n",
        "\n",
        "            # Ensure indices are within the bounds of the text\n",
        "            start_offset = min(start_offset, len(text) - 1)\n",
        "            end_offset = min(end_offset, len(text) - 1)\n",
        "\n",
        "            start_token = tokenizer.encode(text[:start_offset], add_special_tokens=False)\n",
        "            end_token = tokenizer.encode(text[:end_offset], add_special_tokens=False)\n",
        "            if start_token and end_token:\n",
        "\n",
        "                start = len(start_token)\n",
        "                end = len(end_token)\n",
        "                label_id = label2id.get(label, 0)\n",
        "                labels[start:end + 1] = [label_id] * (end - start + 1)\n",
        "\n",
        "\n",
        "        tokenized_data.append({\n",
        "            \"input_ids\": input_ids,\n",
        "            \"labels\": labels,\n",
        "            \"attention_mask\": attention_mask\n",
        "        })\n",
        "\n",
        "    return tokenized_data, label2id\n",
        "\n",
        "\n",
        "# Tokenize the data and add labels\n",
        "tokenized_data, label2id = tokenize_and_add_labels(merged_data)\n",
        "\n",
        "for sample in tokenized_data:\n",
        "    print(\"Input IDs:\", sample[\"input_ids\"])\n",
        "    print(\"Labels:\", sample[\"labels\"])\n",
        "    print(\"Attention Mask:\", sample[\"attention_mask\"])\n",
        "    print()\n",
        "    break\n",
        "id2label = {v: k for k, v in label2id.items()}\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XtStMJYEbj5o"
      },
      "outputs": [],
      "source": [
        "# Collect unique labels across all samples\n",
        "unique_labels = set(label for sample in tokenized_data for label in sample[\"labels\"])\n",
        "id2label = {v: k for k, v in label2id.items()}\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W9X48NP_bj5o",
        "outputId": "f3a61ac4-baab-4e6d-ae9f-54cca3598fb5"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at nlpaueb/bert-base-uncased-contracts and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"nlpaueb/bert-base-uncased-contracts\")\n",
        "model = AutoModelForTokenClassification.from_pretrained(\"nlpaueb/bert-base-uncased-contracts\")\n",
        "# nlp = pipeline(\"ner\", model=model, tokenizer=tokenizer)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nb2SC42Pbj5o"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader, Dataset\n",
        "from sklearn.model_selection import train_test_split\n",
        "import torch\n",
        "# Split the data into training and validation sets\n",
        "train_data, valid_data = train_test_split(tokenized_data, test_size=0.2)\n",
        "\n",
        "num_labels = len(label2id)\n",
        "train_loader = DataLoader(train_data, batch_size=4, shuffle=True)\n",
        "valid_loader = DataLoader(valid_data, batch_size=4, shuffle=False)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Lhk_uos5bj5p"
      },
      "outputs": [],
      "source": [
        "for sample in tokenized_data:\n",
        "    if len(sample[\"labels\"]) != 512:\n",
        "        print(len(sample[\"labels\"]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8UVU5oXWjmR-"
      },
      "outputs": [],
      "source": [
        "# !pip install -q evaluate\n",
        "# !pip install -q accelerate -U\n",
        "# !pip install -q transformers[torch]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4s-pWXuLbj5q",
        "outputId": "50e89c38-b49d-4c47-de7a-8c5552cea4a0"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at nlpaueb/bert-base-uncased-contracts and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "model_name = 'nlpaueb/bert-base-uncased-contracts'\n",
        "\n",
        "from transformers import AutoModelForTokenClassification\n",
        "\n",
        "model = AutoModelForTokenClassification.from_pretrained(\n",
        "    model_name, num_labels=28, id2label=id2label, label2id=label2id, ignore_mismatched_sizes=True\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MKpzMzh-pL0B"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import evaluate\n",
        "\n",
        "metric = evaluate.load(\"accuracy\")\n",
        "\n",
        "def compute_metrics(eval_pred):\n",
        "    logits, labels = eval_pred\n",
        "    logits_flat = np.argmax(logits, axis=-1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return metric.compute(predictions=logits_flat, references=labels_flat)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vn0xwBME-5ux"
      },
      "outputs": [],
      "source": [
        "learning_rate = 0.0000005\n",
        "lr_max = learning_rate * BATCH_SIZES\n",
        "weight_decay = 0.05\n",
        "\n",
        "optimizer = AdamW(\n",
        "    model.parameters(),\n",
        "    lr=lr_max,\n",
        "    weight_decay=weight_decay)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iyrs4lkB_GGs"
      },
      "outputs": [],
      "source": [
        "num_train_samples = len(train_data)\n",
        "warmup_ratio = 0.2 # Percentage of total steps to go from zero to max learning rate\n",
        "num_cycles=0.8 # The cosine exponential rate\n",
        "\n",
        "num_training_steps = num_train_samples*EPOCHS/BATCH_SIZES\n",
        "num_warmup_steps = num_training_steps*warmup_ratio\n",
        "\n",
        "lr_sched = get_cosine_schedule_with_warmup(optimizer=optimizer,\n",
        "                                           num_warmup_steps=num_warmup_steps,\n",
        "                                           num_training_steps = num_training_steps,\n",
        "                                           num_cycles=num_cycles)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OK8AOfYMbj5r"
      },
      "outputs": [],
      "source": [
        "from transformers import TrainingArguments, Trainer\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"test_trainer\",\n",
        "    evaluation_strategy=\"epoch\",\n",
        "    save_strategy='epoch',\n",
        "    learning_rate=2e-5,\n",
        "    weight_decay=0.01,\n",
        "    num_train_epochs=10,\n",
        "    load_best_model_at_end=True,\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_data,\n",
        "    eval_dataset=valid_data,\n",
        "    compute_metrics=compute_metrics,\n",
        "    optimizer=optimizer\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aGRh3UTCmo4c"
      },
      "outputs": [],
      "source": [
        "del tokenized_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 638
        },
        "id": "w2vhyQVsbj5r",
        "outputId": "46548de5-7b64-45ce-c657-8177bd4699dd"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='2350' max='2350' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [2350/2350 36:30, Epoch 10/10]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.123323</td>\n",
              "      <td>0.974630</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.105203</td>\n",
              "      <td>0.977668</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.184300</td>\n",
              "      <td>0.109177</td>\n",
              "      <td>0.978665</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.184300</td>\n",
              "      <td>0.102771</td>\n",
              "      <td>0.977340</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.069200</td>\n",
              "      <td>0.110868</td>\n",
              "      <td>0.977277</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.069200</td>\n",
              "      <td>0.120152</td>\n",
              "      <td>0.976180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.035200</td>\n",
              "      <td>0.118066</td>\n",
              "      <td>0.979276</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.035200</td>\n",
              "      <td>0.117799</td>\n",
              "      <td>0.977826</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.021000</td>\n",
              "      <td>0.124350</td>\n",
              "      <td>0.977863</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.021000</td>\n",
              "      <td>0.124291</td>\n",
              "      <td>0.978644</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "TrainOutput(global_step=2350, training_loss=0.06835059754391934, metrics={'train_runtime': 2194.5996, 'train_samples_per_second': 8.562, 'train_steps_per_second': 1.071, 'total_flos': 4910831392542720.0, 'train_loss': 0.06835059754391934, 'epoch': 10.0})"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "trainer.train()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "txZhPh51s9G4"
      },
      "outputs": [],
      "source": [
        "trainer.save_model(\"Lber2_full_data\")\n",
        "trainer.model.save_pretrained(\"Lber2_full_data\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "JkFxNdO7krNE",
        "outputId": "4b53c3de-e26b-491c-964c-aef5ede34dfd"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "132"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import os\n",
        "os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n",
        "import gc\n",
        "\n",
        "# Run garbage collection (may not always free memory immediately)\n",
        "gc.collect()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "referenced_widgets": [
            "d2559fd3385e43c59a1da6fc8d7804eb"
          ]
        },
        "id": "uBqNPk53nl7w",
        "outputId": "042eff59-4862-42ed-81a3-7a06a14981dc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n",
            "Token is valid (permission: write).\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d2559fd3385e43c59a1da6fc8d7804eb",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/436M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "CommitInfo(commit_url='https://huggingface.co/Cognitus-Stuti/CommercialLBERT/commit/fe68e781979c41c4114fe37893ca9470038c5ca8', commit_message='Upload tokenizer', commit_description='', oid='fe68e781979c41c4114fe37893ca9470038c5ca8', pr_url=None, pr_revision=None, pr_num=None)"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "!huggingface-cli login --token=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\n",
        "new_model =\"Lber2_full_data\"\n",
        "model.push_to_hub(new_model, max_shard_size='2GB')\n",
        "tokenizer.push_to_hub(new_model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QpK8iHy6ohh9"
      },
      "outputs": [],
      "source": [
        "trainer.push_to_hub(\"LambdaX-AI/Lber2_full_data\")\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}