Roger - T5 - Summarization.ipynb

{
 "metadata": {
  "kernelspec": {
   "language": "python",
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.10",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  }
 },
 "nbformat_minor": 4,
 "nbformat": 4,
 "cells": [
  {
   "cell_type": "code",
   "source": [
    "# ! pip install transformers datasets evaluate rouge_score\n",
    "# !pip install --upgrade accelerate"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:11:52.478317Z",
     "iopub.execute_input": "2023-06-06T03:11:52.478797Z",
     "iopub.status.idle": "2023-06-06T03:12:21.190792Z",
     "shell.execute_reply.started": "2023-06-06T03:11:52.478772Z",
     "shell.execute_reply": "2023-06-06T03:12:21.189635Z"
    },
    "trusted": true
   },
   "execution_count": 1,
   "outputs": [
    {
     "name": "stdout",
     "text": "Requirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.29.2)\nRequirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (2.1.0)\nCollecting evaluate\n  Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m81.4/81.4 kB\u001B[0m \u001B[31m3.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n\u001B[?25hCollecting rouge_score\n  Downloading rouge_score-0.1.2.tar.gz (17 kB)\n  Preparing metadata (setup.py) ... \u001B[?25ldone\n\u001B[?25hRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.12.0)\nRequirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.14.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (1.23.5)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (5.4.1)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.5.5)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers) (2.28.2)\nRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.13.3)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers) (4.64.1)\nRequirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (10.0.1)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.6)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (1.5.3)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.2.0)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.14)\nRequirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2023.5.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.8.4)\nRequirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.18.0)\nRequirement already satisfied: absl-py in /opt/conda/lib/python3.10/site-packages (from rouge_score) (1.4.0)\nRequirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (from rouge_score) (3.2.4)\nRequirement already satisfied: six>=1.14.0 in /opt/conda/lib/python3.10/site-packages (from rouge_score) (1.16.0)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (23.1.0)\nRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (2.1.1)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.2)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.1)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.3)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers) (3.0.9)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.4)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2023.5.7)\nRequirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2023.3)\nBuilding wheels for collected packages: rouge_score\n  Building wheel for rouge_score (setup.py) ... \u001B[?25ldone\n\u001B[?25h  Created wheel for rouge_score: filename=rouge_score-0.1.2-py3-none-any.whl size=24954 sha256=b983be36616ed1c59ae7b45bcd596e3f8f58a0788010a35540b0058ce9b23f4a\n  Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\nSuccessfully built rouge_score\nInstalling collected packages: rouge_score, evaluate\nSuccessfully installed evaluate-0.4.0 rouge_score-0.1.2\n\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n\u001B[0mRequirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (0.12.0)\nCollecting accelerate\n  Downloading accelerate-0.19.0-py3-none-any.whl (219 kB)\n\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m219.1/219.1 kB\u001B[0m \u001B[31m12.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n\u001B[?25hRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate) (1.23.5)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (21.3)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.9.3)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.4.1)\nRequirement already satisfied: torch>=1.6.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (2.0.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->accelerate) (3.0.9)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate) (3.12.0)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate) (4.5.0)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate) (3.1.2)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.6.0->accelerate) (2.1.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.6.0->accelerate) (1.3.0)\nInstalling collected packages: accelerate\n  Attempting uninstall: accelerate\n    Found existing installation: accelerate 0.12.0\n    Uninstalling accelerate-0.12.0:\n      Successfully uninstalled accelerate-0.12.0\nSuccessfully installed accelerate-0.19.0\n\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n\u001B[0m",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "from datasets import load_dataset\n",
    "from transformers import AutoTokenizer\n",
    "from transformers import DataCollatorForSeq2Seq\n",
    "import numpy as np\n",
    "import evaluate\n",
    "from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:12:21.193188Z",
     "iopub.execute_input": "2023-06-06T03:12:21.193552Z",
     "iopub.status.idle": "2023-06-06T03:12:34.383609Z",
     "shell.execute_reply.started": "2023-06-06T03:12:21.193514Z",
     "shell.execute_reply": "2023-06-06T03:12:34.382690Z"
    },
    "trusted": true
   },
   "execution_count": 2,
   "outputs": [
    {
     "name": "stderr",
     "text": "/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:98: UserWarning: unable to load libtensorflow_io_plugins.so: unable to open file: libtensorflow_io_plugins.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so']\ncaused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so: undefined symbol: _ZN3tsl6StatusC1EN10tensorflow5error4CodeESt17basic_string_viewIcSt11char_traitsIcEENS_14SourceLocationE']\n  warnings.warn(f\"unable to load libtensorflow_io_plugins.so: {e}\")\n/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:104: UserWarning: file system plugins are not loaded: unable to open file: libtensorflow_io.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so']\ncaused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so: undefined symbol: _ZTVN10tensorflow13GcsFileSystemE']\n  warnings.warn(f\"file system plugins are not loaded: {e}\")\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "billsum = load_dataset(\"billsum\", split=\"ca_test\")"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:12:34.385014Z",
     "iopub.execute_input": "2023-06-06T03:12:34.385365Z",
     "iopub.status.idle": "2023-06-06T03:12:55.725672Z",
     "shell.execute_reply.started": "2023-06-06T03:12:34.385331Z",
     "shell.execute_reply": "2023-06-06T03:12:55.724642Z"
    },
    "trusted": true
   },
   "execution_count": 3,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading builder script:   0%|          | 0.00/1.53k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "ae7dc0a8c1c145769c8c7837790f557d"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading metadata:   0%|          | 0.00/832 [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "fae0586c313745f99652955eabeaffa1"
      }
     },
     "metadata": {}
    },
    {
     "name": "stdout",
     "text": "Downloading and preparing dataset billsum/default (download: 64.14 MiB, generated: 259.80 MiB, post-processed: Unknown size, total: 323.94 MiB) to /root/.cache/huggingface/datasets/billsum/default/3.0.0/d1e95173aed3acb71327864be74ead49b578522e4c7206048b2f2e5351b57959...\n",
     "output_type": "stream"
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading data:   0%|          | 0.00/67.3M [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "59738c594d484cec9c162c03da0086ff"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Generating train split:   0%|          | 0/18949 [00:00<?, ? examples/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": ""
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Generating test split:   0%|          | 0/3269 [00:00<?, ? examples/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": ""
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Generating ca_test split:   0%|          | 0/1237 [00:00<?, ? examples/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": ""
      }
     },
     "metadata": {}
    },
    {
     "name": "stdout",
     "text": "Dataset billsum downloaded and prepared to /root/.cache/huggingface/datasets/billsum/default/3.0.0/d1e95173aed3acb71327864be74ead49b578522e4c7206048b2f2e5351b57959. Subsequent calls will reuse this data.\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "billsum = billsum.train_test_split(test_size=0.2)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:12:55.728197Z",
     "iopub.execute_input": "2023-06-06T03:12:55.728966Z",
     "iopub.status.idle": "2023-06-06T03:12:55.742756Z",
     "shell.execute_reply.started": "2023-06-06T03:12:55.728931Z",
     "shell.execute_reply": "2023-06-06T03:12:55.741843Z"
    },
    "trusted": true
   },
   "execution_count": 4,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "checkpoint = \"t5-small\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:12:55.744101Z",
     "iopub.execute_input": "2023-06-06T03:12:55.744921Z",
     "iopub.status.idle": "2023-06-06T03:12:59.946658Z",
     "shell.execute_reply.started": "2023-06-06T03:12:55.744888Z",
     "shell.execute_reply": "2023-06-06T03:12:59.945641Z"
    },
    "trusted": true
   },
   "execution_count": 5,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading (…)okenizer_config.json:   0%|          | 0.00/2.32k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b9ec9bb0839d45b7b69a221e11ea2571"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading (…)ve/main/spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "659f44f505694e0790b2b89dce0f4929"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.39M [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "050174dcbcba40b38e40183707e345df"
      }
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "prefix = \"summarize: \"\n",
    "\n",
    "\n",
    "def preprocess_function(examples):\n",
    "    inputs = [prefix + doc for doc in examples[\"text\"]]\n",
    "    model_inputs = tokenizer(inputs, max_length=1024, truncation=True)\n",
    "\n",
    "    labels = tokenizer(text_target=examples[\"summary\"], max_length=128, truncation=True)\n",
    "\n",
    "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:12:59.948278Z",
     "iopub.execute_input": "2023-06-06T03:12:59.948648Z",
     "iopub.status.idle": "2023-06-06T03:12:59.955468Z",
     "shell.execute_reply.started": "2023-06-06T03:12:59.948612Z",
     "shell.execute_reply": "2023-06-06T03:12:59.954525Z"
    },
    "trusted": true
   },
   "execution_count": 6,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "tokenized_billsum = billsum.map(preprocess_function, batched=True)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:12:59.956835Z",
     "iopub.execute_input": "2023-06-06T03:12:59.957403Z",
     "iopub.status.idle": "2023-06-06T03:13:09.107012Z",
     "shell.execute_reply.started": "2023-06-06T03:12:59.957369Z",
     "shell.execute_reply": "2023-06-06T03:13:09.106030Z"
    },
    "trusted": true
   },
   "execution_count": 7,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "  0%|          | 0/1 [00:00<?, ?ba/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "4e1bac0ea54a41868a6dab085e34731e"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "  0%|          | 0/1 [00:00<?, ?ba/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "55ed6353e4e04c44b743bb66ecfaadb2"
      }
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=checkpoint)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:13:09.108362Z",
     "iopub.execute_input": "2023-06-06T03:13:09.108803Z",
     "iopub.status.idle": "2023-06-06T03:13:09.116843Z",
     "shell.execute_reply.started": "2023-06-06T03:13:09.108771Z",
     "shell.execute_reply": "2023-06-06T03:13:09.115922Z"
    },
    "trusted": true
   },
   "execution_count": 8,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "rouge = evaluate.load(\"rouge\")"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:13:09.118088Z",
     "iopub.execute_input": "2023-06-06T03:13:09.118888Z",
     "iopub.status.idle": "2023-06-06T03:13:11.230149Z",
     "shell.execute_reply.started": "2023-06-06T03:13:09.118854Z",
     "shell.execute_reply": "2023-06-06T03:13:11.229220Z"
    },
    "trusted": true
   },
   "execution_count": 9,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading builder script:   0%|          | 0.00/6.27k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "7f28e75075cc4d39aaf80334b76dab7b"
      }
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n",
    "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
    "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
    "\n",
    "    result = rouge.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)\n",
    "\n",
    "    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]\n",
    "    result[\"gen_len\"] = np.mean(prediction_lens)\n",
    "\n",
    "    return {k: round(v, 4) for k, v in result.items()}"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:13:11.233314Z",
     "iopub.execute_input": "2023-06-06T03:13:11.233648Z",
     "iopub.status.idle": "2023-06-06T03:13:11.241505Z",
     "shell.execute_reply.started": "2023-06-06T03:13:11.233616Z",
     "shell.execute_reply": "2023-06-06T03:13:11.239287Z"
    },
    "trusted": true
   },
   "execution_count": 10,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:13:11.242960Z",
     "iopub.execute_input": "2023-06-06T03:13:11.243326Z",
     "iopub.status.idle": "2023-06-06T03:13:32.181556Z",
     "shell.execute_reply.started": "2023-06-06T03:13:11.243292Z",
     "shell.execute_reply": "2023-06-06T03:13:32.180641Z"
    },
    "trusted": true
   },
   "execution_count": 11,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.21k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "47d7a5f1af2b4745a5a0af09f1b8ab67"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading model.safetensors:   0%|          | 0.00/242M [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "3c7ff417aea1417d837b1cfac7eed36c"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading (…)neration_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "cb611a0d8fe34cf89d1e660785851697"
      }
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "import torch\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=\"my_awesome_billsum_model\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=2,\n",
    "    per_device_eval_batch_size=2,\n",
    "    weight_decay=0.01,\n",
    "    save_total_limit=3,\n",
    "    num_train_epochs=4,\n",
    "    predict_with_generate=True,\n",
    "    fp16=True,\n",
    "    push_to_hub=False,\n",
    ")\n",
    "\n",
    "trainer = Seq2SeqTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_billsum[\"train\"],\n",
    "    eval_dataset=tokenized_billsum[\"test\"],\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    ")\n",
    "\n",
    "trainer.train()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-06-06T03:13:32.183011Z",
     "iopub.execute_input": "2023-06-06T03:13:32.184019Z",
     "iopub.status.idle": "2023-06-06T03:21:40.789789Z",
     "shell.execute_reply.started": "2023-06-06T03:13:32.183983Z",
     "shell.execute_reply": "2023-06-06T03:21:40.788809Z"
    },
    "trusted": true
   },
   "execution_count": 12,
   "outputs": [
    {
     "name": "stderr",
     "text": "/opt/conda/lib/python3.10/site-packages/transformers/optimization.py:407: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n  warnings.warn(\n\u001B[34m\u001B[1mwandb\u001B[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n\u001B[34m\u001B[1mwandb\u001B[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n\u001B[34m\u001B[1mwandb\u001B[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:",
     "output_type": "stream"
    },
    {
     "output_type": "stream",
     "name": "stdin",
     "text": "  ········································\n"
    },
    {
     "name": "stderr",
     "text": "\u001B[34m\u001B[1mwandb\u001B[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n",
     "output_type": "stream"
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "Tracking run with wandb version 0.15.3"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "Run data is saved locally in <code>/kaggle/working/wandb/run-20230606_031343-e3f833a3</code>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "Syncing run <strong><a href='https://wandb.ai/rogersingh/huggingface/runs/e3f833a3' target=\"_blank\">tough-hill-3</a></strong> to <a href='https://wandb.ai/rogersingh/huggingface' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": " View project at <a href='https://wandb.ai/rogersingh/huggingface' target=\"_blank\">https://wandb.ai/rogersingh/huggingface</a>"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": " View run at <a href='https://wandb.ai/rogersingh/huggingface/runs/e3f833a3' target=\"_blank\">https://wandb.ai/rogersingh/huggingface/runs/e3f833a3</a>"
     },
     "metadata": {}
    },
    {
     "name": "stderr",
     "text": "You're using a T5TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
     "output_type": "stream"
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n    <div>\n      \n      <progress value='1980' max='1980' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [1980/1980 07:24, Epoch 4/4]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Epoch</th>\n      <th>Training Loss</th>\n      <th>Validation Loss</th>\n      <th>Rouge1</th>\n      <th>Rouge2</th>\n      <th>Rougel</th>\n      <th>Rougelsum</th>\n      <th>Gen Len</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>1</td>\n      <td>No log</td>\n      <td>2.465595</td>\n      <td>0.181700</td>\n      <td>0.083900</td>\n      <td>0.153300</td>\n      <td>0.153300</td>\n      <td>19.000000</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>2.950700</td>\n      <td>2.367128</td>\n      <td>0.200700</td>\n      <td>0.098700</td>\n      <td>0.169800</td>\n      <td>0.170200</td>\n      <td>19.000000</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>2.563900</td>\n      <td>2.332553</td>\n      <td>0.202100</td>\n      <td>0.101500</td>\n      <td>0.172000</td>\n      <td>0.172300</td>\n      <td>19.000000</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>2.487300</td>\n      <td>2.321744</td>\n      <td>0.201600</td>\n      <td>0.101900</td>\n      <td>0.172400</td>\n      <td>0.172600</td>\n      <td>19.000000</td>\n    </tr>\n  </tbody>\n</table><p>"
     },
     "metadata": {}
    },
    {
     "execution_count": 12,
     "output_type": "execute_result",
     "data": {
      "text/plain": "TrainOutput(global_step=1980, training_loss=2.6152942387744633, metrics={'train_runtime': 484.1179, 'train_samples_per_second': 8.172, 'train_steps_per_second': 4.09, 'total_flos': 1070812702310400.0, 'train_loss': 2.6152942387744633, 'epoch': 4.0})"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {},
   "execution_count": null,
   "outputs": []
  }
 ]
}