PEFT_enko.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nXC6xmOlFtyz"
      },
      "source": [
        "# 1. Package Installation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vi6ZScMwq0Lj",
        "outputId": "f8b98fc5-f0ea-4077-9144-8a0a9e75ec20"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mon Aug  5 13:27:29 2024       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.54.03              Driver Version: 535.54.03    CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  NVIDIA A100-SXM4-80GB          On  | 00000000:01:00.0 Off |                    0 |\n",
            "| N/A   59C    P0              71W / 275W |    568MiB / 81920MiB |      0%      Default |\n",
            "|                                         |                      |             Disabled |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "|   1  NVIDIA A100-SXM4-80GB          On  | 00000000:47:00.0 Off |                    0 |\n",
            "| N/A   58C    P0              70W / 275W |      4MiB / 81920MiB |      0%      Default |\n",
            "|                                         |                      |             Disabled |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "|   2  NVIDIA A100-SXM4-80GB          On  | 00000000:81:00.0 Off |                    0 |\n",
            "| N/A   64C    P0             298W / 275W |  22082MiB / 81920MiB |     89%      Default |\n",
            "|                                         |                      |             Disabled |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "|   3  NVIDIA DGX Display             On  | 00000000:C1:00.0 Off |                  N/A |\n",
            "| 34%   44C    P8              N/A /  50W |      1MiB /  4096MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "|   4  NVIDIA A100-SXM4-80GB          On  | 00000000:C2:00.0 Off |                    0 |\n",
            "| N/A   67C    P0             302W / 275W |  59742MiB / 81920MiB |     80%      Default |\n",
            "|                                         |                      |             Disabled |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|    0   N/A  N/A   1664868      C   python                                      554MiB |\n",
            "|    2   N/A  N/A   2510846      C   python                                    22068MiB |\n",
            "|    4   N/A  N/A   2556107      C   python                                    59728MiB |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KpkmrFpqYISS",
        "outputId": "101897e1-a939-4c4e-a635-e72d8cf15ff4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting requirements.txt\n"
          ]
        }
      ],
      "source": [
        "%%writefile requirements.txt\n",
        "peft\n",
        "fire\n",
        "accelerator\n",
        "transformers\n",
        "datasets\n",
        "evaluate\n",
        "pyarrow\n",
        "galore-torch\n",
        "pytorch-ignite\n",
        "rouge-score\n",
        "nltk\n",
        "py7zr\n",
        "optimum[exporters]\n",
        "trl\n",
        "lightning\n",
        "jsonargparse[signatures]\n",
        "deepspeed\n",
        "colossalai\n",
        "wandb\n",
        "tensorrt\n",
        "nvidia-modelopt --index https://pypi.nvidia.com"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qt3RP1onC5Uf"
      },
      "outputs": [],
      "source": [
        "!CUDA_EXT=1 DS_BUILD=1 pip install -r requirements.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iCRpjtCdPPRx",
        "outputId": "a4940d58-4e9a-4fc4-c33c-fbb8ceb592c0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
            "Token is valid (permission: fineGrained).\n",
            "Your token has been saved to /user/jonathan/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ],
      "source": [
        "#@title Huggingface Login\n",
        "#@markdown huggingface weight 를 이용하고 싶다면 로그인 필수\n",
        "!huggingface-cli login --token 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\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3i2AiD0181GO",
        "outputId": "93dd1375-91ac-40c1-accc-34ed0df01a55"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /user/jonathan/.netrc\n"
          ]
        }
      ],
      "source": [
        "#@title Weight and Bias Train Logger Login\n",
        "#@markdown weight and bias 로그인\n",
        "!wandb login 801d28a4a889fcb0481bc71315c04c70b346d332"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IfgXZtBZFyVE"
      },
      "source": [
        "# 2. Load Model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RO8x9zxrFb1-",
        "outputId": "1869272e-d84f-423a-85a1-9382ae532904"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting peft_model.py\n"
          ]
        }
      ],
      "source": [
        "%%writefile peft_model.py\n",
        "\n",
        "import os\n",
        "import fire\n",
        "import torch\n",
        "from peft import AutoPeftModelForCausalLM, LoraConfig\n",
        "from peft import (inject_adapter_in_model, prepare_model_for_kbit_training,\n",
        "                  get_peft_model, replace_lora_weights_loftq)\n",
        "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
        "from transformers.utils import PaddingStrategy\n",
        "from transformers.tokenization_utils_base import TruncationStrategy\n",
        "from datasets import load_dataset\n",
        "from random import randint\n",
        "\n",
        "\n",
        "hf_model_list = [\n",
        "    # \"Gunulhona/tb_pretrained_sts\",\n",
        "    # \"Gunulhona/tb_pretrained\",\n",
        "    \"google/flan-t5-xl\",\n",
        "    # \"meta-llama/Meta-Llama-3.1-8B\",\n",
        "    # \"meta-llama/Meta-Llama-3-70B-Instruct\",\n",
        "    \"mistralai/Mistral-7B-Instruct-v0.3\",\n",
        "    \"mistralai/Mistral-Nemo-Instruct-2407\",\n",
        "    \"Qwen/Qwen2-7B-Instruct\",\n",
        "    \"google/gemma-7b\",\n",
        "    \"MLP-KTLim/llama-3-Korean-Bllossom-8B\",\n",
        "    \"EleutherAI/polyglot-ko-12.8b\",\n",
        "    \"vilm/vulture-40b\",\n",
        "    \"tiiuae/falcon-11B\",\n",
        "    \"tiiuae/falcon-7b-instruct\",\n",
        "     \"arcee-ai/Arcee-Spark\",\n",
        "     \"apple/DCLM-7B-8k\",\n",
        "     \"SciPhi/Triplex\", # rag chatbot\n",
        "    # \"TinyLlama/TinyLlama-1.1B-Chat-v1.0\",\n",
        "     \"bigscience/bloomz-7b1-mt\",\n",
        "    # \"bigscience/bloomz-1b1\",\n",
        "    # \"Qwen/Qwen2-1.5B-Instruct\",\n",
        "    # \"Qwen/Qwen2-0.5B-Instruct\",\n",
        "    # \"OuteAI/Lite-Oute-1-65M\",\n",
        "    # \"OuteAI/Lite-Mistral-150M-v2-Instruct\",\n",
        "    # \"google/gemma-2b-it\",\n",
        "    \"jjhsnail0822/danube-ko-1.8b-base\",\n",
        "    # \"OpenBuddy/openbuddy-stablelm-3b-v13\",\n",
        "    \"daekeun-ml/phi-2-ko-v0.1\",\n",
        "    # \"microsoft/Phi-3-mini-128k-instruct\",\n",
        "    # \"HuggingFaceTB/SmolLM-1.7B\",\n",
        "    # \"HuggingFaceTB/SmolLM-360M\",\n",
        "    # \"HuggingFaceTB/SmolLM-135M\",\n",
        "    # \"numind/NuExtract\",\n",
        "    ]\n",
        "base_model_id = \"Qwen/Qwen2-1.5B-Instruct\" # @param [\"Gunulhona/tb_pretrained_sts\", \"Gunulhona/tb_pretrained\", \"google/flan-t5-xxl\", \"meta-llama/Meta-Llama-3-8B\", \"meta-llama/Meta-Llama-3-70B-Instruct\", \"mistralai/Mistral-7B-Instruct-v0.3\", \"Qwen/Qwen2-7B-Instruct\", \"google/gemma-7b\", \"MLP-KTLim/llama-3-Korean-Bllossom-8B\", \"EleutherAI/polyglot-ko-12.8b\", \"vilm/vulture-40b\", \"arcee-ai/Arcee-Spark\", \"Qwen/Qwen2-1.5B-Instruct\", \"OuteAI/Lite-Mistral-150M\", \"google/gemma-2b-it\"] {allow-input: true}\n",
        "\n",
        "def get_model(model_name:str,\n",
        "              r: int = 8,\n",
        "              lora_alpha: int = 32,\n",
        "              lora_dropout: float = 0.05,\n",
        "              init_lora_weights: str = \"gaussian\", #\"gaussian\", \"pissa\", \"pissa_niter_{n}\", \"loftq\", False\n",
        "              use_dora: bool = False,\n",
        "              use_rslora: bool = False,\n",
        "              fan_in_fan_out: bool = False,):\n",
        "    bnb_config = BitsAndBytesConfig(\n",
        "        load_in_8bit=True,\n",
        "        bnb_8bit_quant_type=\"nf8\",\n",
        "        bnb_8bit_compute_dtype=torch.float16\n",
        "    )\n",
        "\n",
        "\n",
        "    peft_model = AutoModelForCausalLM.from_pretrained(\n",
        "        base_model_id,\n",
        "        trust_remote_code=True,\n",
        "        # quantization_config=bnb_config\n",
        "        )\n",
        "\n",
        "    peft_model = prepare_model_for_kbit_training(peft_model)\n",
        "\n",
        "    # adapter configuration\n",
        "    lora_config = LoraConfig(\n",
        "        target_modules=[\"q_proj\", \"k_proj\"],\n",
        "        init_lora_weights=init_lora_weights,\n",
        "        r=r,\n",
        "        lora_alpha=lora_alpha,\n",
        "        lora_dropout=lora_dropout,\n",
        "        inference_mode=False,\n",
        "        use_dora=use_dora,\n",
        "        use_rslora=use_rslora,\n",
        "        fan_in_fan_out=fan_in_fan_out,\n",
        "        task_type=\"CAUSAL_LM\",\n",
        "    )\n",
        "\n",
        "    # peft_model.add_adapter(lora_config, adapter_name=\"adapter_1\")\n",
        "    inject_adapter_in_model(lora_config, peft_model, \"adapter_1\")\n",
        "    peft_model = get_peft_model(peft_model, lora_config)\n",
        "\n",
        "\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\n",
        "        model_name,\n",
        "        add_special_tokens=True,\n",
        "        trust_remote_code=True)\n",
        "    tokenizer.model_input_names=['input_ids', 'attention_mask']\n",
        "    if tokenizer.pad_token is None:\n",
        "        tokenizer.pad_token = tokenizer.eos_token\n",
        "\n",
        "    tokenizer.padding_side = \"right\"\n",
        "    tokenizer.truncation_side = \"right\"\n",
        "    return {\n",
        "        \"model\": peft_model,\n",
        "        \"tokenizer\": tokenizer\n",
        "        }\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wK7h7vdKX0r7"
      },
      "source": [
        "#3. Load Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AQd5nc9cYixp",
        "outputId": "673319cf-c7cd-4208-fc65-0458eba02f80"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting finetuning_datasets.py\n"
          ]
        }
      ],
      "source": [
        "%%writefile finetuning_datasets.py\n",
        "import numpy as np\n",
        "from datasets import load_dataset, concatenate_datasets\n",
        "from transformers import DataCollatorForSeq2Seq, DataCollatorWithPadding, DataCollatorForLanguageModeling\n",
        "\n",
        "from evaluate import load\n",
        "from finetuning_datafunctions import formatting, preprocess_function, SumDataCallator\n",
        "\n",
        "\n",
        "dataset_path = \"jonathankang/ENKO-MEDIQA\" # @param [\"Samsung/samsum\", \"emozilla/soda_synthetic_dialogue\", \"frcp/summary-alpaca-v01\", \"ChuGyouk/Ko-MTS-Dialog\", \"har1/MTS_Dialogue-Clinical_Note\", \"316usman/research_clinical_visit_note_summarization_corpus_mts\", \"jonathankang/MEDICAL-DIALOG-SUMMARY\", \"jonathankang/ENKO-MEDIQA\"] {allow-input: true}\n",
        "def get_dataset(dataset_name: str,\n",
        "                tokenizer):\n",
        "    raw_dataset = load_dataset(\n",
        "    dataset_path,\n",
        "    trust_remote_code=True,\n",
        "    revision=\"main\",  \n",
        "    )\n",
        "\n",
        "    metric = load(\"rouge\")\n",
        "    # full_dataset = concatenate_datasets([raw_dataset[\"train\"], raw_dataset[\"test\"]])\n",
        "    tokenized_inputs = raw_dataset[\"train\"].map(\n",
        "        lambda x: tokenizer(x[\"dialogue\"], truncation=True),\n",
        "        batched=True,\n",
        "        remove_columns=[\"dialogue\", \"summary\"])\n",
        "\n",
        "    input_lenghts = [len(x) for x in tokenized_inputs[\"input_ids\"]]\n",
        "    # take 85 percentile of max length for better utilization\n",
        "    max_source_length = int(np.percentile(input_lenghts, 100)) + int(np.percentile(input_lenghts, 10))\n",
        "    max_source_length = min(4096, max_source_length)\n",
        "\n",
        "    tokenized_targets = raw_dataset[\"train\"].map(\n",
        "        lambda x: tokenizer(x[\"summary\"], truncation=True),\n",
        "        batched=True,\n",
        "        remove_columns=[\"dialogue\", \"summary\"])\n",
        "    target_lenghts = [len(x) for x in tokenized_targets[\"input_ids\"]]\n",
        "    # take 90 percentile of max length for better utilization\n",
        "    max_target_length = int(np.percentile(target_lenghts, 100)) + int(np.percentile(target_lenghts, 10))\n",
        "\n",
        "\n",
        "\n",
        "    en_dataset = raw_dataset.map(\n",
        "        preprocess_function,\n",
        "        batched=True,\n",
        "        remove_columns=[\"dialogue\", \"summary\", \"id\", \"대화\", \"요약\"],\n",
        "        fn_kwargs={\n",
        "            \"tokenizer\": tokenizer,\n",
        "            \"max_source_length\": max_source_length,\n",
        "            \"max_target_length\": max_target_length,\n",
        "            \"context_column\": \"dialogue\",\n",
        "            \"target_column\": \"summary\"\n",
        "            },)\n",
        "    \n",
        "    ko_dataset = raw_dataset.map(\n",
        "        preprocess_function,\n",
        "        batched=True,\n",
        "        remove_columns=[\"dialogue\", \"summary\", \"id\", \"대화\", \"요약\"],\n",
        "        fn_kwargs={\n",
        "            \"tokenizer\": tokenizer,\n",
        "            \"max_source_length\": max_source_length,\n",
        "            \"max_target_length\": max_target_length,\n",
        "            \"context_column\": \"대화\",\n",
        "            \"target_column\": \"요약\"\n",
        "            },)\n",
        "\n",
        "    train_dataset = concatenate_datasets([en_dataset[\"train\"], ko_dataset[\"train\"]])\n",
        "    test_dataset = concatenate_datasets([en_dataset[\"test\"], ko_dataset[\"test\"]])\n",
        "    val_dataset = concatenate_datasets([en_dataset[\"validation\"], ko_dataset[\"validation\"]])\n",
        "    \n",
        "    # dataset = raw_dataset\n",
        "    # if any([d for d in dataset.values() if \"token_type_ids\" in d.features]):\n",
        "    #     dataset = dataset.map(lambda x: x,\n",
        "    #                           batched=True,\n",
        "    #                           remove_columns=[\"token_type_ids\"], )\n",
        "    return {\n",
        "        \"train\":train_dataset,\n",
        "        \"test\":test_dataset,\n",
        "        \"val\":val_dataset,\n",
        "        \"metric\": metric,\n",
        "        \"max_source_length\": max_source_length\n",
        "        }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_6hq8eyYLvCn",
        "outputId": "d1b24240-9537-4e9d-a038-64bb7ddab23e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting finetuning_datafunctions.py\n"
          ]
        }
      ],
      "source": [
        "%%writefile finetuning_datafunctions.py\n",
        "import numpy as np\n",
        "import torch\n",
        "from datasets import load_dataset, concatenate_datasets\n",
        "from transformers import DataCollatorForSeq2Seq, DataCollatorWithPadding, DataCollatorForLanguageModeling\n",
        "\n",
        "from evaluate import load\n",
        "\n",
        "\n",
        "def formatting(sample,\n",
        "               max_source_length,\n",
        "               max_target_length,\n",
        "               tokenizer,\n",
        "               padding:str = \"max_length\",\n",
        "               context_column: str = \"dialogue\",\n",
        "               target_column: str = \"summary\"):\n",
        "    # add prefix to the input for t5\n",
        "    model_inputs, labels = [], []\n",
        "    for dialogue, summary in zip(sample[context_column], sample[target_column]):\n",
        "        chat_template = [\n",
        "            # {\n",
        "            #     \"role\": \"system\",\n",
        "            #     \"content\": \"You are a friendly chatbot who always responds with summary\",\n",
        "            # },\n",
        "            {\n",
        "                \"role\": \"user\",\n",
        "                \"content\": f\"Summarize the following dialogue\\n\\n{dialogue}\"\n",
        "            },\n",
        "\n",
        "        ]\n",
        "        label_template = [{\n",
        "                \"role\": \"assistant\",\n",
        "                \"content\": f\"{summary}\"\n",
        "        }]\n",
        "\n",
        "        chat_message = tokenizer.apply_chat_template(conversation=chat_template,\n",
        "                                                     tokenize=False,\n",
        "                                                     add_generateion_prompt=False, )\n",
        "        bot_message = tokenizer.apply_chat_template(conversation=label_template,\n",
        "                                                    tokenize=False,\n",
        "                                                    add_generateion_prompt=False, )\n",
        "        model_inputs.append(chat_message)\n",
        "        labels.append(bot_message)\n",
        "\n",
        "    # inputs = [\"summarize: \" + item for item in sample[\"dialogue\"]]\n",
        "\n",
        "    # tokenize inputs\n",
        "    # model_inputs = tokenizer(inputs, max_length=max_source_length, padding=padding, truncation=True, )\n",
        "\n",
        "    # Tokenize targets with the `text_target` keyword argument\n",
        "    # labels = tokenizer(text_target=sample[\"summary\"],\n",
        "    #                    max_length=max_target_length,\n",
        "    #                    padding=padding,\n",
        "    #                    truncation=True,)\n",
        "\n",
        "    # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore\n",
        "    # padding in the loss.\n",
        "    # if padding == \"max_length\":\n",
        "    #     labels[\"input_ids\"] = [\n",
        "    #         [(l if l != tokenizer.pad_token_id else 1) for l in label] for label in labels[\"input_ids\"]\n",
        "    #     ]\n",
        "    # model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
        "    return model_inputs, labels\n",
        "\n",
        "def preprocess_function(sample, max_source_length, max_target_length, tokenizer, context_column, target_column):\n",
        "    templated_text, labels = formatting(sample=sample, \n",
        "                                        max_source_length=max_source_length, \n",
        "                                        max_target_length=max_target_length, \n",
        "                                        tokenizer=tokenizer,\n",
        "                                        context_column=context_column,\n",
        "                                        target_column=target_column,)\n",
        "\n",
        "    return {\n",
        "        \"input_ids\": templated_text,\n",
        "        \"labels\": labels\n",
        "    }\n",
        "\n",
        "class CallatorOutput:\n",
        "    def __init__(self, input_ids, attention_mask, labels):\n",
        "        self._input_ids = input_ids\n",
        "        self._attention_mask = attention_mask\n",
        "        self._labels = labels\n",
        "\n",
        "    def __len__(self,):\n",
        "        return len(self._input_ids)\n",
        "\n",
        "    def __getitem__(self, key):\n",
        "        match key:\n",
        "            case \"input_ids\":\n",
        "                return self._input_ids\n",
        "            case \"attention_mask\":\n",
        "                return self._attention_mask\n",
        "            case \"labels\":\n",
        "                return self._labels\n",
        "            case _:\n",
        "                raise KeyError(f\"Key {key} not found\")\n",
        "\n",
        "    def __setitem__(self, key, value):\n",
        "        match key:\n",
        "            case \"input_ids\":\n",
        "                self._input_ids = value\n",
        "            case \"attention_mask\":\n",
        "                self._attention_mask = value\n",
        "            case \"labels\":\n",
        "                self._labels = value\n",
        "            case _:\n",
        "                raise KeyError(f\"Key {key} not found\")\n",
        "\n",
        "    def __iter__(self):\n",
        "        return iter(self.__dict__.keys())\n",
        "\n",
        "    def to_dict(self):\n",
        "        return {\n",
        "            \"input_ids\": self[\"input_ids\"],\n",
        "            \"attention_mask\": self[\"attention_mask\"],\n",
        "            \"labels\": self[\"labels\"]\n",
        "        }\n",
        "    def items(self):\n",
        "        return self.to_dict().items()\n",
        "\n",
        "    def keys(self):\n",
        "        return self.to_dict().keys()\n",
        "\n",
        "    def values(self):\n",
        "        return self.to_dict().values()\n",
        "\n",
        "    @property\n",
        "    def input_ids(self):\n",
        "        return self._input_ids\n",
        "\n",
        "    @input_ids.setter\n",
        "    def input_ids(self, value):\n",
        "        self._input_ids = value\n",
        "\n",
        "    @property\n",
        "    def attention_mask(self):\n",
        "        return self._attention_mask\n",
        "\n",
        "    @attention_mask.setter\n",
        "    def attention_mask(self, value):\n",
        "        self._attention_mask = value\n",
        "\n",
        "    @property\n",
        "    def labels(self):\n",
        "        return self._labels\n",
        "\n",
        "    @labels.setter\n",
        "    def labels(self, value):\n",
        "        self._labels = value\n",
        "\n",
        "\n",
        "\n",
        "class SumDataCallator(DataCollatorForLanguageModeling):\n",
        "    def __init__(self, tokenizer, max_length,):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.max_length = max_length\n",
        "        self.device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "    def _tokenizing(self, text):\n",
        "        return self.tokenizer(text,\n",
        "                              truncation=True,\n",
        "                              padding=\"max_length\",\n",
        "                              max_length=self.max_length,\n",
        "                              return_tensors=\"pt\")\n",
        "\n",
        "    def __call__(self, batch):\n",
        "            input_text = []\n",
        "            labels = []\n",
        "            for b in batch:\n",
        "                input_text += [b[\"input_ids\"]]\n",
        "                labels += [b[\"labels\"]]\n",
        "            input_tokens = self._tokenizing(input_text)\n",
        "            label_tokens = self._tokenizing(labels)\n",
        "\n",
        "            return CallatorOutput(**{\n",
        "                \"input_ids\": input_tokens['input_ids'].to(self.device),\n",
        "                \"attention_mask\": input_tokens['attention_mask'].to(self.device),\n",
        "                \"labels\": label_tokens['input_ids'].to(self.device),\n",
        "            })\n",
        "\n",
        "        # raise Exception(\"STOP\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3C1Fs6eeX4Ri"
      },
      "source": [
        "#4. Train"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wnRyfucpWeK5",
        "outputId": "432ff482-13d4-4ed0-b3b8-436f22c6759a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting train.py\n"
          ]
        }
      ],
      "source": [
        "%%writefile train.py\n",
        "import nltk\n",
        "import numpy as np\n",
        "import torch\n",
        "from torch.utils.data import DataLoader\n",
        "from transformers import TrainingArguments, Trainer, TrainerCallback\n",
        "from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments\n",
        "from transformers import DataCollatorForSeq2Seq, DataCollatorWithPadding, DataCollatorForLanguageModeling\n",
        "from trl import SFTTrainer, SFTConfig\n",
        "from ignite.metrics import Rouge\n",
        "from peft_model import peft_model, tokenizer, lora_config\n",
        "from finetuning_datasets import dataset, metric, max_source_length\n",
        "from finetuning_datafunctions import SumDataCallator, formatting, preprocess_function\n",
        "\n",
        "os.environ[\"WANDB_PROJECT\"] = \"<summ>\"  # Change this to your project name\n",
        "os.environ[\"WANDB_LOG_MODEL\"] = \"checkpoint\"  # Log model checkpoints\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "peft_model.to(device)\n",
        "\n",
        "class ColabTrainer(SFTTrainer):\n",
        "    pass\n",
        "\n",
        "\n",
        "# Callback Class\n",
        "class EarlyStoppingCallback(TrainerCallback):\n",
        "    def __init__(self, num_steps=150):\n",
        "        self.num_steps = num_steps\n",
        "\n",
        "    def on_step_end(self, args, state, control, **kwargs):\n",
        "        if state.global_step >= self.num_steps:\n",
        "            control.should_training_stop = True\n",
        "\n",
        "        return control\n",
        "\n",
        "# metric function\n",
        "def compute_metrics(eval_pred):\n",
        "    predictions, labels = eval_pred\n",
        "    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n",
        "    # Replace -100 in the labels as we can't decode them.\n",
        "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
        "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
        "\n",
        "    # Rouge expects a newline after each sentence\n",
        "    decoded_preds = [\"\\n\".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]\n",
        "    decoded_labels = [\"\\n\".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]\n",
        "\n",
        "    result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True, use_aggregator=True)\n",
        "    # Extract a few results\n",
        "    result = {key: value * 100 for key, value in result.items()}\n",
        "\n",
        "    # Add mean generated length\n",
        "    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]\n",
        "    result[\"gen_len\"] = np.mean(prediction_lens)\n",
        "\n",
        "    return {k: round(v, 4) for k, v in result.items()}\n",
        "\n",
        "data_collator = SumDataCallator(tokenizer, max_length=max_source_length)\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"llms\",\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=1,\n",
        "    per_device_eval_batch_size=1,\n",
        "    num_train_epochs=4,\n",
        "    weight_decay=0.01,\n",
        "    # eval_strategy=\"epoch\",\n",
        "    save_strategy=\"no\",\n",
        "    # use_cpu=True,\n",
        "    # load_best_model_at_end=True,\n",
        "    remove_unused_columns=False,\n",
        "    push_to_hub=True,\n",
        "    logging_steps=1000,\n",
        "    save_steps=1000,\n",
        "    warmup_steps=0.03,\n",
        "    gradient_accumulation_steps=4,\n",
        "    fp16=True,\n",
        "    save_total_limit=3,\n",
        "    logging_dir=\"llms/logs\",\n",
        "    optim=\"paged_adamw_8bit\",\n",
        "    report_to=\"wandb\",\n",
        "    run_name='qwenrun1',\n",
        ")\n",
        "\n",
        "# Initialize W&B\n",
        "wandb.init(\n",
        "    project=\"<summ>\",  # Change to your project name\n",
        "    config=training_args\n",
        ")\n",
        "\n",
        "trainer = SFTTrainer(\n",
        "    model=peft_model,\n",
        "    args=training_args,\n",
        "    train_dataset=dataset[\"train\"],\n",
        "    eval_dataset=dataset[\"test\"],\n",
        "    tokenizer=tokenizer,\n",
        "    data_collator=data_collator,\n",
        "    compute_metrics=compute_metrics,\n",
        "    # callbacks=[EarlyStoppingCallback()],\n",
        "    peft_config=lora_config,\n",
        "    # formatting_func=formatting,\n",
        ")\n",
        "\n",
        "trainer.train()\n",
        "\n",
        "trainer.save_model(\"/user/jonathan/TRAINED\")\n",
        "tokenizer.save_pretrained(\"/user/jonathan/TRAINED\")\n",
        "# # Save the model, tokenizer and push everything to the Hub\n",
        "# trainer.save_model()\n",
        "# tokenizer.save_pretrained(training_args.output_dir)\n",
        "\n",
        "# trainer.push_to_hub()\n",
        "# tokenizer.push_to_hub(training_args.output_dir)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uysQB-wlnmt3",
        "outputId": "57f623e9-516a-4833-e940-9f3d26aa10fd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting deepspeed_config.yaml\n"
          ]
        }
      ],
      "source": [
        "%%writefile deepspeed_config.yaml\n",
        "compute_environment: LOCAL_MACHINE\n",
        "debug: false\n",
        "deepspeed_config:\n",
        "  deepspeed_multinode_launcher: standard\n",
        "  gradient_accumulation_steps: 4\n",
        "  offload_optimizer_device: none\n",
        "  offload_param_device: none\n",
        "  zero3_init_flag: true\n",
        "  zero3_save_16bit_model: true\n",
        "  zero_stage: 2\n",
        "distributed_type: DEEPSPEED\n",
        "downcast_bf16: 'no'\n",
        "machine_rank: 0\n",
        "main_training_function: main\n",
        "mixed_precision: bf16\n",
        "num_machines: 1\n",
        "num_processes: 1\n",
        "rdzv_backend: static\n",
        "same_network: true\n",
        "tpu_env: []\n",
        "tpu_use_cluster: false\n",
        "tpu_use_sudo: false\n",
        "use_cpu: false"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "iX1Vp8-2PQN8",
        "outputId": "0343cb99-1fc9-4d63-e6db-fef02533ed7c"
      },
      "outputs": [],
      "source": [
        "# Cell for setting up the environment and launching training with accelerate\n",
        "!export CUDA_VISIBLE_DEVICES=0\n",
        "!export CUDA_LAUNCH_BLOCKING=1\n",
        "!export TORCH_USE_CUDA_DSA=0\n",
        "!export HF_DATASETS_CACHE='/content/hf_cache/'\n",
        "!accelerate launch --config_file \"deepspeed_config.yaml\" train.py\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d5KC2KSMllGB"
      },
      "source": [
        "## Training code to Lightning module"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YSF7QaH1zhQ9",
        "outputId": "962e6e12-cc7a-45a4-cc95-d7ef45956c62"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting l_datamodule.py\n"
          ]
        }
      ],
      "source": [
        "%%writefile l_datamodule.py\n",
        "#@title Lightning Data Moudle\n",
        "#@markdown Lightning Data Loading Modules\n",
        "\n",
        "import lightning as L\n",
        "import torch\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "from transformers import DataCollatorForLanguageModeling\n",
        "from finetuning_datafunctions import SumDataCallator\n",
        "\n",
        "\n",
        "class FTDataModule(L.LightningDataModule):\n",
        "    def __init__(self,\n",
        "                 train_dataset,\n",
        "                 val_dataset,\n",
        "                 test_dataset,\n",
        "                 data_collator:DataCollatorForLanguageModeling,\n",
        "                 train_batch_size: int = 1,\n",
        "                 eval_batch_size:int = 1,\n",
        "                 training_args: dict = {}):\n",
        "        super().__init__()\n",
        "        self.train_dataset = train_dataset\n",
        "        self.val_dataset = val_dataset\n",
        "        self.test_dataset = test_dataset\n",
        "        self.data_collator = data_collator\n",
        "        self.train_batch_size = train_batch_size\n",
        "        self.eval_batch_size = eval_batch_size\n",
        "        self.training_args = training_args\n",
        "\n",
        "    def _get_dataloader(self, dataset, eval_mode: bool = False):\n",
        "        return DataLoader(dataset=dataset,\n",
        "                          batch_size=self.train_batch_size if eval_mode else self.eval_batch_size,\n",
        "                          shuffle=not eval_mode,\n",
        "                          num_workers=0,\n",
        "                          collate_fn=self.data_collator)\n",
        "\n",
        "    def train_dataloader(self):\n",
        "        return self._get_dataloader(dataset=self.train_dataset)\n",
        "\n",
        "    def val_dataloader(self):\n",
        "        return self._get_dataloader(dataset=self.val_dataset, eval_mode=True)\n",
        "\n",
        "    def test_dataloader(self):\n",
        "        return self._get_dataloader(dataset=self.test_dataset, eval_mode=True)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "agrktrAylkal",
        "outputId": "938b5dbe-0bc9-4d4e-8259-a27ab25c4687"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting l_model.py\n"
          ]
        }
      ],
      "source": [
        "%%writefile l_model.py\n",
        "#@title Lightning Model\n",
        "#@markdown Lightning Modules and Training step\n",
        "\n",
        "import lightning as L\n",
        "import torch\n",
        "\n",
        "from transformers import DataCollatorForSeq2Seq\n",
        "from bitsandbytes.optim import AdamW, Lion, PagedAdamW, PagedLion, LARS\n",
        "from torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingWarmRestarts\n",
        "from torchmetrics.functional.text.rouge import rouge_score\n",
        "\n",
        "\n",
        "class LLamaFTLightningModule(L.LightningModule):\n",
        "    def __init__(self,\n",
        "                 #data_collator,\n",
        "                 peft_model,\n",
        "                 tokenizer,\n",
        "                 learning_rate: float = 2e-5 ):\n",
        "        super().__init__()\n",
        "        self.save_hyperparameters(ignore=['peft_model'])\n",
        "        self.model = peft_model\n",
        "        self.tokenizer = tokenizer\n",
        "        # self.data_collator = DataCollatorForSeq2Seq(tokenizer, model=peft_model)\n",
        "        self.learning_rate = learning_rate\n",
        "\n",
        "    def _get_rouge_score(self, predictions, labels):\n",
        "        generated_tokens = predictions.argmax(dim=-1)\n",
        "        decoded_labels = self.tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
        "        decoded_preds = self.tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)\n",
        "        return rouge_score(preds=decoded_preds, target=decoded_labels)\n",
        "\n",
        "    def _log(self, log_name, value, batch_size):\n",
        "        self.log(\n",
        "            log_name,\n",
        "            value if value.device == self.model.device else value.to(self.model.device),\n",
        "            prog_bar=True,\n",
        "            on_step=True,\n",
        "            on_epoch=True,\n",
        "            batch_size=batch_size,\n",
        "            sync_dist=True)\n",
        "\n",
        "    def _batch_device_correction(self, batch):\n",
        "        for k, v in batch.items():\n",
        "            if v.device != self.model.device:\n",
        "                batch[k] = v.to(self.model.device)\n",
        "        return batch\n",
        "\n",
        "    def forward(self, input_ids, attention_mask, labels):\n",
        "        # print(input_ids.shape, input_ids.min(), input_ids.max())\n",
        "        return self.model(**{\n",
        "            \"input_ids\":input_ids,\n",
        "            \"attention_mask\":attention_mask,\n",
        "            \"labels\": labels\n",
        "            })\n",
        "\n",
        "    def training_step(self, batch, batch_idx):\n",
        "        batch = self._batch_device_correction(batch)\n",
        "        outputs = self(input_ids=batch.input_ids,\n",
        "                       attention_mask=batch.attention_mask,\n",
        "                       labels=batch.labels)\n",
        "        rouge_score = self._get_rouge_score(outputs.logits, batch.labels)\n",
        "        loss = outputs.loss\n",
        "        self._log(\"train_loss\", loss, self.trainer.datamodule.train_batch_size,)\n",
        "        for k, v in rouge_score.items():\n",
        "            self._log(f\"train_{k}\", v, self.trainer.datamodule.train_batch_size,)\n",
        "\n",
        "        return loss\n",
        "\n",
        "    def validation_step(self, batch, batch_idx):\n",
        "        batch = self._batch_device_correction(batch)\n",
        "        outputs = self(input_ids=batch.input_ids,\n",
        "                       attention_mask=batch.attention_mask,\n",
        "                       labels=batch.labels)\n",
        "        rouge_score = self._get_rouge_score(outputs.logits, batch.labels)\n",
        "        val_loss = outputs.loss\n",
        "        self._log(\"val_loss\", val_loss, self.trainer.datamodule.eval_batch_size,)\n",
        "        for k, v in rouge_score.items():\n",
        "            self._log(f\"val_{k}\", v, self.trainer.datamodule.eval_batch_size,)\n",
        "\n",
        "    def configure_optimizers(self):\n",
        "        optimizer = PagedLion(params=self.model.parameters(),\n",
        "                              lr=self.learning_rate,\n",
        "                              weight_decay=0.01,\n",
        "                              optim_bits=32,)\n",
        "        scheduler = CosineAnnealingWarmRestarts(optimizer,\n",
        "                                                T_0=10,\n",
        "                                                T_mult=2,\n",
        "                                                eta_min=0.00001)\n",
        "        # scheduler = ReduceLROnPlateau(optimizer=optimizer, mode=\"min\")\n",
        "        return {\n",
        "            \"optimizer\": optimizer,\n",
        "            \"lr_scheduler\": {\n",
        "                \"scheduler\": scheduler,\n",
        "                \"monitor\": \"val_loss\",\n",
        "                \"interval\": \"step\",\n",
        "                \"frequency\": 1,\n",
        "\n",
        "            },\n",
        "        }\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vFZzH2TmzqP-",
        "outputId": "20c6cb69-a351-4ef6-c8b7-89944aa4cb02"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting l_trainer.py\n"
          ]
        }
      ],
      "source": [
        "%%writefile l_trainer.py\n",
        "#@title Trainer\n",
        "#@markdown Lightning cli trainer\n",
        "\n",
        "import os\n",
        "import lightning as L\n",
        "from lightning.pytorch import Trainer\n",
        "from lightning.pytorch.cli import LightningCLI, LightningArgumentParser\n",
        "from lightning.pytorch.strategies.deepspeed import DeepSpeedStrategy\n",
        "from lightning.pytorch.loggers import WandbLogger, TensorBoardLogger\n",
        "from lightning.pytorch.callbacks import EarlyStopping, LearningRateMonitor, ModelCheckpoint, TQDMProgressBar\n",
        "\n",
        "import ray\n",
        "from ray.train.lightning import (RayTrainReportCallback, RayDeepSpeedStrategy,\n",
        "                                 RayLightningEnvironment, prepare_trainer)\n",
        "from ray.train.torch import TorchTrainer\n",
        "\n",
        "from transformers import DataCollatorForSeq2Seq\n",
        "\n",
        "from l_datamodule import FTDataModule\n",
        "from l_model import LLamaFTLightningModule\n",
        "from peft_model import get_model, base_model_id\n",
        "from finetuning_datasets import get_dataset, dataset_path\n",
        "from finetuning_datafunctions import SumDataCallator\n",
        "\n",
        "\n",
        "# is Lightning able?\n",
        "# L.pytorch.cli_lightning_logo()\n",
        "\n",
        "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"0\"\n",
        "\n",
        "\n",
        "def l2ray_trainer():\n",
        "    model_data = get_model(model_name=base_model_id)\n",
        "    peft_model = model_data[\"model\"]\n",
        "    tokenizer = model_data[\"tokenizer\"]\n",
        "\n",
        "    dataset_data = get_dataset(dataset_name=dataset_path, tokenizer=tokenizer)\n",
        "    dataset = dataset_data[\"dataset\"]\n",
        "    metric = dataset_data[\"metric\"]\n",
        "    max_source_length = dataset_data[\"max_source_length\"]\n",
        "    trainer = Trainer(\n",
        "        # fast_dev_run=True,\n",
        "        max_epochs=10,\n",
        "        devices=\"auto\",\n",
        "        accelerator=\"auto\",\n",
        "        accumulate_grad_batches=4,\n",
        "        precision=\"bf16-mixed\",\n",
        "        default_root_dir=f\"{os.getcwd()}/checkpoints\",\n",
        "        # strategy='deepspeed',\n",
        "        strategy=RayDeepSpeedStrategy(\n",
        "            stage=2,\n",
        "            ),\n",
        "        plugins=[\n",
        "            RayLightningEnvironment()\n",
        "            ],\n",
        "        callbacks=[\n",
        "            RayTrainReportCallback(),\n",
        "            EarlyStopping(monitor=\"val_loss\"),\n",
        "            LearningRateMonitor(),\n",
        "            TQDMProgressBar(refresh_rate=5),\n",
        "            ModelCheckpoint(\n",
        "                save_top_k=3,\n",
        "                monitor='val_loss',\n",
        "                mode='min',\n",
        "                save_weights_only=True,  # 가중치만 저장\n",
        "                save_on_train_epoch_end=True,\n",
        "                dirpath=\"checkpoint\",\n",
        "                filename=f'{base_model_id}_PEFT_ckpt'\n",
        "                )\n",
        "        ],\n",
        "        logger=[\n",
        "            WandbLogger(project=\"LLM-Finetuning\"),\n",
        "            TensorBoardLogger(save_dir=os.getcwd(), version=1, name=\"lightning_logs\")\n",
        "        ],\n",
        "    )\n",
        "    trainer = prepare_trainer(trainer=trainer)\n",
        "    trainer.fit(\n",
        "        model=LLamaFTLightningModule(\n",
        "            peft_model=peft_model,\n",
        "            tokenizer=tokenizer,),\n",
        "        datamodule=FTDataModule(\n",
        "            train_dataset=dataset[\"train\"],\n",
        "            val_dataset=dataset[\"test\"],\n",
        "            test_dataset=dataset[\"test\"],\n",
        "            data_collator=SumDataCallator(tokenizer, max_length=max_source_length),\n",
        "            train_batch_size=2,\n",
        "            eval_batch_size=1))\n",
        "\n",
        "\n",
        "ray_trainer = TorchTrainer(\n",
        "    l2ray_trainer,\n",
        "    scaling_config=ray.train.ScalingConfig(\n",
        "        num_workers=1,\n",
        "        use_gpu=True,\n",
        "        # resources_per_worker={ \"CPU\": 8, \"GPU\": 1, },\n",
        "        # trainer_resources={ \"CPU\": 8, \"GPU\": 1 },\n",
        "        accelerator_type=\"A100\",\n",
        "\n",
        "        ),\n",
        "    run_config=ray.train.RunConfig(\n",
        "        checkpoint_config=ray.train.CheckpointConfig(\n",
        "            num_to_keep=3,\n",
        "            checkpoint_score_attribute=\"val_loss\",\n",
        "            checkpoint_score_order=\"min\",\n",
        "        ),\n",
        "    )\n",
        ")\n",
        "ray.init(\n",
        "    num_cpus=8,\n",
        "    ignore_reinit_error=True,\n",
        ")\n",
        "result = ray_trainer.fit()\n",
        "\n",
        "print(result)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "RYALPHD3PyBD",
        "outputId": "794a31fb-9e15-47a5-a591-8406d06b8626"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting l_sweep.py\n"
          ]
        }
      ],
      "source": [
        "%%writefile l_sweep.py\n",
        "#@title Wandb Sweep\n",
        "#@markdown Lightning cli Sweep\n",
        "\n",
        "import os\n",
        "import lightning as L\n",
        "from lightning.pytorch import Trainer\n",
        "from lightning.pytorch.cli import LightningCLI, LightningArgumentParser\n",
        "from lightning.pytorch.strategies.deepspeed import DeepSpeedStrategy\n",
        "from lightning.pytorch.loggers import WandbLogger, TensorBoardLogger\n",
        "from lightning.pytorch.callbacks import EarlyStopping, LearningRateMonitor, TQDMProgressBar, ModelCheckpoint\n",
        "\n",
        "import ray\n",
        "from ray.train.lightning import (RayTrainReportCallback, RayDeepSpeedStrategy,\n",
        "                                 RayLightningEnvironment, prepare_trainer)\n",
        "from ray.train.torch import TorchTrainer\n",
        "\n",
        "from transformers import DataCollatorForSeq2Seq\n",
        "import wandb\n",
        "import gc\n",
        "\n",
        "from l_datamodule import FTDataModule\n",
        "from l_model import LLamaFTLightningModule\n",
        "from peft_model import get_model, hf_model_list\n",
        "from finetuning_datasets import get_dataset, dataset_path\n",
        "from finetuning_datafunctions import SumDataCallator\n",
        "\n",
        "\n",
        "# is Lightning able?\n",
        "L.pytorch.cli_lightning_logo()\n",
        "\n",
        "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"0\"\n",
        "\n",
        "os.environ[\"WANDB_PROJECT_NAME\"] = \"LLM-Finetuning\"\n",
        "\n",
        "def l2ray_trainer():\n",
        "    wandb.init(\n",
        "        project=\"LLM-Finetuning\")\n",
        "    config = wandb.config\n",
        "\n",
        "    model_data = get_model(\n",
        "        model_name=config.model_name,\n",
        "        r=config.lora_rank,\n",
        "        lora_alpha=config.lora_alpha,\n",
        "        lora_dropout=config.lora_dropout,\n",
        "        init_lora_weights=config.init_lora_weights\n",
        "        )\n",
        "    peft_model = model_data[\"model\"]\n",
        "    tokenizer = model_data[\"tokenizer\"]\n",
        "\n",
        "    dataset = get_dataset(dataset_name=dataset_path, tokenizer=tokenizer)\n",
        "    #dataset = dataset_data[\"dataset\"]\n",
        "    metric = dataset[\"metric\"]\n",
        "    max_source_length = dataset[\"max_source_length\"]\n",
        "    trainer = Trainer(\n",
        "        devices=\"auto\",\n",
        "        accelerator=\"auto\",\n",
        "        max_epochs=config.epochs,\n",
        "        accumulate_grad_batches=config.accumulate_grad_batches,\n",
        "        gradient_clip_val=config.gradient_clip_val,\n",
        "        precision=\"bf16-mixed\",\n",
        "        strategy=\"deepspeed\",\n",
        "        # enable_checkpointing=False,\n",
        "        # strategy=RayDeepSpeedStrategy(\n",
        "        #     stage=2,\n",
        "        #     contiguous_memory_optimization=False\n",
        "        #     ),\n",
        "        plugins=[\n",
        "            # RayLightningEnvironment()\n",
        "        ],\n",
        "        callbacks=[\n",
        "            # RayTrainReportCallback(),\n",
        "            EarlyStopping(monitor=\"val_loss\"),\n",
        "            TQDMProgressBar(refresh_rate=5),\n",
        "            LearningRateMonitor(),\n",
        "            ModelCheckpoint(\n",
        "                save_top_k=3,\n",
        "                monitor='val_loss',\n",
        "                mode='min',\n",
        "                save_weights_only=True,  # 가중치만 저장\n",
        "                save_on_train_epoch_end=True,\n",
        "                dirpath=\"checkpoint\",\n",
        "                filename=f'{config.model_name}_PEFT_ckpt'\n",
        "                )\n",
        "        ],\n",
        "        logger=[\n",
        "            WandbLogger(project=os.getenv(\"WANDB_PROJECT_NAME\")),\n",
        "            TensorBoardLogger(save_dir=os.getcwd(), version=1, name=\"lightning_logs\")\n",
        "        ],\n",
        "    )\n",
        "    # trainer = prepare_trainer(trainer=trainer)\n",
        "    trainer.fit(\n",
        "        model=LLamaFTLightningModule(\n",
        "            peft_model=peft_model,\n",
        "            learning_rate=config.lr,\n",
        "            tokenizer=tokenizer,),\n",
        "        datamodule=FTDataModule(\n",
        "            train_dataset=dataset[\"train\"].shard(num_shards=10, index=0),\n",
        "            val_dataset=dataset[\"val\"].shard(num_shards=10, index=0),\n",
        "            test_dataset=dataset[\"test\"].shard(num_shards=10, index=1),\n",
        "            data_collator=SumDataCallator(tokenizer, max_length=max_source_length),\n",
        "            train_batch_size=2,\n",
        "            eval_batch_size=1))\n",
        "    gc.collect()\n",
        "    del trainer\n",
        "\n",
        "\n",
        "def ray_wrapped_trainer():\n",
        "    ray_trainer = TorchTrainer(\n",
        "        l2ray_trainer,\n",
        "        scaling_config=ray.train.ScalingConfig(\n",
        "            num_workers=1,\n",
        "            use_gpu=True,\n",
        "            resources_per_worker={ \"CPU\": 8, \"GPU\": 1, },\n",
        "            trainer_resources={ \"CPU\": 8, \"GPU\": 1 },\n",
        "            accelerator_type=\"L4\",\n",
        "            ),\n",
        "        run_config=ray.train.RunConfig(\n",
        "            checkpoint_config=ray.train.CheckpointConfig(\n",
        "                num_to_keep=1,\n",
        "                checkpoint_score_attribute=\"val_loss\",\n",
        "                checkpoint_score_order=\"min\",\n",
        "            ),\n",
        "        )\n",
        "    )\n",
        "    ray.init(\n",
        "        num_cpus=8,\n",
        "        ignore_reinit_error=True,\n",
        "    )\n",
        "    result = ray_trainer.fit()\n",
        "\n",
        "wandb.agent(\n",
        "    sweep_id=wandb.sweep(\n",
        "        {\n",
        "            \"method\": \"random\",\n",
        "            \"name\": \"sweep\",\n",
        "            \"metric\": {\n",
        "                \"goal\": \"maximize\",\n",
        "                \"name\": \"val_rouge1_fmeasure_epoch\"\n",
        "                },\n",
        "            \"parameters\": {\n",
        "                \"model_name\": {\n",
        "                    \"values\": hf_model_list\n",
        "                    },\n",
        "                \"epochs\": {\n",
        "                    \"values\": [1]\n",
        "                    },\n",
        "                \"lr\": {\n",
        "                    \"max\": 5e-4,\n",
        "                    \"min\": 5e-5\n",
        "                    },\n",
        "                \"accumulate_grad_batches\": {\n",
        "                    \"min\": 1,\n",
        "                    \"max\": 8\n",
        "                    },\n",
        "                \"gradient_clip_val\": {\n",
        "                    \"min\": 0.1,\n",
        "                    \"max\": 1.0\n",
        "                    },\n",
        "                \"lora_rank\":{\n",
        "                    \"values\": [2, 4, 8, 16, 32]\n",
        "                },\n",
        "                \"lora_alpha\":{\n",
        "                    \"values\": [16, 32, 64, 128]\n",
        "                },\n",
        "                \"lora_dropout\":{\n",
        "                    \"min\": 0.05,\n",
        "                    \"max\": 0.1\n",
        "                },\n",
        "                \"init_lora_weights\":{\n",
        "                    \"values\": [\"gaussian\", \"pissa\", \"pissa_niter_16\",\"pissa_niter_32\", \"loftq\", False]\n",
        "                }\n",
        "            },\n",
        "        },\n",
        "        project=os.getenv(\"WANDB_PROJECT_NAME\"),\n",
        "    ),\n",
        "    function=l2ray_trainer,\n",
        "    # function=ray_wrapped_trainer,\n",
        "    count=len(hf_model_list),\n",
        "    project=os.getenv(\"WANDB_PROJECT_NAME\"))\n",
        "\n",
        "# print(result)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#@title Start Training\n",
        "# %%writefile run_train\n",
        "#@markdown 실험 결과\n",
        "#@markdown\n",
        "#@markdown * batch_size <b>2</b> 넘기는 경우 OOM\n",
        "#@markdown * DeepSpeed의 경우 GPU Ram 20GB로 7B finetuning 가능\n",
        "#@markdown * DeepSpeed의 경우, 7B L4 GPU에서 사용 가능\n",
        "#@markdown * 70B의 경우 RAM에서 Weight 가져오다 OOM\n",
        "#@markdown * 1.5B T4 GPU에서 성공\n",
        "\n",
        "!export CUDA_VISIBLE_DEVICES=0 && export CUDA_LAUNCH_BLOCKING=1 && python l_trainer.py\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[0;35m\n",
            "                    ####\n",
            "                ###########\n",
            "             ####################\n",
            "         ############################\n",
            "    #####################################\n",
            "##############################################\n",
            "#########################  ###################\n",
            "#######################    ###################\n",
            "####################      ####################\n",
            "##################       #####################\n",
            "################        ######################\n",
            "#####################        #################\n",
            "######################     ###################\n",
            "#####################    #####################\n",
            "####################   #######################\n",
            "###################  #########################\n",
            "##############################################\n",
            "    #####################################\n",
            "         ############################\n",
            "             ####################\n",
            "                  ##########\n",
            "                     ####\n",
            "\u001b[0m\n",
            "\n",
            "Create sweep with ID: u0ifdpln\n",
            "Sweep URL: https://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/sweeps/u0ifdpln\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: ojuvfttn with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \taccumulate_grad_batches: 4\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tgradient_clip_val: 0.39285968154137385\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tinit_lora_weights: False\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_alpha: 64\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_dropout: 0.07039886102490088\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_rank: 4\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlr: 0.00034020376426838775\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tmodel_name: MLP-KTLim/llama-3-Korean-Bllossom-8B\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mj0ntendo\u001b[0m (\u001b[33mj0ntendo-yonsei-university\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Ignored wandb.init() arg project when running a sweep.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/user/jonathan/wandb/run-20240802_173715-ojuvfttn\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mconfused-sweep-1\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🧹 View sweep at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/sweeps/u0ifdpln\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/ojuvfttn\u001b[0m\n",
            "Unused kwargs: ['bnb_8bit_quant_type', 'bnb_8bit_compute_dtype']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
            "Repo card metadata block was not found. Setting CardData to empty.\n",
            "[2024-08-02 17:37:24,203] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n",
            "Warning: The default cache directory for DeepSpeed Triton autotune, /user/jonathan/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.\n",
            "\u001b[93m [WARNING] \u001b[0m async_io requires the dev libaio .so object and headers but these were not found.\n",
            "\u001b[93m [WARNING] \u001b[0m async_io: please install the libaio-dev package with apt\n",
            "\u001b[93m [WARNING] \u001b[0m If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found.\n",
            "\u001b[93m [WARNING] \u001b[0m Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH\n",
            "\u001b[93m [WARNING] \u001b[0m sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.4\n",
            "\u001b[93m [WARNING] \u001b[0m using untested triton version (3.0.0), only 1.0.0 is known to be compatible\n",
            "/user/jonathan/jonathan/lib/python3.10/site-packages/deepspeed/runtime/zero/linear.py:49: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.\n",
            "  def forward(ctx, input, weight, bias=None):\n",
            "/user/jonathan/jonathan/lib/python3.10/site-packages/deepspeed/runtime/zero/linear.py:67: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.\n",
            "  def backward(ctx, grad_output):\n",
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "You are using a CUDA device ('NVIDIA A100-SXM4-80GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
            "initializing deepspeed distributed: GLOBAL_RANK: 0, MEMBER: 1/1\n",
            "/user/jonathan/jonathan/lib/python3.10/site-packages/lightning/pytorch/loggers/wandb.py:396: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.\n",
            "2024-08-02 17:37:28.145920: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-08-02 17:37:31.859210: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "/user/jonathan/jonathan/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:652: Checkpoint directory /user/jonathan/checkpoint exists and is not empty.\n",
            "Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "[2024-08-02 17:37:39,749] [WARNING] [engine.py:1179:_do_optimizer_sanity_check] **** You are using ZeRO with an untested optimizer, proceed with caution *****\n",
            "\n",
            "  | Name  | Type                 | Params | Mode \n",
            "-------------------------------------------------------\n",
            "0 | model | PeftModelForCausalLM | 1.5 B  | train\n",
            "-------------------------------------------------------\n",
            "544 K     Trainable params\n",
            "1.5 B     Non-trainable params\n",
            "1.5 B     Total params\n",
            "6,179.215 Total estimated model params size (MB)\n",
            "Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]/user/jonathan/jonathan/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=127` in the `DataLoader` to improve performance.\n",
            "Sanity Checking DataLoader 0:   0%|                       | 0/2 [00:00<?, ?it/s]We detected that you are passing `past_key_values` as a tuple and this is deprecated and will be removed in v4.43. Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/v4.41.3/en/internal/generation_utils#transformers.Cache)\n",
            "/user/jonathan/jonathan/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=127` in the `DataLoader` to improve performance.\n",
            "Epoch 0: 100%|█| 96/96 [00:33<00:00,  2.88it/s, v_num=tn_1, train_loss_step=10.2\n",
            "Validation: |                                             | 0/? [00:00<?, ?it/s]\u001b[A\n",
            "Validation:   0%|                                         | 0/7 [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:   0%|                            | 0/7 [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:  71%|██████████████▎     | 5/7 [00:01<00:00,  4.17it/s]\u001b[A\n",
            "Validation DataLoader 0: 100%|████████████████████| 7/7 [00:01<00:00,  4.47it/s]\u001b[A\n",
            "Epoch 0: 100%|█| 96/96 [00:34<00:00,  2.75it/s, v_num=tn_1, train_loss_step=10.2\u001b[A`Trainer.fit` stopped: `max_epochs=1` reached.\n",
            "Epoch 0: 100%|█| 96/96 [00:58<00:00,  1.63it/s, v_num=tn_1, train_loss_step=10.2\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch ▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step ▁▁▂▂▂▃▃██\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step █▁▆▅▃▇▆\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step ▅▆▃▂▄▁█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step ▅▆▃▂▄▁█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step ▄▄▅▁▆▄█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step ▃▁▃▃▂█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step ▃▁▃▃▁█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step ▄▁▆▂▃█▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step ▃▇▄▂▄▁█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step ▃▇▄▂▄▁█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step ▃▅▇▁▆▅█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step ▅▆▃▂▄▁█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step ▅▆▃▂▄▁█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step ▄▄▅▁▆▄█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch 0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch 10.0862\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch 0.03237\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch 0.01815\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch 0.3138\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch 0.00176\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch 0.00098\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch 0.02337\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch 0.02457\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch 0.01366\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch 0.26257\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch 0.0317\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch 0.01775\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch 0.31036\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step 23\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch 9.61966\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step 9.80807\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch 0.02198\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step 0.03195\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch 0.01142\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step 0.01656\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch 0.34998\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step 0.45455\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch 0.00267\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step 0.00213\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch 0.00138\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step 0.0011\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch 0.05685\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step 0.03125\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch 0.01965\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step 0.02769\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch 0.0102\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step 0.01435\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch 0.32323\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step 0.39394\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch 0.02184\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step 0.03195\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch 0.01135\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step 0.01656\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch 0.34852\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step 0.45455\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mconfused-sweep-1\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/ojuvfttn\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240802_173715-ojuvfttn/logs\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: c36x29vc with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \taccumulate_grad_batches: 7\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tgradient_clip_val: 0.5933794081002868\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tinit_lora_weights: pissa_niter_{n}\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_alpha: 32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_dropout: 0.05508583022963353\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_rank: 32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlr: 0.00027513271512791905\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tmodel_name: apple/DCLM-7B-8k\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Ignored wandb.init() arg project when running a sweep.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/user/jonathan/wandb/run-20240802_173855-c36x29vc\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mhonest-sweep-2\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🧹 View sweep at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/sweeps/u0ifdpln\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/c36x29vc\u001b[0m\n",
            "Unused kwargs: ['bnb_8bit_quant_type', 'bnb_8bit_compute_dtype']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mhonest-sweep-2\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/c36x29vc\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240802_173855-c36x29vc/logs\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n",
            "Run c36x29vc errored:\n",
            "Traceback (most recent call last):\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/wandb/agents/pyagent.py\", line 307, in _run_job\n",
            "    self._function()\n",
            "  File \"/user/jonathan/l_sweep.py\", line 40, in l2ray_trainer\n",
            "    model_data = get_model(\n",
            "  File \"/user/jonathan/peft_model.py\", line 90, in get_model\n",
            "    inject_adapter_in_model(lora_config, peft_model, \"adapter_1\")\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/mapping.py\", line 215, in inject_adapter_in_model\n",
            "    peft_model = tuner_cls(model, peft_config, adapter_name=adapter_name)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 139, in __init__\n",
            "    super().__init__(model, config, adapter_name)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/tuners_utils.py\", line 175, in __init__\n",
            "    self.inject_adapter(self.model, adapter_name)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/tuners_utils.py\", line 431, in inject_adapter\n",
            "    self._create_and_replace(peft_config, adapter_name, target, target_name, parent, current_key=key)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 224, in _create_and_replace\n",
            "    new_module = self._create_new_module(lora_config, adapter_name, target, **kwargs)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 340, in _create_new_module\n",
            "    new_module = dispatcher(target, adapter_name, lora_config=lora_config, **kwargs)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 1088, in dispatch_default\n",
            "    new_module = Linear(target, adapter_name, **kwargs)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 391, in __init__\n",
            "    self.update_layer(\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 128, in update_layer\n",
            "    self.pissa_init(adapter_name, init_lora_weights)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 209, in pissa_init\n",
            "    weight.data, self.r[adapter_name], niter=int(init_lora_weights.split(\"_niter_\")[-1])\n",
            "ValueError: invalid literal for int() with base 10: '{n}'\n",
            "\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Run c36x29vc errored:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Traceback (most recent call last):\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/wandb/agents/pyagent.py\", line 307, in _run_job\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self._function()\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/l_sweep.py\", line 40, in l2ray_trainer\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     model_data = get_model(\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/peft_model.py\", line 90, in get_model\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     inject_adapter_in_model(lora_config, peft_model, \"adapter_1\")\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/mapping.py\", line 215, in inject_adapter_in_model\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     peft_model = tuner_cls(model, peft_config, adapter_name=adapter_name)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 139, in __init__\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     super().__init__(model, config, adapter_name)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/tuners_utils.py\", line 175, in __init__\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self.inject_adapter(self.model, adapter_name)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/tuners_utils.py\", line 431, in inject_adapter\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self._create_and_replace(peft_config, adapter_name, target, target_name, parent, current_key=key)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 224, in _create_and_replace\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     new_module = self._create_new_module(lora_config, adapter_name, target, **kwargs)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 340, in _create_new_module\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     new_module = dispatcher(target, adapter_name, lora_config=lora_config, **kwargs)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 1088, in dispatch_default\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     new_module = Linear(target, adapter_name, **kwargs)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 391, in __init__\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self.update_layer(\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 128, in update_layer\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self.pissa_init(adapter_name, init_lora_weights)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 209, in pissa_init\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     weight.data, self.r[adapter_name], niter=int(init_lora_weights.split(\"_niter_\")[-1])\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m ValueError: invalid literal for int() with base 10: '{n}'\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: mu3v4yom with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \taccumulate_grad_batches: 7\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tgradient_clip_val: 0.3875582505870758\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tinit_lora_weights: False\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_alpha: 16\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_dropout: 0.07319482511910012\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_rank: 8\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlr: 0.0004840553746457767\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tmodel_name: meta-llama/Meta-Llama-3-70B-Instruct\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Ignored wandb.init() arg project when running a sweep.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/user/jonathan/wandb/run-20240802_173914-mu3v4yom\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33melated-sweep-3\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🧹 View sweep at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/sweeps/u0ifdpln\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/mu3v4yom\u001b[0m\n",
            "Unused kwargs: ['bnb_8bit_quant_type', 'bnb_8bit_compute_dtype']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
            "Repo card metadata block was not found. Setting CardData to empty.\n",
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "initializing deepspeed distributed: GLOBAL_RANK: 0, MEMBER: 1/1\n",
            "Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "[2024-08-02 17:39:31,222] [WARNING] [engine.py:1179:_do_optimizer_sanity_check] **** You are using ZeRO with an untested optimizer, proceed with caution *****\n",
            "\n",
            "  | Name  | Type                 | Params | Mode \n",
            "-------------------------------------------------------\n",
            "0 | model | PeftModelForCausalLM | 1.5 B  | train\n",
            "-------------------------------------------------------\n",
            "1.1 M     Trainable params\n",
            "1.5 B     Non-trainable params\n",
            "1.5 B     Total params\n",
            "6,183.574 Total estimated model params size (MB)\n",
            "Epoch 0: 100%|█| 96/96 [00:31<00:00,  3.09it/s, v_num=om_1, train_loss_step=0.95\n",
            "Validation: |                                             | 0/? [00:00<?, ?it/s]\u001b[A\n",
            "Validation:   0%|                                         | 0/7 [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:   0%|                            | 0/7 [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:  71%|██████████████▎     | 5/7 [00:00<00:00,  5.30it/s]\u001b[A\n",
            "Validation DataLoader 0: 100%|████████████████████| 7/7 [00:01<00:00,  5.59it/s]\u001b[A\n",
            "Epoch 0: 100%|█| 96/96 [00:32<00:00,  2.97it/s, v_num=om_1, train_loss_step=0.95\u001b[A`Trainer.fit` stopped: `max_epochs=1` reached.\n",
            "Epoch 0: 100%|█| 96/96 [00:55<00:00,  1.73it/s, v_num=om_1, train_loss_step=0.95\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch ▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step ▁▂▂▃▃▄▄█▇\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step █▆▂▃▂▁▃\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step ▄▂█▄▄█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step ▅█▅▁▂▃▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step ▄▁█▄▄█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step ▄▁█▄▃█▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step ▅█▅▄▁▅█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step ▄▁█▄▃█▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step ▃▂▇▄▄█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step ▃█▃▃▁▄▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step ▄▁▇▄▃█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step ▄▂█▄▃█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step ▆█▆▃▁▄▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step ▄▁█▄▃█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch 0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch 1.73731\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch 0.21843\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch 0.27337\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch 0.37456\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch 0.11576\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch 0.15584\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch 0.17614\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch 0.18288\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch 0.23734\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch 0.30372\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch 0.21028\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch 0.26445\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch 0.3634\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step 12\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch 0.17636\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step 0.16247\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch 0.31305\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step 0.2\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch 0.62727\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step 0.57143\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch 0.22837\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step 0.12121\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch 0.22506\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step 0.15789\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch 0.46173\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step 0.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch 0.16428\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step 0.09375\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch 0.30207\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step 0.2\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch 0.60048\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step 0.57143\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch 0.22112\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step 0.12121\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch 0.30812\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step 0.2\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch 0.61834\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step 0.57143\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch 0.22497\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step 0.12121\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33melated-sweep-3\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/mu3v4yom\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240802_173914-mu3v4yom/logs\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Sweep Agent: Waiting for job.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Job received.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: exz75fz5 with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \taccumulate_grad_batches: 8\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tgradient_clip_val: 0.7175648070377266\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tinit_lora_weights: gaussian\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_alpha: 16\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_dropout: 0.05251386716236513\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_rank: 32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlr: 6.306093060981509e-05\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tmodel_name: meta-llama/Meta-Llama-3.1-8B\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Ignored wandb.init() arg project when running a sweep.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/user/jonathan/wandb/run-20240802_174048-exz75fz5\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33msuper-sweep-4\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🧹 View sweep at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/sweeps/u0ifdpln\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/exz75fz5\u001b[0m\n",
            "Unused kwargs: ['bnb_8bit_quant_type', 'bnb_8bit_compute_dtype']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
            "Repo card metadata block was not found. Setting CardData to empty.\n",
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "initializing deepspeed distributed: GLOBAL_RANK: 0, MEMBER: 1/1\n",
            "Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "[2024-08-02 17:41:08,846] [WARNING] [engine.py:1179:_do_optimizer_sanity_check] **** You are using ZeRO with an untested optimizer, proceed with caution *****\n",
            "\n",
            "  | Name  | Type                 | Params | Mode \n",
            "-------------------------------------------------------\n",
            "0 | model | PeftModelForCausalLM | 1.6 B  | train\n",
            "-------------------------------------------------------\n",
            "4.4 M     Trainable params\n",
            "1.5 B     Non-trainable params\n",
            "1.6 B     Total params\n",
            "6,209.722 Total estimated model params size (MB)\n",
            "Epoch 0: 100%|█| 96/96 [00:35<00:00,  2.72it/s, v_num=z5_1, train_loss_step=11.4\n",
            "Validation: |                                             | 0/? [00:00<?, ?it/s]\u001b[A\n",
            "Validation:   0%|                                         | 0/7 [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:   0%|                            | 0/7 [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:  71%|██████████████▎     | 5/7 [00:01<00:00,  3.93it/s]\u001b[A\n",
            "Validation DataLoader 0: 100%|████████████████████| 7/7 [00:01<00:00,  4.26it/s]\u001b[A\n",
            "Epoch 0: 100%|█| 96/96 [00:37<00:00,  2.59it/s, v_num=z5_1, train_loss_step=11.4\u001b[A`Trainer.fit` stopped: `max_epochs=1` reached.\n",
            "Epoch 0: 100%|█| 96/96 [01:01<00:00,  1.57it/s, v_num=z5_1, train_loss_step=11.4\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch ▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step ▁▂▂▃▄▄▅██\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step ▇▇▅▄▂▁█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step █▇▁▄▃▂▆\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step █▇▁▄▃▂▆\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step █▁▇▅▇█▄\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step ▆▅▁▇▇▅█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step ▆▅▁▇▇▅█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step ▇▁▇▅▆█▃\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step ▆█▁▃▅▂▇\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step ▆█▁▃▅▂▇\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step ▆▁▇▃██▄\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step █▇▁▄▄▂▅\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step █▇▁▄▃▂▅\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step █▁█▅▇█▃\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch 0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch 10.80291\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch 0.03823\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch 0.02031\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch 0.62763\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch 0.0115\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch 0.00602\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch 0.27186\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch 0.02615\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch 0.0138\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch 0.47998\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch 0.03752\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch 0.01993\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch 0.61845\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step 11\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch 10.4673\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step 10.83466\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch 0.02692\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step 0.0308\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch 0.01389\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step 0.01585\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch 0.5907\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step 0.54545\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch 0.01068\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step 0.012\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch 0.00548\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step 0.00617\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch 0.28641\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step 0.21875\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch 0.02159\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step 0.02566\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch 0.01112\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step 0.0132\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch 0.49293\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step 0.45455\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch 0.02647\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step 0.02908\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch 0.01365\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step 0.01496\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch 0.58404\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step 0.51515\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33msuper-sweep-4\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/exz75fz5\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240802_174048-exz75fz5/logs\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: o5sikb4c with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \taccumulate_grad_batches: 7\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tgradient_clip_val: 0.13996474554724866\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tinit_lora_weights: gaussian\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_alpha: 16\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_dropout: 0.0930317818924258\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_rank: 16\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlr: 5.988396432142071e-05\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tmodel_name: vilm/vulture-40b\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Ignored wandb.init() arg project when running a sweep.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/user/jonathan/wandb/run-20240802_174227-o5sikb4c\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mtreasured-sweep-5\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🧹 View sweep at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/sweeps/u0ifdpln\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/o5sikb4c\u001b[0m\n",
            "Unused kwargs: ['bnb_8bit_quant_type', 'bnb_8bit_compute_dtype']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
            "Repo card metadata block was not found. Setting CardData to empty.\n",
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "initializing deepspeed distributed: GLOBAL_RANK: 0, MEMBER: 1/1\n",
            "Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "[2024-08-02 17:42:45,271] [WARNING] [engine.py:1179:_do_optimizer_sanity_check] **** You are using ZeRO with an untested optimizer, proceed with caution *****\n",
            "\n",
            "  | Name  | Type                 | Params | Mode \n",
            "-------------------------------------------------------\n",
            "0 | model | PeftModelForCausalLM | 1.5 B  | train\n",
            "-------------------------------------------------------\n",
            "2.2 M     Trainable params\n",
            "1.5 B     Non-trainable params\n",
            "1.5 B     Total params\n",
            "6,192.290 Total estimated model params size (MB)\n",
            "Epoch 0: 100%|█| 96/96 [00:34<00:00,  2.76it/s, v_num=4c_1, train_loss_step=10.7\n",
            "Validation: |                                             | 0/? [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:   0%|                            | 0/7 [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:  71%|██████████████▎     | 5/7 [00:01<00:00,  3.92it/s]\u001b[A\n",
            "Validation DataLoader 0: 100%|████████████████████| 7/7 [00:01<00:00,  4.24it/s]\u001b[A\n",
            "Epoch 0: 100%|█| 96/96 [00:36<00:00,  2.63it/s, v_num=4c_1, train_loss_step=10.7\u001b[A`Trainer.fit` stopped: `max_epochs=1` reached.\n",
            "Epoch 0: 100%|█| 96/96 [00:59<00:00,  1.60it/s, v_num=4c_1, train_loss_step=10.7\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch ▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step ▁▂▂▃▃▄▄█▇\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step ▇▇▅▄▂▁█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step █▇▁▄▃▂▆\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step █▇▁▄▃▂▆\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step █▁▇▅▆█▄\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step ▅▅▁█▆▄▇\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step ▅▅▁█▆▄▇\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step ▇▁▇▅▆█▃\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step ▆█▁▃▄▂▆\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step ▆█▁▃▄▂▆\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step ▆▁▇▃▇█▃\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step █▇▁▄▃▂▅\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step █▇▁▄▃▂▅\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step █▁█▅▆█▃\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch 0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch 10.39726\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch 0.03822\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch 0.02031\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch 0.62649\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch 0.01157\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch 0.00606\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch 0.27149\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch 0.02596\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch 0.0137\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch 0.47885\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch 0.03741\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch 0.01988\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch 0.61615\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step 12\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch 9.68868\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step 10.03406\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch 0.0269\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step 0.03087\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch 0.01388\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step 0.01589\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch 0.58806\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step 0.54545\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch 0.01086\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step 0.01203\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch 0.00557\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step 0.00618\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch 0.28864\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step 0.21875\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch 0.02182\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step 0.02573\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch 0.01124\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step 0.01324\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch 0.49339\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step 0.45455\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch 0.02644\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step 0.02916\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch 0.01364\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step 0.015\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch 0.5814\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step 0.51515\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mtreasured-sweep-5\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/o5sikb4c\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240802_174227-o5sikb4c/logs\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: 58tl060f with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \taccumulate_grad_batches: 4\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tgradient_clip_val: 0.3635988562370968\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tinit_lora_weights: gaussian\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_alpha: 32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_dropout: 0.06640386442487946\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_rank: 8\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlr: 0.0003682017393527149\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tmodel_name: arcee-ai/Arcee-Spark\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Ignored wandb.init() arg project when running a sweep.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/user/jonathan/wandb/run-20240802_174401-58tl060f\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mfearless-sweep-6\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🧹 View sweep at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/sweeps/u0ifdpln\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/58tl060f\u001b[0m\n",
            "Unused kwargs: ['bnb_8bit_quant_type', 'bnb_8bit_compute_dtype']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
            "Repo card metadata block was not found. Setting CardData to empty.\n",
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "initializing deepspeed distributed: GLOBAL_RANK: 0, MEMBER: 1/1\n",
            "Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "[2024-08-02 17:44:19,888] [WARNING] [engine.py:1179:_do_optimizer_sanity_check] **** You are using ZeRO with an untested optimizer, proceed with caution *****\n",
            "\n",
            "  | Name  | Type                 | Params | Mode \n",
            "-------------------------------------------------------\n",
            "0 | model | PeftModelForCausalLM | 1.5 B  | train\n",
            "-------------------------------------------------------\n",
            "1.1 M     Trainable params\n",
            "1.5 B     Non-trainable params\n",
            "1.5 B     Total params\n",
            "6,183.574 Total estimated model params size (MB)\n",
            "Epoch 0: 100%|█| 96/96 [00:31<00:00,  3.04it/s, v_num=0f_1, train_loss_step=0.86\n",
            "Validation: |                                             | 0/? [00:00<?, ?it/s]\u001b[A\n",
            "Validation:   0%|                                         | 0/7 [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:   0%|                            | 0/7 [00:00<?, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:  71%|██████████████▎     | 5/7 [00:00<00:00,  5.28it/s]\u001b[A\n",
            "Validation DataLoader 0: 100%|████████████████████| 7/7 [00:01<00:00,  5.58it/s]\u001b[A\n",
            "Epoch 0: 100%|█| 96/96 [00:32<00:00,  2.92it/s, v_num=0f_1, train_loss_step=0.86\u001b[A`Trainer.fit` stopped: `max_epochs=1` reached.\n",
            "Epoch 0: 100%|█| 96/96 [00:56<00:00,  1.71it/s, v_num=0f_1, train_loss_step=0.86\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch ▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step ▁▁▂▂▂▃▃██\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step █▆▂▃▂▁▄\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step ▁▃▅▄▅█▃\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step ▃▄▄▅▇█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step ▅▁▇▄▄█▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step ▁▂▅▄▄█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step ▄▃▅▅▆█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step ▄▁█▄▃█▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step ▁▂▅▄▄█▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step ▃▄▄▅▆█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step ▅▁▇▄▃█▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step ▁▃▅▄▄█▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step ▃▅▄▅▆█▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step ▅▁▇▄▃█▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                           epoch 0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train_loss_epoch 1.52179\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge1_fmeasure_epoch 0.27436\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge1_precision_epoch 0.30445\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge1_recall_epoch 0.46365\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rouge2_fmeasure_epoch 0.18046\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rouge2_precision_epoch 0.2144\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rouge2_recall_epoch 0.26934\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train_rougeL_fmeasure_epoch 0.24386\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeL_precision_epoch 0.27932\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       train_rougeL_recall_epoch 0.39029\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  train_rougeLsum_fmeasure_epoch 0.27106\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_rougeLsum_precision_epoch 0.30203\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    train_rougeLsum_recall_epoch 0.45697\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             trainer/global_step 23\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  val_loss_epoch 0.17149\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                   val_loss_step 0.16215\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_fmeasure_epoch 0.36872\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge1_fmeasure_step 0.31034\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge1_precision_epoch 0.52683\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge1_precision_step 0.36\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge1_recall_epoch 0.35369\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge1_recall_step 0.27273\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_fmeasure_epoch 0.27651\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rouge2_fmeasure_step 0.17857\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rouge2_precision_epoch 0.40189\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rouge2_precision_step 0.20833\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rouge2_recall_epoch 0.27381\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rouge2_recall_step 0.15625\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_fmeasure_epoch 0.35125\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        val_rougeL_fmeasure_step 0.27586\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeL_precision_epoch 0.50299\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeL_precision_step 0.32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         val_rougeL_recall_epoch 0.33964\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          val_rougeL_recall_step 0.24242\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_fmeasure_epoch 0.35539\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     val_rougeLsum_fmeasure_step 0.27586\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_rougeLsum_precision_epoch 0.51013\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:    val_rougeLsum_precision_step 0.32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:      val_rougeLsum_recall_epoch 0.34255\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:       val_rougeLsum_recall_step 0.24242\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mfearless-sweep-6\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/58tl060f\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240802_174401-58tl060f/logs\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: j42594zl with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \taccumulate_grad_batches: 5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tgradient_clip_val: 0.3767296999984159\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tinit_lora_weights: pissa_niter_{n}\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_alpha: 64\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_dropout: 0.07994524109092549\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_rank: 32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlr: 7.323902761087395e-05\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tmodel_name: meta-llama/Meta-Llama-3.1-8B\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Ignored wandb.init() arg project when running a sweep.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/user/jonathan/wandb/run-20240802_174532-j42594zl\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mclean-sweep-7\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🧹 View sweep at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/sweeps/u0ifdpln\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/j42594zl\u001b[0m\n",
            "Unused kwargs: ['bnb_8bit_quant_type', 'bnb_8bit_compute_dtype']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mclean-sweep-7\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/j42594zl\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240802_174532-j42594zl/logs\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n",
            "Run j42594zl errored:\n",
            "Traceback (most recent call last):\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/wandb/agents/pyagent.py\", line 307, in _run_job\n",
            "    self._function()\n",
            "  File \"/user/jonathan/l_sweep.py\", line 40, in l2ray_trainer\n",
            "    model_data = get_model(\n",
            "  File \"/user/jonathan/peft_model.py\", line 90, in get_model\n",
            "    inject_adapter_in_model(lora_config, peft_model, \"adapter_1\")\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/mapping.py\", line 215, in inject_adapter_in_model\n",
            "    peft_model = tuner_cls(model, peft_config, adapter_name=adapter_name)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 139, in __init__\n",
            "    super().__init__(model, config, adapter_name)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/tuners_utils.py\", line 175, in __init__\n",
            "    self.inject_adapter(self.model, adapter_name)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/tuners_utils.py\", line 431, in inject_adapter\n",
            "    self._create_and_replace(peft_config, adapter_name, target, target_name, parent, current_key=key)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 224, in _create_and_replace\n",
            "    new_module = self._create_new_module(lora_config, adapter_name, target, **kwargs)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 340, in _create_new_module\n",
            "    new_module = dispatcher(target, adapter_name, lora_config=lora_config, **kwargs)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 1088, in dispatch_default\n",
            "    new_module = Linear(target, adapter_name, **kwargs)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 391, in __init__\n",
            "    self.update_layer(\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 128, in update_layer\n",
            "    self.pissa_init(adapter_name, init_lora_weights)\n",
            "  File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 209, in pissa_init\n",
            "    weight.data, self.r[adapter_name], niter=int(init_lora_weights.split(\"_niter_\")[-1])\n",
            "ValueError: invalid literal for int() with base 10: '{n}'\n",
            "\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Run j42594zl errored:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Traceback (most recent call last):\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/wandb/agents/pyagent.py\", line 307, in _run_job\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self._function()\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/l_sweep.py\", line 40, in l2ray_trainer\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     model_data = get_model(\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/peft_model.py\", line 90, in get_model\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     inject_adapter_in_model(lora_config, peft_model, \"adapter_1\")\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/mapping.py\", line 215, in inject_adapter_in_model\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     peft_model = tuner_cls(model, peft_config, adapter_name=adapter_name)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 139, in __init__\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     super().__init__(model, config, adapter_name)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/tuners_utils.py\", line 175, in __init__\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self.inject_adapter(self.model, adapter_name)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/tuners_utils.py\", line 431, in inject_adapter\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self._create_and_replace(peft_config, adapter_name, target, target_name, parent, current_key=key)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 224, in _create_and_replace\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     new_module = self._create_new_module(lora_config, adapter_name, target, **kwargs)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/model.py\", line 340, in _create_new_module\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     new_module = dispatcher(target, adapter_name, lora_config=lora_config, **kwargs)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 1088, in dispatch_default\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     new_module = Linear(target, adapter_name, **kwargs)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 391, in __init__\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self.update_layer(\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 128, in update_layer\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     self.pissa_init(adapter_name, init_lora_weights)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m   File \"/user/jonathan/jonathan/lib/python3.10/site-packages/peft/tuners/lora/layer.py\", line 209, in pissa_init\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m     weight.data, self.r[adapter_name], niter=int(init_lora_weights.split(\"_niter_\")[-1])\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m ValueError: invalid literal for int() with base 10: '{n}'\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: fflvrgzn with config:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \taccumulate_grad_batches: 4\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tepochs: 1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tgradient_clip_val: 0.94137068521588\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tinit_lora_weights: pissa\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_alpha: 64\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_dropout: 0.0866309695651454\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlora_rank: 32\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tlr: 0.00040971473012344926\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \tmodel_name: mistralai/Mistral-Nemo-Instruct-2407\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Ignored wandb.init() arg project when running a sweep.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/user/jonathan/wandb/run-20240802_174552-fflvrgzn\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mbumbling-sweep-8\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🧹 View sweep at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/sweeps/u0ifdpln\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/j0ntendo-yonsei-university/LLM-Finetuning/runs/fflvrgzn\u001b[0m\n",
            "Unused kwargs: ['bnb_8bit_quant_type', 'bnb_8bit_compute_dtype']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
            "^C\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Ctrl + C detected. Stopping sweep.\n"
          ]
        }
      ],
      "source": [
        "!export CUDA_VISIBLE_DEVICES=0 && export CUDA_LAUNCH_BLOCKING=1 && python l_sweep.py\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NIbLGhFzxyNR"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OXFgDjet3ScB"
      },
      "source": [
        "# Model Saving"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 393
        },
        "id": "2z7GPQJv28QO",
        "outputId": "cc250298-bb5c-4dd0-a65a-935c87ba8ecb"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "from optimum.onnxruntime import ORTModelForCausalLM\n",
        "\n",
        "# Define paths\n",
        "model_checkpoint = \"./saved_model\"  # Directory where your PyTorch model is saved\n",
        "save_directory = \"./onnx_model\"     # Directory where you want to save the ONNX model\n",
        "\n",
        "# Load the PyTorch model and tokenizer\n",
        "model = AutoModelForCausalLM.from_pretrained(model_checkpoint)\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)\n",
        "\n",
        "# Convert the PyTorch model to ONNX\n",
        "ort_model = ORTModelForCausalLM.from_pretrained(model_checkpoint, export=True)\n",
        "\n",
        "# Save the ONNX model\n",
        "ort_model.save_pretrained(save_directory)\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}