FineTuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/miniconda3/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1 GPU(s) available.\n",
      "We will use the GPU: NVIDIA GeForce RTX 3090\n"
     ]
    }
   ],
   "source": [
    "if torch.cuda.is_available():\n",
    "\n",
    "    # Tell PyTorch to use the GPU.\n",
    "    device = torch.device(\"cuda\")\n",
    "\n",
    "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
    "\n",
    "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
    "\n",
    "\n",
    "else:\n",
    "    print('No GPU available, using the CPU instead.')\n",
    "    device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df_train = pd.read_csv(\"./fine_tuning.csv\")\n",
    "\n",
    "train_data_domain = df_train.domain.values\n",
    "train_data_label = df_train.label.values\n",
    "train_data_label = train_data_label.tolist()\n",
    "train_data_label = [0 if item == 2 else 1 for item in train_data_label]\n",
    "train_data_label = np.array(train_data_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer\n",
    "\n",
    "tokenizer = BertTokenizer(vocab_file=\"./bert_tokenizer/vocab.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
      "/usr/local/miniconda3/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:2606: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "input_ids_train = []\n",
    "attention_masks_train = []\n",
    "\n",
    "for sent in train_data_domain:\n",
    "    encoded_dict = tokenizer.encode_plus(\n",
    "        sent,                      # Sentence to encode.\n",
    "        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
    "        max_length = 64,           # Pad & truncate all sentences.\n",
    "        pad_to_max_length = True,\n",
    "        return_attention_mask = True,   # Construct attn. masks.\n",
    "        return_tensors = 'pt',     # Return pytorch tensors.\n",
    "    )\n",
    "    # Add the encoded sentence to the list.\n",
    "    input_ids_train.append(encoded_dict['input_ids'])\n",
    "\n",
    "    # And its attention mask (simply differentiates padding from non-padding).\n",
    "    attention_masks_train.append(encoded_dict['attention_mask'])\n",
    "\n",
    "# Convert the lists into tensors.\n",
    "input_ids_train = torch.cat(input_ids_train, dim=0)\n",
    "attention_masks_train = torch.cat(attention_masks_train, dim=0)\n",
    "labels_train = torch.tensor(train_data_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "111,998 training samples\n",
      "48,000 test samples\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import TensorDataset, random_split\n",
    "\n",
    "dataset_train = TensorDataset(input_ids_train, attention_masks_train, labels_train)\n",
    "# Calculate the number of samples to include in each set.\n",
    "train_size = int(0.7 * len(dataset_train))\n",
    "test_size = len(dataset_train) - train_size\n",
    "\n",
    "train_dataset, test_dataset = random_split(dataset_train, [train_size, test_size])\n",
    "print('{:>5,} training samples'.format(train_size))\n",
    "print('{:>5,} test samples'.format(test_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "train_dataloader = DataLoader(\n",
    "    train_dataset,  # The training samples.\n",
    "    sampler = RandomSampler(train_dataset), # Select batches randomly\n",
    "    batch_size = batch_size # Trains with this batch size.\n",
    ")\n",
    "\n",
    "test_dataloader = DataLoader(\n",
    "    test_dataset, # The test samples.\n",
    "    sampler = SequentialSampler(test_dataset), # Pull out batches sequentially.\n",
    "    batch_size = batch_size # Test with this batch size.\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "\n",
    "EmbeddingPath = \"./FedBert/FedTransformer.pt\"\n",
    "TransformerPath = \"./FedBert/FedEmbedding.pt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertConfig {\n",
      "  \"_name_or_path\": \"./bert-base-uncased-model/\",\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.2,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.34.1\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 1000\n",
      "}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Embedding(1000, 768, padding_idx=0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import (\n",
    "    AutoConfig,\n",
    "    AutoModelForMaskedLM,\n",
    "    AutoTokenizer,DataCollatorForLanguageModeling,HfArgumentParser,Trainer,TrainingArguments,set_seed,\n",
    ")\n",
    "\n",
    "config_kwargs = {\n",
    "    \"cache_dir\": None,\n",
    "    \"revision\": 'main',\n",
    "    \"use_auth_token\": None,\n",
    "    \"hidden_dropout_prob\": 0.2,\n",
    "    \"vocab_size\": 1000\n",
    "}\n",
    "\n",
    "config = AutoConfig.from_pretrained('./bert-base-uncased-model/', **config_kwargs)\n",
    "print(config)\n",
    "\n",
    "model = AutoModelForMaskedLM.from_config(\n",
    "    config=config,\n",
    ")\n",
    "model.resize_token_embeddings(config_kwargs[\"vocab_size\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import copy\n",
    "\n",
    "embedding = model.bert.embeddings\n",
    "\n",
    "class Bert_Embedding(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Bert_Embedding, self).__init__()\n",
    "        self.embeddings = copy.deepcopy(embedding)\n",
    "\n",
    "    def forward(self, input_ids, mask):\n",
    "        embedding_output = self.embeddings(input_ids, mask)\n",
    "        return embedding_output\n",
    "\n",
    "embedding_model = Bert_Embedding()\n",
    "embedding_model.load_state_dict(torch.load(EmbeddingPath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder = model.bert.encoder\n",
    "cls = model.cls\n",
    "\n",
    "class Bert_Encoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Bert_Encoder, self).__init__()\n",
    "        self.encoder = copy.deepcopy(encoder)\n",
    "        self.cls = copy.deepcopy(cls)\n",
    "\n",
    "    def forward(self, embedding_output):\n",
    "        output_encoder = self.encoder(embedding_output).last_hidden_state\n",
    "        return output_encoder\n",
    "encoder_model = Bert_Encoder()\n",
    "encoder_model.load_state_dict(torch.load(TransformerPath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertPooler(\n",
      "  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (activation): Tanh()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from transformers.models.bert.modeling_bert import BertPooler\n",
    "class Pooler_Config:\n",
    "    def __init__(self, entries: dict={}):\n",
    "        for k, v in entries.items():\n",
    "            if isinstance(v, dict):\n",
    "                self.__dict__[k] = Pooler_Config(v)\n",
    "            else:\n",
    "                self.__dict__[k] = v\n",
    "\n",
    "config_pooler = {\"hidden_size\": 768}\n",
    "config_pooler = Pooler_Config(config_pooler)\n",
    "pooler = BertPooler(config_pooler)\n",
    "print(pooler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "class MyModel(nn.Module):\n",
    "    def __init__(self, hidden_size=768, num_classes=2, freeze_bert=False):\n",
    "        super(MyModel, self).__init__()\n",
    "        self.embedding = Bert_Embedding()\n",
    "        self.encoder = Bert_Encoder()\n",
    "        self.pooler = copy.deepcopy(pooler)\n",
    "        if freeze_bert:\n",
    "            for p in self.embedding.parameters():\n",
    "                p.requires_grad = False\n",
    "            for p in self.encoder.parameters():\n",
    "                p.requires_grad = False\n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Dropout(p=0.5),\n",
    "            nn.Linear(hidden_size, num_classes, bias=False),\n",
    "            nn.Sigmoid(),\n",
    "        )\n",
    "\n",
    "    def forward(self, input_ids, mask):\n",
    "        embedding_outputs = self.embedding(input_ids, mask)\n",
    "        encoder_outputs = self.encoder(embedding_outputs)\n",
    "        pooler_outputs = self.pooler(encoder_outputs)\n",
    "\n",
    "        logits = self.fc(pooler_outputs)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/miniconda3/lib/python3.8/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import get_linear_schedule_with_warmup\n",
    "from transformers import AdamW\n",
    "\n",
    "model = MyModel()\n",
    "model.encoder.cls = nn.Sequential()\n",
    "\n",
    "optimizer = AdamW(model.parameters(),\n",
    "                  lr = 2e-5,\n",
    "                  eps = 1e-8\n",
    "                  )\n",
    "\n",
    "\n",
    "epochs = 30\n",
    "\n",
    "\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "\n",
    "# Create the learning rate scheduler.\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
    "                                            num_warmup_steps = 0,\n",
    "                                            num_training_steps = total_steps)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MyModel(\n",
       "  (embedding): Bert_Embedding(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(1000, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.2, inplace=False)\n",
       "    )\n",
       "  )\n",
       "  (encoder): Bert_Encoder(\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.2, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cls): Sequential()\n",
       "  )\n",
       "  (pooler): BertPooler(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (activation): Tanh()\n",
       "  )\n",
       "  (fc): Sequential(\n",
       "    (0): Dropout(p=0.5, inplace=False)\n",
       "    (1): Linear(in_features=768, out_features=2, bias=False)\n",
       "    (2): Sigmoid()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\",category=DeprecationWarning)\n",
    "import time\n",
    "import datetime\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "def flat_accuracy(preds, labels):\n",
    "    return np.sum(preds == labels) / len(labels)\n",
    "\n",
    "def tpr_calculate(preds, labels):\n",
    "    return recall_score(labels, preds, zero_division=1)\n",
    "\n",
    "def fpr_calculate(preds, labels):\n",
    "    conf_matrix = confusion_matrix(labels, preds)\n",
    "    fp = conf_matrix[0, 1]\n",
    "    tn = conf_matrix[0, 0]\n",
    "    fpr = fp / (fp + tn)\n",
    "    return fpr\n",
    "\n",
    "def f1_score_calculate(preds, labels):\n",
    "    return f1_score(labels, preds)\n",
    "\n",
    "def AUC_calculate(preds, labels):\n",
    "    return roc_auc_score(labels, preds)\n",
    "\n",
    "def roc_curve_calculate(preds, labels):\n",
    "    return roc_curve(labels, preds)\n",
    "\n",
    "def format_time(elapsed):\n",
    "    '''\n",
    "    Takes a time in seconds and returns a string hh:mm:ss\n",
    "    '''\n",
    "    # Round to the nearest second.\n",
    "    elapsed_rounded = int(round((elapsed)))\n",
    "\n",
    "    # Format as hh:mm:ss\n",
    "    return str(datetime.timedelta(seconds=elapsed_rounded))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "======== Epoch 1 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:56.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.27645259898315582836\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1870613099\tAcc:0.9285208333\tTpr:0.9298040527\tFpr:0.0732483519\tF1:0.9263041437\tAuc:0.9282778504\n",
      "  test Loss: 0.18706130990261832236\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 2 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.19068384832476398261\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1729152758\tAcc:0.9351041667\tTpr:0.9242910315\tFpr:0.0538937902\tF1:0.9322277073\tAuc:0.9351986207\n",
      "  test Loss: 0.17291527576309939640\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 3 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:56.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:40.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.16718729745104377038\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1577332267\tAcc:0.9422500000\tTpr:0.9388715523\tFpr:0.0540783226\tF1:0.9401981961\tAuc:0.9423966148\n",
      "  test Loss: 0.15773322669727107836\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 4 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.15326702476904860428\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1674501174\tAcc:0.9439166667\tTpr:0.9467629979\tFpr:0.0588737301\tF1:0.9422959368\tAuc:0.9439446339\n",
      "  test Loss: 0.16745011740674575762\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 5 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.14286560968070158295\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1576676361\tAcc:0.9450833333\tTpr:0.9428100422\tFpr:0.0520825397\tF1:0.9433402160\tAuc:0.9453637513\n",
      "  test Loss: 0.15766763612907380709\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 6 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.13345251958078838128\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1563385256\tAcc:0.9448333333\tTpr:0.9361005875\tFpr:0.0458194727\tF1:0.9424999779\tAuc:0.9451405574\n",
      "  test Loss: 0.15633852557465433009\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 7 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.12457566116085010133\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1872226912\tAcc:0.9481666667\tTpr:0.9648439862\tFpr:0.0684368745\tF1:0.9473450501\tAuc:0.9482035558\n",
      "  test Loss: 0.18722269120377799112\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 8 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.11713442694608654226\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1845184432\tAcc:0.9486875000\tTpr:0.9668891523\tFpr:0.0696317770\tF1:0.9480236481\tAuc:0.9486286877\n",
      "  test Loss: 0.18451844315494719817\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 9 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.11049075301257627046\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1780920135\tAcc:0.9495833333\tTpr:0.9499121632\tFpr:0.0505735905\tF1:0.9479097384\tAuc:0.9496692864\n",
      "  test Loss: 0.17809201352655265604\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 10 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:56.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:40.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.10424794694335598466\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1768129648\tAcc:0.9500416667\tTpr:0.9477179902\tFpr:0.0477446924\tF1:0.9482244127\tAuc:0.9499866489\n",
      "  test Loss: 0.17681296481704339429\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 11 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.09671416631547201381\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2244535082\tAcc:0.9388541667\tTpr:0.9106111446\tFpr:0.0326844575\tF1:0.9349835603\tAuc:0.9389633436\n",
      "  test Loss: 0.22445350824474977203\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 12 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.09159944870908345860\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1875828313\tAcc:0.9501458333\tTpr:0.9480476462\tFpr:0.0474303717\tF1:0.9483716352\tAuc:0.9503086373\n",
      "  test Loss: 0.18758283128822222929\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 13 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.08514756120249096505\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1943275781\tAcc:0.9506250000\tTpr:0.9596136448\tFpr:0.0579828589\tF1:0.9495152223\tAuc:0.9508153929\n",
      "  test Loss: 0.19432757805093811565\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 14 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.08072507665452680059\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1880619439\tAcc:0.9514791667\tTpr:0.9567418266\tFpr:0.0534980607\tF1:0.9502612469\tAuc:0.9516218829\n",
      "  test Loss: 0.18806194387694510906\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 15 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.07688611139556658647\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2037659048\tAcc:0.9513750000\tTpr:0.9606202416\tFpr:0.0577735724\tF1:0.9500627576\tAuc:0.9514233346\n",
      "  test Loss: 0.20376590480329467847\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 16 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.07281023040799690371\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.1671017130\tAcc:0.9522916667\tTpr:0.9591923841\tFpr:0.0545051805\tF1:0.9509078979\tAuc:0.9523436018\n",
      "  test Loss: 0.16710171302586482311\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 17 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.06547405945679306283\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2057358562\tAcc:0.9506041667\tTpr:0.9520409744\tFpr:0.0511048527\tF1:0.9491725973\tAuc:0.9504680609\n",
      "  test Loss: 0.20573585618597764468\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 18 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.06354970795777625803\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2064718402\tAcc:0.9521875000\tTpr:0.9566879943\tFpr:0.0523829853\tF1:0.9508417191\tAuc:0.9521525045\n",
      "  test Loss: 0.20647184024145825987\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 19 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.05969818572726633132\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2253442731\tAcc:0.9502291667\tTpr:0.9443437079\tFpr:0.0437981488\tF1:0.9482538658\tAuc:0.9502727795\n",
      "  test Loss: 0.22534427312497670726\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 20 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.05514983731875795075\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2249430889\tAcc:0.9490000000\tTpr:0.9439207485\tFpr:0.0457965957\tF1:0.9471218200\tAuc:0.9490620764\n",
      "  test Loss: 0.22494308894787293229\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 21 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.05145027860541761394\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2216629148\tAcc:0.9513750000\tTpr:0.9521845104\tFpr:0.0495569242\tF1:0.9498210672\tAuc:0.9513137931\n",
      "  test Loss: 0.22166291483305394783\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 22 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.04828683434313695011\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2462320267\tAcc:0.9523125000\tTpr:0.9550928506\tFpr:0.0505038421\tF1:0.9509422767\tAuc:0.9522945042\n",
      "  test Loss: 0.24623202667362056872\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 23 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.04416143023537006523\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2444400065\tAcc:0.9525833333\tTpr:0.9538820533\tFpr:0.0487673593\tF1:0.9510179629\tAuc:0.9525573470\n",
      "  test Loss: 0.24444000652191849587\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 24 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.04228036090282590032\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2416684976\tAcc:0.9508750000\tTpr:0.9514607167\tFpr:0.0496676752\tF1:0.9492928025\tAuc:0.9508965208\n",
      "  test Loss: 0.24166849756974262409\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 25 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.03855120925656852632\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2543500578\tAcc:0.9529791667\tTpr:0.9582474109\tFpr:0.0523319710\tF1:0.9515468572\tAuc:0.9529577199\n",
      "  test Loss: 0.25435005775066871214\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 26 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.03889049141221247174\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2457600355\tAcc:0.9523541667\tTpr:0.9553089671\tFpr:0.0506527231\tF1:0.9508484645\tAuc:0.9523281220\n",
      "  test Loss: 0.24576003548324418246\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 27 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.03420645576604042920\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2637360933\tAcc:0.9517708333\tTpr:0.9520287351\tFpr:0.0486672431\tF1:0.9500332935\tAuc:0.9516807460\n",
      "  test Loss: 0.26373609325363456257\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 28 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.03338510418097887650\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2731623703\tAcc:0.9522291667\tTpr:0.9534129908\tFpr:0.0489298260\tF1:0.9506105486\tAuc:0.9522415824\n",
      "  test Loss: 0.27316237030244278072\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 29 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:12.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.03254292196059499487\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2735513841\tAcc:0.9525833333\tTpr:0.9563883300\tFpr:0.0513575070\tF1:0.9510616861\tAuc:0.9525154115\n",
      "  test Loss: 0.27355138408532364558\n",
      "  test took: 0:00:44\n",
      "\n",
      "======== Epoch 30 / 30 ========\n",
      "Training...\n",
      "  Batch   500  of  3,500.    Elapsed: 0:00:44.\n",
      "  Batch 1,000  of  3,500.    Elapsed: 0:01:28.\n",
      "  Batch 1,500  of  3,500.    Elapsed: 0:02:13.\n",
      "  Batch 2,000  of  3,500.    Elapsed: 0:02:57.\n",
      "  Batch 2,500  of  3,500.    Elapsed: 0:03:41.\n",
      "  Batch 3,000  of  3,500.    Elapsed: 0:04:25.\n",
      "\n",
      "  Average training loss: 0.03132217087057818233\n",
      "  Training epcoh took: 0:05:09\n",
      "\n",
      "Running test...\n",
      "Loss:0.2736965752\tAcc:0.9531041667\tTpr:0.9571011772\tFpr:0.0510282438\tF1:0.9516046052\tAuc:0.9530364667\n",
      "  test Loss: 0.27369657517618423137\n",
      "  test took: 0:00:44\n",
      "\n",
      "Training complete!\n",
      "Total training took 2:56:34 (h:mm:ss)\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "seed_val = 42\n",
    "\n",
    "random.seed(seed_val)\n",
    "np.random.seed(seed_val)\n",
    "torch.manual_seed(seed_val)\n",
    "torch.cuda.manual_seed_all(seed_val)\n",
    "\n",
    "training_stats = []\n",
    "\n",
    "total_t0 = time.time()\n",
    "\n",
    "criterion = torch.nn.BCELoss()\n",
    "\n",
    "test_save = {}\n",
    "test_save[\"loss\"] = []\n",
    "test_save[\"acc\"] = []\n",
    "test_save[\"tpr\"] = []\n",
    "test_save[\"fpr\"] = []\n",
    "test_save[\"f1\"] = []\n",
    "test_save[\"auc\"] = []\n",
    "\n",
    "for epoch_i in range(0, epochs):\n",
    "    print(\"\")\n",
    "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
    "    print('Training...')\n",
    "\n",
    "    loss_batch = []\n",
    "    acc_batch = []\n",
    "    tpr_batch = []\n",
    "    fpr_batch = []\n",
    "    f1_batch = []\n",
    "    auc_batch = []\n",
    "\n",
    "    t0 = time.time()\n",
    "\n",
    "    total_train_loss = 0\n",
    "\n",
    "    model.train()\n",
    "    \n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "        if step % 500 == 0 and not step == 0:\n",
    "            elapsed = format_time(time.time() - t0)\n",
    "\n",
    "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
    "\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "\n",
    "        model.zero_grad()\n",
    "\n",
    "        logits = model.forward(b_input_ids, b_input_mask)\n",
    "\n",
    "        b_labels = b_labels.unsqueeze(1)\n",
    "        b_labels = b_labels.repeat(1,2)\n",
    "        for i in range(len(b_labels)):\n",
    "            b_labels[i][1] = 1-b_labels[i][0]\n",
    "        BCELoss = criterion(logits, b_labels.float())\n",
    "\n",
    "        total_train_loss += BCELoss.item()\n",
    "\n",
    "        BCELoss.backward()\n",
    "\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "        \n",
    "    # Calculate the average loss over all of the batches.\n",
    "    avg_train_loss = total_train_loss / len(train_dataloader)\n",
    "\n",
    "    # Measure how long this epoch took.\n",
    "    training_time = format_time(time.time() - t0)\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"  Average training loss: {0:.20f}\".format(avg_train_loss))\n",
    "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
    "\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Running test...\")\n",
    "\n",
    "    t0 = time.time()\n",
    "\n",
    "    model.eval()\n",
    "\n",
    "    # Tracking variables\n",
    "    total_test_accuracy = 0\n",
    "    total_test_loss = 0\n",
    "    nb_test_steps = 0\n",
    "\n",
    "    # Test data for one epoch\n",
    "    for batch in test_dataloader:\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        with torch.no_grad():\n",
    "            logits = model.forward(b_input_ids, b_input_mask)\n",
    "\n",
    "        b_labels = b_labels.unsqueeze(1)\n",
    "        b_labels = b_labels.repeat(1,2)\n",
    "        for i in range(len(b_labels)):\n",
    "            b_labels[i][1] = 1-b_labels[i][0]\n",
    "\n",
    "        BCELoss = criterion(logits, b_labels.float())\n",
    "        total_test_loss += BCELoss.item()\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        logits = np.argmax(logits, axis=1).flatten()\n",
    "        label_ids = np.argmax(label_ids,axis=1).flatten()\n",
    "\n",
    "        accuracy = flat_accuracy(logits, label_ids)\n",
    "        tpr = tpr_calculate(logits, label_ids)\n",
    "        fpr = fpr_calculate(logits, label_ids)\n",
    "        f1 = f1_score_calculate(logits, label_ids)\n",
    "        if len(set(label_ids)) == 1:\n",
    "            pass\n",
    "        else:\n",
    "            auc = AUC_calculate(logits, label_ids)\n",
    "            auc_batch.append(auc)\n",
    "        \n",
    "        loss_batch.append(BCELoss.item())\n",
    "        acc_batch.append(accuracy)\n",
    "        tpr_batch.append(tpr)\n",
    "        fpr_batch.append(fpr)\n",
    "        f1_batch.append(f1)\n",
    "        \n",
    "        total_test_accuracy += flat_accuracy(logits, label_ids)\n",
    "    \n",
    "    test_save[\"loss\"].append(sum(loss_batch)/len(loss_batch))\n",
    "    test_save[\"acc\"].append(sum(acc_batch)/len(acc_batch))\n",
    "    test_save[\"tpr\"].append(sum(tpr_batch)/len(tpr_batch))\n",
    "    test_save[\"fpr\"].append(sum(fpr_batch)/len(fpr_batch))\n",
    "    test_save[\"f1\"].append(sum(f1_batch)/len(f1_batch))\n",
    "    test_save[\"auc\"].append(sum(auc_batch)/len(auc_batch))\n",
    "    \n",
    "    avg_test_accuracy = total_test_accuracy / len(test_dataloader)\n",
    "    print(\"Loss:{:.10f}\\tAcc:{:.10f}\\tTpr:{:.10f}\\tFpr:{:.10f}\\tF1:{:.10f}\\tAuc:{:.10f}\".format(sum(loss_batch)/len(loss_batch), sum(acc_batch)/len(acc_batch), sum(tpr_batch)/len(tpr_batch), sum(fpr_batch)/len(fpr_batch), sum(f1_batch)/len(f1_batch), sum(auc_batch)/len(auc_batch)))\n",
    "\n",
    "    # Calculate the average loss over all of the batches.\n",
    "    avg_test_loss = total_test_loss / len(test_dataloader)\n",
    "\n",
    "    # Measure how long the test run took.\n",
    "    test_time = format_time(time.time() - t0)\n",
    "\n",
    "    print(\"  test Loss: {0:.20f}\".format(avg_test_loss))\n",
    "    print(\"  test took: {:}\".format(test_time))\n",
    "\n",
    "    # Record all statistics from this epoch.\n",
    "    training_stats.append(\n",
    "        {\n",
    "            'epoch': epoch_i + 1,\n",
    "            'Training Loss': avg_train_loss,\n",
    "            'Valid. Loss': avg_test_loss,\n",
    "            'Valid. Accur.': avg_test_accuracy,\n",
    "            'Training Time': training_time,\n",
    "            'test Time': test_time\n",
    "        }\n",
    "    )\n",
    "\n",
    "print(\"\")\n",
    "print(\"Training complete!\")\n",
    "\n",
    "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import xlwt\n",
    "f = xlwt.Workbook('encoding = utf-8')\n",
    "sheet1 = f.add_sheet('sheet1',cell_overwrite_ok=True)\n",
    "for i in range(len(test_save[\"loss\"])):\n",
    "    sheet1.write(i+1,0,test_save[\"loss\"][i])\n",
    "for i in range(len(test_save[\"acc\"])):\n",
    "    sheet1.write(i+1,1,test_save[\"acc\"][i])\n",
    "for i in range(len(test_save[\"tpr\"])):\n",
    "    sheet1.write(i+1,2,test_save[\"tpr\"][i])\n",
    "for i in range(len(test_save[\"fpr\"])):\n",
    "    sheet1.write(i+1,3,test_save[\"fpr\"][i])\n",
    "for i in range(len(test_save[\"f1\"])):\n",
    "    sheet1.write(i+1,4,test_save[\"f1\"][i])\n",
    "for i in range(len(test_save[\"auc\"])):\n",
    "    sheet1.write(i+1,5,test_save[\"auc\"][i])\n",
    "\n",
    "f.save('test_save.xls')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}