news_decoder_language_model.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aburkov/theLMbook/blob/main/news_decoder_language_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div style=\"display: flex; justify-content: center;\">\n",
        "    <div style=\"background-color: #f4f6f7; padding: 15px; width: 80%;\">\n",
        "        <table style=\"width: 100%\">\n",
        "            <tr>\n",
        "                <td style=\"vertical-align: middle;\">\n",
        "                    <span style=\"font-size: 14px;\">\n",
        "                        A notebook for <a href=\"https://www.thelmbook.com\" target=\"_blank\" rel=\"noopener\">The Hundred-Page Language Models Book</a> by Andriy Burkov<br><br>\n",
        "                        Code repository: <a href=\"https://github.com/aburkov/theLMbook\" target=\"_blank\" rel=\"noopener\">https://github.com/aburkov/theLMbook</a>\n",
        "                    </span>\n",
        "                </td>\n",
        "                <td style=\"vertical-align: middle;\">\n",
        "                    <a href=\"https://www.thelmbook.com\" target=\"_blank\" rel=\"noopener\">\n",
        "                        <img src=\"https://thelmbook.com/img/book.png\" width=\"80px\" alt=\"The Hundred-Page Language Models Book\">\n",
        "                    </a>\n",
        "                </td>\n",
        "            </tr>\n",
        "        </table>\n",
        "    </div>\n",
        "</div>"
      ],
      "metadata": {
        "id": "dQ-tX7JpAdMB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Decoder-based language model\n",
        "\n",
        "## Utility functions and classes\n",
        "\n",
        "In the cell below, we import the dependencies and define the utility functions and the model class:"
      ],
      "metadata": {
        "id": "kb9Akwe7xttX"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yy0zjL_2ouOU"
      },
      "outputs": [],
      "source": [
        "# Import required libraries\n",
        "import os               # For file and path operations (check_file_exists, extract_dataset)\n",
        "import urllib.request   # For downloading dataset files from URLs\n",
        "import tarfile          # For extracting .tar.gz dataset archives\n",
        "import torch            # Main PyTorch library for tensor operations and deep learning\n",
        "import torch.nn as nn   # Neural network modules, layers, and utilities\n",
        "import torch.nn.functional as F  # For softmax\n",
        "from torch.utils.data import DataLoader, IterableDataset  # For efficient data loading\n",
        "import random           # For setting random seeds\n",
        "from tqdm import tqdm   # For progress bars\n",
        "import math             # For computing perplexity using exp()\n",
        "import re               # For preprocessing text (replacing numbers with placeholders)\n",
        "from transformers import AutoTokenizer  # For loading pre-trained tokenizer\n",
        "#import tempfile         # For temporary file handling during extraction\n",
        "#import shutil           # For file operations during extraction\n",
        "\n",
        "# ----------------------------\n",
        "# Utility Functions\n",
        "# ----------------------------\n",
        "\n",
        "def set_seed(seed):\n",
        "    \"\"\"\n",
        "    Sets random seeds for reproducibility across different Python libraries.\n",
        "    This ensures that random operations give the same results across runs.\n",
        "\n",
        "    Args:\n",
        "        seed (int): Seed value for random number generation\n",
        "    \"\"\"\n",
        "    # Set seed for Python's built-in random module\n",
        "    random.seed(seed)\n",
        "    # Set seed for PyTorch's CPU random number generator\n",
        "    torch.manual_seed(seed)\n",
        "    # Set seed for PyTorch's GPU random number generator\n",
        "    torch.cuda.manual_seed_all(seed)\n",
        "    # Requests cuDNN to use deterministic algorithms when possible\n",
        "    # Note: This may impact performance and might not guarantee determinism in all cases\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    # Disables cuDNN's auto-tuner which finds the best algorithm for your specific input size\n",
        "    # Ensures consistent behavior but might be slower as it doesn't optimize for input sizes\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "\n",
        "# ----------------------------\n",
        "# Dataset Class\n",
        "# ----------------------------\n",
        "\n",
        "class IterableTextDataset(IterableDataset):\n",
        "    \"\"\"\n",
        "    An iterable dataset for processing text data in a memory-efficient way.\n",
        "    Instead of loading all data into memory, it streams data from disk.\n",
        "    Inherits from PyTorch's IterableDataset for streaming support.\n",
        "\n",
        "    Args:\n",
        "        file_path (str): Path to the text file containing sentences\n",
        "        tokenizer: Tokenizer object for converting text to tokens\n",
        "        max_length (int): Maximum sequence length to process (default: 30)\n",
        "    \"\"\"\n",
        "    def __init__(self, file_path, tokenizer, max_length=30):\n",
        "        # Store file path for reading data\n",
        "        self.file_path = file_path\n",
        "        # Store tokenizer for text processing\n",
        "        self.tokenizer = tokenizer\n",
        "        # Set maximum sequence length to truncate long sequences\n",
        "        self.max_length = max_length\n",
        "        self._count_sentences()\n",
        "\n",
        "    def __iter__(self):\n",
        "        \"\"\"\n",
        "        Creates an iterator over the dataset.\n",
        "        This method is called when iterating over the dataset.\n",
        "\n",
        "        Yields:\n",
        "            tuple: (input_sequence, target_sequence) pairs for language modeling\n",
        "                  input_sequence is the sequence up to the last token\n",
        "                  target_sequence is the sequence shifted one position right\n",
        "        \"\"\"\n",
        "        # Open file in read mode with UTF-8 encoding\n",
        "        with open(self.file_path, 'r', encoding=\"utf-8\") as f:\n",
        "            # Process each line (sentence) in the file\n",
        "            for line in f:\n",
        "                # Remove leading/trailing whitespace\n",
        "                sentence = line.strip()\n",
        "                # Replace all numbers with ### placeholder\n",
        "                # This reduces vocabulary size and helps model generalize\n",
        "                sentence = re.sub(r\"\\d+\", \"###\", sentence)\n",
        "\n",
        "                # Convert sentence to token IDs\n",
        "                encoded_sentence = self.tokenizer.encode(\n",
        "                    sentence,\n",
        "                    max_length=self.max_length,\n",
        "                    truncation=True\n",
        "                )\n",
        "\n",
        "                # Only use sequences with at least 2 tokens\n",
        "                # (need at least one input and one target token)\n",
        "                if len(encoded_sentence) >= 2:\n",
        "                    # Input is all tokens except last\n",
        "                    input_seq = encoded_sentence[:-1]\n",
        "                    # Target is all tokens except first\n",
        "                    target_seq = encoded_sentence[1:]\n",
        "                    # Convert to PyTorch tensors and yield\n",
        "                    yield torch.tensor(input_seq, dtype=torch.long), torch.tensor(target_seq, dtype=torch.long)\n",
        "    def __len__(self):\n",
        "        return self._num_sentences\n",
        "\n",
        "    def _count_sentences(self):\n",
        "        print(f\"\\nCounting sentences in {self.file_path}...\")\n",
        "        with open(self.file_path, 'r', encoding=\"utf-8\") as f:\n",
        "            self._num_sentences = sum(1 for _ in f)\n",
        "        print(f\"\\nFound {self._num_sentences} sentences in {self.file_path}.\")\n",
        "\n",
        "## ----------------------------\n",
        "## Download and prepare data\n",
        "## ----------------------------\n",
        "\n",
        "def create_collate_fn(tokenizer):\n",
        "    \"\"\"\n",
        "    Creates a collate function for batching sequences of different lengths.\n",
        "    This function pads shorter sequences to match the longest sequence in the batch.\n",
        "\n",
        "    Args:\n",
        "        tokenizer: Tokenizer object containing padding token information\n",
        "\n",
        "    Returns:\n",
        "        function: Collate function that handles padding in batches\n",
        "    \"\"\"\n",
        "    def collate_fn(batch):\n",
        "        # Separate inputs and targets from batch\n",
        "        input_seqs, target_seqs = zip(*batch)\n",
        "        # Get padding token ID from tokenizer\n",
        "        pad_index = tokenizer.pad_token_id\n",
        "        # Pad input sequences to same length\n",
        "        input_padded = nn.utils.rnn.pad_sequence(input_seqs, batch_first=True, padding_value=pad_index)\n",
        "        # Pad target sequences to same length\n",
        "        target_padded = nn.utils.rnn.pad_sequence(target_seqs, batch_first=True, padding_value=pad_index)\n",
        "        return input_padded, target_padded\n",
        "    return collate_fn\n",
        "\n",
        "def check_file_exists(filename):\n",
        "    \"\"\"\n",
        "    Checks if a file exists in the current directory.\n",
        "    Args:\n",
        "        filename (str): Name of the file to check\n",
        "    Returns:\n",
        "        bool: True if file exists, False otherwise\n",
        "    \"\"\"\n",
        "    return os.path.exists(filename)\n",
        "\n",
        "def download_file(url):\n",
        "    \"\"\"\n",
        "    Downloads a file from the given URL if it doesn't exist locally.\n",
        "    Uses a custom User-Agent to help prevent download blocks.\n",
        "\n",
        "    Args:\n",
        "        url (str): URL of the file to download\n",
        "    Returns:\n",
        "        str: Name of the downloaded file (\"news.tar.gz\")\n",
        "    \"\"\"\n",
        "    # Always use news.tar.gz as the filename, regardless of URL\n",
        "    filename = \"news.tar.gz\"\n",
        "\n",
        "    if not check_file_exists(filename):\n",
        "        print(f\"\\nDownloading dataset from {url}...\")\n",
        "        req = urllib.request.Request(\n",
        "            url,\n",
        "            headers={\"User-Agent\": \"Mozilla/5.0\"}\n",
        "        )\n",
        "        with urllib.request.urlopen(req) as response:\n",
        "            with open(filename, \"wb\") as out_file:\n",
        "                out_file.write(response.read())\n",
        "        print(\"\\nDownload completed.\")\n",
        "    else:\n",
        "        print(f\"\\n{filename} already downloaded.\")\n",
        "    return filename\n",
        "\n",
        "def is_within_directory(directory, target):\n",
        "    \"\"\"\n",
        "    Checks if a target path is within a specified directory by comparing absolute paths.\n",
        "\n",
        "    Args:\n",
        "        directory (str): Base directory path\n",
        "        target (str): Target path to check\n",
        "    Returns:\n",
        "        bool: True if target's absolute path starts with directory's absolute path\n",
        "    \"\"\"\n",
        "    abs_directory = os.path.abspath(directory)\n",
        "    abs_target = os.path.abspath(target)\n",
        "    prefix = os.path.commonprefix([abs_directory, abs_target])\n",
        "    return prefix == abs_directory\n",
        "\n",
        "def extract_dataset(filename):\n",
        "    \"\"\"\n",
        "    Extracts train.txt and test.txt from the downloaded archive.\n",
        "    Includes debug information about archive contents.\n",
        "\n",
        "    Args:\n",
        "        filename (str): Name of the archive file\n",
        "    Returns:\n",
        "        tuple: Paths to extracted train and test files\n",
        "    \"\"\"\n",
        "    data_dir = os.path.join(os.path.dirname(filename), \"news\")\n",
        "    train_path = os.path.join(data_dir, \"train.txt\")\n",
        "    test_path = os.path.join(data_dir, \"test.txt\")\n",
        "\n",
        "    if check_file_exists(train_path) and check_file_exists(test_path):\n",
        "        print(\"\\nData files already extracted.\")\n",
        "        return train_path, test_path\n",
        "\n",
        "    print(\"\\nListing archive contents:\")\n",
        "    with tarfile.open(filename, \"r:gz\") as tar:\n",
        "        for member in tar.getmembers():\n",
        "            print(f\"\\nArchive member: {member.name}\")\n",
        "\n",
        "        print(\"\\nExtracting files...\")\n",
        "        # Extract to current directory first\n",
        "        tar.extractall('.')\n",
        "\n",
        "    if not (check_file_exists(train_path) and check_file_exists(test_path)):\n",
        "        raise FileNotFoundError(f\"\\nRequired files not found in the archive. Please check the paths above.\")\n",
        "\n",
        "    print(\"\\nExtraction completed.\")\n",
        "    return train_path, test_path\n",
        "\n",
        "def create_datasets(train_file, test_file, tokenizer, max_length=30):\n",
        "    \"\"\"\n",
        "    Creates IterableTextDataset objects for training and testing.\n",
        "    These datasets will stream data from disk instead of loading it all into memory.\n",
        "\n",
        "    Args:\n",
        "        train_file (str): Path to training data file\n",
        "        test_file (str): Path to test data file\n",
        "        tokenizer: Tokenizer object for text processing\n",
        "\n",
        "    Returns:\n",
        "        tuple: (train_dataset, test_dataset) - Dataset objects for training and testing\n",
        "    \"\"\"\n",
        "    # Create training dataset\n",
        "    train_dataset = IterableTextDataset(train_file, tokenizer, max_length)\n",
        "    # Create test dataset\n",
        "    test_dataset = IterableTextDataset(test_file, tokenizer, max_length)\n",
        "\n",
        "    # Print dataset sizes\n",
        "    print(f\"\\nTraining sentences: {len(train_dataset)}\")\n",
        "    print(f\"\\nTest sentences: {len(test_dataset)}\")\n",
        "\n",
        "    return train_dataset, test_dataset\n",
        "\n",
        "def create_dataloaders(train_dataset, test_dataset, batch_size, collate_fn):\n",
        "    \"\"\"\n",
        "    Creates DataLoader objects for efficient data iteration.\n",
        "\n",
        "    Args:\n",
        "        train_dataset: Training dataset\n",
        "        test_dataset: Test dataset\n",
        "        batch_size (int): Number of sequences per batch\n",
        "        collate_fn: Function to handle padding and batch creation\n",
        "\n",
        "    Returns:\n",
        "        tuple: (train_dataloader, test_dataloader) - DataLoader objects for\n",
        "               iterating over batches of data with proper padding\n",
        "    \"\"\"\n",
        "    # Create training data loader\n",
        "    train_dataloader = DataLoader(\n",
        "        train_dataset,\n",
        "        batch_size=batch_size,\n",
        "        collate_fn=collate_fn,    # Function to handle padding\n",
        "        num_workers=0             # Number of worker processes (0 = single process)\n",
        "    )\n",
        "    # Create test data loader\n",
        "    test_dataloader = DataLoader(\n",
        "        test_dataset,\n",
        "        batch_size=batch_size,\n",
        "        collate_fn=collate_fn,\n",
        "        num_workers=0\n",
        "    )\n",
        "    return train_dataloader, test_dataloader\n",
        "\n",
        "def download_and_prepare_data(url, batch_size, tokenizer, max_length=30):\n",
        "    \"\"\"\n",
        "    Main function to handle the complete data preparation pipeline.\n",
        "    Downloads data, extracts it, and creates necessary dataset objects.\n",
        "\n",
        "    Args:\n",
        "        url (str): URL where the dataset archive can be downloaded\n",
        "        batch_size (int): Batch size for data loading\n",
        "        tokenizer: Tokenizer object for text processing\n",
        "        max_length (int): Maximum sequence length for tokenization (default: 30)\n",
        "\n",
        "    Returns:\n",
        "        tuple: (train_dataloader, test_dataloader) - Ready-to-use data loaders\n",
        "    \"\"\"\n",
        "    # Step 1: Download dataset archive from URL\n",
        "    filename = download_file(url)\n",
        "\n",
        "    # Step 2: Extract training and test files from archive\n",
        "    train_file, test_file = extract_dataset(filename)\n",
        "\n",
        "    # Step 3: Create dataset objects for streaming data\n",
        "    train_dataset, test_dataset = create_datasets(train_file, test_file, tokenizer, max_length)\n",
        "\n",
        "    # Step 4: Create function to handle batch creation\n",
        "    collate_fn = create_collate_fn(tokenizer)\n",
        "\n",
        "    # Step 5: Create and return data loaders\n",
        "    return create_dataloaders(train_dataset, test_dataset, batch_size, collate_fn)\n",
        "\n",
        "# ----------------------------\n",
        "# Evaluation Functions\n",
        "# ----------------------------\n",
        "\n",
        "def compute_loss_and_perplexity(model, dataloader, tokenizer, criterion, device, max_sentences=1000):\n",
        "    \"\"\"\n",
        "    Evaluates model performance by computing loss and perplexity on data.\n",
        "\n",
        "    Args:\n",
        "        model (nn.Module): The language model to evaluate\n",
        "        dataloader (DataLoader): Data loader containing batched sequences\n",
        "        tokenizer: Tokenizer for handling special tokens like padding\n",
        "        criterion: Loss function (usually CrossEntropyLoss)\n",
        "        device: Device to run computation on (cuda/cpu)\n",
        "        max_sentences (int): Maximum number of sentences to evaluate (default: 1000)\n",
        "                           Limits evaluation to a subset for faster validation\n",
        "\n",
        "    Returns:\n",
        "        tuple: (average_loss, perplexity, sentences_processed)\n",
        "               - average_loss: Mean loss per token (excluding padding)\n",
        "               - perplexity: exp(average_loss), lower is better\n",
        "    \"\"\"\n",
        "    # Set model to evaluation mode (disables dropout, etc.)\n",
        "    model.eval()\n",
        "\n",
        "    # Initialize counters for loss calculation\n",
        "    total_loss = 0.0          # Accumulator for total loss across all batches\n",
        "    total_tokens = 0          # Counter for total number of tokens (excluding padding)\n",
        "    sentences_processed = 0    # Counter for number of sentences processed\n",
        "\n",
        "    # Disable gradient computation for efficiency\n",
        "    with torch.no_grad():\n",
        "        # Iterate through data with progress bar\n",
        "        for input_seq, target_seq in tqdm(dataloader, desc=\"Evaluating\", leave=False):\n",
        "            # Move input and target sequences to specified device\n",
        "            input_seq = input_seq.to(device)      # Shape: (batch_size, seq_len)\n",
        "            target_seq = target_seq.to(device)    # Shape: (batch_size, seq_len)\n",
        "\n",
        "            # Get current batch size (might be smaller for last batch)\n",
        "            batch_size_current = input_seq.size(0)\n",
        "\n",
        "            # Forward pass through the model\n",
        "            logits = model(input_seq)             # Shape: (batch_size, seq_len, vocab_size)\n",
        "\n",
        "            # Reshape logits and target for loss calculation\n",
        "            logits = logits.reshape(-1, logits.size(-1))  # Shape: (batch_size * seq_len, vocab_size)\n",
        "            target = target_seq.reshape(-1)              # Shape: (batch_size * seq_len)\n",
        "\n",
        "            # Create mask to exclude padding tokens\n",
        "            mask = target != tokenizer.pad_token_id\n",
        "\n",
        "            # Compute loss only on non-padded tokens\n",
        "            loss = criterion(logits[mask], target[mask])\n",
        "\n",
        "            # Update counters\n",
        "            loss_value = loss.item() * mask.sum().item()  # Total loss for this batch\n",
        "            total_loss += loss_value                      # Accumulate batch loss\n",
        "            total_tokens += mask.sum().item()             # Count non-padding tokens\n",
        "\n",
        "            # Update sentence counter and check if we've reached maximum\n",
        "            sentences_processed += batch_size_current\n",
        "            if sentences_processed >= max_sentences:\n",
        "                break\n",
        "\n",
        "    # Calculate final metrics\n",
        "    average_loss = total_loss / total_tokens           # Normalize loss by number of tokens\n",
        "    perplexity = math.exp(average_loss)               # Convert loss to perplexity\n",
        "\n",
        "    return average_loss, perplexity\n",
        "\n",
        "def generate_text(model, start_string, tokenizer, device, max_length=50):\n",
        "    \"\"\"\n",
        "    Generates text continuation from a given start string using greedy decoding.\n",
        "\n",
        "    Args:\n",
        "        model (nn.Module): Trained language model\n",
        "        start_string (str): Initial text to continue from\n",
        "        tokenizer: Tokenizer for text processing\n",
        "        device: Device to run generation on (cuda/cpu)\n",
        "        max_length (int): Maximum length of generated sequence\n",
        "\n",
        "    Returns:\n",
        "        str: Generated text continuation\n",
        "    \"\"\"\n",
        "    # Set model to evaluation mode to disable dropout and other training-specific behaviors\n",
        "    model.eval()\n",
        "\n",
        "    # Convert input string to token indices\n",
        "    input_indices = tokenizer.encode(start_string, add_special_tokens=False)\n",
        "\n",
        "    # Convert indices to tensor and move to specified device (GPU/CPU)\n",
        "    input_tensor = torch.tensor([input_indices], dtype=torch.long).to(device)\n",
        "\n",
        "    # Keep track of all generated tokens, starting with input sequence\n",
        "    generated_indices = input_indices.copy()\n",
        "\n",
        "    # Generate tokens until we hit max length or end-of-sequence token\n",
        "    for _ in range(max_length - len(input_indices)):\n",
        "        # Get model predictions for the entire sequence\n",
        "        logits = model(input_tensor)\n",
        "        # Only take predictions for the last token position\n",
        "        logits = logits[:, -1, :]\n",
        "\n",
        "        # Prevent the model from generating unknown tokens by setting their probability to negative infinity\n",
        "        if tokenizer.unk_token_id is not None:\n",
        "            logits[:, tokenizer.unk_token_id] = float(\"-inf\")\n",
        "\n",
        "        # Greedy decoding: select the token with highest probability\n",
        "        next_token = torch.argmax(logits, dim=-1)\n",
        "\n",
        "        # Add the chosen token to our generated sequence\n",
        "        generated_indices.append(next_token.item())\n",
        "\n",
        "        # If we generate an end-of-sequence token, stop generation\n",
        "        if next_token.item() == tokenizer.eos_token_id:\n",
        "            break\n",
        "\n",
        "        # Add the new token to input tensor for next iteration\n",
        "        input_tensor = torch.cat([input_tensor, next_token.unsqueeze(0)], dim=1)\n",
        "\n",
        "    # Convert token indices back to text, removing any special tokens\n",
        "    return tokenizer.decode(generated_indices, skip_special_tokens=True)\n",
        "\n",
        "def save_model(model, tokenizer, model_name):\n",
        "    \"\"\"\n",
        "    Saves the model state dictionary and tokenizer using the specified model name.\n",
        "\n",
        "    Args:\n",
        "        model (nn.Module): The trained model to save\n",
        "        tokenizer: The tokenizer used with the model\n",
        "        model_name (str): Name to use for the saved model files\n",
        "    \"\"\"\n",
        "    # Create the models directory if it doesn't exist\n",
        "    save_dir = os.path.join(\"models\", model_name)\n",
        "    os.makedirs(save_dir, exist_ok=True)\n",
        "\n",
        "    # Save the model state dictionary and configuration\n",
        "    model_path = os.path.join(save_dir, f\"{model_name}.pth\")\n",
        "    torch.save({\n",
        "        \"model_state_dict\": model.state_dict(),\n",
        "        \"model_config\": {\n",
        "            \"vocab_size\": len(tokenizer),\n",
        "            \"emb_dim\": model.embedding.embedding_dim,\n",
        "            \"num_heads\": len(model.layers[0].attn.heads),\n",
        "            \"num_blocks\": len(model.layers),\n",
        "            \"pad_idx\": model.embedding.padding_idx\n",
        "        }\n",
        "    }, model_path)\n",
        "\n",
        "    # Save the tokenizer\n",
        "    tokenizer_path = os.path.join(save_dir, \"tokenizer\")\n",
        "    tokenizer.save_pretrained(tokenizer_path)\n",
        "\n",
        "    print(f\"Model and tokenizer saved as '{model_name}'\")\n",
        "\n",
        "def load_model(model_name, device=None):\n",
        "    \"\"\"\n",
        "    Loads a saved model and tokenizer using the model name.\n",
        "\n",
        "    Args:\n",
        "        model_name (str): Name of the model to load\n",
        "        device: Device to load the model onto (if None, uses available device)\n",
        "\n",
        "    Returns:\n",
        "        tuple: (loaded_model, loaded_tokenizer)\n",
        "    \"\"\"\n",
        "    if device is None:\n",
        "        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "    save_dir = os.path.join(\"models\", model_name)\n",
        "\n",
        "    # Check if model exists\n",
        "    if not os.path.exists(save_dir):\n",
        "        raise FileNotFoundError(f\"No saved model found with name '{model_name}'\")\n",
        "\n",
        "    # Load the tokenizer\n",
        "    tokenizer_path = os.path.join(save_dir, \"tokenizer\")\n",
        "    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)\n",
        "\n",
        "    # Load the model state and config\n",
        "    model_path = os.path.join(save_dir, f\"{model_name}.pth\")\n",
        "    checkpoint = torch.load(model_path, map_location=device, weights_only=False)\n",
        "\n",
        "    # Create a new model instance with the saved configuration\n",
        "    model = DecoderLanguageModel(\n",
        "        vocab_size=checkpoint[\"model_config\"][\"vocab_size\"],\n",
        "        emb_dim=checkpoint[\"model_config\"][\"emb_dim\"],\n",
        "        num_heads=checkpoint[\"model_config\"][\"num_heads\"],\n",
        "        num_blocks=checkpoint[\"model_config\"][\"num_blocks\"],\n",
        "        pad_idx=checkpoint[\"model_config\"][\"pad_idx\"]\n",
        "    )\n",
        "\n",
        "    # Load the saved state dictionary\n",
        "    model.load_state_dict(checkpoint[\"model_state_dict\"])\n",
        "    model.to(device)\n",
        "    model.eval()\n",
        "\n",
        "    print(f\"\\nModel '{model_name}' loaded successfully\")\n",
        "    return model, tokenizer\n",
        "\n",
        "def get_hyperparameters():\n",
        "    emb_dim = 128\n",
        "    num_heads = 8\n",
        "    num_blocks = 2\n",
        "    batch_size = 128\n",
        "    learning_rate = 0.001\n",
        "    num_epochs = 1\n",
        "    context_size = 30\n",
        "    return emb_dim, num_heads, num_blocks, batch_size, learning_rate, num_epochs, context_size"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model Classes\n",
        "\n",
        "Decoder Transformer language model classes and the initialization method:"
      ],
      "metadata": {
        "id": "7dSK1bTyku7F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# ----------------------------\n",
        "# Weight Initialization and Core Functions\n",
        "# This section contains utility functions for weight initialization\n",
        "# and core computational functions used throughout the model\n",
        "# ----------------------------\n",
        "\n",
        "def initialize_weights(model):\n",
        "    \"\"\"\n",
        "    Initialize the weights of different model components using appropriate schemes.\n",
        "    Each layer type receives specialized initialization for optimal training.\n",
        "    \"\"\"\n",
        "    for module in model.modules():\n",
        "        if isinstance(module, nn.Linear):\n",
        "            # Xavier uniform initialization for linear layers\n",
        "            # Helps maintain variance across network layers\n",
        "            nn.init.xavier_uniform_(module.weight)\n",
        "            if module.bias is not None:\n",
        "                nn.init.zeros_(module.bias)  # Initialize biases to zero\n",
        "        elif isinstance(module, nn.Embedding):\n",
        "            # Initialize embedding layers with normal distribution\n",
        "            nn.init.normal_(module.weight, mean=0, std=0.02)\n",
        "            if module.padding_idx is not None:\n",
        "                # Ensure padding tokens have zero embeddings\n",
        "                with torch.no_grad():\n",
        "                    module.weight[module.padding_idx].fill_(0)\n",
        "        elif isinstance(module, AttentionHead):\n",
        "            # Initialize query, key, and value projection matrices\n",
        "            # Xavier uniform helps maintain good gradient flow\n",
        "            nn.init.xavier_uniform_(module.W_Q)\n",
        "            nn.init.xavier_uniform_(module.W_K)\n",
        "            nn.init.xavier_uniform_(module.W_V)\n",
        "        elif isinstance(module, MultiHeadAttention):\n",
        "            # Initialize output projection matrix for attention mechanism\n",
        "            nn.init.xavier_uniform_(module.W_O)\n",
        "        elif isinstance(module, DecoderLanguageModel):\n",
        "            # Initialize final output projection layer\n",
        "            nn.init.xavier_uniform_(module.output)\n",
        "        elif isinstance(module, RMSNorm):\n",
        "            # Initialize RMSNorm scale parameters to ones\n",
        "            # This starts with identity transformation\n",
        "            nn.init.ones_(module.scale)\n",
        "        elif isinstance(module, MLP):\n",
        "            # Initialize feed-forward network parameters\n",
        "            nn.init.xavier_uniform_(module.W_1)\n",
        "            nn.init.xavier_uniform_(module.W_2)\n",
        "            nn.init.zeros_(module.B_1)\n",
        "            nn.init.zeros_(module.B_2)\n",
        "\n",
        "def rope(x, theta_base=10000.0):\n",
        "    \"\"\"\n",
        "    Implements Rotary Position Embedding (RoPE) for transformer attention.\n",
        "    RoPE encodes position information through rotation matrices applied to pairs of dimensions.\n",
        "\n",
        "    Args:\n",
        "        x: Input tensor of shape (batch_size, seq_len, emb_dim)\n",
        "        theta_base: Base for computing rotation frequencies (default: 10000.0)\n",
        "\n",
        "    Returns:\n",
        "        Tensor with position information encoded through rotations\n",
        "    \"\"\"\n",
        "    batch_size, seq_len, emb_dim = x.size()\n",
        "    assert emb_dim % 2 == 0, \"Embedding dimensionality must be even for RoPE\"\n",
        "\n",
        "    # Generate sequence position indices\n",
        "    pos = torch.arange(0, seq_len, dtype=torch.float32, device=x.device)\n",
        "    pos = pos.unsqueeze(0).expand(batch_size, seq_len)\n",
        "\n",
        "    # Compute frequency bands for each dimension pair\n",
        "    # Modified: frequencies start from p=1 and use (p-1) in exponent\n",
        "    p = torch.arange(1, emb_dim // 2 + 1, dtype=torch.float32, device=x.device)\n",
        "    theta_p = 1.0 / (theta_base ** (2 * (p - 1) / emb_dim))\n",
        "\n",
        "    # Compute rotation angles for each position and frequency\n",
        "    pos = pos.unsqueeze(-1)\n",
        "    theta = pos * theta_p\n",
        "\n",
        "    # Compute rotation components\n",
        "    sin_theta = torch.sin(theta)\n",
        "    cos_theta = torch.cos(theta)\n",
        "\n",
        "    # Split input into alternating dimensions\n",
        "    x1 = x[..., 0::2]  # Dimensions at indices 0,2,4,...\n",
        "    x2 = x[..., 1::2]  # Dimensions at indices 1,3,5,...\n",
        "\n",
        "    # Apply 2D rotations to each pair\n",
        "    x_rotated_1 = x1 * cos_theta - x2 * sin_theta\n",
        "    x_rotated_2 = x1 * sin_theta + x2 * cos_theta\n",
        "\n",
        "    # Recombine rotated pairs into final output\n",
        "    x_rotated = torch.stack((x_rotated_1, x_rotated_2), dim=-1).reshape(batch_size, seq_len, emb_dim)\n",
        "\n",
        "    return x_rotated\n",
        "\n",
        "# ----------------------------\n",
        "# Model Components\n",
        "# This section contains the building blocks of the transformer decoder\n",
        "# including normalization, attention, and feed-forward layers\n",
        "# ----------------------------\n",
        "\n",
        "class RMSNorm(nn.Module):\n",
        "    \"\"\"\n",
        "    Root Mean Square Layer Normalization\n",
        "    A simplified alternative to Layer Normalization that only uses RMS statistics\n",
        "    \"\"\"\n",
        "    def __init__(self, emb_dim, epsilon=1e-8):\n",
        "        super().__init__()\n",
        "        self.scale = nn.Parameter(torch.ones(emb_dim))  # Learnable scale parameter\n",
        "        self.epsilon = epsilon  # Small constant for numerical stability\n",
        "\n",
        "    def forward(self, x):\n",
        "        # Compute root mean square normalization\n",
        "        squared_x = x ** 2\n",
        "        mean_squared = torch.mean(squared_x, dim=-1, keepdim=True)\n",
        "        rms = torch.sqrt(mean_squared + self.epsilon)\n",
        "\n",
        "        # Normalize and scale\n",
        "        x_normalized = x / rms\n",
        "        output = x_normalized * self.scale\n",
        "        return output\n",
        "\n",
        "class AttentionHead(nn.Module):\n",
        "    \"\"\"\n",
        "    Single head of self-attention\n",
        "    Transforms input using learned projections and computes scaled dot-product attention\n",
        "    \"\"\"\n",
        "    def __init__(self, emb_dim, d_h):\n",
        "        super().__init__()\n",
        "        # Initialize projection matrices for queries, keys, and values\n",
        "        self.W_Q = nn.Parameter(torch.rand(emb_dim, d_h))\n",
        "        self.W_K = nn.Parameter(torch.rand(emb_dim, d_h))\n",
        "        self.W_V = nn.Parameter(torch.rand(emb_dim, d_h))\n",
        "        self.d_h = d_h  # Dimensionality of attention head\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        # Project input into query, key, and value spaces\n",
        "        Q = x @ self.W_Q\n",
        "        K = x @ self.W_K\n",
        "        V = x @ self.W_V\n",
        "\n",
        "        # Apply rotary position embeddings to queries and keys\n",
        "        Q, K = rope(Q), rope(K)\n",
        "\n",
        "        # Compute attention scores with scaling factor\n",
        "        scores = Q @ K.transpose(-2, -1) / math.sqrt(self.d_h)\n",
        "\n",
        "        # Apply causal mask and attention weights\n",
        "        masked_scores = scores.masked_fill(mask == 0, float(\"-inf\"))\n",
        "        attention_weights = torch.softmax(masked_scores, dim=-1)\n",
        "\n",
        "        return attention_weights @ V\n",
        "\n",
        "class MultiHeadAttention(nn.Module):\n",
        "    \"\"\"\n",
        "    Multi-head attention mechanism\n",
        "    Allows the model to jointly attend to information from different positions\n",
        "    \"\"\"\n",
        "    def __init__(self, emb_dim, num_heads):\n",
        "        super().__init__()\n",
        "        d_h = emb_dim // num_heads  # Dimensionality of each attention head\n",
        "\n",
        "        # Create multiple attention heads\n",
        "        self.heads = nn.ModuleList([\n",
        "            AttentionHead(emb_dim, d_h)\n",
        "            for _ in range(num_heads)\n",
        "        ])\n",
        "\n",
        "        # Output projection matrix\n",
        "        self.W_O = nn.Parameter(torch.rand(emb_dim, emb_dim))\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        # Process input through each attention head\n",
        "        head_outputs = [head(x, mask) for head in self.heads]\n",
        "\n",
        "        # Concatenate outputs and project to final dimensionality\n",
        "        x = torch.cat(head_outputs, dim=-1)\n",
        "        return x @ self.W_O\n",
        "\n",
        "class MLP(nn.Module):\n",
        "    \"\"\"\n",
        "    Multi-Layer Perceptron for transformer feed-forward network\n",
        "    Uses a larger intermediate dimensionality (4x) with ReLU activation\n",
        "    \"\"\"\n",
        "    def __init__(self, emb_dim):\n",
        "        super().__init__()\n",
        "        # Initialize weights and biases for two-layer feed-forward network\n",
        "        self.W_1 = nn.Parameter(torch.rand(emb_dim, emb_dim * 4))\n",
        "        self.B_1 = nn.Parameter(torch.rand(emb_dim * 4))\n",
        "        self.W_2 = nn.Parameter(torch.rand(emb_dim * 4, emb_dim))\n",
        "        self.B_2 = nn.Parameter(torch.rand(emb_dim))\n",
        "\n",
        "    def forward(self, x):\n",
        "        # First linear transformation and activation\n",
        "        x = x @ self.W_1 + self.B_1\n",
        "        x = torch.relu(x)\n",
        "\n",
        "        # Second linear transformation\n",
        "        x = x @ self.W_2 + self.B_2\n",
        "        return x\n",
        "\n",
        "class DecoderBlock(nn.Module):\n",
        "    \"\"\"\n",
        "    Single transformer decoder block\n",
        "    Combines self-attention and feed-forward layers with residual connections\n",
        "    \"\"\"\n",
        "    def __init__(self, emb_dim, num_heads):\n",
        "        super().__init__()\n",
        "        # Layer components\n",
        "        self.norm1 = RMSNorm(emb_dim)\n",
        "        self.attn = MultiHeadAttention(emb_dim, num_heads)\n",
        "        self.norm2 = RMSNorm(emb_dim)\n",
        "        self.mlp = MLP(emb_dim)\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        # Self-attention sub-block with residual connection\n",
        "        attn_out = self.attn(self.norm1(x), mask)\n",
        "        x = x + attn_out\n",
        "\n",
        "        # Feed-forward sub-block with residual connection\n",
        "        mlp_out = self.mlp(self.norm2(x))\n",
        "        x = x + mlp_out\n",
        "        return x\n",
        "\n",
        "class DecoderLanguageModel(nn.Module):\n",
        "    \"\"\"\n",
        "    Complete decoder-only transformer language model\n",
        "    Processes input sequences using multiple decoder blocks and projects to vocabulary\n",
        "    \"\"\"\n",
        "    def __init__(self, vocab_size, emb_dim, num_heads, num_blocks, pad_idx):\n",
        "        super().__init__()\n",
        "        # Token embedding layer\n",
        "        self.embedding = nn.Embedding(vocab_size, emb_dim, padding_idx=pad_idx)\n",
        "\n",
        "        # Stack of decoder blocks\n",
        "        self.layers = nn.ModuleList([\n",
        "            DecoderBlock(emb_dim, num_heads) for _ in range(num_blocks)\n",
        "        ])\n",
        "\n",
        "        # Output projection to vocabulary size\n",
        "        self.output = nn.Parameter(torch.rand(emb_dim, vocab_size))\n",
        "\n",
        "    def forward(self, x):\n",
        "        # Embed input tokens\n",
        "        x = self.embedding(x)\n",
        "\n",
        "        # Create causal attention mask\n",
        "        _, seq_len, _ = x.size()\n",
        "        mask = torch.tril(torch.ones(seq_len, seq_len, device=x.device))\n",
        "\n",
        "        # Process through decoder blocks\n",
        "        for layer in self.layers:\n",
        "            x = layer(x, mask)\n",
        "\n",
        "        # Project to vocabulary distribution\n",
        "        return x @ self.output"
      ],
      "metadata": {
        "id": "Iz7RlKWvk0Wk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training the language model\n",
        "\n",
        "In the cell below, we load the data, train, and save the language model:"
      ],
      "metadata": {
        "id": "RqQxwedtxESQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# ----------------------------\n",
        "# Main training loop for a Decoder Language Model\n",
        "# This script handles the entire training process including data loading,\n",
        "# model training, validation, and text generation\n",
        "# ----------------------------\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    # Initialize random seeds to ensure reproducible results\n",
        "    set_seed(42)\n",
        "\n",
        "    # Retrieve model architecture and training hyperparameters from configuration\n",
        "    # emb_dim: dimensionality of input token and intermediary embeddings\n",
        "    # num_heads: number of attention heads in each transformer block\n",
        "    # num_blocks: number of transformer blocks in the model\n",
        "    # batch_size: mini-batch size\n",
        "    # learning_rate: step size for optimizer updates\n",
        "    # num_epochs: number of complete passes through the training dataset\n",
        "    # context_size: maximum input sequence length\n",
        "    emb_dim, num_heads, num_blocks, batch_size, learning_rate, num_epochs, context_size = get_hyperparameters()\n",
        "\n",
        "    # Initialize the tokenizer using Microsoft's Phi-3.5-mini model\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\"microsoft/Phi-3.5-mini-instruct\")\n",
        "    # Get padding token index for padding shorter sequences\n",
        "    pad_idx = tokenizer.pad_token_id\n",
        "\n",
        "    # Check for CUDA-capable GPU and set the device accordingly\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "    # Download the news dataset and create DataLoader objects for training and testing\n",
        "    # DataLoaders handle batching and shuffling\n",
        "    data_url = \"https://www.thelmbook.com/data/news\"\n",
        "    train_dataloader, test_dataloader = download_and_prepare_data(\n",
        "        data_url, batch_size, tokenizer, context_size\n",
        "    )\n",
        "\n",
        "    # Get the size of the vocabulary that the model needs to handle\n",
        "    vocab_size = len(tokenizer)\n",
        "    print(f\"\\nVocabulary size: {vocab_size}\\n\")\n",
        "\n",
        "    # Initialize the Decoder language model with specified architecture parameters\n",
        "    # vocab_size: determines output layer dimensionality\n",
        "    # emb_dim: size of token embeddings and intermediary embeddings\n",
        "    # num_heads: number of attention heads per transformer block\n",
        "    # num_blocks: number of transformer blocks in the model\n",
        "    # pad_idx: special token ID used for padding shorter sequences\n",
        "    model = DecoderLanguageModel(\n",
        "        vocab_size, emb_dim, num_heads, num_blocks, pad_idx\n",
        "    )\n",
        "\n",
        "    # Move the model to GPU if available\n",
        "    model.to(device)\n",
        "\n",
        "    # Initialize model weights using custom initialization scheme\n",
        "    # This is important for stable training of deep neural networks\n",
        "    initialize_weights(model)\n",
        "\n",
        "    # Initialize the AdamW optimizer with specified learning rate\n",
        "    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
        "\n",
        "    # Initialize the loss function (Cross Entropy) for training\n",
        "    # ignore_index=pad_idx ensures that padding tokens don't contribute to the loss\n",
        "    criterion = nn.CrossEntropyLoss(ignore_index=pad_idx)\n",
        "\n",
        "    # Calculate and display the total number of trainable parameters in the model\n",
        "    total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "    print(f\"\\nTotal trainable parameters: {total_params}\\n\")\n",
        "\n",
        "    # Set evaluation interval (number of examples after which to perform validation)\n",
        "    # 200,000 examples provides a good balance between training time and monitoring frequency\n",
        "    eval_interval = 200_000\n",
        "    examples_processed = 0  # Counter for tracking progress toward next evaluation\n",
        "\n",
        "    # Define test contexts for generating sample text during evaluation\n",
        "    contexts = [\n",
        "        \"Moscow\",\n",
        "        \"New York\",\n",
        "        \"A hurricane\",\n",
        "        \"The President\"\n",
        "    ]\n",
        "\n",
        "    # Main training loop - iterate through specified number of epochs\n",
        "    for epoch in range(num_epochs):\n",
        "        # Set model to training mode\n",
        "        model.train()\n",
        "\n",
        "        # Initialize tracking variables for this epoch\n",
        "        total_loss = 0.0      # Accumulator for loss across all batches\n",
        "        total_tokens = 0      # Counter for actual tokens processed (excluding padding)\n",
        "\n",
        "        # Create progress bar for monitoring training progress\n",
        "        progress_bar = tqdm(train_dataloader, desc=f\"Epoch {epoch+1}/{num_epochs}\")\n",
        "\n",
        "        # Iterate through batches in the training data\n",
        "        for batch_idx, (input_seq, target_seq) in enumerate(progress_bar):\n",
        "            # Move input and target sequences to GPU if available\n",
        "            input_seq = input_seq.to(device)\n",
        "            target_seq = target_seq.to(device)\n",
        "\n",
        "            # Clear gradients from previous batch\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            # Forward pass: get model predictions for this batch\n",
        "            # output shape: (batch_size, seq_len, vocab_size)\n",
        "            logits = model(input_seq)\n",
        "\n",
        "            # Reshape logits and target tensors for loss computation\n",
        "            logits = logits.reshape(-1, logits.size(-1))\n",
        "            target = target_seq.reshape(-1)\n",
        "\n",
        "            # Create mask to exclude padding tokens from loss calculation\n",
        "            mask = target != pad_idx\n",
        "\n",
        "            # Compute loss between model predictions and actual targets\n",
        "            # Using masked versions to ignore padding tokens\n",
        "            loss = criterion(logits[mask], target[mask])\n",
        "\n",
        "            # Backward pass: compute gradients of loss with respect to model parameters\n",
        "            loss.backward()\n",
        "\n",
        "            # Update model parameters using calculated gradients\n",
        "            optimizer.step()\n",
        "\n",
        "            # Calculate actual loss value for this batch accounting for padding\n",
        "            loss_value = loss.item() * mask.sum().item()\n",
        "\n",
        "            # Accumulate total loss and tokens for epoch statistics\n",
        "            total_loss += loss_value\n",
        "            total_tokens += mask.sum().item()\n",
        "            examples_processed += input_seq.size(0)\n",
        "\n",
        "            # Update progress bar with current batch loss\n",
        "            progress_bar.set_postfix({\"loss\": f\"{loss.item():.4f}\"})\n",
        "\n",
        "            # Periodic evaluation after processing specified number of examples\n",
        "            if examples_processed >= eval_interval:\n",
        "                # Calculate average loss over the last eval_interval examples\n",
        "                avg_loss = total_loss / total_tokens\n",
        "                print(f\"\\nAfter {examples_processed} examples, Average Loss: {avg_loss:.4f}\")\n",
        "\n",
        "                # Switch to evaluation mode\n",
        "                model.eval()\n",
        "\n",
        "                # Compute validation metrics\n",
        "                average_loss, perplexity = compute_loss_and_perplexity(\n",
        "                    model, test_dataloader, tokenizer, criterion, device, max_sentences=1000\n",
        "                )\n",
        "                # Record validation\n",
        "                print(f\"\\nValidation Average Loss: {average_loss:.4f}, Perplexity: {perplexity:.2f}\")\n",
        "\n",
        "                model.eval()\n",
        "\n",
        "                # Generate sample texts to qualitatively assess model performance\n",
        "                for context in contexts:\n",
        "                    # Generate text continuation for each test context\n",
        "                    generated_text = generate_text(\n",
        "                        model=model,\n",
        "                        start_string=context,\n",
        "                        tokenizer=tokenizer,\n",
        "                        device=device,\n",
        "                        max_length=50\n",
        "                    )\n",
        "                    print(f\"\\nContext: {context}\")\n",
        "                    print(f\"\\nGenerated text: {generated_text}\\n\")\n",
        "\n",
        "                # Switch back to training mode for continued training\n",
        "                model.train()\n",
        "\n",
        "                # Reset counters for next evaluation interval\n",
        "                examples_processed = 0\n",
        "                total_loss = 0.0\n",
        "                total_tokens = 0\n",
        "\n",
        "        # End-of-epoch reporting\n",
        "        if total_tokens > 0:\n",
        "            # Calculate and display average loss for the epoch\n",
        "            avg_loss = total_loss / total_tokens\n",
        "            print(f\"\\nEpoch {epoch+1}/{num_epochs}, Average Loss: {avg_loss:.4f}\")\n",
        "        else:\n",
        "            # Handle edge case where no tokens were processed\n",
        "            print(f\"\\nEpoch {epoch+1}/{num_epochs} completed.\")\n",
        "\n",
        "        # Perform end-of-epoch validation\n",
        "        model.eval()\n",
        "\n",
        "        # Generate sample texts for qualitative assessment\n",
        "        print(\"\\nGenerating text based on contexts using generate_text:\\n\")\n",
        "        for context in contexts:\n",
        "            generated_text = generate_text(\n",
        "                model=model,\n",
        "                start_string=context,\n",
        "                tokenizer=tokenizer,\n",
        "                device=device,\n",
        "                max_length=50\n",
        "            )\n",
        "            print(f\"\\nContext: {context}\")\n",
        "            print(f\"\\nGenerated text: {generated_text}\\n\")\n",
        "\n",
        "        average_loss, perplexity = compute_loss_and_perplexity(\n",
        "            model, test_dataloader, tokenizer, criterion, device, max_sentences=1000\n",
        "        )\n",
        "        print(f\"\\nValidation Average Loss: {average_loss:.4f}, Perplexity: {perplexity:.2f}\")\n",
        "\n",
        "        # Reset to training mode for next epoch\n",
        "        model.train()\n",
        "\n",
        "    # Save the trained model and tokenizer for later use\n",
        "    # This includes model architecture, weights, and tokenizer configuration\n",
        "    model_name = \"Decoder_LM\"\n",
        "    save_model(model, tokenizer, model_name)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "9a8699a145a94ed89b8b11d715067c3f",
            "b576e000e0d549b089431943ad050eba",
            "181b36d4f4e14c10adc7b249717e21e4",
            "2226577264e14049b08244ce001e4e31",
            "d7f2588a852742afa54c616961176830",
            "c36c028aeee14b95901e1fb005c7201a",
            "cd63f0fa54284235bb73c59bf5cb2a92",
            "ae3f1435c9714894810273ab6cc8fb39",
            "188a4756346f45f796f49a44e694ad29",
            "8873103295c3485ca41a3347f8766a9d",
            "2f54ab7c292246db81dc7cbab21c7cbb",
            "e41117d4fed1445daf6fb85a01cbc81e",
            "fefd7c010fdf4735aa9b2513db0497d0",
            "f48e51c77737475fb4aaafce03380fb7",
            "82778a1b41a0417b86fc1e29455fb383",
            "5b6201ac09a64fa79f255ac5a14b3a5a",
            "f46a10202f08457393536590b5084b3e",
            "32dc1d47b7a545ff9759d3897eee59d2",
            "12db66482d064d7e831767ccbcef1bec",
            "ee332c51eee94dcbb5586f463e39afdf",
            "ec1dd8303a3b4f5e97b3d218f3de8613",
            "09222bf385da4ce0be09da2e03b74f74",
            "6ba6646a5d5347059b14acdd79d6da07",
            "756c21cc62c14083a6a6729f54232d79",
            "0b2e8ac291734ae39059110f96493a18",
            "cf27f80668ff40c9ac07a93197b34640",
            "c65097f7cf4a4722a186cfb1822f0315",
            "beb1e66b6cc44e7490fb207bff6fbd2c",
            "3daf9a7b54124787b732fdb108089e00",
            "2adb1d17c4404860b49575ea15dbff13",
            "08718b87ba204a34a9449fc9a60f97ad",
            "72a5c19cce364fed959f66f6521a3cf0",
            "57c3488609f046c8852641b73b83e678",
            "00647dd5e8034abcb0217cc30e02177f",
            "57ef6e22ee014ce0b467d6ae328d37ec",
            "26c3ff709f804abcacac8861c235b6f4",
            "ae4d86bfa9914f6e81886ab56a011ec9",
            "677fb8e2b421433fa412f9d75e319081",
            "156cc956117a4ed984446f3d1a3aebe3",
            "260bd2215a284268b2920be0c458393c",
            "fadf515115e64ebeb234df1c3c9dce71",
            "20370596c8824147a0aab279ad6241d8",
            "bf8a3680974742c4962608eaac628a6e",
            "b9d3e3c74d644a27a6bfb7b374683f02",
            "6466f91527ee437ebd69ce5cb90bdf6f",
            "ecf37b3cecb8405685d622d1f3e05bcf",
            "3277a976d1034d12b12ec767ae80e3c5",
            "6a1bda5b60a1449caf1a7b1e9ef6e844",
            "1fa0d48eb7624fc8806942941ca4b411",
            "f17c35f573354ce78baf719054af4732",
            "6415e99c230c463298a92d761875fd2e",
            "0f34f0115c644766a1c6cc5b61485608",
            "6044a91e47104cb9b18a8add2a0e4c6f",
            "d808e2efb7b54ae79e431f56ab9fc871",
            "6aa8bfe1c8864a679e8634806ec46dac"
          ]
        },
        "id": "QCs-n1WaxFf9",
        "outputId": "7a39a591-0f98-4753-c2f9-cbb717a518f6"
      },
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9a8699a145a94ed89b8b11d715067c3f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/3.98k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e41117d4fed1445daf6fb85a01cbc81e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer.model:   0%|          | 0.00/500k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6ba6646a5d5347059b14acdd79d6da07",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.84M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "00647dd5e8034abcb0217cc30e02177f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "added_tokens.json:   0%|          | 0.00/306 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6466f91527ee437ebd69ce5cb90bdf6f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Downloading dataset from https://www.thelmbook.com/data/news...\n",
            "\n",
            "Download completed.\n",
            "\n",
            "Listing archive contents:\n",
            "\n",
            "Archive member: news\n",
            "\n",
            "Archive member: news/train.txt\n",
            "\n",
            "Archive member: news/test.txt\n",
            "\n",
            "Extracting files...\n",
            "\n",
            "Extraction completed.\n",
            "\n",
            "Counting sentences in news/train.txt...\n",
            "\n",
            "Found 22034911 sentences in news/train.txt.\n",
            "\n",
            "Counting sentences in news/test.txt...\n",
            "\n",
            "Found 449693 sentences in news/test.txt.\n",
            "\n",
            "Training sentences: 22034911\n",
            "\n",
            "Test sentences: 449693\n",
            "\n",
            "Vocabulary size: 32011\n",
            "\n",
            "\n",
            "Total trainable parameters: 8589824\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   1%|          | 1561/172148 [01:47<3:13:16, 14.71it/s, loss=5.0692]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 5.3709\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:26, 23.90it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:23, 24.50it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.8423, Perplexity: 126.77\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's office said the government has been a `` new '' . '' '' 's `` new '' and the government has been a new role in the UK . '' '' 's `` new '' . '' '' '' 's `` new role in\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , ## , #### , #### , #### , #### , #### , #### , #### , #### , #### , #### , #### , #### , #### . 's # . 's # . 's # . 's # . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane with the most important thing , the most important thing , the most important thing , ' he said . ' '' 's `` I 'm not a good thing . '' '' '' '' . '' '' 's `` the United States\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   1%|          | 1565/172148 [01:54<35:22:01,  1.34it/s, loss=4.8445]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President Barack Obama has been a `` very good '' for the first time . '' '' 's newborn , the former president of the ####s . '' '' 's `` The United States and the United States '' . '' '' ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   2%|▏         | 3125/172148 [03:45<3:19:52, 14.09it/s, loss=4.8313]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.7426\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.56it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.42it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.6261, Perplexity: 102.11\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's office said the government had been `` a very strong '' . '' ' '' Aston Villa 's `` <rare> '' . '' '' . '' ' '' `` Weather . '' ' '' . '' 's `` <rare>\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City boss Jose Mourinho ( left ) and his wife of the . ) . ) . 's wife , ## , and the ##-year-old . ) . 's . 's , . 's . '' .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane Sandy Hook Elementary School , which was found in the airport , was found in the airport . ' '' 's office said . ' '' 's Office . ' '' . ' '' 's statement said . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   2%|▏         | 3127/172148 [03:49<30:42:07,  1.53it/s, loss=4.5720]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President Barack Obama has been charged with a murder investigation into the attack . '' 's death . ' '' 's statement . ' '' 's statement . ' '' 's statement . 's . 's . 's . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   3%|▎         | 4687/172148 [05:39<3:28:43, 13.37it/s, loss=4.6308]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.5903\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.86it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.24it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.5210, Perplexity: 91.92\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been re-elected by the U.S. military . '' 's official figures are not yet to be made . ' '' 's decision to . 's . 's report . '' 's . 's\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York : The couple of years ago , who was born in #### , was arrested in the early hours of Friday . 's #-# . 's #-#-# . 's # , and #-# . 's #\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a key part of the new film , which is a big deal . '' 's <rare> . ' '' 's website . ' '' 's website . ' '' <rare> . 's <rare> .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   3%|▎         | 4691/172148 [05:43<22:22:57,  2.08it/s, loss=4.4753]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military , which is a key part of the state . '' 's decision to take place in the area . '' 's . ' '' 's official report . ' '' <rare> said . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   4%|▎         | 6251/172148 [07:34<3:33:55, 12.93it/s, loss=4.5005]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.5015\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.69it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:26, 23.99it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.4343, Perplexity: 84.29\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a long-term economic crisis since #### . '' '' 'The FBI said . '' 'We 're not going to be a `` unacceptable '' . '' '' . '' '' . '' '' . '' '' . '' ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the company 's decision to be `` a `` thorough investigation '' . '' '' `` The FBI 's Office '' . '' '' `` Weather 's `` The FBI '' . '' '' . '' '' . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane has been a long-term problem with the sun , which is the most important thing that 's the most important thing to do . '' '' 'The <rare> said . '' . '' '' . '' '' . '' ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   4%|▎         | 6253/172148 [07:38<30:21:09,  1.52it/s, loss=4.5450]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been a `` significant concern '' for the presidential election campaign . '' '' `` The FBI 's decision to be the first time . '' '' '' . '' '' Ahmadinejad 's `` <rare> ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   5%|▍         | 7813/172148 [09:29<3:18:22, 13.81it/s, loss=4.4301]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4431\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:32, 23.01it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.65it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.3915, Perplexity: 80.76\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's military commander , said the government 's decision to be `` unacceptable '' . '' '' . '' '' . ' '' 's official statement . '' . 's the U.S. said . 's `` . '' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York : The ##-year-old was arrested in connection with the incident . ' '' . ' '' 'The .###-acre . ' '' 'The . 's I 's . '' '' . 's . 's I\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a very good thing . ' '' 'The <rare> ' . ' '' 's <rare> . ' '' 'The <rare> 's . ' '' ) . ' '' . ' '' 's . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   5%|▍         | 7817/172148 [09:33<22:09:05,  2.06it/s, loss=4.4911]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's decision to be the first minister , and the Senate 's decision to vote on the Senate . ' '' . ' '' . ' '' 's decision to be a `` warrant . '' '' . '' . '' '' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   5%|▌         | 9377/172148 [11:24<3:13:58, 13.99it/s, loss=4.4043]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4034\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.47it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.36it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.3506, Perplexity: 77.53\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with the offence of the US . ' '' . ' '' 's official . ' '' 's official said . ' '' 'We 's decision to `` the United States . '' '' . '' '' . ' '' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Mayor Mike Hutchinson said : 'We are not aware of the situation . ' '' 's decision to be the first time . ' '' 's decision to . 's decision to . 's . 's . 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been a huge problem with the sunset of the Earth 's surface . ' '' 's surface . ' '' 's surface . ' '' 's surface . 's surface . 's temperature of <rare>\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   5%|▌         | 9379/172148 [11:28<30:00:05,  1.51it/s, loss=4.3991]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President said the president 's office had been `` a `` very strongest '' . '' '' . ' '' 's decision to `` be a `` very important '' . '' '' . '' 's decision to . '' 's decision to be\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   6%|▋         | 10939/172148 [13:19<3:09:01, 14.21it/s, loss=4.3177]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3703\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.72it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.86it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.3255, Perplexity: 75.60\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major blow to the continent 's largest ever-sized country . '' 's ## % of the population . '' 's ## % . '' ) . 's ## . 's ## % . 's most of the\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Times reported that the ##-year-old was a `` very good '' . '' 's `` The New York Times '' . '' '' . '' 's `` The New York Times . '' '' . '' '' . '' ) . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a bit of a bit of a bit of a bit of a bit of a bit of a bit of a bit of a bit of a bit of a bit of a bit of a . ' '' . ' '' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   6%|▋         | 10943/172148 [13:23<21:29:11,  2.08it/s, loss=4.3868]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government was `` a `` very good '' . '' '' . '' 's `` the first time the president is a `` very good '' . '' '' . '' 's `` the president of . '' '' . ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   7%|▋         | 12503/172148 [15:14<3:10:46, 13.95it/s, loss=4.4349]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3388\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.78it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.79it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.3032, Perplexity: 73.94\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official said the `` unanimous '' of the government 's `` `` `` The United States '' '' is not a `` problem . '' '' . '' '' `` The U.S. government . '' '' . '' `` . ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Boris Johnson said the ##-year-old was 'the first-class ' . ' ' ' and 'fighting ' . ' 's `` No '' . '' `` The Late Late Late Late Late Late Late Late\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major blow to the moon , and the atmosphere is a bit more likely to be a bit more likely to be a bit more likely to be a bit more . 's atmosphere . '' 's atmosphere . 's atmosphere\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   7%|▋         | 12505/172148 [15:18<29:09:32,  1.52it/s, loss=4.4265]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government 's `` unanimous '' . '' '' `` The White House 's `` `` `` The White House '' '' . '' '' `` We 's `` `` `` No '' and IRS 's ``\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   8%|▊         | 14065/172148 [17:10<3:06:47, 14.10it/s, loss=4.3300]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3159\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.53it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.69it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.2833, Perplexity: 72.48\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has repeatedly denied the accusation against the United States , saying that the United States has been `` deeply concerned '' . '' . '' '' . '' ' I 'm not sure . '' 's . '' . 's position . '' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill Clinton , who has been accused of murdering the president of `` a ##-year-old son '' and was jailed for ## years . '' . '' 's . '' ) . '' . '' and the president\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a huge part of the world 's largest ever-known country . ' '' The <rare> . '' ' The <rare> ' . ' '' . '' 's <rare> of <rare> toddit\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   8%|▊         | 14069/172148 [17:14<21:49:45,  2.01it/s, loss=4.3073]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been criticised by the party 's presidential nominee for the first time . ' '' . ' '' I am not a member of the United States . '' 's . 's . 's . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   9%|▉         | 15629/172148 [19:05<3:03:01, 14.25it/s, loss=4.2351]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.2993\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:26, 23.89it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.78it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.2593, Perplexity: 70.76\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's presidential candidate , who has been accused of being a member of the U.S. military . 's office said the U.S. government has not been in the country . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York-based company , which is based on the number of people in the UK . 's <rare> . ' . 's . ' '' 'S . 's . 's . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a huge part of the world 's most popularity . ' '' 'Sullen . ' '' ' A . '' 's 'Sullen . 's . 's . 's of the world 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:   9%|▉         | 15631/172148 [19:09<28:21:05,  1.53it/s, loss=4.2543]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government was `` a very serious issue '' . '' ' '' . ' '' 'S. President Obama said . ' '' 'S. Secretary of State Hillary Clinton . 's `` . 's not\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  10%|▉         | 17191/172148 [21:00<2:59:28, 14.39it/s, loss=4.2538]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.2840\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:26, 23.90it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:26, 23.88it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.2486, Perplexity: 70.00\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's Foreign Ministry spokesman said the government 's government had not been `` in the country '' . '' '' . '' 's `` the U.S. government '' . '' '' '' . '' `` said . '' `` inev\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police said the suspect was not a `` unacceptable '' and `` the only person '' . '' '' '' . '' '' . '' '' . '' 's . '' 's . '' 's . '' 's . '' '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major city of the city of the city of the city of the city of the city of the city of <rare> . '' 's <rare> . '' . 's . 's <rare> . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  10%|▉         | 17195/172148 [21:04<20:33:39,  2.09it/s, loss=4.1595]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had been `` very close '' to the White House . '' '' 's decision to be a `` unacceptable '' . '' '' . '' 's `` a `` <rare> '' . '' '' is\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  11%|█         | 18755/172148 [22:55<3:01:59, 14.05it/s, loss=4.2031]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.2690\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.76it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.40it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.2345, Perplexity: 69.03\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow and the United States have been linked to the Soviet Union . ' '' 's . ' '' 's . ' '' 's . ' '' 's . 's . 's president , '' said . 's chief executive of the president\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Mayor Michael Bloomberg said the `` right-wing '' was `` very friendly '' . '' '' . '' '' 's `` `` The '' and `` The '' '' . '' '' '' . '' '' was the . '' `` 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be seen in the southern city of Kangar , where the city is expected to be in the city of Kiev . 's . 's . 's . 's . 's . ' '' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  11%|█         | 18757/172148 [22:59<28:53:38,  1.47it/s, loss=4.2755]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been accused of trying to protect the country 's economy . ' '' 's . ' '' 's . ' '' 's . ' '' 's . 's . 's . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  12%|█▏        | 20317/172148 [24:51<3:02:55, 13.83it/s, loss=4.1744]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.2525\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.59it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.56it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.2276, Perplexity: 68.55\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's Foreign Ministry of Defence , which is a major political crisis , is a major political crisis . '' ] . ' '' ' A . ' '' ' . ' '' ' . ' '' 's . 's a . 's .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Times , which is the most important thing to do is to be . ' '' ' . ' '' ' . ' '' ' . ' '' ' I 've got a lot of things . '' ) . 's . '' '' . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be seen in the city of the city of the city of the city of the city of the city of the city of the city of <rare> . ' . ' . ' . ' . ' . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  12%|█▏        | 20321/172148 [24:55<20:43:22,  2.04it/s, loss=4.2535]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had been `` very difficult '' . '' '' . ' '' 's decision to stop the attack . ' '' ' I 'm not going to be a . '' 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  13%|█▎        | 21881/172148 [26:47<3:05:04, 13.53it/s, loss=4.2825]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.2444\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 2/3514 [00:00<03:02, 19.28it/s]\u001b[A\n",
            "Evaluating:   0%|          | 5/3514 [00:00<02:45, 21.14it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.2101, Perplexity: 67.36\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of killing the ##-year-old . ' '' 's name . ' '' 's name . ' '' 's name . ' '' 's . 's apartment . 's a . 's . 's\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Boris Johnson said the government had `` no chance '' of the `` terrorist attack . '' '' . '' '' . '' 's `` <rare> '' . '' '' . '' '' . '' 's . '' 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major blow to the sea of the sea of the sea . '### . ' '' 's <rare> . ' '' 's <rare> . 's . 's . 's <rare> .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  13%|█▎        | 21883/172148 [26:51<28:03:25,  1.49it/s, loss=4.3492]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been accused of `` the first time in the ####s '' . '' 's `` <rare> '' . '' '' . '' '' . '' 's `` <rare> '' . '' '' . '' 's . '' 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  14%|█▎        | 23443/172148 [28:42<2:51:04, 14.49it/s, loss=4.1522]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.2329\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:25, 24.09it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:25, 24.18it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.2024, Perplexity: 66.84\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's nuclear weapons program has been used in the United States , and has been used to be a major threat to the country 's nuclear weapons . ' '' 's statement said . 's . '' 's . 's . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the government 's decision to take on the issue . ' '' 's decision to 's decision . ' '' 's decision to 's . 's 's . 's . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a huge blow to the sea . '' '###°C . ' '' 's . '' '' 's . '' '' 's ISIL 's . '' 's . '' 's a <rare>\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  14%|█▎        | 23447/172148 [28:46<20:16:13,  2.04it/s, loss=4.3136]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's decision to take on the president 's decision to re-election campaign . '' 's decision to . ' '' 's decision to . ' '' 's decision to . 's decision to . 's decision to .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  15%|█▍        | 25007/172148 [30:38<2:56:35, 13.89it/s, loss=4.1860]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.2203\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.86it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.20it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1981, Perplexity: 66.56\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been criticised for the `` unacceptable '' of the U.S. government . '' ) . '' '' 's report on the U.S. 's . '' 's . 's <rare> .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government 's policies are 'cold ' and 'cold ' . ' '' 'Sometimes the . ' '' 's . 's `` `` `` . '' '' '' '' is a few\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be seen as a Category # storm . '' ] . '' ] . '' 's <rare> . '' 's <rare> . ' '' 's a . ' '' ' I am a big . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  15%|█▍        | 25009/172148 [30:42<27:48:29,  1.47it/s, loss=4.2120]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been accused of using the `` <rare> '' and `` the best of the world . '' '' '' . '' '' 's `` The Washington Post '' . '' '' . '' '' ) . '' 's `` <rare> ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  15%|█▌        | 26569/172148 [32:34<2:51:51, 14.12it/s, loss=4.3145]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.2120\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.87it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.76it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1828, Perplexity: 65.55\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's military base is a major target for the U.S. military . '' ] . ' '' 's ##-year-old . '' 's ##-year-old . 's . 's . 's . 's\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said : ' I 'm sorry for the . ' '' 's decision to be a 'good job . ' '' ' . ' '' 's . 's 's . 's . 's '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a huge hit by a wave of snowstorms , which is a huge hit by the weather . '' '### . ' '' ' . ' '' 's . 's ##-year-e . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  15%|█▌        | 26573/172148 [32:38<19:36:57,  2.06it/s, loss=4.1841]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States has been in the country for ## years . '' 's ##-year-old son of the U.S. Army . '' ] . 's . 's . 's . 's ##-year-\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  16%|█▋        | 28133/172148 [34:30<2:50:53, 14.05it/s, loss=4.1260]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.2057\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:35, 22.54it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.31it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1801, Perplexity: 65.37\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has said it is `` a very dangerous situation '' . ' '' 's `` `` The United States is not a problem . '' '' '' . '' 's `` a `` <rare> '' . '' '' . '' '' is\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , the world 's most famous , is the most popular destination for the world 's most famous . '' '## . ' '' 's <rare> . ' '' 's <rare> . 's best . 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a tsunami , and the sun is a bit more complex than the sunshine . '' '## . ' '' ' I 'm a bit more . ' '' 's . ' '' 's . ' ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  16%|█▋        | 28135/172148 [34:34<26:21:15,  1.52it/s, loss=4.1731]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had `` no idea '' . 's `` a lot of people . '' '' '' . '' 's `` the . '' '' . '' '' . '' '' . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  17%|█▋        | 29695/172148 [36:25<2:47:45, 14.15it/s, loss=3.9899]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1957\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.59it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.59it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1744, Perplexity: 65.00\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major concern since the ####s . ' '' 'S . ' '' 's . ' '' 's . ' '' 's `` The ##-year-old '' . '' '' . '' '' . 's . '' ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police said the man was 'very angry ' . ' ' '' . ' '' ' : `` I 've been a good thing . '' '' ' . 's `` I 's . '' '' . '' '' . 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be the first to be the first to be the first to be the first to be the first to be the first to be the first to be the .###-hee . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  17%|█▋        | 29699/172148 [36:30<19:14:58,  2.06it/s, loss=4.3300]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has said that the country 's most recent political party has been a `` very strong '' . ' '' . ' '' . ' '' `` I 've been a `` very strong '' . '' '' . '' '' . '' '' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  18%|█▊        | 31259/172148 [38:21<2:47:00, 14.06it/s, loss=4.3159]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1944\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:25, 24.13it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:26, 23.96it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1682, Perplexity: 64.60\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major problem with the U.S. military and the U.S. military . '' ] . '' 's statement . '' 's statement . '' 's statement said . 's spokesman . 's spokes\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police Chief Michael Brown said : 'We are investigating the circumstances surrounding the incident . ' '' . ' '' . ' '' 's . ' '' 's . 's . 's . 's . 's . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a huge storm of the city 's east coast . '' ' A . '' ' A . ' '' ' A . '' 's statement . '' 's . 's . 's . 's . '' 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  18%|█▊        | 31261/172148 [38:25<25:48:15,  1.52it/s, loss=4.2346]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government would not be able to `` stop '' . '' . '' 's `` the first time the government is to be held in the UK . '' '' . '' 's . '' `` . '' `` inability\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  19%|█▉        | 32821/172148 [40:17<2:52:46, 13.44it/s, loss=4.0578]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1843\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:31, 23.17it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.10it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1577, Perplexity: 63.92\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been fighting for the past few years , and the United States has been fighting for the past few years . ' '' 's . '' 's . 's . 's . 's . 's . 's . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , Florida , Florida , was killed in the ####s , and the ##-year-old was killed by a gunman . '' 's . 's . 's . '' 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be reached for a week . '' ' A . ' '' ' A. <rare> . ' '' ' A . ' '' 's . ' '' 's 's . ' '' 's in the first\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  19%|█▉        | 32825/172148 [40:21<19:16:29,  2.01it/s, loss=4.2439]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has said that the government has not been able to do so . '' 's decision to take a long time . ' '' 's decision to . ' '' 's . 's . 's . 's . 's . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  20%|█▉        | 34385/172148 [42:13<2:45:36, 13.86it/s, loss=4.2394]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1768\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.55it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.44it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1568, Perplexity: 63.87\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been working on the US to help the country 's economy . '' 's official said . ' '' I 've been in the world . ' '' . ' '' . 's . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Times : The story is the first time the U.S. president has been charged with murder . ' '' I 'm not a U.S. citizen . '' 's statement said . 's . 's . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be the first to be in the world to be in the world . '' ' I 'm not going to be a very different point . '' 's . ' '' . 's I-in-like .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  20%|█▉        | 34387/172148 [42:17<25:00:03,  1.53it/s, loss=4.1129]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's first-choice goalkeeper , who has been in the world 's first-team squad . '' 's . '' 's . ' '' I 've been a great player . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  21%|██        | 35947/172148 [44:08<2:40:49, 14.11it/s, loss=4.1404]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1721\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:26, 23.92it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.60it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1540, Perplexity: 63.69\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has also criticised the U.S. military and the United States . ' '' 's statement said . ' '' 's statement . ' '' 's statement . ' '' 's . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said : ' I 'm not sure what I 'm doing . ' '' ' I 've done . ' '' 's `` and I 's . '' '' '' . ' '' 's ``\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center in the western city of the city of the city of the city of the city of the city of the city of <rare> . ' . ' '' 's . '' 's . ' . ' '' . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  21%|██        | 35951/172148 [44:12<18:10:22,  2.08it/s, loss=4.1024]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has said that the country 's most importantly support of the country 's economy . ' '' ' I 've been working with the government . ' '' 's `` . '' '' . 's `` . '' '' is a ``\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  22%|██▏       | 37511/172148 [46:04<2:41:28, 13.90it/s, loss=4.0550]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1687\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:37, 22.33it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:39, 21.94it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1477, Perplexity: 63.29\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major blow since the ####s . '###-###-#### . ' '' 'Sintelligent the U.S. Embassy . 's report . 's report in . 's . ' ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government would not be able to use the `` <rare> '' . '' `` We 're not going to be a good idea . '' '' `` `` will . '' '' . '' '' . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be closed . ' '' I 'm not sure if it 's not the first time . ' '' ' I 'm not a man . 's . ' '' . 's . 's . ' ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  22%|██▏       | 37513/172148 [46:08<25:00:20,  1.50it/s, loss=4.2787]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's decision to take a new presidential election . ' '' `` I 'm not a leader . '' '' '' . '' 's `` The White House '' is a `` <rare> . '' '' is austerity . ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  23%|██▎       | 39073/172148 [48:00<2:34:48, 14.33it/s, loss=4.1124]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1624\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.84it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.21it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1361, Perplexity: 62.56\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of making a `` significant contribution '' of the `` <rare> '' and `` the United States '' . '' '' . '' 's `` The U.S. 's `` a very high-security '' . '' '' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Mayor Michael Bloomberg said the government 's decision to take part in the investigation into the incident . ' '' 's decision to . ' '' 's decision . 's . 's said . 's . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-like storms are expected to be closed . '' 's the airport . ' '' I 've been in the area . '' ' '' I 've been in-sh . ' '' 's I a man .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  23%|██▎       | 39077/172148 [48:04<17:53:46,  2.07it/s, loss=3.9941]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government 's decision to `` stop the bill '' . '' 's decision to `` be made '' . '' '' . '' ' '' . 's `` a `` <rare> '' is . '' '' is .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  24%|██▎       | 40637/172148 [49:56<2:36:19, 14.02it/s, loss=4.0990]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1568\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.52it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.65it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1381, Perplexity: 62.69\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's military base is the latest in a series of attacks on the country 's military bases . '' ' . ' '' . ' '' . ' '' . ' '' 's . 's . 's . 's . 's .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the `` unfortunate '' of the `` unfortunate '' of the ####s . '' '' . '' 's `` The New York City '' . '' '' . '' '' `` The . '' ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been seen in the past few days . ' '' 's . '' 's . ' '' 's . ' '' 's . ' '' 's . ' '' 's . 's . ' '' ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  24%|██▎       | 40639/172148 [50:00<23:57:19,  1.52it/s, loss=4.1321]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's decision to withdraw the death penalty , which is expected to be held at the same time . '' ' . ' '' . ' '' . ' '' . 's . 's . 's . ' . ' . ' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  25%|██▍       | 42199/172148 [51:53<2:41:41, 13.39it/s, loss=4.0335]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1553\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:35, 22.54it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:41, 21.67it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1302, Perplexity: 62.19\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has also been accused of killing the two men and women in the past . ' '' . ' '' . ' '' ' . ' '' ' : . ' '' 's . 's . 's I 's a little . 's '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Times , and the . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' I 've got a little bit of a little bit . '' 's <rare> . ' '' 's ' I '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds of ##mph , the largest in the world , was the most popular destination for the world . '' ' . ' '' . ' '' ' . ' '' . ' '' ' . ' '' ' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  25%|██▍       | 42203/172148 [51:57<18:15:57,  1.98it/s, loss=4.1217]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the ####s , the first of the ####s , was a .##-caliber rifle . '' ) . ' '' . ' '' . ' '' 's . 's . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  25%|██▌       | 43763/172148 [53:49<2:34:02, 13.89it/s, loss=4.2219]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1512\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:36, 22.48it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:32, 22.98it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1367, Perplexity: 62.60\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major problem with the world 's largest city . ' '' 's . '' 's . '' 's . '' 's . '' 's . 's . 's a `` <rare> '' . 's ``\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old daughter , who was born in the UK , was a ##-year-old son . ' '' ' I 'm not to be able to . '' 's . '' 's . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major city of the world 's largest city of <rare> , which is the largest city of <rare> . '' ' . ' '' 's . ' '' 's . ' '' 's <rare>\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  25%|██▌       | 43765/172148 [53:53<23:34:21,  1.51it/s, loss=4.1210]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. government has been a major issue with the U.S. government . '' 's statement . ' '' . ' '' . ' '' . 's . 's . 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  26%|██▋       | 45325/172148 [55:44<2:29:25, 14.15it/s, loss=4.0737]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1470\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:38, 22.19it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.36it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1213, Perplexity: 61.64\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow , which is the first to be a major issue in the country . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's a <rare> . 's . 's a .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Times Square , which is the first time the . 's . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's The New Yorker . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds of ##mph , which is the first time the storm is expected to be a Category # storm . '###m . ' . ' . ' . ' . ' . ' . ' . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  26%|██▋       | 45329/172148 [55:48<16:53:56,  2.08it/s, loss=4.0375]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States , which is a major issue in the United States , is a major issue . '' 's . ' '' . ' '' . ' '' . 's . 's . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  27%|██▋       | 46889/172148 [57:39<2:26:20, 14.27it/s, loss=4.2514]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1426\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:39, 22.07it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:36, 22.36it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1247, Perplexity: 61.85\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major issue in the region . ' '' I 'm sorry to the United States . '' ' '' I 've been a very close relationship with the U.S. . 's the same . '' 's ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government 's decision to allow the government to take a `` step forward '' . ' '' . 's `` `` The '' . '' '' . '' '' . '' '' . '' '' . '' '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major concern for the country 's most recent wildlife . ' '' . '' ' . '' 's . ' '' . ' '' . 's . ' '' ' . ' '' ' . ' . ' '' '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  27%|██▋       | 46891/172148 [57:43<22:54:16,  1.52it/s, loss=4.2775]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The Presidential Medal of Freedom Party has been a `` great opportunity '' for the president . 's `` The Dark Knight '' . '' '' . '' 's `` The White House '' . '' '' is . '' . 's . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  28%|██▊       | 48451/172148 [59:35<2:26:31, 14.07it/s, loss=4.1754]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1393\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.85it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.69it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1114, Perplexity: 61.03\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow , where he was killed in the attack . ' '' 'Sentinel . ' '' 's . ' '' 's . ' '' 'Sentinel . 's . 's . 's a man . 's . 's\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said : 'The police are not aware of the incident . ' '' ' . ' '' 's . ' '' 's . 's . 's . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a tropical storm , and it is a tropical storm . '' ' . ' '' ' . ' '' ' . ' '' 'Sentinel . 's . 's . ' '' ' . ' '' ' . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  28%|██▊       | 48455/172148 [59:39<16:41:24,  2.06it/s, loss=4.1460]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said it was `` a very difficult situation '' . ' '' . ' '' `` I 'm not going to be a very good job . '' '' . 's `` <rare> '' is . '' 's 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  29%|██▉       | 50015/172148 [1:01:31<2:25:50, 13.96it/s, loss=4.1088]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1352\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.44it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.37it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1165, Perplexity: 61.34\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of being a member of the U.S. military . ' '' . ' '' . ' '' . ' '' . ' '' 's . 's . 's . 's a terrorist group . 's ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government 's decision to `` stop the attack '' . '' `` I 'm not a terrorist organization . '' '' '' . '' 's . '' 's . '' '' `` . '' '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is seen in the sky , and the sky is seen as the sun 's atmosphere . ' '' ' . ' '' ' . ' '' ' . ' '' 's . ' '' ' . 's . ' '' ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  29%|██▉       | 50017/172148 [1:01:35<22:13:56,  1.53it/s, loss=4.0820]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government 's decision to `` stop the attack '' . '' `` I 'm not a terrorist organization . '' '' . '' '' . 's `` a . '' `` a `` <rare> '' '' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  30%|██▉       | 51577/172148 [1:03:27<2:23:06, 14.04it/s, loss=4.1683]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1333\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.49it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.44it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1121, Perplexity: 61.08\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major blow for the past two years , but it has been a long way to the region . ' '' 's . ' '' 's . 's a . 's `` a long way to '' . 's . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said he was `` shocked '' and `` the . '' '' . '' '' `` I 've been in the area . '' '' . '' 's . '' 's `` Icelest '' . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be seen in the early hours of Sunday . ' . ' '' ' I 'm not going to be a bit more . ' '' ' . ' '' ' . ' '' ' . 's of the U.\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  30%|██▉       | 51581/172148 [1:03:31<16:18:01,  2.05it/s, loss=4.0689]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government 's decision to take place in the next election . ' '' . ' '' ' I 'm not sure . ' '' 's decision to . 's decision . 's decision was . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  31%|███       | 53141/172148 [1:05:23<2:18:41, 14.30it/s, loss=4.1115]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1282\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.62it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:32, 22.93it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1167, Perplexity: 61.36\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major threat to the United States since #### , but it has been a long-awaited move to the United States . '' ' . ' '' 's a . 's presidential . 's 's . 'rex\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , who has been charged with murder . ' '' It 's not a case of a murder . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's . 's ## ,\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to rise to #.# million .# million miles from the city of <rare> .### miles from the capital . '### . 's <rare> . ' . 's-a . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  31%|███       | 53143/172148 [1:05:27<22:08:46,  1.49it/s, loss=4.0496]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been working with the government to help the public to help the public to help the public . ' '' . ' '' . ' '' . 's . 's . 's . 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  32%|███▏      | 54703/172148 [1:07:18<2:19:30, 14.03it/s, loss=4.1919]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1298\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.43it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.84it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1055, Perplexity: 60.67\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised for the `` unprecedented '' of the United States , which has been a major . '' 's official statement . '' 's statement . . 's . 's a spokeswyned . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's top-up shower , which is the first to be a major event in the world . '' ' . ' '' . ' '' . '' 's . 's . '' 's . 's <rare> .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been hit by a massive explosion in the area . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . 's . '' 's . '' . 's I. ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  32%|███▏      | 54707/172148 [1:07:22<15:37:14,  2.09it/s, loss=4.0970]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said it was `` not clear '' . '' 's request was a `` very important step '' . '' '' . '' ' '' The . 's `` a `` <rare> '' . '' '' was . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  33%|███▎      | 56267/172148 [1:09:13<2:21:56, 13.61it/s, loss=4.0903]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1227\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.72it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.23it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0975, Perplexity: 60.19\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has said it has been a `` very difficult '' . ' ''### . ' '' `` I 'm not going to be a problem . '' '' . '' 's `` I 's not . '' '' '' . ' '' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government had been `` very open '' . ' '' . ' '' `` I 'm not going to be a very good job . '' '' `` I 's . '' '' '' . '' ''s\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be delayed by a week later . ' '' ' I 'm not going to be a problem . '' ' '' I 'm not going to be a problem . ' '' . 's ofhero . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  33%|███▎      | 56269/172148 [1:09:17<21:11:26,  1.52it/s, loss=4.1839]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has said that the government has not yet responded to the issue . ' '' . ' '' ' I 'm not sure . ' '' 's . ' '' . 's . 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  34%|███▎      | 57829/172148 [1:11:09<2:14:24, 14.18it/s, loss=4.0432]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1202\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.82it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.34it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.1014, Perplexity: 60.42\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the past since #### , with the United States and the United States . '' ' Ahmadinejad . '' ' Ahmadinejad of the U.N. 's 's ' . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government 's `` unacceptable '' of the government 's policy . '' 's statement was `` a `` <rare> '' . '' '' . '' . 's `` a year '' .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane has been seen by a large cloud of smoke in the sky . '' '### . ' '' ' A . ' '' 'Sentinel . 's . 's 's . 's 'herocious ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  34%|███▎      | 57833/172148 [1:11:13<15:33:44,  2.04it/s, loss=4.1278]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the #### election , which was the first president to be elected president of the Senate . '' ) . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's 's . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  35%|███▍      | 59393/172148 [1:13:06<2:14:47, 13.94it/s, loss=4.1885]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1192\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.58it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.37it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0985, Perplexity: 60.25\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised for the `` unacceptable '' and `` a `` very important step '' to ensure the United States is not a `` very important step '' . '' '' . 's `` a `` a `` a `` a `` in ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police Department officials said the suspects were not involved in the attack . 's arrest . ' '' ' . ' '' ' . ' '' 's . 's . 's . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to hit the ground in the southeast of the capital , where the storm is expected to be delayed . '' 'The .### . 's . 's . ' '' . 's 's 's second\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  35%|███▍      | 59395/172148 [1:13:10<20:49:00,  1.50it/s, loss=4.0960]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been in the past . ' '' 's office said . ' '' 'Jod said . ' '' ' I 'm not going to be a `` very good '' . 's `` a `` a `` <rare> '' ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  35%|███▌      | 60955/172148 [1:15:01<2:14:04, 13.82it/s, loss=4.0482]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1175\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:31, 23.22it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.77it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0966, Perplexity: 60.14\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been trying to reach the U.S. government . ' '' 's chief executive , '' he said . 's chief executive of the U.S. . 's chief of the U.S. . 's chief of the\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government had been `` very close to the world 's most powerful and unbelievable '' . ' '' `` I 'm not going . '' '' . '' ''s . '' ''s .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major blow to the south of the country . '' 's . ' '' ' A-levels , '' he said . ' '' 's . 's a <rare> . 's that ofe 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  35%|███▌      | 60959/172148 [1:15:05<14:51:52,  2.08it/s, loss=4.0808]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had no plans to use the `` <rare> '' . '' 's statement . ' '' . 's aired statement . 's statement . 's statement said . 's . 's statement .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  36%|███▋      | 62519/172148 [1:16:57<2:11:47, 13.86it/s, loss=4.2047]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1115\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:36, 22.48it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:33, 22.85it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0935, Perplexity: 59.95\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been a major source of the country 's economic crisis . ' '' Weibo . ' '' ' . ' '' We 're not going to be a country . 's . 's `` a `` <rare>\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill Gates said the government had `` no idea what happened to the president '' . ' '' `` I 've had to be a `` very stronghold '' . '' '' . '' `` `` I '' . '' '' . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major blow to the Atlantic coastline . ' '' Weiner said . ' '' We 're not going to be a `` very strong '' . ' '' . ' '' . 's . 's . 's '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  36%|███▋      | 62521/172148 [1:17:01<20:13:12,  1.51it/s, loss=4.1015]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had `` no idea what happened to the people '' . ' '' `` I 've had to be a `` very stronghold '' . '' '' . 's `` <rare> '' . 's `` <\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  37%|███▋      | 64081/172148 [1:18:52<2:08:43, 13.99it/s, loss=4.1080]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1087\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.48it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.42it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0938, Perplexity: 59.97\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised for failing to be a member of the United Nations Security Council . ' '' . ' '' I 'm not going to be in the process . ' '' . 's `` . '' . ' . 's ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government would be `` a very difficult situation '' . '' 'We are not going to be in the process . '' 's `` <rare> '' . '' '' . ''s . '' `` ``\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been forced to evacuate . ' '' . ' '' I 'm not going to be a big part of the city . '' ' I 'll be able to . ' . ' '' . ' '' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  37%|███▋      | 64085/172148 [1:18:57<14:31:11,  2.07it/s, loss=4.0985]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been in the process of negotiating the deal . '' ' I 'm not going to be in the process . ' '' . ' '' . ' '' . 's `` the . '' ' . '' ' '' is the . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  38%|███▊      | 65645/172148 [1:20:48<2:06:45, 14.00it/s, loss=4.0689]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1070\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:35, 22.60it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:33, 22.88it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0922, Perplexity: 59.87\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised for its `` huge '' and `` the United States '' and `` the United States '' and `` the United States '' of the United States . '' '' . '' ) . '' ''s to the U.S. policy .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , where the ##-year-old was born in #### , was born in the ####s . '' ) . ' '' . '' . ' '' . '' ) . 's . '' 's . 's . 's <\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center in the city of <rare> , the city of <rare> , is a <rare> . ' '' . ' '' . ' '' . 's . '' <rare> . '' ) <rare> . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  38%|███▊      | 65647/172148 [1:20:52<19:37:39,  1.51it/s, loss=4.2062]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States is a major issue with the United States . '' ) . '' . '' . '' . '' . '' . ' '' . ' '' . 's . 's . 's . 's . 's . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  39%|███▉      | 67207/172148 [1:22:43<2:03:52, 14.12it/s, loss=4.0752]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1035\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:30, 23.32it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:34, 22.72it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0901, Perplexity: 59.74\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised for its `` economic and economic '' and the `` <rare> '' of the United States . '' '' 's `` <rare> '' is the `` most important '' . '' '' . '' `` a long time '' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the government was `` not going to be a good job '' . ' '' 'We 're not going to be . ' '' 's . ' '' 's . 's . 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be seen as a result of the storm . ' '' 's statement . ' '' ' . ' '' ' I 'm not sure . ' '' 's . ' '' ' . ' '' . ' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  39%|███▉      | 67211/172148 [1:22:47<14:05:22,  2.07it/s, loss=4.1724]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the White House has said it is not clear if the president has a `` right to be a political issue '' . ' '' ' I 'm not . ' '' 's . 's . 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  40%|███▉      | 68771/172148 [1:24:39<2:00:57, 14.24it/s, loss=4.2127]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1053\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:31, 23.22it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.11it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0796, Perplexity: 59.12\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major international international international . ' '' ' I 'm not a player . ' '' ' '' I 'm not a player . '' ' '' I 'm not a player . 's a very good . ' '' 're\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , where the ##-year-old was born in #### , was born in #### . ' '' ' I . ' '' ' I 'm not a child . 's . '' 's . 's . '' 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major storm of rain and winds of about ##mph in the north of the country . ' '' ' . ' '' ' . ' '' ' . ' '' ' . ' '' 'or-off-up .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  40%|███▉      | 68773/172148 [1:24:43<18:49:53,  1.52it/s, loss=4.0310]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the ####s , the president of the U.S. , has been a major political issue in the Middle East . '' 's . ' '' 's . 's . 's . 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  41%|████      | 70333/172148 [1:26:34<2:01:02, 14.02it/s, loss=4.0916]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.1022\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.84it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:34, 22.77it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0747, Perplexity: 58.83\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major source of concern about the country 's economic crisis . ' '' ' . ' '' ' . ' '' ' . ' '' 's . 's . 's . 's a long . 's . 're-\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the government had been `` very strong '' . '' '' . ' '' `` I 've been in the area '' . '' '' . 's . 's . 's . '' '' `` s\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds have been seen in the area . '###m . ' '' ' . ' '' ' . ' '' ' . ' '' ' . ' '' ' . ' . ' . ' . ' . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  41%|████      | 70337/172148 [1:26:38<13:35:37,  2.08it/s, loss=3.9764]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States is a key issue in the country 's history . '' ' . ' '' ' . ' '' ' . ' '' ' I . 's . 's . 's . '' 's . 's . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  42%|████▏     | 71897/172148 [1:28:29<1:58:53, 14.05it/s, loss=4.0568]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0997\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.66it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:38, 22.15it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0759, Perplexity: 58.90\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has said it is unlikely that the government would be able to make the country 's economy . '' ' . ' '' `` I 'm not going to be a problem . '' '' . 's . 's . 's\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police Chief Michael Brown said the investigation was `` deeply saddened '' . ' '' . ' '' `` I 'm not sure . '' '' . '' 's . 's . 's . '' `` is . '' . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major blow to the coastal coastal coastal city of <rare> , which is the first time the country has been in the country . '' 's .xal . 's . ' . ' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  42%|████▏     | 71899/172148 [1:28:33<18:15:22,  1.53it/s, loss=4.2145]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's decision to allow the government to make a decision to allow the government to make a decision . '' ' . ' '' . ' '' . ' '' . 's . 's . '' 's . '' . 's . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  43%|████▎     | 73459/172148 [1:30:25<1:55:41, 14.22it/s, loss=4.1151]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0980\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:25, 24.12it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:33, 22.90it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0797, Perplexity: 59.13\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's presidential candidate , Mr Trump , said the .### was `` a . '' '' . '' '' . ' '' `` I 'm not going to be a . '' '' '' . ' '' `` `` is a `` and ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the government had been `` very concerned '' and `` the government was `` not going to be in the right direction '' . '' '' . ' '' `` `` is '' . '' '' . '' '' ``\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane warning was issued by the National Oceanic , which was the first to be seen in the region of the Pacific Ocean . ' '' ' . ' '' ' . ' '' . ' . ' . ' . ' . ' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  43%|████▎     | 73463/172148 [1:30:29<13:11:15,  2.08it/s, loss=4.2046]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been trying to get the Senate 's nomination . '' ' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  44%|████▎     | 75023/172148 [1:32:21<1:57:45, 13.75it/s, loss=4.1072]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0957\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:32, 22.98it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:37, 22.29it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0783, Perplexity: 59.05\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised for the `` unacceptable '' and the United States . '' 's aunt said . ' '' 's . ' '' 's . 's `` a . '' '' '' . 's a `` a `` a\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police Department said the investigation was `` not a crime '' . '' ' '' . ' '' 's office was not immediately available . ' '' ' . 's . 's . 's . 's a . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center , where the storm was hit by a storm , washed down the coast of the city . ' '' ' . ' '' ' . ' '' . 's . ' . 's . 'staking . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  44%|████▎     | 75025/172148 [1:32:25<17:53:32,  1.51it/s, loss=4.0381]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The Presidential Medal of Freedom of Information Act , the National Security Agency , said the government would not comment on the case . '' ' . ' '' ' . 's . 's . 's . 's . 's . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  44%|████▍     | 76585/172148 [1:34:16<1:52:49, 14.12it/s, loss=3.9817]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0943\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 2/3514 [00:00<02:55, 19.97it/s]\u001b[A\n",
            "Evaluating:   0%|          | 5/3514 [00:00<02:39, 22.02it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0715, Perplexity: 58.65\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a long since the ####s , and it is a long way to the United States . '' ' I . ' '' ' I 'm not going to be . 's . 's a long . 's . 'thank\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government had been `` a very good '' . '' ' I 've been in the United States . '' ' I 've been . 's . 's . 's . 's aides\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be a Category # storm , but the storm is expected to be expected to be expected to be expected . '' ' said . ' '' ' . 's . ' . ' . 's . ' . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  44%|████▍     | 76589/172148 [1:34:20<12:44:41,  2.08it/s, loss=4.0339]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The Presidential candidate , who has been a member of the U.S. government , has been a member of the U.S. government . 's statement . 's statement .i .i . 's .i . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  45%|████▌     | 78149/172148 [1:36:11<1:52:03, 13.98it/s, loss=4.0926]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0937\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:31, 23.25it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.32it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0717, Perplexity: 58.66\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has also been criticised for the `` significant '' and the U.S. government . ' '' `` I 'm not going to be a threat to the United States . '' '' . 's a `` a long-de . '' ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , who has been in the United States since #### , has been accused of . ' '' 'The . ' '' . ' '' . ' '' . ' '' . 's a <rare> . 's the U.S. president\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been hit by a tornado in the north of the city of San Diego .## .### . '### . ' . ' . ' . ' . ' . ' . 's ' . ' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  45%|████▌     | 78151/172148 [1:36:15<17:13:11,  1.52it/s, loss=4.1526]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the bill was `` a significant step toward the law . '' '' . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  46%|████▋     | 79711/172148 [1:38:07<1:52:31, 13.69it/s, loss=4.0990]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0901\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.60it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.28it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0678, Perplexity: 58.43\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major problem with the United States since #### , and the United States has been in the United States . ' '' '' . ' '' ' I 've been . 's . 's . 's . 's 'reled\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomfield said the company was `` a very good person '' . ' '' `` I 'm not sure what happened . '' '' '' . '' '' 's . 's . '' '' 's <rare> .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major problem with the storm . ' '' ' I . ' '' ' I . ' '' ' I . ' '' ' I . ' '' 's . ' '' . ' '' ' . '' 's spokesw\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  46%|████▋     | 79715/172148 [1:38:11<12:42:58,  2.02it/s, loss=4.1323]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said it was `` a very difficult decision '' . ' '' `` I 'm not sure what happened . '' '' '' . '' ' '' I 've been . 's . 's . '' `` I.Jet\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  47%|████▋     | 81275/172148 [1:40:03<1:48:55, 13.90it/s, loss=4.0383]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0911\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:42, 21.59it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:32, 22.94it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0616, Perplexity: 58.06\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised for its `` great-grandfather '' and `` a great deal of the world 's most famous . '' '' . '' ' . '' 's . '' 's . '' 's the . '' 's . ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said he was `` shocked '' by the incident . '' 's . ' '' `` I 've been in the process of the investigation . '' '' . '' '' . '' 's . '' 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be cancelled . ' '' ' The . '' ' . ' '' ' I . ' '' ' I . ' '' ' I . 's . '' ' . ' . ' . 'st of me .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  47%|████▋     | 81277/172148 [1:40:07<16:39:20,  1.52it/s, loss=4.0259]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been accused of `` a very serious crime '' and said he had been `` a very good person . '' '' . '' ' '' . ' '' 's . 's . '' 's . 's . '' 's 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  48%|████▊     | 82837/172148 [1:41:59<1:45:17, 14.14it/s, loss=4.0905]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0865\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.82it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.28it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0708, Perplexity: 58.60\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major problem for the United States since #### . ' '' `` I 'm not going to be a good thing . '' '' . '' `` I 'm not a Republican . '' '' . 's `` a long time '' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the new government would be `` a very good thing to do '' . '' ' '' . ' '' `` I 'm not . '' `` I 'm not . '' '' . '' '' . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to reach the ##th anniversary of the storm . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's-like . ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  48%|████▊     | 82841/172148 [1:42:03<12:00:22,  2.07it/s, loss=4.0883]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The Presidential Medal of Freedom , the first time in #### , has been to be a member of the European Union . '' 'The . 's . 's . '' . 's . '' . 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  49%|████▉     | 84401/172148 [1:43:55<1:46:01, 13.79it/s, loss=4.2083]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0849\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:35, 22.59it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:33, 22.81it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0679, Perplexity: 58.43\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has also said it has not been able to comment on the issue . ' '' ' I 've been . ' '' ' . ' '' `` I 've been a very good . '' '' . 's . 's . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company has not yet responded to the report . ' . ' '' `` I 'm not sure what the company has . '' '' . '' '' . 's . 's . '' . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds of the storm fell in the middle of the weekend , the National Weather Service said . ' '' . ' '' . ' '' . 's . 's . ' . 'staking . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  49%|████▉     | 84403/172148 [1:43:59<16:06:51,  1.51it/s, loss=3.9369]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has said that the country has not been able to control the country 's economic sanctions . ' '' `` I 'm not going to be a problem '' . '' 's `` <rare> . '' '' . '' '' . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  50%|████▉     | 85963/172148 [1:45:51<1:43:12, 13.92it/s, loss=4.0335]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0837\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:35, 22.63it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.31it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0663, Perplexity: 58.34\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow , the country 's capital of the capital , has been accused of . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company 's chief executive of the National Park Service , which has been a major concern for the company . 's decision to payments . 's . 's . 's . 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major storm of snow . 's forecast for the winter storms . ' '' ' I 've been in the area . ' '' . 's . 's . ' . 'stow . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  50%|████▉     | 85967/172148 [1:45:55<11:41:40,  2.05it/s, loss=4.1035]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the plan was `` a `` very important step forward '' . '' '' . 's `` The FBI '' . '' `` The FBI '' is not the first . '' `` <rare> '' . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  51%|█████     | 87527/172148 [1:47:46<1:39:32, 14.17it/s, loss=4.0950]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0845\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.58it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.09it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0590, Perplexity: 57.92\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised for its involvement in the attacks . ' '' 's . ' '' 's . ' '' . ' '' . ' '' . ' '' 's . ' '' 's the . 's . 's ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government would not be able to confirm the report . ' '' ' I 'm not sure that the president is not in the case . ' '' . 's . 's . 's the way\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane warning has been issued for the past two years . ' '' 's . ' '' ' I 'm not sure that the U.S. is not the most commonplace in-like . 'stank . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  51%|█████     | 87529/172148 [1:47:50<15:28:12,  1.52it/s, loss=4.1391]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the president 's office said the president was `` a very good thing '' . ' '' `` I 'm not going to be a good thing . '' '' '' . ' '' 's . 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  52%|█████▏    | 89089/172148 [1:49:42<1:40:00, 13.84it/s, loss=4.1239]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0819\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.66it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.74it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0591, Perplexity: 57.92\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major problem with the United States since the ####s . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' . 's . ' . '' . 's . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government would be `` a good '' . ' '' `` I 'm not going to do anything . '' '' . '' '' . ' '' . 's . '' 's . '' '' . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center in the U.S. , where the ##-year-old was killed in the crash . '' ' . ' . ' . ' '' . ' . ' '' . ' . ' . '' <rare> . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  52%|█████▏    | 89093/172148 [1:49:46<11:07:51,  2.07it/s, loss=4.1311]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had `` no '' that the government would be `` inappropriate '' . '' '' '' . ' '' 'The . 's . 's . 's . 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  53%|█████▎    | 90653/172148 [1:51:38<1:35:40, 14.20it/s, loss=4.0792]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0811\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.72it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:32, 22.99it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0551, Perplexity: 57.69\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has not yet been able to confirm the agreement . ' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's . ’ s . 's . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the `` is a very difficult decision '' . ' '' . 's `` a very big step in the future . '' '' . 's `` a <rare> '' . '' '' . 's ``\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane has been seen as a Category # storm . ' '' '### . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's of ##-th-\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  53%|█████▎    | 90655/172148 [1:51:42<15:16:50,  1.48it/s, loss=4.0991]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` is a very difficult decision '' . ' '' . 's `` a very good idea '' . '' ' '' . ' '' `` I 've been inability to . '' `` is '' . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  54%|█████▎    | 92215/172148 [1:53:34<1:33:47, 14.20it/s, loss=4.0667]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0790\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.63it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.45it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0559, Perplexity: 57.74\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's military presence in the region is a major threat to the country 's nuclear program . ' '' ' . ' '' ' . ' '' ' . 's . ' '' . ' . 's . ' . 's . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police Chief Constable John McCain , ## , said : 'The . ' '' ' I . ' '' ' I . ' ' '' 's . 's statement was the #### . ' . ' . ' . ' . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major concern for the airline , and it is not clear whether the plane was in the air . ' '' ' . ' '' ' . ' . ' . ' '' ' . ' '' ' . ' . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  54%|█████▎    | 92219/172148 [1:53:38<10:44:13,  2.07it/s, loss=4.0499]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said it was `` a very difficult decision '' . ' '' `` I 've been in the process of the process . '' '' '' . ' '' 's . 's . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  54%|█████▍    | 93779/172148 [1:55:30<1:33:12, 14.01it/s, loss=4.0964]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0776\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:30, 23.39it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.23it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0613, Perplexity: 58.05\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major political issue since the #### election . ' '' ' I 've been in the past . '' ' '' I 've been in the past . 's . '' 's . 's-like . 'ed insp\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill deals to the House of Commons . 's decision to take a long time . ' '' 'The . ' '' 's statement . '' 's statement said . 's . 's . '' 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds hit the coast of the South Pacific coast of England , the highest in the world .###m wide .###mph winds and winds of ##mphins .###s the .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  54%|█████▍    | 93781/172148 [1:55:34<14:21:09,  1.52it/s, loss=3.9851]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the plan was `` a very difficult situation '' . ' '' `` I 'm not going to get the right to the right . '' '' '' . 's `` <rare> '' . 's 's . '\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  55%|█████▌    | 95341/172148 [1:57:26<1:30:07, 14.20it/s, loss=4.0654]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0732\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.69it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.58it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0595, Perplexity: 57.95\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a key target for the United States since #### . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's the . 's . '-ed\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , California , and the state of California are not the only ones who are not . '' ' . ' '' . ' '' . ' '' . 's . '' 's . '' . 's the U.S.. and Texas\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been seen as a result of the storm . 's storms . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 'st ? ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  55%|█████▌    | 95345/172148 [1:57:30<10:16:58,  2.07it/s, loss=4.1072]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States has been a key ally of the United States since #### , according to the U.S. Department of Homeland Security . '' ) . '' 's . 's .i . 's . 's .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  56%|█████▋    | 96905/172148 [1:59:21<1:29:10, 14.06it/s, loss=4.0722]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0778\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.73it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.70it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0523, Perplexity: 57.53\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of a `` very serious '' and the government has been forced to withdraw from the country 's military . '' ' '' The . 's . '' 's . 's . '' . 's the . '' . 's\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the company was `` very proud of the idea of the project '' . ' '' . ' '' . ' '' . 's . 's . 's . 's . . . . . .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been hit by a stormy weather in the southern hemisphere .### miles north of the city . '### . ' '' . 's . 's . ' . 'staking . ' .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  56%|█████▋    | 96907/172148 [1:59:25<13:54:59,  1.50it/s, loss=4.1391]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States has been accused of a `` very serious '' in the United States . '' ) . '' 's . '' ' A.J . '' 's . 's . '' 's . '' ] . '' . ''\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  57%|█████▋    | 98467/172148 [2:01:16<1:26:42, 14.16it/s, loss=3.9674]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0748\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:25, 24.12it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:26, 24.01it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Validation Average Loss: 4.0494, Perplexity: 57.36\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major target of the conflict since the ####s . ' '' ' I . ' '' ' I . ' '' ' I . ' '' 's . ' '' 's . '' 's the . 's . 'rex\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police Department officials said the investigation was `` very unstable '' . ' '' ' I 've been in the area . ' '' ' I 've been . 's . 's . 's . 's . '' 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been seen in the past two years , with the largest snowfall in the north of the country . ' '' ' I . ' '' ' . 's . 's . 's . 'ed-control of .\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch 1/1:  57%|█████▋    | 98471/172148 [2:01:20<9:43:27,  2.10it/s, loss=4.1187] "
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the administration 's actions were `` very serious '' . ' '' 'but that the president 's actions were `` very serious '' . '' '' . ' '' `` IRS '' . '' `` I-General '' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  58%|█████▊    | 100031/172148 [2:03:11<1:24:56, 14.15it/s, loss=3.9867]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0717\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.82it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.11it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0532, Perplexity: 57.58\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major threat to the United States since the ####s . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' 's . 's . 's-based . 's <rare\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company 's `` most important '' is to be `` a very important thing to do '' . '' '' . ' '' 's `` is the <rare> . '' '' '' . '' '' ``\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be a major milestone in the Pacific Ocean , which is expected to be the first to be the first to be a major destination for the world . 's .xing . 's tot . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  58%|█████▊    | 100033/172148 [2:03:15<13:11:28,  1.52it/s, loss=4.0568]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has said that the government has been trying to get the power to the right . '' ' I 'm not sure . ' '' ' . ' '' 's . 's . 's . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  59%|█████▉    | 101593/172148 [2:05:07<1:24:55, 13.85it/s, loss=4.0608]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0703\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:30, 23.29it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.65it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0435, Perplexity: 57.03\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been trying to fight the strikes against the rebels in the region . ' '' 's . ' '' ' I . ' '' ' I . 's . ' '' . 's . ' . ' . 's . 's\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , which is the most popular destination for people , is the most popular destination for people . '' ' The . ' '' . ' '' . ' '' . 's . '' 's . 's the . 's . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be a Category # . '' 's . ' '' ' I 'm not going to be a big problem . ' '' ' I 'm not . ' '' ' . ' '' 'staking . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  59%|█████▉    | 101597/172148 [2:05:11<9:40:38,  2.03it/s, loss=4.1032] "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has said that the government has been trying to establish a `` significant '' of the government 's government . ' '' `` I 'm not sure what we 're doing . '' '' . '' '' . 's . 's . ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  60%|█████▉    | 103157/172148 [2:07:03<1:24:12, 13.66it/s, loss=4.0793]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0715\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.92it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.43it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0509, Perplexity: 57.45\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow denies the accusations . ' '' ' I have . ' '' ' I 'm not a member of the U.S. . ' '' 's . ' '' `` . '' '' `` “ `` <rare> ” of . ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government had no intention of being able to get the money from the country . '' ' I . ' '' ' I . 's . 's . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been named as the first of the year , but the weather has been delayed by the storm . ' '' ' I . ' '' ' I . 's . ' '' . ' . ' '' . ' '' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  60%|█████▉    | 103159/172148 [2:07:07<12:44:34,  1.50it/s, loss=3.9285]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States has been in the past , but the U.S. has been in the country since the ####s . '' ' I . ' '' . '' 's . '' 's . 's . ' . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  61%|██████    | 104719/172148 [2:08:59<1:20:56, 13.88it/s, loss=3.9755]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0696\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:30, 23.31it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.72it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0434, Perplexity: 57.02\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been trying to force the rebels to force the rebels to the rebels . ' '' ' said . ' '' ' . ' '' ' . ' '' ' . ' '' ' . ' . ' . ' . ' . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the `` unacceptable '' of the U.S. Embassy in the U.S. Embassy in the U.S. Embassy . 's . 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane has been seen in the past few days , with the weather forecasting forecast for the storm . ' '' '### . ' '' ' . 's . 'staking . ' . ' '' . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  61%|██████    | 104723/172148 [2:09:03<9:01:26,  2.08it/s, loss=4.0461] "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` very important '' of the `` very important '' . '' '' `` I 've been in the process of the process . '' '' '' . '' 's . 's . 's . 's . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  62%|██████▏   | 106283/172148 [2:10:56<1:20:00, 13.72it/s, loss=4.0420]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0682\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:35, 22.56it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.33it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0437, Perplexity: 57.03\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of killing the U.S. government in #### . ' '' ' I 'm not sure if it 's not a problem . ' '' . ' '' 's . 's the . ' . 's ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the `` unacceptable '' of the U.S. Embassy in the U.S. Embassy in the U.S. Embassy .### said the U.N\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be a major factor in the storm , which is expected to be a major factor in the storm . '' ' I . ' '' ' I 'm not . ' '' ' . ' '' ' '' . ' ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  62%|██████▏   | 106285/172148 [2:11:00<12:13:11,  1.50it/s, loss=4.0833]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government was `` deeply concerned '' and `` the `` very serious '' and the administration is not a viable option . '' '' . '' 's . '' 's . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  63%|██████▎   | 107845/172148 [2:12:53<1:17:00, 13.92it/s, loss=4.0694]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0657\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.84it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:34, 22.70it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0441, Perplexity: 57.06\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major target for the past few years . ' '' ' I . ' '' ' I . ' '' ' I 'm not a member of the public . ' '' 's . 's the . 's . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Times , the company 's chief executive , said the company 's decision to . ' '' ' . ' '' ' I . ' '' ' I . 's . 's . 's . 's . 's . ' .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds are expected to rise in the number of people in the region . '' 's . ' '' ' I . ' '' 's . 's . 's . ' '' ' . ' '' ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  63%|██████▎   | 107849/172148 [2:12:57<8:43:19,  2.05it/s, loss=4.0561] "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States has been in the past since #### , and the president has been in the country . '' ' I . ' '' ' I . ' '' 's . '' 's . 's . 's . ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  64%|██████▎   | 109409/172148 [2:14:50<1:13:37, 14.20it/s, loss=3.8702]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0668\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.82it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.46it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0364, Perplexity: 56.62\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticising the United States for its nuclear program , which has been widely criticised for its nuclear program . ' '' . ' '' ' I 've been a . ' '' 's . ' . 's . ' . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company was `` not a big-budget movie '' . ' '' ' I 've never seen a movie . 's `` The Falklands '' . '' 's <rare> . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been closed for several months . ' '' ' . ' '' ' I 've been a very quiet . ' '' ' I 've never seen it . ' '' 's . ' '' . 'er ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  64%|██████▎   | 109411/172148 [2:14:54<11:31:07,  1.51it/s, loss=4.0064]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the White House has said that the government has been committed to the transition . '' ' I 've never been able to . ' '' ' . ' '' 's . 's . '' 's . '' . 's . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  64%|██████▍   | 110971/172148 [2:16:47<1:13:59, 13.78it/s, loss=3.9269]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0631\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.68it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.30it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0416, Perplexity: 56.92\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a key part of the country 's nuclear program . 's . ' '' 's . ' '' 's . ' '' ' I . 's . 's . 's . 's the . 's . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company was `` not a good thing '' . ' '' 's `` a `` very good '' . '' '' . 's `` is a `` <rare> . '' '' '' . 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane warning is that the storm is still being hit by the storm . ' '' 's . ' '' ' . ' '' ' . ' '' 's . 's . . . . . . . . . . ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  64%|██████▍   | 110975/172148 [2:16:51<8:09:14,  2.08it/s, loss=4.0722] "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's decision to take place in the next election . '' ' I . ' '' ' I . ' '' ' I . ' '' ' I . 's . 's . '' 's . 's . 's . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  65%|██████▌   | 112535/172148 [2:18:43<1:13:23, 13.54it/s, loss=4.1254]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0650\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:30, 23.36it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:34, 22.75it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0329, Perplexity: 56.42\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been pushing for a new deal to bring in a new deal . ' '' 'The . ' '' ' I 've never been able to do anything . ' '' 's . 's . ' . ' . ' . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company 's decision to take the next step . ' '' 'The . ' '' ' I . ' '' 's . 's . 's . 's . 's . 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be expected to be expected to be expected to be expected to be expected to be expected to be expected to be completed . '' 'The next day . ' '' 's time ' . ' '' ' . ' ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  65%|██████▌   | 112537/172148 [2:18:47<10:54:40,  1.52it/s, loss=4.0837]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been in the process of being called the U.S. and the U.S. and the U.S. . '' 's . 's . 's . 's . ' . ' . ' '' . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  66%|██████▋   | 114097/172148 [2:20:39<1:09:22, 13.95it/s, loss=4.0461]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0649\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.78it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:32, 23.04it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0418, Perplexity: 56.93\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been a major target for the rebels in the region . ' '' ' . ' '' ' I 'm not sure . ' '' 's . ' '' . ' '' . ' . 's the . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government was `` not a very good idea '' . ' '' ' I 'm not sure the government is going to be . ' '' 's . 's . 's . '' 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane warning is that the airport is not a way to the airport . ' '' 's . ' '' ' I . ' '' ' I. <rare> . ' '' . ' '' . ' '' . ' '' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  66%|██████▋   | 114101/172148 [2:20:43<7:54:53,  2.04it/s, loss=3.8734] "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the administration had `` no intention of negotiating '' . ' '' `` I 'm not going to do anything '' . '' '' '' . '' 's office said . '' `` I havehtagues '' . ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  67%|██████▋   | 115661/172148 [2:22:35<1:07:34, 13.93it/s, loss=4.1192]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0643\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.69it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.66it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0374, Perplexity: 56.68\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been fighting for the rebels in the region since #### . ' '' ' Ibrahimovic . ' '' ' Ibrahimovic . 's . ' '' 's . ' . ' . 's ' . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's new home is a new home . ' '' ' I 'm not sure what you 're going to do . ' '' ' I 've been . ' '' 's statement . ' . ' . ' '' ' .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane has been seen since the storm hit the surface of the snowfall . ' '' ' I 've been in the sky . ' '' ' ITVU . 's . ' '' ' . ' '' ' . ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  67%|██████▋   | 115663/172148 [2:22:39<10:15:38,  1.53it/s, loss=3.9713]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The Presidential Medal of Honorary Committee , the first of the ####s , was the first to be released by the U.S. and the United States . 's name . 's name . ' . ' '' . ' '' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  68%|██████▊   | 117223/172148 [2:24:32<1:05:43, 13.93it/s, loss=4.1483]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0623\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 2/3514 [00:00<02:58, 19.64it/s]\u001b[A\n",
            "Evaluating:   0%|          | 5/3514 [00:00<02:39, 22.06it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0374, Perplexity: 56.68\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been fighting the rebels in the country . ' '' ' I 'm not going to be in the country . ' '' ' I 'm not going to be . ' '' . ' '' . ' '' . ' '' ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company was `` a very good '' . ' '' '' . ' '' `` I 'm not going to do it . '' ' '' . '' `` I ' . '' '' '' . '' `` ``\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been hit by a storm in the area . ' '' ' I 've been in the area . ' '' ' I 've been told . 's . ' '' ' . ' '' ' ' '' . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  68%|██████▊   | 117227/172148 [2:24:37<7:26:10,  2.05it/s, loss=4.1124] "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The Presidential Medal of Honor ceremony , which is the first time in the country , has been in the country . '' ' I . ' '' ' I . ' '' 's . 's . 's . 's ' '' . ' ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  69%|██████▉   | 118787/172148 [2:26:29<1:03:52, 13.92it/s, loss=4.0498]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0616\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 2/3514 [00:00<03:00, 19.49it/s]\u001b[A\n",
            "Evaluating:   0%|          | 5/3514 [00:00<02:41, 21.72it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0306, Perplexity: 56.30\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has said it has been a long time for the country to be able to make a decision . ' '' ' I 'm not going to be . ' '' ' . ' '' 's . ' . ' . ' . ' '' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the government would not be able to comment on the issue . ' '' ' I . ' '' ' I . ' '' 's . 's . 's . 's . . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane season has been a major hit by the storm . ' '' ' I 'm not going to be . ' '' ' I 'm not going to be . ' . ' '' ' . ' . 'er #- . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  69%|██████▉   | 118789/172148 [2:26:33<9:56:28,  1.49it/s, loss=4.0116]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government would not be able to provide a `` full-time job '' . '' ' '' . ' '' ' I 'm not a member of the family . 's . 's . 's . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  70%|██████▉   | 120349/172148 [2:28:25<1:01:25, 14.05it/s, loss=4.0902]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0591\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.77it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:34, 22.67it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0343, Perplexity: 56.50\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the region since #### . ' '' ' I 'm not going to be a good place . ' '' ' I 'm not going to be a good job . ' '' 's . ' '' ' . ' '' ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police Department spokesman Michael Gove said the case was `` a very serious case '' . ' '' '' . ' '' ' I 'm not sure the suspects are not . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be in the southern hemisphere . '' 's a . ' '' ' I . ' '' ' I 'm not sure . ' '' 's . ' '' ' . ' '' ' '' ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  70%|██████▉   | 120353/172148 [2:28:30<7:05:28,  2.03it/s, loss=4.0904]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has said that the government has `` no need to be able to do this '' . ' '' 'We are not going to be a good way to the country . ' '' 's . 's . 's . '' 's .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  71%|███████   | 121913/172148 [2:30:23<59:45, 14.01it/s, loss=4.0176]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0569\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.83it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.13it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0353, Perplexity: 56.56\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the U.S. since #### . ' '' ' I am not a member of the U.S. military . '' ' I . '' ' . '' ' . '' ' . '' ' . '' ) . '' ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , where the ##-year-old was in the hospital , was . ' '' ' I 'm not a member of the family . ' '' 's . '' 's . '' ' . 's <rare> . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds of the storm was delayed after the storm was delayed . ' '' ' I . ' '' ' I . ' '' ' I . ' '' 's . ' . ' '' ' . ' '' ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  71%|███████   | 121915/172148 [2:30:27<9:16:32,  1.50it/s, loss=4.1528]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the attack was `` very important '' . ' '' ' I 'm not sure what happened . ' '' ' I 'm not sure if he was going to . 's . 's . 's . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  72%|███████▏  | 123475/172148 [2:32:19<59:15, 13.69it/s, loss=3.9406]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0577\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.76it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.21it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0317, Perplexity: 56.36\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been trying to retake the country in the past . ' '' ' I . ' '' ' I 'm not a member of the U.N. Security Council . ' '' 's . ' . ' . 's ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , where the ##-year-old was born in #### , was born in #### . ' '' ' I 've been in the United States . ' '' 's . 's . 's ast . 's #### . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center in the city of <rare> , the largest of the world 's largest city , was the largest in the world . ' '' 's . 's . ' '' <rare> . ' . ' '' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  72%|███████▏  | 123479/172148 [2:32:24<6:34:54,  2.05it/s, loss=4.0573]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` unfortunate '' was `` a `` very important step '' . '' '' '' . ' '' ' I 've been in the process of the investigation . 's . 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  73%|███████▎  | 125039/172148 [2:34:16<55:39, 14.11it/s, loss=4.0175]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0553\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:30, 23.32it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.36it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0337, Perplexity: 56.47\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of killing a ##-year-old man in the .##-###-### . '### . ' '' 's . 's . ' '' ' . ' . ' . ' . ' ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the `` unprecedented '' of the city 's `` most importantly . '' '' ' '' . ' '' 's statement was the way . 's . 's . 's a day\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watches on the coast of the city 's coastal coastal city of the city of Kensington , where the city 's airport is in the city 's top . ' . 'ering of the .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  73%|███████▎  | 125041/172148 [2:34:20<8:38:20,  1.51it/s, loss=4.0952]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` unprecedented '' of the U.S. government , the U.S. military said . `` is a `` very important step '' . '' '' . '' 's a . 's `` in\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  74%|███████▎  | 126601/172148 [2:36:12<53:21, 14.23it/s, loss=4.0824]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0590\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:35, 22.54it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.12it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0260, Perplexity: 56.03\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's new government has been working with the United States since #### . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ) . ' . 's a . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Mayor Michael Bloomberg said : 'The . ' '' ' I 'm not going to be a . ' '' ' I 'm not going to be a . ' '' . 's . 's . 's . ' ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watchdog has been criticised for its `` high-speed '' strategy for the past ## years . ' '' ' I 'm not going to be a factor . 's .taking . ' .t . '' ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  74%|███████▎  | 126605/172148 [2:36:16<6:13:12,  2.03it/s, loss=4.0180]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States has been working with the United States and the United States . '' ' I 'm not going to be in the United States . ' '' 's . 's . 's . ' . ' . ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  74%|███████▍  | 128165/172148 [2:38:08<51:27, 14.25it/s, loss=3.9976]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0574\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:30, 23.30it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.55it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0318, Perplexity: 56.36\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been fighting for the past ## months . ' '' ' I 'm not going to be a very good candidate . ' '' ' I 'm not going to . ' '' 's the . ' . ' . ' '' ' '' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , which is the first to be a major event in the world , is expected to be a major event in the summer . ' '' ' I 've seen . 's . '' 's . 's the . ’ he said\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane season is expected to be the first to be a Category # storm . ' '' ' I 'm not going to be a big part of the day . 's . ' '' ' . ' '' 's to- . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  74%|███████▍  | 128167/172148 [2:38:12<7:56:08,  1.54it/s, loss=4.1632]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` unprecedented '' was `` a `` very good '' . '' '' . '' ' '' . ' '' ' I 'm not going to be . '' 's . 's . 's name .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  75%|███████▌  | 129727/172148 [2:40:05<51:12, 13.81it/s, loss=4.0865]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0559\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:35, 22.62it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:36, 22.36it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0310, Perplexity: 56.32\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a key target for the country 's national security forces since the ####s . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' . ' . ' . ' . 's ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill Clinton said the `` unacceptable '' of the `` unacceptable '' . '' ' '' . ' '' ' I 've been in the past . 's . '' 's . 's the . '' ``\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center of the storms is expected to be closed . ' '' ' I . ' '' ' I . ' '' ' I . ' '' ' I . 's . ' '' ' . ' '' . ' . ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  75%|███████▌  | 129730/172148 [2:40:09<6:39:34,  1.77it/s, loss=4.0139]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` unacceptable '' of the `` unacceptable '' . '' ' '' . ' '' ' I 've been in the process . ' '' 's office 's . 's .-s . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  76%|███████▋  | 131290/172148 [2:42:01<49:08, 13.86it/s, loss=4.0354]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0505\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:31, 23.14it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.27it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0251, Perplexity: 55.99\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of the `` unacceptable '' of the `` unacceptable '' of the `` <rare> '' and `` the U.S. , '' is a key part of a `` a `` a . ''et ' . ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company was `` a very good idea '' . ' '' ' I 'm not sure what the company is doing . '' 's . '' 's . '' 's . '' 's . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds are expected to be the most likely to be the most popular in the world . ' '' ' I 've been in the past . ' '' 's time forged . ' '' . ' . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  76%|███████▋  | 131294/172148 [2:42:05<5:28:41,  2.07it/s, loss=4.1396]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government was `` a very good idea '' . ' '' ' I 'm not sure what the government is doing . '' ' '' . ' '' 's office is . '' 's . 's . 's\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  77%|███████▋  | 132854/172148 [2:43:58<46:44, 14.01it/s, loss=3.9637]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0563\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.77it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.32it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0260, Perplexity: 56.03\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major target for the country 's nuclear power plant . ' '' ' I 'm not going to be a good idea . ' '' 's . '' ' . '' . '' . '' . ' . ' . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Times-based <rare> , ## , was found guilty of murder and murder . ' '' . ' '' . ' '' . ' '' . ' '' . ' . ' '' . 's . ' '' . ' . ' . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watchdog has been set up for a weekly storm . '' 'SJ . ' '' . ' '' . ' '' . ' '' . ' '' . ' . ' '' . ' '' . ' . ' . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  77%|███████▋  | 132856/172148 [2:44:03<7:13:22,  1.51it/s, loss=4.1105]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the administration had `` no intention of '' . '' ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . '' . 's . '' ``\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  78%|███████▊  | 134416/172148 [2:45:55<45:32, 13.81it/s, loss=4.1364]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0501\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.54it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.80it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0296, Perplexity: 56.24\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major threat to the country 's economy , and the UK has been a major problem . '' ' I . ' '' ' I . ' '' '' . ' '' 's a <rare> . ' '' ' . 'f\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company was `` a very good '' . ' '' 'The . ' '' ' I 've been in the process . '' 's a <rare> . 's . '' 's a .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds are seen as a result of the storm . ' '' ' I . ' '' ' I . ' '' ' I . ' '' 's . ' '' 's . ' '' ' `` s. ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  78%|███████▊  | 134420/172148 [2:45:59<5:02:19,  2.08it/s, loss=4.1260]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the House of Commons , which is a major issue in the House of Commons , said the government would be `` a very clear '' . '' '' '' . 's . 's . '' 's . '' . ' .x .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  79%|███████▉  | 135980/172148 [2:47:52<43:42, 13.79it/s, loss=4.0467]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0523\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.86it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.39it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0338, Perplexity: 56.48\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of 'inspiring ' and 'happy ' . ' '' ' I 'm not a very nice person . ' '' ' I 'm not . ' '' ' . ' . ' '' ' . ' . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Boris Johnson said : 'We are not going to have a great day for the next day . ' '' ' I 'm not going to be a good job . ' '' 's . ' '' 's . ' ’\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center , a ##-year-old man , was killed by a gunshot wound to the ground . ' '' ' I . ' '' ' . ' . ' . ' '' . ' '' . ' ' . ' ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  79%|███████▉  | 135982/172148 [2:47:56<6:35:59,  1.52it/s, loss=4.0682]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` unprecedented '' of the #### presidential election . ' '' ' I 'm not sure . ' '' ' I 'm not a very good person . '' `` . '' '' . '' . ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  80%|███████▉  | 137542/172148 [2:49:48<41:59, 13.74it/s, loss=3.9968]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0486\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.84it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.69it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0264, Perplexity: 56.06\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a long-term partner for the past two years . ' '' ' I . ' '' ' I ’ m not a <rare> . ' '' ' . ' '' ' . '' ' . ' '' . ' . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company 's `` most important '' of the company 's economy is not a priority . '' ' '' . ' '' 's . 's . 's . 's <rare> . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds of ##mph winds of ##mph in the ##mph coast , with ##mph winds of ##mph .###mph### .## calfest .###\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  80%|███████▉  | 137546/172148 [2:49:52<4:42:26,  2.04it/s, loss=4.2025]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the bill was `` very important '' . ' '' 'We 've been in the House of Commons . ' '' ' I 've been in the House of Representatives . 's . '' `` 's 's .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  81%|████████  | 139106/172148 [2:51:44<38:37, 14.26it/s, loss=4.2003]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0492\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.77it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:33, 22.87it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0274, Perplexity: 56.11\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a key target for the Syrian regime , which has been a key target for the Syrian regime . ' '' ' I . ' '' 's . ' '' 's . ' . .t . 's . 'reh\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Mayor Michael Bloomberg said the `` unnamed '' was `` a `` very good '' and `` very good '' . '' '' '' `` The FBI . '' `` I 's the U.N . '' ) . '' ) .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center in the city of <rare> , the city of <rare> , was closed to the city of <rare> , where the city 's . 'iscalfed oficles . 'isal . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  81%|████████  | 139108/172148 [2:51:49<6:08:54,  1.49it/s, loss=4.0754]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said it was `` a very difficult decision '' . '' ' '' . ' '' ' I 'm not sure if the president is not in the case . ' '' 's statement . '' 's . 's . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  82%|████████▏ | 140668/172148 [2:53:41<37:05, 14.15it/s, loss=4.0797]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0493\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.93it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.24it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0223, Perplexity: 55.83\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of trying to kill the U.S. government . ' '' ' Ibrahim . ' '' ' Ibrahim . ' . 's . ' '' ' . . . . . ' . . ' . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the incident was `` very difficult '' . ' '' ' . ' '' ' I 'm not sure the president is the same . '' 's statement . 's . . . . ' . . .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watches on the U.S. Capitol Hill street in the city of <rare> , the city 's largest city of the city .t . 's time toileverord . 'ering . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  82%|████████▏ | 140672/172148 [2:53:45<4:12:10,  2.08it/s, loss=4.0044]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` unfortunate '' was `` a `` very good '' . '' '' '' . '' ' '' . ' '' ' I 'm not a member of the family . ' '' `` . '' '' . 's\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  83%|████████▎ | 142232/172148 [2:55:37<35:37, 14.00it/s, loss=4.0530]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0472\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:26, 23.95it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.76it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0271, Perplexity: 56.10\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major political crisis since the end of #### . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' . ' . ' . ' ” ? ’ ” forge\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company 's decision to take a `` good '' . ' '' `` I 'm not going to be a good job . '' ' '' . '' ' . 's . '' 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center in the southernmost city of the city of Lt.A. , the National Weather Service said . ' '' 's . ' '' ' . 's . ' . ' . ' . 'er- . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  83%|████████▎ | 142234/172148 [2:55:41<5:26:56,  1.52it/s, loss=4.0274]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government was `` very concerned '' about the `` very important '' . ' '' ' I 've been in the area . ' '' 's . 's . 's . 's . 's . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  84%|████████▎ | 143794/172148 [2:57:34<36:17, 13.02it/s, loss=4.0725]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0468\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:31, 23.22it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:35, 22.56it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0331, Perplexity: 56.43\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the past since the ####s , but has been in the past . '' ' . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' . ' '' . ' . ' '' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , where the ##-year-old was born in #### , was found guilty of the murder of a ##-year-old woman . 's mother . 's . ' '' . ' '' . ' '' . ' '' '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watches on as the storms are expected to hit the coastline . '' ' . ' '' ' I . ' '' ' . ' '' ' . ' . ' '' ' `` ' '' . '' ''P . '' )\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  84%|████████▎ | 143798/172148 [2:57:38<3:49:23,  2.06it/s, loss=4.1177]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had `` no idea '' the government would be able to do it . '' ' '' . ' '' ' I 'm not the only president . '' 's . 's . 's . 's .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  84%|████████▍ | 145358/172148 [2:59:30<31:39, 14.11it/s, loss=3.9876]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0450\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:30, 23.35it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:36, 22.36it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0229, Perplexity: 55.86\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a long-term target for the U.S. and has been a major target for the country . '' ' I . ' '' . ' '' . ' '' . ' . ' . ' .t. ' . ' .t\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the incident was `` a very difficult decision '' . ' '' ' I 'm not sure the president is . ' '' 's . 's . 's . 's . 's . ' .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watchdog has warned that the government is not able to take a .### to the ground . ' '' . ' '' . ' '' . ' . ' . ' . ' . ' . ' . ' . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  84%|████████▍ | 145360/172148 [2:59:34<4:57:42,  1.50it/s, loss=4.2119]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had `` no guarantees '' . ' '' `` The . '' 'This is a very difficult decision . '' ' I 'm not going to be . '' 's . '' 's . '' ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  85%|████████▌ | 146920/172148 [3:01:26<29:43, 14.14it/s, loss=4.0193]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0482\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.59it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.73it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0266, Perplexity: 56.07\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major target for the U.S. military since #### . ' '' . ' '' . ' '' . ' '' . ' '' . . . ' . ' . ' . . . ' . . ' '' ? . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company was `` a very good idea '' . ' '' ' I 'm not sure . '' ' . ' '' . ' . 's . '' ' . . . . . . . . .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watcher has been killed in the past two years , according to the U.S. . ' '' ' I . ' . ' '' . ' . 's . ' '' . ' . ' . ' . ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  85%|████████▌ | 146924/172148 [3:01:30<3:23:49,  2.06it/s, loss=4.1521]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had been `` very careful '' . ' '' ' I have no doubt . ' '' ' I 'm not sure . 's . ' '' 's . 's . . . . . . . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  86%|████████▋ | 148484/172148 [3:03:23<28:07, 14.03it/s, loss=4.0223]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0450\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:39, 21.95it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:36, 22.46it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0323, Perplexity: 56.39\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major target for the U.S. military since #### . ' '' . ' '' . ' '' . ' '' . ' '' . ' . ' . ' . ' . ' . ' .. ' . ' . 'f\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the incident was `` a very serious matter '' . ' '' ' I 'm not going to be a problem . ' . ' . ' . ' '' . 's . 's . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to reach ##.# inches of snow in the Pacific Ocean , according to the National Park Service . ' '' ' AAA . ' . ' . 's .taking of . ' . . . . . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  86%|████████▋ | 148486/172148 [3:03:27<4:24:51,  1.49it/s, loss=3.9392]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` unprecedented '' of the `` unprecedented '' . '' ' '' . ' '' ' I 'm not going to be a matter of . 's . 's 's . 's\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  87%|████████▋ | 150046/172148 [3:05:19<26:18, 14.00it/s, loss=4.1127]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0433\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.70it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.29it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0319, Perplexity: 56.37\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major target for the United States since #### . ' '' ' I 'm a member of the United States . ' '' . ' '' . '' . '' . '' . '' . '' .i . 's . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company was `` a very good man '' . ' '' `` I 'm not going to be a good thing . '' 's . '' 's . '' `` I '' . '' `` . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is a major city of about #.# million miles ( #.# million km ) of rain .### .### .###m . 'stsyest . 's .##PJeg .##\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  87%|████████▋ | 150050/172148 [3:05:23<2:57:28,  2.08it/s, loss=3.8726]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been in the process of a new deal to make a decision on the issue . '' ' I . ' '' . ' '' . ' '' . ' '' . 's . '' . '' . ' . ' . ' . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  88%|████████▊ | 151610/172148 [3:07:16<24:10, 14.16it/s, loss=4.0604]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0434\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:27, 23.83it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:26, 23.93it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0170, Perplexity: 55.53\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of failing to protect the country 's military and the .###-### . ' '' ' I . ' '' ' I . '' ' . ' '' ' . . . . ' . ' . ' . ' .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said he was `` disappointed '' and `` the most important thing is that the people who have been in the world '' . 's `` is the best . '' `` . '' ) '' . '' ) .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been called to the area for a week of the storm . ' '' 's . ' '' 's . ' '' 's . ' '' 's . ' '' . ' . ' '' . ' `` . ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  88%|████████▊ | 151612/172148 [3:07:20<3:44:53,  1.52it/s, loss=4.0006]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said he was `` very concerned '' about the situation . ' '' ' I 'm not going to do so . ' '' ' I 'm not going to be . '' 's . '' ' . ' . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  89%|████████▉ | 153172/172148 [3:09:12<22:27, 14.09it/s, loss=3.9705]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0435\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:32, 23.05it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.63it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0212, Perplexity: 55.77\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the country since #### , when it was announced that it was a `` very difficult decision '' . ' '' ' I am not going to be a . '' ' . '' ' . ' . 'ir . ' . ' .er\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , where the ##-year-old was born in #### , was born in #### . 's . ' '' ' I ’ m not a <rare> . ' '' . 's . 's a.th . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center is expected to be a Category # storm in the Pacific Ocean , the largest city of ## million people in the world . '' 's most popular destination is . 'stalking of . ' tob . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  89%|████████▉ | 153176/172148 [3:09:16<2:32:44,  2.07it/s, loss=3.8735]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the situation was `` very difficult '' . ' '' ' I am not sure . ' '' ' I am not sure . ' '' 's . ' '' 's . '' ' . '' 's . ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  90%|████████▉ | 154736/172148 [3:11:09<20:55, 13.87it/s, loss=3.8592]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0432\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:30, 23.30it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.30it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0198, Perplexity: 55.69\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a key part of the country 's history of the country 's history . '' ' I 'm not going to be a big part of the world . '' ' . '' . '' . ''et . 's <rarere\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police Chief Michael Carrick said the incident was a `` very serious '' . ' '' `` I 'm not going to have to be a member of the public . '' `` I '' . '' . '' `` . '' ) . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane warning is that the storms are expected to rise by #.# per cent . '' '### . ' '' ' A . ' '' . ' '' . ' '' . '' ' . '' . '' . '' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  90%|████████▉ | 154738/172148 [3:11:13<3:11:55,  1.51it/s, loss=4.0279]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had been `` very disappointed '' . ' '' `` I 've got to be a great player . '' ' '' . '' `` I am not . '' `` `` I '' . '' `` is , ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  91%|█████████ | 156298/172148 [3:13:05<19:15, 13.72it/s, loss=4.1038]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0448\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 2/3514 [00:00<03:01, 19.37it/s]\u001b[A\n",
            "Evaluating:   0%|          | 5/3514 [00:00<02:38, 22.07it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0280, Perplexity: 56.15\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a key figure for the United States since #### . '' ' I . ' '' . ' '' . ' '' . ' '' . ' '' . '' ' . ' '' . '' . ' . ' . ' '' ? ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police Chief Inspector Steve <rare> said : 'The . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' . ' '' . ' '' . ' . ' . ' '' '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center in the city of <rare> , the city 's largest city of the city of <rare> , is a popular destination for the city 's .oratehead of . '' `` <rare> . '' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  91%|█████████ | 156302/172148 [3:13:09<2:08:26,  2.06it/s, loss=3.9650]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States has been a key figure for the first time in the past ## years . '' ' I . ' '' . ' '' . ' '' . 's . '' ) . ' '' ? ' . ' . ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  92%|█████████▏| 157862/172148 [3:15:01<16:43, 14.23it/s, loss=3.9977]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0418\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:39, 22.06it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:34, 22.70it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0212, Perplexity: 55.77\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of trying to force the government to help the government to help the government to help the government to help the government to help . ' '' . ' '' . ' '' . 's . ' . ' . ' . ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the government would not be able to make the decision . ' '' ' I 've been in the process of a new way . 's . 's . 's . 's . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center has been forced to evacuate the area , but the area has been closed . ' '' ' I 've been told . ' '' 's . 's . ' '' . ' . 'er of . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  92%|█████████▏| 157864/172148 [3:15:05<2:38:25,  1.50it/s, loss=3.8866]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has said that the government has been trying to help the government to protect the economy . ' '' ' I 've been in the process of negotiating the situation . ' '' 's . 's . 'sgaster . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  93%|█████████▎| 159424/172148 [3:16:57<15:31, 13.65it/s, loss=4.0040]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0386\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:25, 24.09it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:27, 23.86it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0219, Perplexity: 55.81\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the region since the ####s , with a ##-year-old man from the United States . ' '' . ' '' ' I 'm not . '' 's . '' 's the . '' 's the . ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , where the ##-year-old was arrested on suspicion of murder . ' '' . ' '' ' I 'm not a member of the family . '' 's . 's . 's . ' '' . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watches on as the storm hit the Pacific Ocean hit the Pacific Ocean . 's ##.# feet . ' '' ' A . ' '' . ' '' . '' 's . '' 's spoketer . '' '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  93%|█████████▎| 159428/172148 [3:17:01<1:42:32,  2.07it/s, loss=3.9597]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government would not comment on the matter . ' '' ' I was not aware of the incident . ' '' ' I 'm not sure . ' '' . '' 's . '' `` . '' `` . '' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  94%|█████████▎| 160988/172148 [3:18:53<14:35, 12.75it/s, loss=4.1613]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0403\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:29, 23.49it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.53it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0184, Perplexity: 55.61\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has a strong presence in the region , and the world 's largest city of the world . '' ' I . ' '' ' I 'm not a member of the . '' 's . ''isph . ' ''esweet- .\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York : The ##-year-old was found dead in a car crash in the city of <rare> , near the city of San Diego . 's apartment . '' 's . 's . ' . ' '' 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane is expected to be expected to be reached for comment on Monday . '' 's statement . ' '' ' I 'm not sure if the weather is a . 's . ' '' ' . ' '' ' said to . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  94%|█████████▎| 160990/172148 [3:18:57<2:03:35,  1.50it/s, loss=3.9362]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the president 's decision to `` make the decision '' . '' 'This is a `` very important issue '' . '' ' '' . 's `` I am . '' `` I am '' . '' `` I . ``\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  94%|█████████▍| 162550/172148 [3:20:49<11:30, 13.89it/s, loss=4.0773]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0387\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:34, 22.75it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:30, 23.26it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0202, Perplexity: 55.71\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major contributor to the U.S. government . '' ' I . ' '' ' I 'm not sure the president is in the way of the . ' '' . '' ' .kle . ' '' . ' ”\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company 's `` most importantly , '' and `` the best way to the world '' . '' ' '' . '' `` I . '' ) `` I 'm. '' . '' ) . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watches on as the storms are expected to fall below the surface . '' ' I . ' '' ' I 'm not sure if you 're going to distance . ' '' ' . '' ' . '' ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  94%|█████████▍| 162554/172148 [3:20:54<1:18:17,  2.04it/s, loss=4.0650]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States , the United States , the United States and the United States , the United States and the United States . '' ' I . '' 's . '' 's . ''es ? ” . ' ” said said said said\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  95%|█████████▌| 164114/172148 [3:22:46<09:29, 14.11it/s, loss=3.9948]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0407\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:28, 23.63it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:28, 23.60it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0257, Perplexity: 56.02\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the country since #### , and the country has been in the country since #### . '' 's . '' ' I 'm not sure . '' ' . '' ' . '' 's . ' '' . ' . ' ''es\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's newest of the world 's most famous , which is the world 's most famous for the world 's history . '' 's . '' ' . '' 's . '' . '' . '' . '' . ''\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane center in the city of Bali , where the ##-year-old was killed by a fireman . ' '' ' . ' '' . ' '' . '' . '' ' . ' '' . '' ' . '' ? ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  95%|█████████▌| 164116/172148 [3:22:50<1:27:04,  1.54it/s, loss=4.0456]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said that the government was `` notified '' . '' ' '' . ' '' ' I 'm not sure the president is going to be in the United States . '' `` . '' `` . '' `` . '' '' to\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  96%|█████████▌| 165676/172148 [3:24:42<07:44, 13.93it/s, loss=3.9422]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0407\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:40, 21.87it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:35, 22.50it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0248, Perplexity: 55.97\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of conspiracy to commit to the attack . ' '' . ' '' . ' '' I . ' '' . ' '' ' I . '' . '' ' . '' . '' . '' ' . . '' . '' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company 's decision to pay the $ #.# million . ' '' ' I 'm not going to be a big-name player . 's . 's . 's . 's\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane-force winds and winds of ##mph winds were seen in the sky and the winds were seen in the sky .###m .###mins . 's .###-sh . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  96%|█████████▌| 165680/172148 [3:24:46<52:18,  2.06it/s, loss=3.9099]  "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the administration had `` no reason '' . '' 'We have been investigating . ' '' ' I 'm not going to be a member of the . 's . 's . 's . 's . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  97%|█████████▋| 167240/172148 [3:26:38<05:56, 13.76it/s, loss=4.0184]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0389\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.92it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:31, 23.13it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0197, Perplexity: 55.69\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of trying to kill the ##-year-old . ' '' ' I 'm not a fan of the <rare> . ' '' . '' ' . ' '' . '' ' . ' . ' . ' . ' '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Police said the man was shot and killed by a man in the car . ' '' ' I 'm not sure . ' '' ' . ' '' . ' . ' . ' . ' . ' . ' . ' . ' .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane season is expected to be delayed for a week after the storm . ' '' ' I 'm not sure . ' '' ' I 'm not sure . ' '' 's . ' '' . ' '' ' . ' . '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  97%|█████████▋| 167242/172148 [3:26:43<54:31,  1.50it/s, loss=3.9002]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the `` unacceptable '' of the government 's `` right-wing government '' is `` a `` very important issue '' . '' '' . '' ' '' and `` Im not . '' `` `` Iak ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  98%|█████████▊| 168802/172148 [3:28:35<03:58, 14.02it/s, loss=4.0960]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0402\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:33, 22.93it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:29, 23.48it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0251, Perplexity: 55.98\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major political figure in the region since #### . ' '' . ' '' . ' '' . ' '' . ' '' . ' '' . ' . ' . ' '' . ' . '' .thgaster ' . ' .th\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company was `` very concerned '' . ' '' . ' '' . ' '' . ' '' . ' '' . 's . 's . 's . 's . 's . 's .\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane for the first time in the past year , the U.S. has been in the midst of a war in the region . 's easternmost east of .over .thingent , .### . ' `` .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  98%|█████████▊| 168806/172148 [3:28:39<26:56,  2.07it/s, loss=4.0919]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had `` no intention of making any mistakes '' . ' '' . ' '' 'We 're not sure that the government is not in the case . 's . 's . 's .- to .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  99%|█████████▉| 170366/172148 [3:30:32<02:08, 13.85it/s, loss=4.0480]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0366\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:26, 23.91it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:26, 23.87it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0154, Perplexity: 55.45\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major problem for the country 's most populous country . '' ' I 'm not a terrorist . ' '' ' I 'm not a terrorist . '' ' . '' ' .f . ' '' ' . '\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New Yorker , who was a ##-year-old man , was arrested on suspicion of murdering a ##-year-old girl . 's mother . 's . 's . ' ''escoelew. #### . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane for the first time since the storm was hit by a storm of the storm . ' '' ' . ' '' ' I 'm not sure . ' '' 's . '' ' `` . '' ' . '' ' . '' '\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  99%|█████████▉| 170368/172148 [3:30:36<19:30,  1.52it/s, loss=4.1219]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government had `` no plans to get the necessary necessary to ensure that the government is aware of the situation '' . ' '' ' . 's . ' '' 's . '' 's . '' ' . '' ''\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1: 100%|█████████▉| 171928/172148 [3:32:28<00:16, 13.74it/s, loss=3.9812]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.0359\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:39, 22.08it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:35, 22.60it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0179, Perplexity: 55.59\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the region of the country since #### . '' ' I . ' '' ' I 'm not a member of the Arabian community . ' '' 's . ' '' . '' 'ed the . ' '' ' . ' ''\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Bill de Blasio said the company was `` very concerned '' . ' '' 'We are not going to do . ' '' ' I 'm not . '' 's . 's . 's . 's . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane watches on as the storm hit the coast of the city of Lt.A. . '' 's . ' '' ' I . ' '' 's . 's .thoged . ' `` <reable .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1: 100%|█████████▉| 171932/172148 [3:32:33<01:44,  2.07it/s, loss=4.1245]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the administration 's decision to address the issue of the president 's actions . '' 'Yes . ' '' ' I . ' '' 's . 's . 's . 's . 's . ' .\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1: 100%|██████████| 172148/172148 [3:32:48<00:00, 13.48it/s, loss=4.0147]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Epoch 1/1, Average Loss: 4.0339\n",
            "\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow , which has been the first of its ##,###-strong .###-###-###-### .### .### . 's . 's the . 'ed . ' '' . 's the\n",
            "\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the company 's decision to take a breakdown of the new deal . '' 's . ' '' ' I . 's . 's . ' '' . 's . ' . ' . '\n",
            "\n",
            "\n",
            "Context: A hurricane\n",
            "\n",
            "Generated text: A hurricane season is expected to be completed in #### . '' ' I 'm not going to be a major factor . ' '' ' I 'm not going to be a matter of . ' '' ' . ' '' ' . ' ''\n",
            "\n",
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the House of Representatives has said that the bill would be a `` very important step '' . '' ' '' . ' '' ' I 'm not sure that the president of the House of Representatives of the House of Representatives . '' `` the\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "                                                            "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Validation Average Loss: 4.0108, Perplexity: 55.19\n",
            "Model and tokenizer saved as 'Decoder_LM'\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "s61ovCwawq3f"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Testing the model\n",
        "\n",
        "In the cell below, we load and test the language model:"
      ],
      "metadata": {
        "id": "Bze_jce90ZJg"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ezbawsEzZQVj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a9efdcc4-8f81-479e-e208-5089bfe092c6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Model 'Decoder_LM' loaded successfully\n",
            "\n",
            "Testing the model:\n",
            "\n",
            "\n",
            "Prompt: Moscow\n",
            "\n",
            "Generated response: Moscow , which has been the first of its ##,###-strong .###-###-###-### .### .### . 's . 's the . 'ed . ' '' . 's the\n",
            "\n",
            "\n",
            "Prompt: New York\n",
            "\n",
            "Generated response: New York City Mayor Michael Bloomberg said the company 's decision to take a breakdown of the new deal . '' 's . ' '' ' I . 's . 's . ' '' . 's . ' . ' . '\n",
            "\n",
            "\n",
            "Prompt: A hurricane\n",
            "\n",
            "Generated response: A hurricane season is expected to be completed in #### . '' ' I 'm not going to be a major factor . ' '' ' I 'm not going to be a matter of . ' '' ' . ' '' ' . ' ''\n",
            "\n",
            "\n",
            "Prompt: The President\n",
            "\n",
            "Generated response: The President of the House of Representatives has said that the bill would be a `` very important step '' . '' ' '' . ' '' ' I 'm not sure that the president of the House of Representatives of the House of Representatives . '' `` the\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# ----------------------------\n",
        "# Model tests\n",
        "# ----------------------------\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "\n",
        "    model_name = \"Decoder_LM\"\n",
        "\n",
        "    # Load the previously saved model and tokenizer from disk\n",
        "    # This recreates the exact model state from after training\n",
        "    model, tokenizer = load_model(model_name)\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    # Print header for test section\n",
        "    print(\"\\nTesting the model:\\n\")\n",
        "\n",
        "    # Define a list of test prompts to evaluate model performance\n",
        "    contexts = [\n",
        "        \"Moscow\",\n",
        "        \"New York\",\n",
        "        \"A hurricane\",\n",
        "        \"The President\"\n",
        "    ]\n",
        "\n",
        "    # Iterate through each test prompt and generate text\n",
        "    for context in contexts:\n",
        "        # Generate text using greedy decoding (most likely tokens)\n",
        "        generated_text = generate_text(\n",
        "            model=model,          # The loaded language model\n",
        "            start_string=context, # Text to continue\n",
        "            tokenizer=tokenizer,  # Tokenizer for text conversion\n",
        "            device=device,        # CPU or GPU device\n",
        "            max_length=50         # Maximum length of generated sequence\n",
        "        )\n",
        "        # Print the original prompt and model's response\n",
        "        print(f\"\\nPrompt: {context}\")\n",
        "        print(f\"\\nGenerated response: {generated_text}\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import runtime\n",
        "runtime.unassign()"
      ],
      "metadata": {
        "id": "x-ZB0pUgEZXA"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "A100",
      "provenance": [],
      "authorship_tag": "ABX9TyP8DX5jT6kcvT1Sm5bOodq4",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9a8699a145a94ed89b8b11d715067c3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b576e000e0d549b089431943ad050eba",
              "IPY_MODEL_181b36d4f4e14c10adc7b249717e21e4",
              "IPY_MODEL_2226577264e14049b08244ce001e4e31"
            ],
            "layout": "IPY_MODEL_d7f2588a852742afa54c616961176830"
          }
        },
        "b576e000e0d549b089431943ad050eba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c36c028aeee14b95901e1fb005c7201a",
            "placeholder": "​",
            "style": "IPY_MODEL_cd63f0fa54284235bb73c59bf5cb2a92",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "181b36d4f4e14c10adc7b249717e21e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae3f1435c9714894810273ab6cc8fb39",
            "max": 3984,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_188a4756346f45f796f49a44e694ad29",
            "value": 3984
          }
        },
        "2226577264e14049b08244ce001e4e31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8873103295c3485ca41a3347f8766a9d",
            "placeholder": "​",
            "style": "IPY_MODEL_2f54ab7c292246db81dc7cbab21c7cbb",
            "value": " 3.98k/3.98k [00:00&lt;00:00, 344kB/s]"
          }
        },
        "d7f2588a852742afa54c616961176830": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c36c028aeee14b95901e1fb005c7201a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd63f0fa54284235bb73c59bf5cb2a92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ae3f1435c9714894810273ab6cc8fb39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "188a4756346f45f796f49a44e694ad29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8873103295c3485ca41a3347f8766a9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2f54ab7c292246db81dc7cbab21c7cbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e41117d4fed1445daf6fb85a01cbc81e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fefd7c010fdf4735aa9b2513db0497d0",
              "IPY_MODEL_f48e51c77737475fb4aaafce03380fb7",
              "IPY_MODEL_82778a1b41a0417b86fc1e29455fb383"
            ],
            "layout": "IPY_MODEL_5b6201ac09a64fa79f255ac5a14b3a5a"
          }
        },
        "fefd7c010fdf4735aa9b2513db0497d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f46a10202f08457393536590b5084b3e",
            "placeholder": "​",
            "style": "IPY_MODEL_32dc1d47b7a545ff9759d3897eee59d2",
            "value": "tokenizer.model: 100%"
          }
        },
        "f48e51c77737475fb4aaafce03380fb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12db66482d064d7e831767ccbcef1bec",
            "max": 499723,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ee332c51eee94dcbb5586f463e39afdf",
            "value": 499723
          }
        },
        "82778a1b41a0417b86fc1e29455fb383": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec1dd8303a3b4f5e97b3d218f3de8613",
            "placeholder": "​",
            "style": "IPY_MODEL_09222bf385da4ce0be09da2e03b74f74",
            "value": " 500k/500k [00:00&lt;00:00, 7.61MB/s]"
          }
        },
        "5b6201ac09a64fa79f255ac5a14b3a5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f46a10202f08457393536590b5084b3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "32dc1d47b7a545ff9759d3897eee59d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "12db66482d064d7e831767ccbcef1bec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee332c51eee94dcbb5586f463e39afdf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ec1dd8303a3b4f5e97b3d218f3de8613": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "09222bf385da4ce0be09da2e03b74f74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6ba6646a5d5347059b14acdd79d6da07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_756c21cc62c14083a6a6729f54232d79",
              "IPY_MODEL_0b2e8ac291734ae39059110f96493a18",
              "IPY_MODEL_cf27f80668ff40c9ac07a93197b34640"
            ],
            "layout": "IPY_MODEL_c65097f7cf4a4722a186cfb1822f0315"
          }
        },
        "756c21cc62c14083a6a6729f54232d79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_beb1e66b6cc44e7490fb207bff6fbd2c",
            "placeholder": "​",
            "style": "IPY_MODEL_3daf9a7b54124787b732fdb108089e00",
            "value": "tokenizer.json: 100%"
          }
        },
        "0b2e8ac291734ae39059110f96493a18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2adb1d17c4404860b49575ea15dbff13",
            "max": 1844408,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_08718b87ba204a34a9449fc9a60f97ad",
            "value": 1844408
          }
        },
        "cf27f80668ff40c9ac07a93197b34640": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72a5c19cce364fed959f66f6521a3cf0",
            "placeholder": "​",
            "style": "IPY_MODEL_57c3488609f046c8852641b73b83e678",
            "value": " 1.84M/1.84M [00:00&lt;00:00, 11.3MB/s]"
          }
        },
        "c65097f7cf4a4722a186cfb1822f0315": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "beb1e66b6cc44e7490fb207bff6fbd2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3daf9a7b54124787b732fdb108089e00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2adb1d17c4404860b49575ea15dbff13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08718b87ba204a34a9449fc9a60f97ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "72a5c19cce364fed959f66f6521a3cf0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "57c3488609f046c8852641b73b83e678": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "00647dd5e8034abcb0217cc30e02177f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_57ef6e22ee014ce0b467d6ae328d37ec",
              "IPY_MODEL_26c3ff709f804abcacac8861c235b6f4",
              "IPY_MODEL_ae4d86bfa9914f6e81886ab56a011ec9"
            ],
            "layout": "IPY_MODEL_677fb8e2b421433fa412f9d75e319081"
          }
        },
        "57ef6e22ee014ce0b467d6ae328d37ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_156cc956117a4ed984446f3d1a3aebe3",
            "placeholder": "​",
            "style": "IPY_MODEL_260bd2215a284268b2920be0c458393c",
            "value": "added_tokens.json: 100%"
          }
        },
        "26c3ff709f804abcacac8861c235b6f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fadf515115e64ebeb234df1c3c9dce71",
            "max": 306,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_20370596c8824147a0aab279ad6241d8",
            "value": 306
          }
        },
        "ae4d86bfa9914f6e81886ab56a011ec9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf8a3680974742c4962608eaac628a6e",
            "placeholder": "​",
            "style": "IPY_MODEL_b9d3e3c74d644a27a6bfb7b374683f02",
            "value": " 306/306 [00:00&lt;00:00, 28.4kB/s]"
          }
        },
        "677fb8e2b421433fa412f9d75e319081": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "156cc956117a4ed984446f3d1a3aebe3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "260bd2215a284268b2920be0c458393c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fadf515115e64ebeb234df1c3c9dce71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20370596c8824147a0aab279ad6241d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bf8a3680974742c4962608eaac628a6e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b9d3e3c74d644a27a6bfb7b374683f02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6466f91527ee437ebd69ce5cb90bdf6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ecf37b3cecb8405685d622d1f3e05bcf",
              "IPY_MODEL_3277a976d1034d12b12ec767ae80e3c5",
              "IPY_MODEL_6a1bda5b60a1449caf1a7b1e9ef6e844"
            ],
            "layout": "IPY_MODEL_1fa0d48eb7624fc8806942941ca4b411"
          }
        },
        "ecf37b3cecb8405685d622d1f3e05bcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f17c35f573354ce78baf719054af4732",
            "placeholder": "​",
            "style": "IPY_MODEL_6415e99c230c463298a92d761875fd2e",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "3277a976d1034d12b12ec767ae80e3c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0f34f0115c644766a1c6cc5b61485608",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6044a91e47104cb9b18a8add2a0e4c6f",
            "value": 665
          }
        },
        "6a1bda5b60a1449caf1a7b1e9ef6e844": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d808e2efb7b54ae79e431f56ab9fc871",
            "placeholder": "​",
            "style": "IPY_MODEL_6aa8bfe1c8864a679e8634806ec46dac",
            "value": " 665/665 [00:00&lt;00:00, 53.1kB/s]"
          }
        },
        "1fa0d48eb7624fc8806942941ca4b411": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f17c35f573354ce78baf719054af4732": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6415e99c230c463298a92d761875fd2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f34f0115c644766a1c6cc5b61485608": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6044a91e47104cb9b18a8add2a0e4c6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d808e2efb7b54ae79e431f56ab9fc871": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6aa8bfe1c8864a679e8634806ec46dac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}