diff --git a/.ipynb_checkpoints/visualization-checkpoint.ipynb b/.ipynb_checkpoints/visualization-checkpoint.ipynb new file mode 100644 index 0000000..2d5ee44 --- /dev/null +++ b/.ipynb_checkpoints/visualization-checkpoint.ipynb @@ -0,0 +1,343 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "lovely-budapest", + "metadata": {}, + "source": [ + "# This is a notebook that shows how to produce Grad-CAM visualizations for ALBEF" + ] + }, + { + "cell_type": "markdown", + "id": "czech-surprise", + "metadata": {}, + "source": [ + "# 1. Set the paths for model checkpoint and configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "institutional-sarah", + "metadata": {}, + "outputs": [], + "source": [ + "model_path = '../VL/Example/refcoco.pth'\n", + "bert_config_path = 'configs/config_bert.json'\n", + "use_cuda = False" + ] + }, + { + "cell_type": "markdown", + "id": "lovely-passage", + "metadata": {}, + "source": [ + "# 2. Model defination" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "documented-symbol", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial\n", + "from models.vit import VisionTransformer\n", + "from models.xbert import BertConfig, BertModel\n", + "from models.tokenization_bert import BertTokenizer\n", + "\n", + "import torch\n", + "from torch import nn\n", + "from torchvision import transforms\n", + "\n", + "import json\n", + "\n", + "class VL_Transformer_ITM(nn.Module):\n", + " def __init__(self, \n", + " text_encoder = None,\n", + " config_bert = ''\n", + " ):\n", + " super().__init__()\n", + " \n", + " bert_config = BertConfig.from_json_file(config_bert)\n", + "\n", + " self.visual_encoder = VisionTransformer(\n", + " img_size=384, patch_size=16, embed_dim=768, depth=12, num_heads=12, \n", + " mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) \n", + "\n", + " self.text_encoder = BertModel.from_pretrained(text_encoder, config=bert_config, add_pooling_layer=False) \n", + " \n", + " self.itm_head = nn.Linear(768, 2) \n", + "\n", + " \n", + " def forward(self, image, text):\n", + " image_embeds = self.visual_encoder(image) \n", + "\n", + " image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)\n", + "\n", + " output = self.text_encoder(text.input_ids, \n", + " attention_mask = text.attention_mask,\n", + " encoder_hidden_states = image_embeds,\n", + " encoder_attention_mask = image_atts, \n", + " return_dict = True,\n", + " ) \n", + " \n", + " vl_embeddings = output.last_hidden_state[:,0,:]\n", + " vl_output = self.itm_head(vl_embeddings) \n", + " return vl_output" + ] + }, + { + "cell_type": "markdown", + "id": "renewable-eight", + "metadata": {}, + "source": [ + "# 3. Text Preprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "optional-brooklyn", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "\n", + "def pre_caption(caption,max_words=30):\n", + " caption = re.sub(\n", + " r\"([,.'!?\\\"()*#:;~])\",\n", + " '',\n", + " caption.lower(),\n", + " ).replace('-', ' ').replace('/', ' ')\n", + "\n", + " caption = re.sub(\n", + " r\"\\s{2,}\",\n", + " ' ',\n", + " caption,\n", + " )\n", + " caption = caption.rstrip('\\n') \n", + " caption = caption.strip(' ')\n", + "\n", + " #truncate caption\n", + " caption_words = caption.split(' ')\n", + " if len(caption_words)>max_words:\n", + " caption = ' '.join(caption_words[:max_words]) \n", + " return caption" + ] + }, + { + "cell_type": "markdown", + "id": "based-roads", + "metadata": {}, + "source": [ + "# 4. Image Preprocessing and Postpressing" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "subsequent-flesh", + "metadata": {}, + "outputs": [], + "source": [ + "from PIL import Image\n", + "\n", + "import cv2\n", + "import numpy as np\n", + "\n", + "from skimage import transform as skimage_transform\n", + "from scipy.ndimage import filters\n", + "from matplotlib import pyplot as plt\n", + "\n", + "def getAttMap(img, attMap, blur = True, overlap = True):\n", + " attMap -= attMap.min()\n", + " if attMap.max() > 0:\n", + " attMap /= attMap.max()\n", + " attMap = skimage_transform.resize(attMap, (img.shape[:2]), order = 3, mode = 'constant')\n", + " if blur:\n", + " attMap = filters.gaussian_filter(attMap, 0.02*max(img.shape[:2]))\n", + " attMap -= attMap.min()\n", + " attMap /= attMap.max()\n", + " cmap = plt.get_cmap('jet')\n", + " attMapV = cmap(attMap)\n", + " attMapV = np.delete(attMapV, 3, 2)\n", + " if overlap:\n", + " attMap = 1*(1-attMap**0.7).reshape(attMap.shape + (1,))*img + (attMap**0.7).reshape(attMap.shape+(1,)) * attMapV\n", + " return attMap\n", + "\n", + "\n", + "normalize = transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))\n", + "\n", + "transform = transforms.Compose([\n", + " transforms.Resize((384,384),interpolation=Image.BICUBIC),\n", + " transforms.ToTensor(),\n", + " normalize,\n", + "]) " + ] + }, + { + "cell_type": "markdown", + "id": "occasional-trace", + "metadata": {}, + "source": [ + "# 5. Load model and tokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "qualified-sleep", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['bert.pooler.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'bert.pooler.dense.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of BertModel were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['bert.encoder.layer.10.crossattention.self.key.weight', 'bert.encoder.layer.6.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.10.crossattention.output.dense.bias', 'bert.encoder.layer.8.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.9.crossattention.output.dense.bias', 'bert.encoder.layer.6.crossattention.output.dense.bias', 'bert.encoder.layer.9.crossattention.self.value.bias', 'bert.encoder.layer.9.crossattention.self.key.bias', 'bert.encoder.layer.8.crossattention.self.key.weight', 'bert.encoder.layer.6.crossattention.self.query.weight', 'bert.encoder.layer.8.crossattention.self.value.weight', 'bert.encoder.layer.9.crossattention.self.query.weight', 'bert.encoder.layer.9.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.11.crossattention.self.value.weight', 'bert.encoder.layer.7.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.8.crossattention.self.key.bias', 'bert.encoder.layer.7.crossattention.self.key.weight', 'bert.encoder.layer.7.crossattention.output.dense.weight', 'bert.encoder.layer.11.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.9.crossattention.self.key.weight', 'bert.encoder.layer.9.crossattention.self.value.weight', 'bert.encoder.layer.8.crossattention.self.query.bias', 'bert.encoder.layer.11.crossattention.self.key.bias', 'bert.encoder.layer.10.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.8.crossattention.output.dense.bias', 'bert.encoder.layer.7.crossattention.output.dense.bias', 'bert.encoder.layer.7.crossattention.self.value.weight', 'bert.encoder.layer.11.crossattention.self.query.bias', 'bert.encoder.layer.9.crossattention.self.query.bias', 'bert.encoder.layer.6.crossattention.output.dense.weight', 'bert.encoder.layer.10.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.6.crossattention.self.value.bias', 'bert.encoder.layer.8.crossattention.self.query.weight', 'bert.encoder.layer.11.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.11.crossattention.self.value.bias', 'bert.encoder.layer.6.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.9.crossattention.output.dense.weight', 'bert.encoder.layer.6.crossattention.self.query.bias', 'bert.encoder.layer.8.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.11.crossattention.self.key.weight', 'bert.encoder.layer.7.crossattention.self.key.bias', 'bert.encoder.layer.6.crossattention.self.value.weight', 'bert.encoder.layer.7.crossattention.self.query.bias', 'bert.encoder.layer.8.crossattention.self.value.bias', 'bert.encoder.layer.6.crossattention.self.key.bias', 'bert.encoder.layer.7.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.10.crossattention.self.value.weight', 'bert.encoder.layer.10.crossattention.self.query.weight', 'bert.encoder.layer.11.crossattention.output.dense.weight', 'bert.encoder.layer.11.crossattention.output.dense.bias', 'bert.encoder.layer.9.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.7.crossattention.self.query.weight', 'bert.encoder.layer.10.crossattention.output.dense.weight', 'bert.encoder.layer.6.crossattention.self.key.weight', 'bert.encoder.layer.10.crossattention.self.key.bias', 'bert.encoder.layer.8.crossattention.output.dense.weight', 'bert.encoder.layer.7.crossattention.self.value.bias', 'bert.encoder.layer.11.crossattention.self.query.weight', 'bert.encoder.layer.10.crossattention.self.value.bias', 'bert.encoder.layer.10.crossattention.self.query.bias']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + } + ], + "source": [ + "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n", + "\n", + "model = VL_Transformer_ITM(text_encoder='bert-base-uncased', config_bert=bert_config_path)\n", + "\n", + "checkpoint = torch.load(model_path, map_location='cpu') \n", + "msg = model.load_state_dict(checkpoint,strict=False)\n", + "model.eval()\n", + "\n", + "block_num = 8\n", + "\n", + "model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.save_attention = True\n", + "\n", + "if use_cuda:\n", + " model.cuda() " + ] + }, + { + "cell_type": "markdown", + "id": "apparent-captain", + "metadata": {}, + "source": [ + "# 6. Load Image and Text" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "finite-angle", + "metadata": {}, + "outputs": [], + "source": [ + "image_path = 'examples/image0.jpg'\n", + "image_pil = Image.open(image_path).convert('RGB') \n", + "image = transform(image_pil).unsqueeze(0) \n", + "\n", + "caption = 'the woman is working on her computer at the desk'\n", + "text = pre_caption(caption)\n", + "text_input = tokenizer(text, return_tensors=\"pt\")\n", + "\n", + "if use_cuda:\n", + " image = image.cuda()\n", + " text_input = text_input.to(image.device)" + ] + }, + { + "cell_type": "markdown", + "id": "gorgeous-matrix", + "metadata": {}, + "source": [ + "# 7. Compute GradCAM" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "driven-termination", + "metadata": {}, + "outputs": [], + "source": [ + "output = model(image, text_input)\n", + "loss = output[:,1].sum()\n", + "\n", + "model.zero_grad()\n", + "loss.backward() \n", + "\n", + "with torch.no_grad():\n", + " mask = text_input.attention_mask.view(text_input.attention_mask.size(0),1,-1,1,1)\n", + "\n", + " grads=model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.get_attn_gradients()\n", + " cams=model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.get_attention_map()\n", + "\n", + " cams = cams[:, :, :, 1:].reshape(image.size(0), 12, -1, 24, 24) * mask\n", + " grads = grads[:, :, :, 1:].clamp(0).reshape(image.size(0), 12, -1, 24, 24) * mask\n", + "\n", + " gradcam = cams * grads\n", + " gradcam = gradcam[0].mean(0).cpu().detach()" + ] + }, + { + "cell_type": "markdown", + "id": "abroad-northern", + "metadata": {}, + "source": [ + "# 8. Visualize GradCam for each word" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fourth-cache", + "metadata": {}, + "outputs": [], + "source": [ + "num_image = len(text_input.input_ids[0]) \n", + "fig, ax = plt.subplots(num_image, 1, figsize=(15,5*num_image))\n", + "\n", + "rgb_image = cv2.imread(image_path)[:, :, ::-1]\n", + "rgb_image = np.float32(rgb_image) / 255\n", + "\n", + "ax[0].imshow(rgb_image)\n", + "ax[0].set_yticks([])\n", + "ax[0].set_xticks([])\n", + "ax[0].set_xlabel(\"Image\")\n", + " \n", + "for i,token_id in enumerate(text_input.input_ids[0][1:]):\n", + " word = tokenizer.decode([token_id])\n", + " gradcam_image = getAttMap(rgb_image, gradcam[i+1])\n", + " ax[i+1].imshow(gradcam_image)\n", + " ax[i+1].set_yticks([])\n", + " ax[i+1].set_xticks([])\n", + " ax[i+1].set_xlabel(word)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/Grounding.py b/Grounding.py new file mode 100644 index 0000000..6cc9bbd --- /dev/null +++ b/Grounding.py @@ -0,0 +1,295 @@ +import argparse +import os +import ruamel_yaml as yaml +import numpy as np +import random +import time +import datetime +import json +from pathlib import Path + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +import torch.distributed as dist +from torch.utils.data import DataLoader + +from models.model_retrieval import ALBEF +from models.vit import interpolate_pos_embed +from models.tokenization_bert import BertTokenizer + +import utils +from dataset import create_dataset, create_sampler, create_loader +from dataset.utils import collect_result, grounding_eval +from scheduler import create_scheduler +from optim import create_optimizer + +from refTools.refer_python3 import REFER + +from pdb import set_trace as breakpoint + +def train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, scheduler, config): + # train + model.train() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}')) + metric_logger.add_meter('loss_itm', utils.SmoothedValue(window_size=1, fmt='{value:.4f}')) + metric_logger.add_meter('loss_ita', utils.SmoothedValue(window_size=1, fmt='{value:.4f}')) + header = 'Train Epoch: [{}]'.format(epoch) + print_freq = 50 + step_size = 100 + warmup_iterations = warmup_steps*step_size + + for i,(image, text, idx) in enumerate(metric_logger.log_every(data_loader, print_freq, header)): + image = image.to(device,non_blocking=True) + idx = idx.to(device,non_blocking=True) + text_input = tokenizer(text, padding='longest', max_length=30, return_tensors="pt").to(device) + + if epoch>0 or not config['warm_up']: + alpha = config['alpha'] + else: + alpha = config['alpha']*min(1,i/len(data_loader)) + + loss_ita, loss_itm = model(image, text_input,alpha=alpha, idx=idx) + loss = loss_ita + loss_itm + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + metric_logger.update(loss_itm=loss_itm.item()) + metric_logger.update(loss_ita=loss_ita.item()) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + if epoch==0 and i%step_size==0 and i<=warmup_iterations: + scheduler.step(i//step_size) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.3f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + +def val(model, data_loader, tokenizer, device, gradcam_mode, block_num): + # test + model.eval() + + metric_logger = utils.MetricLogger(delimiter=" ") + header = 'Evaluation:' + print_freq = 50 + + if gradcam_mode=='itm': + model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.save_attention = True + + result = [] + for image, text, ref_ids in metric_logger.log_every(data_loader, print_freq, header): + image = image.to(device) + text_input = tokenizer(text, padding='longest', return_tensors="pt").to(device) + + if gradcam_mode=='itm': + image_embeds = model.visual_encoder(image) + image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device) + output = model.text_encoder(text_input.input_ids, + attention_mask = text_input.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True, + ) + + vl_embeddings = output.last_hidden_state[:,0,:] + vl_output = model.itm_head(vl_embeddings) + loss = vl_output[:,1].sum() + + model.zero_grad() + loss.backward() + + with torch.no_grad(): + mask = text_input.attention_mask.view(text_input.attention_mask.size(0),1,-1,1,1) + + grads = model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.get_attn_gradients().detach() + cams = model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.get_attention_map().detach() + + cams = cams[:, :, :, 1:].reshape(image.size(0), 12, -1, 24, 24) * mask + grads = grads[:, :, :, 1:].clamp(min=0).reshape(image.size(0), 12, -1, 24, 24) * mask + + gradcam = cams * grads + gradcam = gradcam.mean(1).mean(1) + + elif gradcam_mode=='itc': + image_embeds = model.visual_encoder(image, register_blk=block_num) + image_feat = F.normalize(model.vision_proj(image_embeds[:,0,:]),dim=-1) + text_output = model.text_encoder(text_input.input_ids, attention_mask = text_input.attention_mask, + return_dict = True, mode = 'text') + text_embeds = text_output.last_hidden_state + text_feat = F.normalize(model.text_proj(text_embeds[:,0,:]),dim=-1) + sim = image_feat@text_feat.t()/model.temp + loss = sim.diag().sum() + + model.zero_grad() + loss.backward() + + with torch.no_grad(): + grad = model.visual_encoder.blocks[block_num].attn.get_attn_gradients().detach() + cam = model.visual_encoder.blocks[block_num].attn.get_attention_map().detach() + cam = cam[:, :, 0, 1:].reshape(image.size(0), -1, 24, 24) + grad = grad[:, :, 0, 1:].reshape(image.size(0), -1, 24, 24).clamp(0) + gradcam = (cam * grad).mean(1) + + for r_id, cam in zip(ref_ids, gradcam): + result.append({'ref_id':r_id.item(), 'pred':cam}) + + if gradcam_mode=='itm': + model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.save_attention = False + return result + + +def main(args, config): + utils.init_distributed_mode(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + #### Dataset #### + print("Creating dataset") + grd_train_dataset, grd_test_dataset = create_dataset('grounding', config) + datasets = [grd_train_dataset, grd_test_dataset] + + if args.distributed: + num_tasks = utils.get_world_size() + global_rank = utils.get_rank() + samplers = create_sampler(datasets, [True, False], num_tasks, global_rank) + else: + samplers = [None, None] + + train_loader, test_loader = create_loader(datasets,samplers,batch_size=[config['batch_size'],config['batch_size']], + num_workers=[4,4],is_trains=[True, False], collate_fns=[None,None]) + + tokenizer = BertTokenizer.from_pretrained(args.text_encoder) + + ## refcoco evaluation tools + refer = REFER(config['refcoco_data'], 'refcoco+', 'unc') + dets = json.load(open(config['det_file'],'r')) + cocos = json.load(open(config['coco_file'],'r')) + + #### Model #### + print("Creating model") + model = ALBEF(config = config, text_encoder=args.text_encoder, tokenizer=tokenizer) + + if args.checkpoint: + checkpoint = torch.load(args.checkpoint, map_location='cpu') + state_dict = checkpoint['model'] + pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) + state_dict['visual_encoder.pos_embed'] = pos_embed_reshaped + m_pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder_m.pos_embed'],model.visual_encoder_m) + state_dict['visual_encoder_m.pos_embed'] = m_pos_embed_reshaped + + for key in list(state_dict.keys()): + if 'bert' in key: + encoder_key = key.replace('bert.','') + state_dict[encoder_key] = state_dict[key] + del state_dict[key] + msg = model.load_state_dict(state_dict,strict=False) + + print('load checkpoint from %s'%args.checkpoint) + print(msg) + + model = model.to(device) + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + + arg_opt = utils.AttrDict(config['optimizer']) + optimizer = create_optimizer(arg_opt, model) + arg_sche = utils.AttrDict(config['schedular']) + lr_scheduler, _ = create_scheduler(arg_sche, optimizer) + + max_epoch = config['schedular']['epochs'] + warmup_steps = config['schedular']['warmup_epochs'] + best = 0 + + print("Start training") + start_time = time.time() + for epoch in range(0, max_epoch): + if not args.evaluate: + if args.distributed: + train_loader.sampler.set_epoch(epoch) + train_stats = train(model, train_loader, optimizer, tokenizer, epoch, warmup_steps, device, lr_scheduler, config) + + result = val(model_without_ddp, test_loader, tokenizer, device, args.gradcam_mode, args.block_num) + + results = collect_result(result, args.result_dir, 'epoch%d'%epoch, is_json=False, is_list=True) + + if utils.is_main_process(): + + grounding_acc = grounding_eval(results, dets, cocos, refer, alpha=0.5, mask_size=24) + + if args.evaluate: + log_stats = {**{f'{k}': v for k, v in grounding_acc.items()}, + 'epoch': epoch, + } + else: + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + **{f'{k}': v for k, v in grounding_acc.items()}, + 'epoch': epoch, + } + if grounding_acc['val_d']>best: + save_obj = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'config': config, + 'epoch': epoch, + } + torch.save(save_obj, os.path.join(args.output_dir, 'checkpoint_best.pth')) + best = grounding_acc['val_d'] + + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + + if args.evaluate: + break + + lr_scheduler.step(epoch+warmup_steps+1) + dist.barrier() + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', default='./configs/Grounding.yaml') + parser.add_argument('--checkpoint', default='') + parser.add_argument('--output_dir', default='output/RefCOCO') + parser.add_argument('--gradcam_mode', default='itm', choices=['itm','itc']) + parser.add_argument('--block_num', default=8, type=int) + parser.add_argument('--text_encoder', default='bert-base-uncased') + parser.add_argument('--evaluate', action='store_true') + parser.add_argument('--device', default='cuda') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--distributed', default=True, type=bool) + args = parser.parse_args() + + config = yaml.load(open(args.config, 'r'), Loader=yaml.Loader) + + args.result_dir = os.path.join(args.output_dir, 'result') + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + Path(args.result_dir).mkdir(parents=True, exist_ok=True) + + yaml.dump(config, open(os.path.join(args.output_dir, 'config.yaml'), 'w')) + + main(args, config) \ No newline at end of file diff --git a/NLVR.py b/NLVR.py new file mode 100644 index 0000000..3865d94 --- /dev/null +++ b/NLVR.py @@ -0,0 +1,228 @@ +import argparse +import os +import ruamel_yaml as yaml +import numpy as np +import random +import time +import datetime +import json +from pathlib import Path +import json +import pickle + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +import torch.backends.cudnn as cudnn +import torch.distributed as dist + +from models.model_retrieval import ALBEF +from models.vit import interpolate_pos_embed +from models.tokenization_bert import BertTokenizer + +import utils +from dataset import create_dataset, create_sampler, create_loader +from scheduler import create_scheduler +from optim import create_optimizer + + +def train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, scheduler, config): + # train + model.train() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=50, fmt='{value:.6f}')) + metric_logger.add_meter('loss', utils.SmoothedValue(window_size=50, fmt='{value:.4f}')) + + header = 'Train Epoch: [{}]'.format(epoch) + print_freq = 50 + step_size = 100 + warmup_iterations = warmup_steps*step_size + + for i,(image0, image1, text, targets) in enumerate(metric_logger.log_every(data_loader, print_freq, header)): + images = torch.cat([image0, image1], dim=0) + images, targets = images.to(device), targets.to(device) + + text_inputs = tokenizer(text, padding='longest', return_tensors="pt").to(device) + + if epoch>0 or not config['warm_up']: + alpha = config['alpha'] + else: + alpha = config['alpha']*min(1,i/len(data_loader)) + + loss = model(images, text_inputs, targets=targets, train=True, alpha=alpha) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + metric_logger.update(loss=loss.item()) + + if epoch==0 and i%step_size==0 and i<=warmup_iterations: + scheduler.step(i//step_size) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.4f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def evaluate(model, data_loader, tokenizer, device, config): + # test + model.eval() + + metric_logger = utils.MetricLogger(delimiter=" ") + + header = 'Evaluation:' + print_freq = 50 + + for image0, image1, text, targets in metric_logger.log_every(data_loader, print_freq, header): + images = torch.cat([image0, image1], dim=0) + images, targets = images.to(device), targets.to(device) + + text_inputs = tokenizer(text, padding='longest', return_tensors="pt").to(device) + + prediction = model(images, text_inputs, targets=targets, train=False) + + _, pred_class = prediction.max(1) + accuracy = (targets==pred_class).sum() / targets.size(0) + + metric_logger.meters['acc'].update(accuracy.item(), n=image0.size(0)) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.4f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + +def main(args, config): + utils.init_distributed_mode(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + #### Dataset #### + print("Creating dataset") + datasets = create_dataset('nlvr', config) + + if args.distributed: + num_tasks = utils.get_world_size() + global_rank = utils.get_rank() + samplers = create_sampler(datasets, [True, False, False], num_tasks, global_rank) + else: + samplers = [None, None, None] + + train_loader, val_loader, test_loader = create_loader(datasets,samplers,batch_size=[config['batch_size']]*3, + num_workers=[4,4,4],is_trains=[True,False,False], collate_fns=[None,None,None]) + + tokenizer = BertTokenizer.from_pretrained(args.text_encoder) + + #### Model #### + print("Creating model") + model = ALBEF(config=config, text_encoder=args.text_encoder, tokenizer=tokenizer) + + if args.checkpoint: + checkpoint = torch.load(args.checkpoint, map_location='cpu') + state_dict = checkpoint['model'] + + # reshape positional embedding to accomodate for image resolution change + pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) + state_dict['visual_encoder.pos_embed'] = pos_embed_reshaped + + if config['distill']: + model.copy_params() + + msg = model.load_state_dict(state_dict,strict=False) + print('load checkpoint from %s'%args.checkpoint) + print(msg) + + model = model.to(device) + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + + arg_opt = utils.AttrDict(config['optimizer']) + optimizer = create_optimizer(arg_opt, model) + arg_sche = utils.AttrDict(config['schedular']) + lr_scheduler, _ = create_scheduler(arg_sche, optimizer) + + max_epoch = config['schedular']['epochs'] + warmup_steps = config['schedular']['warmup_epochs'] + + print("Start training") + start_time = time.time() + best = 0 + best_epoch = 0 + + for epoch in range(0, max_epoch): + if args.distributed: + train_loader.sampler.set_epoch(epoch) + + train_stats = train(model, train_loader, optimizer, tokenizer, epoch, warmup_steps, device, lr_scheduler, config) + val_stats = evaluate(model, val_loader, tokenizer, device, config) + test_stats = evaluate(model, test_loader, tokenizer, device, config) + + if utils.is_main_process(): + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + **{f'val_{k}': v for k, v in val_stats.items()}, + **{f'test_{k}': v for k, v in test_stats.items()}, + 'epoch': epoch, + } + + if float(val_stats['acc'])>best: + save_obj = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'config': config, + 'epoch': epoch, + } + torch.save(save_obj, os.path.join(args.output_dir, 'checkpoint_%02d.pth'%epoch)) + best = float(val_stats['acc']) + best_epoch = epoch + + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + + lr_scheduler.step(epoch+warmup_steps+1) + dist.barrier() + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + if utils.is_main_process(): + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write("best epoch: %d"%best_epoch) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', default='./configs/NLVR.yaml') + parser.add_argument('--checkpoint', default='') + parser.add_argument('--output_dir', default='output/NLVR') + parser.add_argument('--text_encoder', default='bert-base-uncased') + parser.add_argument('--device', default='cuda') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--distributed', default=True, type=bool) + args = parser.parse_args() + + config = yaml.load(open(args.config, 'r'), Loader=yaml.Loader) + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + yaml.dump(config, open(os.path.join(args.output_dir, 'config.yaml'), 'w')) + + main(args, config) \ No newline at end of file diff --git a/Pretrain.py b/Pretrain.py new file mode 100644 index 0000000..e5deb6e --- /dev/null +++ b/Pretrain.py @@ -0,0 +1,196 @@ +import argparse +import os +import ruamel_yaml as yaml +import numpy as np +import random +import time +import datetime +import json +from pathlib import Path + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +import torch.backends.cudnn as cudnn +import torch.distributed as dist + +from models.model_pretrain import ALBEF +from models.vit import interpolate_pos_embed +from models.tokenization_bert import BertTokenizer + +import utils +from dataset import create_dataset, create_sampler, create_loader +from scheduler import create_scheduler +from optim import create_optimizer + + +def train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, scheduler, config): + # train + model.train() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=50, fmt='{value:.6f}')) + metric_logger.add_meter('loss_mlm', utils.SmoothedValue(window_size=50, fmt='{value:.4f}')) + metric_logger.add_meter('loss_ita', utils.SmoothedValue(window_size=50, fmt='{value:.4f}')) + metric_logger.add_meter('loss_itm', utils.SmoothedValue(window_size=50, fmt='{value:.4f}')) + + header = 'Train Epoch: [{}]'.format(epoch) + print_freq = 50 + step_size = 100 + warmup_iterations = warmup_steps*step_size + + if args.distributed: + data_loader.sampler.set_epoch(epoch) + + for i, (image, text) in enumerate(metric_logger.log_every(data_loader, print_freq, header)): + + optimizer.zero_grad() + + image = image.to(device,non_blocking=True) + + text_input = tokenizer(text, padding='longest', truncation=True, max_length=25, return_tensors="pt").to(device) + + if epoch>0: + alpha = config['alpha'] + else: + alpha = config['alpha']*min(1,i/len(data_loader)) + + loss_mlm, loss_ita, loss_itm = model(image, text_input, alpha = alpha) + + loss = loss_mlm + loss_ita + loss_itm + + loss.backward() + optimizer.step() + + metric_logger.update(loss_mlm=loss_mlm.item()) + metric_logger.update(loss_ita=loss_ita.item()) + metric_logger.update(loss_itm=loss_itm.item()) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + + if epoch==0 and i%step_size==0 and i<=warmup_iterations: + scheduler.step(i//step_size) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.3f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + +def main(args, config): + utils.init_distributed_mode(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + start_epoch = 0 + max_epoch = config['schedular']['epochs'] + warmup_steps = config['schedular']['warmup_epochs'] + + #### Dataset #### + print("Creating dataset") + datasets = [create_dataset('pretrain', config)] + + if args.distributed: + num_tasks = utils.get_world_size() + global_rank = utils.get_rank() + samplers = create_sampler(datasets, [True], num_tasks, global_rank) + else: + samplers = [None] + + data_loader = create_loader(datasets,samplers,batch_size=[config['batch_size']], num_workers=[4], is_trains=[True], collate_fns=[None])[0] + + tokenizer = BertTokenizer.from_pretrained(args.text_encoder) + + #### Model #### + print("Creating model") + model = ALBEF(config=config, text_encoder=args.text_encoder, tokenizer=tokenizer, init_deit=True) + + model = model.to(device) + + arg_opt = utils.AttrDict(config['optimizer']) + optimizer = create_optimizer(arg_opt, model) + arg_sche = utils.AttrDict(config['schedular']) + lr_scheduler, _ = create_scheduler(arg_sche, optimizer) + + + if args.checkpoint: + checkpoint = torch.load(args.checkpoint, map_location='cpu') + state_dict = checkpoint['model'] + if args.resume: + optimizer.load_state_dict(checkpoint['optimizer']) + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + start_epoch = checkpoint['epoch']+1 + else: + pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) + m_pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder_m.pos_embed'],model.visual_encoder_m) + state_dict['visual_encoder.pos_embed'] = pos_embed_reshaped + state_dict['visual_encoder_m.pos_embed'] = m_pos_embed_reshaped + model.load_state_dict(state_dict) + print('load checkpoint from %s'%args.checkpoint) + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + + print("Start training") + start_time = time.time() + + for epoch in range(start_epoch, max_epoch): + + if epoch>0: + lr_scheduler.step(epoch+warmup_steps) + + train_stats = train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, lr_scheduler, config) + if utils.is_main_process(): + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch, + } + save_obj = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'config': config, + 'epoch': epoch, + } + torch.save(save_obj, os.path.join(args.output_dir, 'checkpoint_%02d.pth'%epoch)) + + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + + dist.barrier() + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', default='./configs/Pretrain.yaml') + parser.add_argument('--checkpoint', default='') + parser.add_argument('--resume', default=False, type=bool) + parser.add_argument('--output_dir', default='Pretrain/') + parser.add_argument('--text_encoder', default='bert-base-uncased') + parser.add_argument('--device', default='cuda') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--distributed', default=True, type=bool) + args = parser.parse_args() + + config = yaml.load(open(args.config, 'r'), Loader=yaml.Loader) + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + yaml.dump(config, open(os.path.join(args.output_dir, 'config.yaml'), 'w')) + + main(args, config) \ No newline at end of file diff --git a/Pretrain_nlvr.py b/Pretrain_nlvr.py new file mode 100644 index 0000000..8fb5a1c --- /dev/null +++ b/Pretrain_nlvr.py @@ -0,0 +1,200 @@ +import argparse +import os +import ruamel_yaml as yaml +import numpy as np +import random +import time +import datetime +import json +from pathlib import Path + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +import torch.backends.cudnn as cudnn +import torch.distributed as dist + +from models.model_retrieval import ALBEF +from models.vit import interpolate_pos_embed +from models.tokenization_bert import BertTokenizer + +import utils +from dataset import create_dataset, create_sampler, create_loader +from scheduler import create_scheduler +from optim import create_optimizer + + +def train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, scheduler, config): + # train + model.train() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=50, fmt='{value:.6f}')) + metric_logger.add_meter('loss', utils.SmoothedValue(window_size=50, fmt='{value:.4f}')) + + header = 'Train Epoch: [{}]'.format(epoch) + print_freq = 50 + step_size = 100 + warmup_iterations = warmup_steps*step_size + + if args.distributed: + data_loader.sampler.set_epoch(epoch) + + for i, (image, text) in enumerate(metric_logger.log_every(data_loader, print_freq, header)): + + optimizer.zero_grad() + + image = image.to(device,non_blocking=True) + text_input = tokenizer(text, padding='longest', truncation=True, max_length=25, return_tensors="pt").to(device) + + loss = model(image, text_input) + loss.backward() + + optimizer.step() + + metric_logger.update(loss=loss.item()) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + + if epoch==0 and i%step_size==0 and i<=warmup_iterations: + scheduler.step(i//step_size) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.3f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + +def main(args, config): + utils.init_distributed_mode(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + start_epoch = 0 + max_epoch = config['schedular']['epochs'] + warmup_steps = config['schedular']['warmup_epochs'] + + #### Dataset #### + print("Creating dataset") + datasets = [create_dataset('pretrain', config)] + + if args.distributed: + num_tasks = utils.get_world_size() + global_rank = utils.get_rank() + samplers = create_sampler(datasets, [True], num_tasks, global_rank) + else: + samplers = [None] + + data_loader = create_loader(datasets,samplers,batch_size=[config['batch_size']], num_workers=[4], is_trains=[True], collate_fns=[None])[0] + + tokenizer = BertTokenizer.from_pretrained(args.text_encoder) + + #### Model #### + print("Creating model") + model = ALBEF(config=config, text_encoder=args.text_encoder, tokenizer=tokenizer) + + model = model.to(device) + + arg_opt = utils.AttrDict(config['optimizer']) + optimizer = create_optimizer(arg_opt, model) + arg_sche = utils.AttrDict(config['schedular']) + lr_scheduler, _ = create_scheduler(arg_sche, optimizer) + + if args.checkpoint: + checkpoint = torch.load(args.checkpoint, map_location='cpu') + state_dict = checkpoint['model'] + pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) + state_dict['visual_encoder.pos_embed'] = pos_embed_reshaped + + for key in list(state_dict.keys()): + if 'bert' in key: + new_key = key.replace('bert.','') + + if 'layer' in new_key: + keys = new_key.split('.') + layer_num = int(keys[3]) + # replicate the multimodal encoder's blocks for two images + if layer_num>=6: + new_layer_num = (layer_num-6)*2+6 + keys[3] = str(new_layer_num) + new_key_0 = '.'.join(keys) + state_dict[new_key_0] = state_dict[key] + keys[3] = str(new_layer_num+1) + new_key_1 = '.'.join(keys) + state_dict[new_key_1] = state_dict[key] + else: + state_dict[new_key] = state_dict[key] + else: + state_dict[new_key] = state_dict[key] + del state_dict[key] + + msg = model.load_state_dict(state_dict,strict=False) + print('load checkpoint from %s'%args.checkpoint) + print(msg) + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True) + model_without_ddp = model.module + + print("Start training") + start_time = time.time() + + for epoch in range(start_epoch, max_epoch): + + if epoch>0: + lr_scheduler.step(epoch+warmup_steps) + + train_stats = train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, lr_scheduler, config) + + if utils.is_main_process(): + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch, + } + save_obj = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'config': config, + 'epoch': epoch, + } + torch.save(save_obj, os.path.join(args.output_dir, 'checkpoint_%02d.pth'%epoch)) + + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + + dist.barrier() + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', default='./configs/NLVR_pretrain.yaml') + parser.add_argument('--checkpoint', default='') + parser.add_argument('--output_dir', default='output/NLVR_pretrain') + parser.add_argument('--text_encoder', default='bert-base-uncased') + parser.add_argument('--device', default='cuda') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--distributed', default=True, type=bool) + args = parser.parse_args() + + config = yaml.load(open(args.config, 'r'), Loader=yaml.Loader) + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + yaml.dump(config, open(os.path.join(args.output_dir, 'config.yaml'), 'w')) + + main(args, config) \ No newline at end of file diff --git a/Pretrain_resnet.py b/Pretrain_resnet.py new file mode 100644 index 0000000..b4a8fad --- /dev/null +++ b/Pretrain_resnet.py @@ -0,0 +1,190 @@ +import argparse +import os +import ruamel_yaml as yaml +import numpy as np +import random +import time +import datetime +import json +from pathlib import Path + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +import torch.backends.cudnn as cudnn +import torch.distributed as dist + +from models.model_pretrain_resnet import ALBEF +from models.tokenization_bert import BertTokenizer + +import utils +from dataset import create_dataset, create_sampler, create_loader +from scheduler import create_scheduler +from optim import create_optimizer + + +def train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, scheduler, config): + # train + model.train() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=50, fmt='{value:.6f}')) + metric_logger.add_meter('loss_mlm', utils.SmoothedValue(window_size=50, fmt='{value:.4f}')) + metric_logger.add_meter('loss_ita', utils.SmoothedValue(window_size=50, fmt='{value:.4f}')) + metric_logger.add_meter('loss_itm', utils.SmoothedValue(window_size=50, fmt='{value:.4f}')) + + header = 'Train Epoch: [{}]'.format(epoch) + print_freq = 50 + step_size = 100 + warmup_iterations = warmup_steps*step_size + + if args.distributed: + data_loader.sampler.set_epoch(epoch) + + for i, (image, text) in enumerate(metric_logger.log_every(data_loader, print_freq, header)): + + optimizer.zero_grad() + + image = image.to(device,non_blocking=True) + + text_input = tokenizer(text, padding='longest', truncation=True, max_length=25, return_tensors="pt").to(device) + + if epoch>0: + alpha = config['alpha'] + else: + alpha = config['alpha']*min(1,i/len(data_loader)) + + loss_mlm, loss_ita, loss_itm = model(image, text_input, alpha = alpha) + + loss = loss_mlm + loss_ita + loss_itm + + loss.backward() + optimizer.step() + + metric_logger.update(loss_mlm=loss_mlm.item()) + metric_logger.update(loss_ita=loss_ita.item()) + metric_logger.update(loss_itm=loss_itm.item()) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + + if epoch==0 and i%step_size==0 and i<=warmup_iterations: + scheduler.step(i//step_size) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.3f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + +def main(args, config): + utils.init_distributed_mode(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + start_epoch = 0 + max_epoch = config['schedular']['epochs'] + warmup_steps = config['schedular']['warmup_epochs'] + + #### Dataset #### + print("Creating dataset") + datasets = [create_dataset('pretrain', config)] + + if args.distributed: + num_tasks = utils.get_world_size() + global_rank = utils.get_rank() + samplers = create_sampler(datasets, [True], num_tasks, global_rank) + else: + samplers = [None] + + data_loader = create_loader(datasets,samplers,batch_size=[config['batch_size']], num_workers=[4], is_trains=[True], collate_fns=[None])[0] + + tokenizer = BertTokenizer.from_pretrained(args.text_encoder) + + #### Model #### + print("Creating model") + model = ALBEF(config=config, text_encoder=args.text_encoder, tokenizer=tokenizer) + + model = model.to(device) + + arg_opt = utils.AttrDict(config['optimizer']) + optimizer = create_optimizer(arg_opt, model) + arg_sche = utils.AttrDict(config['schedular']) + lr_scheduler, _ = create_scheduler(arg_sche, optimizer) + + + if args.checkpoint: + checkpoint = torch.load(args.checkpoint, map_location='cpu') + state_dict = checkpoint['model'] + if args.resume: + optimizer.load_state_dict(checkpoint['optimizer']) + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + start_epoch = checkpoint['epoch']+1 + model.load_state_dict(state_dict) + print('load checkpoint from %s'%args.checkpoint) + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + + print("Start training") + start_time = time.time() + + for epoch in range(start_epoch, max_epoch): + + if epoch>0: + lr_scheduler.step(epoch+warmup_steps) + + train_stats = train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, lr_scheduler, config) + if utils.is_main_process(): + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch, + } + save_obj = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'config': config, + 'epoch': epoch, + } + torch.save(save_obj, os.path.join(args.output_dir, 'checkpoint_%02d.pth'%epoch)) + + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + + dist.barrier() + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', default='./configs/Pretrain.yaml') + parser.add_argument('--checkpoint', default='') + parser.add_argument('--resume', default=False, type=bool) + parser.add_argument('--output_dir', default='Pretrain/resnet101') + parser.add_argument('--text_encoder', default='bert-base-uncased') + parser.add_argument('--device', default='cuda') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--distributed', default=True, type=bool) + args = parser.parse_args() + + config = yaml.load(open(args.config, 'r'), Loader=yaml.Loader) + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + yaml.dump(config, open(os.path.join(args.output_dir, 'config.yaml'), 'w')) + + main(args, config) \ No newline at end of file diff --git a/Retrieval.py b/Retrieval.py new file mode 100644 index 0000000..eb4df4e --- /dev/null +++ b/Retrieval.py @@ -0,0 +1,379 @@ +import argparse +import os +import ruamel_yaml as yaml +import numpy as np +import random +import time +import datetime +import json +from pathlib import Path + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +import torch.distributed as dist +from torch.utils.data import DataLoader + +from models.model_retrieval import ALBEF +from models.vit import interpolate_pos_embed +from models.tokenization_bert import BertTokenizer + +import utils +from dataset import create_dataset, create_sampler, create_loader +from scheduler import create_scheduler +from optim import create_optimizer + + +def train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, scheduler, config): + # train + model.train() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}')) + metric_logger.add_meter('loss_itm', utils.SmoothedValue(window_size=1, fmt='{value:.4f}')) + metric_logger.add_meter('loss_ita', utils.SmoothedValue(window_size=1, fmt='{value:.4f}')) + header = 'Train Epoch: [{}]'.format(epoch) + print_freq = 50 + step_size = 100 + warmup_iterations = warmup_steps*step_size + + for i,(image, text, idx) in enumerate(metric_logger.log_every(data_loader, print_freq, header)): + image = image.to(device,non_blocking=True) + idx = idx.to(device,non_blocking=True) + text_input = tokenizer(text, padding='longest', max_length=30, return_tensors="pt").to(device) + + if epoch>0 or not config['warm_up']: + alpha = config['alpha'] + else: + alpha = config['alpha']*min(1,i/len(data_loader)) + + loss_ita, loss_itm = model(image, text_input,alpha=alpha, idx=idx) + loss = loss_ita + loss_itm + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + metric_logger.update(loss_itm=loss_itm.item()) + metric_logger.update(loss_ita=loss_ita.item()) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + if epoch==0 and i%step_size==0 and i<=warmup_iterations: + scheduler.step(i//step_size) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.3f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + + +@torch.no_grad() +def evaluation(model, data_loader, tokenizer, device, config): + # test + model.eval() + + metric_logger = utils.MetricLogger(delimiter=" ") + header = 'Evaluation:' + + print('Computing features for evaluation...') + start_time = time.time() + + texts = data_loader.dataset.text + num_text = len(texts) + text_bs = 256 + text_feats = [] + text_embeds = [] + text_atts = [] + for i in range(0, num_text, text_bs): + text = texts[i: min(num_text, i+text_bs)] + text_input = tokenizer(text, padding='max_length', truncation=True, max_length=30, return_tensors="pt").to(device) + text_output = model.text_encoder(text_input.input_ids, attention_mask = text_input.attention_mask, mode='text') + text_feat = text_output.last_hidden_state + text_embed = F.normalize(model.text_proj(text_feat[:,0,:])) + text_embeds.append(text_embed) + text_feats.append(text_feat) + text_atts.append(text_input.attention_mask) + text_embeds = torch.cat(text_embeds,dim=0) + text_feats = torch.cat(text_feats,dim=0) + text_atts = torch.cat(text_atts,dim=0) + + image_feats = [] + image_embeds = [] + for image, img_id in data_loader: + image = image.to(device) + image_feat = model.visual_encoder(image) + image_embed = model.vision_proj(image_feat[:,0,:]) + image_embed = F.normalize(image_embed,dim=-1) + + image_feats.append(image_feat) + image_embeds.append(image_embed) + + image_feats = torch.cat(image_feats,dim=0) + image_embeds = torch.cat(image_embeds,dim=0) + + sims_matrix = image_embeds @ text_embeds.t() + score_matrix_i2t = torch.full((len(data_loader.dataset.image),len(texts)),-100.0).to(device) + + num_tasks = utils.get_world_size() + rank = utils.get_rank() + step = sims_matrix.size(0)//num_tasks + 1 + start = rank*step + end = min(sims_matrix.size(0),start+step) + + for i,sims in enumerate(metric_logger.log_every(sims_matrix[start:end], 50, header)): + topk_sim, topk_idx = sims.topk(k=config['k_test'], dim=0) + + encoder_output = image_feats[start+i].repeat(config['k_test'],1,1) + encoder_att = torch.ones(encoder_output.size()[:-1],dtype=torch.long).to(device) + output = model.text_encoder(encoder_embeds = text_feats[topk_idx], + attention_mask = text_atts[topk_idx], + encoder_hidden_states = encoder_output, + encoder_attention_mask = encoder_att, + return_dict = True, + mode = 'fusion' + ) + score = model.itm_head(output.last_hidden_state[:,0,:])[:,1] + score_matrix_i2t[start+i,topk_idx] = score + + sims_matrix = sims_matrix.t() + score_matrix_t2i = torch.full((len(texts),len(data_loader.dataset.image)),-100.0).to(device) + + step = sims_matrix.size(0)//num_tasks + 1 + start = rank*step + end = min(sims_matrix.size(0),start+step) + + for i,sims in enumerate(metric_logger.log_every(sims_matrix[start:end], 50, header)): + + topk_sim, topk_idx = sims.topk(k=config['k_test'], dim=0) + encoder_output = image_feats[topk_idx] + encoder_att = torch.ones(encoder_output.size()[:-1],dtype=torch.long).to(device) + output = model.text_encoder(encoder_embeds = text_feats[start+i].repeat(config['k_test'],1,1), + attention_mask = text_atts[start+i].repeat(config['k_test'],1), + encoder_hidden_states = encoder_output, + encoder_attention_mask = encoder_att, + return_dict = True, + mode = 'fusion' + ) + score = model.itm_head(output.last_hidden_state[:,0,:])[:,1] + score_matrix_t2i[start+i,topk_idx] = score + + if args.distributed: + dist.barrier() + torch.distributed.all_reduce(score_matrix_i2t, op=torch.distributed.ReduceOp.SUM) + torch.distributed.all_reduce(score_matrix_t2i, op=torch.distributed.ReduceOp.SUM) + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Evaluation time {}'.format(total_time_str)) + + return score_matrix_i2t.cpu().numpy(), score_matrix_t2i.cpu().numpy() + + + +@torch.no_grad() +def itm_eval(scores_i2t, scores_t2i, txt2img, img2txt): + + #Images->Text + ranks = np.zeros(scores_i2t.shape[0]) + for index,score in enumerate(scores_i2t): + inds = np.argsort(score)[::-1] + # Score + rank = 1e20 + for i in img2txt[index]: + tmp = np.where(inds == i)[0][0] + if tmp < rank: + rank = tmp + ranks[index] = rank + + # Compute metrics + tr1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks) + tr5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks) + tr10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks) + + #Text->Images + ranks = np.zeros(scores_t2i.shape[0]) + + for index,score in enumerate(scores_t2i): + inds = np.argsort(score)[::-1] + ranks[index] = np.where(inds == txt2img[index])[0][0] + + # Compute metrics + ir1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks) + ir5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks) + ir10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks) + + tr_mean = (tr1 + tr5 + tr10) / 3 + ir_mean = (ir1 + ir5 + ir10) / 3 + r_mean = (tr_mean + ir_mean) / 2 + + eval_result = {'txt_r1': tr1, + 'txt_r5': tr5, + 'txt_r10': tr10, + 'txt_r_mean': tr_mean, + 'img_r1': ir1, + 'img_r5': ir5, + 'img_r10': ir10, + 'img_r_mean': ir_mean, + 'r_mean': r_mean} + return eval_result + + + + +def main(args, config): + utils.init_distributed_mode(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + #### Dataset #### + print("Creating retrieval dataset") + train_dataset, val_dataset, test_dataset = create_dataset('re', config) + + if args.distributed: + num_tasks = utils.get_world_size() + global_rank = utils.get_rank() + samplers = create_sampler([train_dataset], [True], num_tasks, global_rank) + [None, None] + else: + samplers = [None, None, None] + + train_loader, val_loader, test_loader = create_loader([train_dataset, val_dataset, test_dataset],samplers, + batch_size=[config['batch_size_train']]+[config['batch_size_test']]*2, + num_workers=[4,4,4], + is_trains=[True, False, False], + collate_fns=[None,None,None]) + + tokenizer = BertTokenizer.from_pretrained(args.text_encoder) + + #### Model #### + print("Creating model") + model = ALBEF(config=config, text_encoder=args.text_encoder, tokenizer=tokenizer) + + if args.checkpoint: + checkpoint = torch.load(args.checkpoint, map_location='cpu') + state_dict = checkpoint['model'] + + # reshape positional embedding to accomodate for image resolution change + pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) + state_dict['visual_encoder.pos_embed'] = pos_embed_reshaped + m_pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder_m.pos_embed'],model.visual_encoder_m) + state_dict['visual_encoder_m.pos_embed'] = m_pos_embed_reshaped + + for key in list(state_dict.keys()): + if 'bert' in key: + encoder_key = key.replace('bert.','') + state_dict[encoder_key] = state_dict[key] + del state_dict[key] + msg = model.load_state_dict(state_dict,strict=False) + + print('load checkpoint from %s'%args.checkpoint) + print(msg) + + + model = model.to(device) + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + + arg_opt = utils.AttrDict(config['optimizer']) + optimizer = create_optimizer(arg_opt, model) + arg_sche = utils.AttrDict(config['schedular']) + lr_scheduler, _ = create_scheduler(arg_sche, optimizer) + + max_epoch = config['schedular']['epochs'] + warmup_steps = config['schedular']['warmup_epochs'] + best = 0 + best_epoch = 0 + + print("Start training") + start_time = time.time() + for epoch in range(0, max_epoch): + if not args.evaluate: + if args.distributed: + train_loader.sampler.set_epoch(epoch) + train_stats = train(model, train_loader, optimizer, tokenizer, epoch, warmup_steps, device, lr_scheduler, config) + + score_val_i2t, score_val_t2i, = evaluation(model_without_ddp, val_loader, tokenizer, device, config) + score_test_i2t, score_test_t2i = evaluation(model_without_ddp, test_loader, tokenizer, device, config) + + if utils.is_main_process(): + + val_result = itm_eval(score_val_i2t, score_val_t2i, val_loader.dataset.txt2img, val_loader.dataset.img2txt) + print(val_result) + test_result = itm_eval(score_test_i2t, score_test_t2i, test_loader.dataset.txt2img, test_loader.dataset.img2txt) + print(test_result) + + if args.evaluate: + log_stats = {**{f'val_{k}': v for k, v in val_result.items()}, + **{f'test_{k}': v for k, v in test_result.items()}, + 'epoch': epoch, + } + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + else: + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + **{f'val_{k}': v for k, v in val_result.items()}, + **{f'test_{k}': v for k, v in test_result.items()}, + 'epoch': epoch, + } + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + + if val_result['r_mean']>best: + save_obj = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'config': config, + 'epoch': epoch, + } + torch.save(save_obj, os.path.join(args.output_dir, 'checkpoint_best.pth')) + best = val_result['r_mean'] + best_epoch = epoch + + if args.evaluate: + break + + lr_scheduler.step(epoch+warmup_steps+1) + dist.barrier() + torch.cuda.empty_cache() + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + if utils.is_main_process(): + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write("best epoch: %d"%best_epoch) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', default='./configs/Retrieval_flickr.yaml') + parser.add_argument('--output_dir', default='output/Retrieval_flickr') + parser.add_argument('--checkpoint', default='') + parser.add_argument('--text_encoder', default='bert-large-uncased') + parser.add_argument('--evaluate', action='store_true') + parser.add_argument('--device', default='cuda') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--distributed', default=True, type=bool) + args = parser.parse_args() + + config = yaml.load(open(args.config, 'r'), Loader=yaml.Loader) + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + yaml.dump(config, open(os.path.join(args.output_dir, 'config.yaml'), 'w')) + + main(args, config) \ No newline at end of file diff --git a/VE.py b/VE.py new file mode 100644 index 0000000..bf4642c --- /dev/null +++ b/VE.py @@ -0,0 +1,251 @@ +import argparse +import os +import ruamel_yaml as yaml +import numpy as np +import random +import time +import datetime +import json +from pathlib import Path +import json +import pickle + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +import torch.backends.cudnn as cudnn +import torch.distributed as dist + +from models.model_ve import ALBEF +from models.vit import interpolate_pos_embed +from models.tokenization_bert import BertTokenizer + +import utils +from dataset import create_dataset, create_sampler, create_loader +from scheduler import create_scheduler +from optim import create_optimizer + + +def train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, scheduler, config): + # train + model.train() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=50, fmt='{value:.6f}')) + metric_logger.add_meter('loss', utils.SmoothedValue(window_size=50, fmt='{value:.4f}')) + + header = 'Train Epoch: [{}]'.format(epoch) + print_freq = 50 + step_size = 100 + warmup_iterations = warmup_steps*step_size + + for i,(images, text, targets) in enumerate(metric_logger.log_every(data_loader, print_freq, header)): + + images, targets = images.to(device,non_blocking=True), targets.to(device,non_blocking=True) + + text_inputs = tokenizer(text, padding='longest', return_tensors="pt").to(device) + + if epoch>0 or not config['warm_up']: + alpha = config['alpha'] + else: + alpha = config['alpha']*min(1,i/len(data_loader)) + + loss = model(images, text_inputs, targets=targets, train=True, alpha=alpha) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + metric_logger.update(loss=loss.item()) + + if epoch==0 and i%step_size==0 and i<=warmup_iterations: + scheduler.step(i//step_size) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.4f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def evaluate(model, data_loader, tokenizer, device, config): + # test + model.eval() + + metric_logger = utils.MetricLogger(delimiter=" ") + + header = 'Evaluation:' + print_freq = 50 + + for images, text, targets in metric_logger.log_every(data_loader, print_freq, header): + + images, targets = images.to(device,non_blocking=True), targets.to(device,non_blocking=True) + + text_inputs = tokenizer(text, padding='longest', return_tensors="pt").to(device) + + prediction = model(images, text_inputs, targets=targets, train=False) + + _, pred_class = prediction.max(1) + accuracy = (targets==pred_class).sum() / targets.size(0) + + metric_logger.meters['acc'].update(accuracy.item(), n=images.size(0)) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.4f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + +def main(args, config): + utils.init_distributed_mode(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + #### Dataset #### + print("Creating dataset") + datasets = create_dataset('ve', config) + + if args.distributed: + num_tasks = utils.get_world_size() + global_rank = utils.get_rank() + samplers = create_sampler(datasets, [True, False, False], num_tasks, global_rank) + else: + samplers = [None, None, None] + + train_loader, val_loader, test_loader = create_loader(datasets,samplers, + batch_size=[config['batch_size_train']]+[config['batch_size_test']]*2, + num_workers=[4,4,4],is_trains=[True,False,False], + collate_fns=[None,None,None]) + + tokenizer = BertTokenizer.from_pretrained(args.text_encoder) + + #### Model #### + print("Creating model") + model = ALBEF(config=config, text_encoder=args.text_encoder, tokenizer=tokenizer) + + if args.checkpoint: + checkpoint = torch.load(args.checkpoint, map_location='cpu') + state_dict = checkpoint['model'] + + # reshape positional embedding to accomodate for image resolution change + pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) + state_dict['visual_encoder.pos_embed'] = pos_embed_reshaped + + if not args.evaluate: + if config['distill']: + m_pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder_m.pos_embed'],model.visual_encoder_m) + state_dict['visual_encoder_m.pos_embed'] = m_pos_embed_reshaped + + for key in list(state_dict.keys()): + if 'bert' in key: + new_key = key.replace('bert.','') + state_dict[new_key] = state_dict[key] + del state_dict[key] + + msg = model.load_state_dict(state_dict,strict=False) + print('load checkpoint from %s'%args.checkpoint) + print(msg) + + model = model.to(device) + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + + arg_opt = utils.AttrDict(config['optimizer']) + optimizer = create_optimizer(arg_opt, model) + arg_sche = utils.AttrDict(config['schedular']) + lr_scheduler, _ = create_scheduler(arg_sche, optimizer) + + max_epoch = config['schedular']['epochs'] + warmup_steps = config['schedular']['warmup_epochs'] + best = 0 + best_epoch = 0 + + print("Start training") + start_time = time.time() + + for epoch in range(0, max_epoch): + if not args.evaluate: + if args.distributed: + train_loader.sampler.set_epoch(epoch) + train_stats = train(model, train_loader, optimizer, tokenizer, epoch, warmup_steps, device, lr_scheduler, config) + + val_stats = evaluate(model, val_loader, tokenizer, device, config) + test_stats = evaluate(model, test_loader, tokenizer, device, config) + + if utils.is_main_process(): + if args.evaluate: + log_stats = {**{f'val_{k}': v for k, v in val_stats.items()}, + **{f'test_{k}': v for k, v in test_stats.items()}, + 'epoch': epoch, + } + + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + else: + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + **{f'val_{k}': v for k, v in val_stats.items()}, + **{f'test_{k}': v for k, v in test_stats.items()}, + 'epoch': epoch, + } + + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + + if float(val_stats['acc'])>best: + save_obj = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'config': config, + 'epoch': epoch, + } + torch.save(save_obj, os.path.join(args.output_dir, 'checkpoint_best.pth')) + best = float(val_stats['acc']) + + if args.evaluate: + break + lr_scheduler.step(epoch+warmup_steps+1) + dist.barrier() + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + if utils.is_main_process(): + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write("best epoch: %d"%best_epoch) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', default='./configs/VE.yaml') + parser.add_argument('--output_dir', default='output/VE') + parser.add_argument('--checkpoint', default='') + parser.add_argument('--text_encoder', default='bert-base-uncased') + parser.add_argument('--evaluate', action='store_true') + parser.add_argument('--device', default='cuda') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--distributed', default=True, type=bool) + args = parser.parse_args() + + config = yaml.load(open(args.config, 'r'), Loader=yaml.Loader) + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + yaml.dump(config, open(os.path.join(args.output_dir, 'config.yaml'), 'w')) + + main(args, config) \ No newline at end of file diff --git a/VQA.py b/VQA.py new file mode 100644 index 0000000..181100c --- /dev/null +++ b/VQA.py @@ -0,0 +1,260 @@ +import argparse +import os +import ruamel_yaml as yaml +import numpy as np +import random +import time +import datetime +import json +from pathlib import Path + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +import torch.backends.cudnn as cudnn +import torch.distributed as dist + +from models.model_vqa import ALBEF +from models.vit import interpolate_pos_embed +from models.tokenization_bert import BertTokenizer + +import utils +from dataset.utils import save_result +from dataset import create_dataset, create_sampler, create_loader, vqa_collate_fn + +from scheduler import create_scheduler +from optim import create_optimizer + + +def train(model, data_loader, optimizer, tokenizer, epoch, warmup_steps, device, scheduler, config): + # train + model.train() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}')) + metric_logger.add_meter('loss', utils.SmoothedValue(window_size=1, fmt='{value:.4f}')) + + header = 'Train Epoch: [{}]'.format(epoch) + print_freq = 50 + step_size = 100 + warmup_iterations = warmup_steps*step_size + + for i,(image, question, answer, weights, n) in enumerate(metric_logger.log_every(data_loader, print_freq, header)): + image, weights = image.to(device,non_blocking=True), weights.to(device,non_blocking=True) + question_input = tokenizer(question, padding='longest', truncation=True, max_length=25, return_tensors="pt").to(device) + answer_input = tokenizer(answer, padding='longest', return_tensors="pt").to(device) + + if epoch>0 or not config['warm_up']: + alpha = config['alpha'] + else: + alpha = config['alpha']*min(1,i/len(data_loader)) + + loss = model(image, question_input, answer_input, train=True, alpha=alpha, k=n, weights=weights) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + metric_logger.update(loss=loss.item()) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + + if epoch==0 and i%step_size==0 and i<=warmup_iterations: + scheduler.step(i//step_size) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger.global_avg()) + return {k: "{:.3f}".format(meter.global_avg) for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def evaluation(model, data_loader, tokenizer, device, config) : + # test + model.eval() + + metric_logger = utils.MetricLogger(delimiter=" ") + header = 'Generate VQA test result:' + print_freq = 50 + + result = [] + + answer_list = [answer+config['eos'] for answer in data_loader.dataset.answer_list] + answer_input = tokenizer(answer_list, padding='longest', return_tensors='pt').to(device) + + for n, (image, question, question_id) in enumerate(metric_logger.log_every(data_loader, print_freq, header)): + image = image.to(device,non_blocking=True) + question_input = tokenizer(question, padding='longest', return_tensors="pt").to(device) + + topk_ids, topk_probs = model(image, question_input, answer_input, train=False, k=config['k_test']) + + for ques_id, topk_id, topk_prob in zip(question_id, topk_ids, topk_probs): + ques_id = int(ques_id.item()) + _, pred = topk_prob.max(dim=0) + result.append({"question_id":ques_id, "answer":data_loader.dataset.answer_list[topk_id[pred]]}) + + return result + + + + +def main(args, config): + utils.init_distributed_mode(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + cudnn.benchmark = True + + start_epoch = 0 + max_epoch = config['schedular']['epochs'] + warmup_steps = config['schedular']['warmup_epochs'] + + + #### Dataset #### + print("Creating vqa datasets") + datasets = create_dataset('vqa', config) + + if args.distributed: + num_tasks = utils.get_world_size() + global_rank = utils.get_rank() + samplers = create_sampler(datasets, [True, False], num_tasks, global_rank) + else: + samplers = [None, None] + + train_loader, test_loader = create_loader(datasets,samplers, + batch_size=[config['batch_size_train'],config['batch_size_test']], + num_workers=[4,4],is_trains=[True, False], + collate_fns=[vqa_collate_fn,None]) + + tokenizer = BertTokenizer.from_pretrained(args.text_encoder) + + #### Model #### + print("Creating model") + model = ALBEF(config=config, text_encoder=args.text_encoder, text_decoder=args.text_decoder, tokenizer=tokenizer) + model = model.to(device) + + arg_opt = utils.AttrDict(config['optimizer']) + optimizer = create_optimizer(arg_opt, model) + arg_sche = utils.AttrDict(config['schedular']) + lr_scheduler, _ = create_scheduler(arg_sche, optimizer) + + if args.checkpoint: + checkpoint = torch.load(args.checkpoint, map_location='cpu') + state_dict = checkpoint['model'] + + # reshape positional embedding to accomodate for image resolution change + pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) + state_dict['visual_encoder.pos_embed'] = pos_embed_reshaped + + if not args.evaluate: + if config['distill']: + m_pos_embed_reshaped = interpolate_pos_embed(state_dict['visual_encoder_m.pos_embed'],model.visual_encoder_m) + state_dict['visual_encoder_m.pos_embed'] = m_pos_embed_reshaped + + for key in list(state_dict.keys()): + if 'bert' in key: + encoder_key = key.replace('bert.','') + state_dict[encoder_key] = state_dict[key] + # intialize text decoder as multimodal encoder (last 6 layers of model.text_encoder) + if 'text_encoder' in key: + if 'layer' in key: + encoder_keys = key.split('.') + layer_num = int(encoder_keys[4]) + if layer_num<6: + del state_dict[key] + continue + else: + decoder_layer_num = (layer_num-6) + encoder_keys[4] = str(decoder_layer_num) + encoder_key = '.'.join(encoder_keys) + else: + encoder_key = key + decoder_key = encoder_key.replace('text_encoder','text_decoder') + state_dict[decoder_key] = state_dict[key] + + del state_dict[key] + + msg = model.load_state_dict(state_dict,strict=False) + print('load checkpoint from %s'%args.checkpoint) + print(msg) + + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + + + print("Start training") + start_time = time.time() + + for epoch in range(start_epoch, max_epoch): + if epoch>0: + lr_scheduler.step(epoch+warmup_steps) + + if not args.evaluate: + if args.distributed: + train_loader.sampler.set_epoch(epoch) + + train_stats = train(model, train_loader, optimizer, tokenizer, epoch, warmup_steps, device, lr_scheduler, config) + + if args.evaluate: + break + + if utils.is_main_process(): + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch, + } + with open(os.path.join(args.output_dir, "log.txt"),"a") as f: + f.write(json.dumps(log_stats) + "\n") + + save_obj = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'config': config, + 'epoch': epoch, + } + torch.save(save_obj, os.path.join(args.output_dir, 'checkpoint_%02d.pth'%epoch)) + + dist.barrier() + + vqa_result = evaluation(model, test_loader, tokenizer, device, config) + result_file = save_result(vqa_result, args.result_dir, 'vqa_result_epoch%d'%epoch) + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', default='./configs/VQA.yaml') + parser.add_argument('--checkpoint', default='') + parser.add_argument('--output_dir', default='output/vqa') + parser.add_argument('--evaluate', action='store_true') + parser.add_argument('--text_encoder', default='bert-base-uncased') + parser.add_argument('--text_decoder', default='bert-base-uncased') + parser.add_argument('--device', default='cuda') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--distributed', default=True, type=bool) + args = parser.parse_args() + + config = yaml.load(open(args.config, 'r'), Loader=yaml.Loader) + + args.result_dir = os.path.join(args.output_dir, 'result') + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + Path(args.result_dir).mkdir(parents=True, exist_ok=True) + + yaml.dump(config, open(os.path.join(args.output_dir, 'config.yaml'), 'w')) + + main(args, config) \ No newline at end of file diff --git a/configs/Grounding.yaml b/configs/Grounding.yaml new file mode 100644 index 0000000..d90bd5b --- /dev/null +++ b/configs/Grounding.yaml @@ -0,0 +1,33 @@ +train_file: ['data/refcoco+_train.json'] +test_file: ['data/refcoco+_val.json','data/refcoco+_test.json'] + +refcoco_data: 'data' +det_file: 'data/refcoco+/dets.json' +coco_file: 'data/refcoco+/cocos.json' + +image_root: '/export/share/datasets/vision/coco/images/' + +bert_config: 'configs/config_bert.json' + +image_res: 384 +batch_size: 32 + +queue_size: 65536 +momentum: 0.995 +vision_width: 768 +embed_dim: 256 +temp: 0.07 + +alpha: 0.4 +distill: True +warm_up: True + +optimizer: {opt: adamW, lr: 1e-5, weight_decay: 0.02} +schedular: {sched: cosine, lr: 1e-5, epochs: 5, min_lr: 1e-6, decay_rate: 1, warmup_lr: 1e-5, warmup_epochs: 1, cooldown_epochs: 0} + + + + + + + diff --git a/configs/NLVR.yaml b/configs/NLVR.yaml new file mode 100644 index 0000000..9baca1d --- /dev/null +++ b/configs/NLVR.yaml @@ -0,0 +1,25 @@ +train_file: ['data/nlvr_train.json'] +val_file: ['data/nlvr_dev.json'] +test_file: ['data/nlvr_test.json'] + +image_root: '/export/share/datasets/vision/NLVR2/' + +image_res: 384 +batch_size: 16 + +bert_config: 'configs/config_bert.json' + +alpha: 0.4 +distill: True +warm_up: True +eval_ema: False + +optimizer: {opt: adamW, lr: 2e-5, weight_decay: 0.02} +schedular: {sched: cosine, lr: 2e-5, epochs: 10, min_lr: 1e-6, decay_rate: 1, warmup_lr: 1e-5, warmup_epochs: 1, cooldown_epochs: 0} + + + + + + + diff --git a/configs/NLVR_pretrain.yaml b/configs/NLVR_pretrain.yaml new file mode 100644 index 0000000..34ea117 --- /dev/null +++ b/configs/NLVR_pretrain.yaml @@ -0,0 +1,25 @@ +train_file: ['/export/home/project/VL/dataset/caption/coco_karpathy_train.json', + '/export/home/project/VL/dataset/caption/vg_caption.json', + '/export/home/project/VL/dataset/pretrain_caption/conceptual_caption_train.json', + '/export/home/project/VL/dataset/pretrain_caption/conceptual_caption_val.json', + '/export/home/project/VL/dataset/pretrain_caption/sbu_caption.json' + ] + +# each train_file (json) contains a python list where each item is {'image': img_path, 'caption': text or list_of_text } + +bert_config: 'configs/config_bert.json' + +image_res: 256 +vision_width: 768 +embed_dim: 256 +batch_size: 64 + +optimizer: {opt: adamW, lr: 2e-5, weight_decay: 0.02} +schedular: {sched: cosine, lr: 2e-5, epochs: 1, min_lr: 1e-5, decay_rate: 1, warmup_lr: 1e-5, warmup_epochs: 1, cooldown_epochs: 0} + + + + + + + diff --git a/configs/Pretrain.yaml b/configs/Pretrain.yaml new file mode 100644 index 0000000..37e0bb1 --- /dev/null +++ b/configs/Pretrain.yaml @@ -0,0 +1,29 @@ +train_file: ['data/coco_karpathy_train.json', + 'data/vg_caption.json', + 'data/cc12m.json', + 'data/conceptual_caption_train.json', + 'data/conceptual_caption_val.json', + 'data/sbu_caption.json' + ] +# each train_file (json) contains a python list where each item is {'image': img_path, 'caption': text or list_of_text } +bert_config: 'configs/config_bert.json' + +image_res: 256 +vision_width: 768 +embed_dim: 256 +batch_size: 64 +temp: 0.07 +mlm_probability: 0.15 +queue_size: 65536 +momentum: 0.995 +alpha: 0.4 + +optimizer: {opt: adamW, lr: 1e-4, weight_decay: 0.02} +schedular: {sched: cosine, lr: 1e-4, epochs: 30, min_lr: 1e-5, decay_rate: 1, warmup_lr: 1e-5, warmup_epochs: 20, cooldown_epochs: 0} + + + + + + + diff --git a/configs/Retrieval_coco.yaml b/configs/Retrieval_coco.yaml new file mode 100644 index 0000000..30e0a6b --- /dev/null +++ b/configs/Retrieval_coco.yaml @@ -0,0 +1,31 @@ +train_file: ['data/coco_train.json'] +val_file: 'data/coco_val.json' +test_file: 'data/coco_test.json' +image_root: '/export/share/datasets/vision/coco/images/' + +bert_config: 'configs/config_bert.json' + +image_res: 384 +batch_size_train: 32 +batch_size_test: 64 + +queue_size: 65536 +momentum: 0.995 +vision_width: 768 +embed_dim: 256 +temp: 0.07 +k_test: 256 + +alpha: 0.4 +distill: True +warm_up: True + +optimizer: {opt: adamW, lr: 1e-5, weight_decay: 0.02} +schedular: {sched: cosine, lr: 1e-5, epochs: 5, min_lr: 1e-6, decay_rate: 1, warmup_lr: 1e-5, warmup_epochs: 1, cooldown_epochs: 0} + + + + + + + diff --git a/configs/Retrieval_flickr.yaml b/configs/Retrieval_flickr.yaml new file mode 100644 index 0000000..4b7d626 --- /dev/null +++ b/configs/Retrieval_flickr.yaml @@ -0,0 +1,31 @@ +train_file: ['data/flickr30k_train.json'] +val_file: 'data/flickr30k_val.json' +test_file: 'data/flickr30k_test.json' +image_root: '/export/share/datasets/vision/flickr30k/' #flickr30k-images/ + +bert_config: 'configs/config_bert.json' + +image_res: 384 +batch_size_train: 32 +batch_size_test: 64 + +queue_size: 65536 +momentum: 0.995 +vision_width: 768 +embed_dim: 256 +temp: 0.07 +k_test: 128 + +alpha: 0.4 +distill: True +warm_up: True + +optimizer: {opt: adamW, lr: 1e-5, weight_decay: 0.02} +schedular: {sched: cosine, lr: 1e-5, epochs: 10, min_lr: 1e-6, decay_rate: 1, warmup_lr: 1e-5, warmup_epochs: 1, cooldown_epochs: 0} + + + + + + + diff --git a/configs/VE.yaml b/configs/VE.yaml new file mode 100644 index 0000000..baf70a6 --- /dev/null +++ b/configs/VE.yaml @@ -0,0 +1,25 @@ +train_file: 'data/ve_train.json' +val_file: 'data/ve_dev.json' +test_file: 'data/ve_test.json' + +image_root: '/export/home/project/SNLI-VE/data/images' + +image_res: 384 +batch_size_train: 32 +batch_size_test: 64 + +alpha: 0.4 +distill: True +warm_up: False + +bert_config: 'configs/config_bert.json' + +optimizer: {opt: adamW, lr: 2e-5, weight_decay: 0.02} +schedular: {sched: cosine, lr: 2e-5, epochs: 5, min_lr: 1e-6, decay_rate: 1, warmup_lr: 1e-5, warmup_epochs: 1, cooldown_epochs: 0} + + + + + + + diff --git a/configs/VQA.yaml b/configs/VQA.yaml new file mode 100644 index 0000000..f379338 --- /dev/null +++ b/configs/VQA.yaml @@ -0,0 +1,32 @@ +train_file: ['data/vqa_train.json', + 'data/vqa_val.json', + 'data/vg_qa.json'] + +test_file: ['data/vqa_test.json'] +answer_list: 'data/answer_list.json' + +vqa_root: '/export/share/datasets/vision/VQA/Images/mscoco/' #train2014/ +vg_root: '/export/share/datasets/vision/visual-genome/' #image/ + +image_res: 384 +batch_size_train: 32 +batch_size_test: 16 +k_test: 128 + +alpha: 0.4 +distill: True +warm_up: True + +eos: '[SEP]' + +bert_config: 'configs/config_bert.json' + +optimizer: {opt: adamW, lr: 2e-5, weight_decay: 0.02} +schedular: {sched: cosine, lr: 2e-5, epochs: 8, min_lr: 1e-6, decay_rate: 1, warmup_lr: 1e-5, warmup_epochs: 4, cooldown_epochs: 0} + + + + + + + diff --git a/configs/config_bert.json b/configs/config_bert.json new file mode 100644 index 0000000..48a8449 --- /dev/null +++ b/configs/config_bert.json @@ -0,0 +1,21 @@ +{ + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 512, + "model_type": "bert", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "pad_token_id": 0, + "type_vocab_size": 2, + "vocab_size": 30522, + "fusion_layer": 6, + "encoder_width": 768 +} diff --git a/dataset/__init__.py b/dataset/__init__.py new file mode 100644 index 0000000..7dedda1 --- /dev/null +++ b/dataset/__init__.py @@ -0,0 +1,120 @@ +import torch +from torch.utils.data import DataLoader +from torchvision import transforms +from PIL import Image + +from dataset.caption_dataset import re_train_dataset, re_eval_dataset, pretrain_dataset +from dataset.nlvr_dataset import nlvr_dataset +from dataset.ve_dataset import ve_dataset +from dataset.vqa_dataset import vqa_dataset +from dataset.grounding_dataset import grounding_dataset + +from dataset.randaugment import RandomAugment + +def create_dataset(dataset, config): + + normalize = transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)) + + pretrain_transform = transforms.Compose([ + transforms.RandomResizedCrop(config['image_res'],scale=(0.2, 1.0), interpolation=Image.BICUBIC), + transforms.RandomHorizontalFlip(), + RandomAugment(2,7,isPIL=True,augs=['Identity','AutoContrast','Equalize','Brightness','Sharpness', + 'ShearX', 'ShearY', 'TranslateX', 'TranslateY', 'Rotate']), + transforms.ToTensor(), + normalize, + ]) + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(config['image_res'],scale=(0.5, 1.0), interpolation=Image.BICUBIC), + transforms.RandomHorizontalFlip(), + RandomAugment(2,7,isPIL=True,augs=['Identity','AutoContrast','Equalize','Brightness','Sharpness', + 'ShearX', 'ShearY', 'TranslateX', 'TranslateY', 'Rotate']), + transforms.ToTensor(), + normalize, + ]) + test_transform = transforms.Compose([ + transforms.Resize((config['image_res'],config['image_res']),interpolation=Image.BICUBIC), + transforms.ToTensor(), + normalize, + ]) + + if dataset=='pretrain': + dataset = pretrain_dataset(config['train_file'], pretrain_transform) + return dataset + + elif dataset=='re': + train_dataset = re_train_dataset(config['train_file'], train_transform, config['image_root']) + val_dataset = re_eval_dataset(config['val_file'], test_transform, config['image_root']) + test_dataset = re_eval_dataset(config['test_file'], test_transform, config['image_root']) + return train_dataset, val_dataset, test_dataset + + elif dataset=='vqa': + train_dataset = vqa_dataset(config['train_file'], train_transform, config['vqa_root'], config['vg_root'], split='train') + vqa_test_dataset = vqa_dataset(config['test_file'], test_transform, config['vqa_root'], config['vg_root'], split='test', answer_list=config['answer_list']) + return train_dataset, vqa_test_dataset + + elif dataset=='nlvr': + train_dataset = nlvr_dataset(config['train_file'], train_transform, config['image_root']) + val_dataset = nlvr_dataset(config['val_file'], test_transform, config['image_root']) + test_dataset = nlvr_dataset(config['test_file'], test_transform, config['image_root']) + return train_dataset, val_dataset, test_dataset + + elif dataset=='ve': + train_dataset = ve_dataset(config['train_file'], train_transform, config['image_root']) + val_dataset = ve_dataset(config['val_file'], test_transform, config['image_root']) + test_dataset = ve_dataset(config['test_file'], test_transform, config['image_root']) + return train_dataset, val_dataset, test_dataset + + elif dataset=='grounding': + train_transform = transforms.Compose([ + transforms.Resize((config['image_res'],config['image_res']),interpolation=Image.BICUBIC), + transforms.RandomHorizontalFlip(), + RandomAugment(2,7,isPIL=True,augs=['Identity','AutoContrast','Equalize','Brightness','Sharpness', + 'ShearX', 'ShearY', 'TranslateX', 'TranslateY', 'Rotate']), + transforms.ToTensor(), + normalize, + ]) + train_dataset = grounding_dataset(config['train_file'], train_transform, config['image_root'], mode='train') + test_dataset = grounding_dataset(config['test_file'], test_transform, config['image_root'], mode='test') + return train_dataset, test_dataset + + +def vqa_collate_fn(batch): + image_list, question_list, answer_list, weight_list, n = [], [], [], [], [] + for image, question, answer, weights in batch: + image_list.append(image) + question_list.append(question) + weight_list += weights + answer_list += answer + n.append(len(answer)) + return torch.stack(image_list,dim=0), question_list, answer_list, torch.Tensor(weight_list), n + + +def create_sampler(datasets, shuffles, num_tasks, global_rank): + samplers = [] + for dataset,shuffle in zip(datasets,shuffles): + sampler = torch.utils.data.DistributedSampler(dataset, num_replicas=num_tasks, rank=global_rank, shuffle=shuffle) + samplers.append(sampler) + return samplers + + +def create_loader(datasets, samplers, batch_size, num_workers, is_trains, collate_fns): + loaders = [] + for dataset,sampler,bs,n_worker,is_train,collate_fn in zip(datasets,samplers,batch_size,num_workers,is_trains,collate_fns): + if is_train: + shuffle = (sampler is None) + drop_last = True + else: + shuffle = False + drop_last = False + loader = DataLoader( + dataset, + batch_size=bs, + num_workers=n_worker, + pin_memory=True, + sampler=sampler, + shuffle=shuffle, + collate_fn=collate_fn, + drop_last=drop_last, + ) + loaders.append(loader) + return loaders \ No newline at end of file diff --git a/dataset/__pycache__/__init__.cpython-38.pyc b/dataset/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..0ac04b1 Binary files /dev/null and b/dataset/__pycache__/__init__.cpython-38.pyc differ diff --git a/dataset/__pycache__/caption_dataset.cpython-38.pyc b/dataset/__pycache__/caption_dataset.cpython-38.pyc new file mode 100644 index 0000000..0cd5212 Binary files /dev/null and b/dataset/__pycache__/caption_dataset.cpython-38.pyc differ diff --git a/dataset/__pycache__/grounding_dataset.cpython-38.pyc b/dataset/__pycache__/grounding_dataset.cpython-38.pyc new file mode 100644 index 0000000..d1f18e7 Binary files /dev/null and b/dataset/__pycache__/grounding_dataset.cpython-38.pyc differ diff --git a/dataset/__pycache__/nlvr_dataset.cpython-38.pyc b/dataset/__pycache__/nlvr_dataset.cpython-38.pyc new file mode 100644 index 0000000..4370298 Binary files /dev/null and b/dataset/__pycache__/nlvr_dataset.cpython-38.pyc differ diff --git a/dataset/__pycache__/randaugment.cpython-38.pyc b/dataset/__pycache__/randaugment.cpython-38.pyc new file mode 100644 index 0000000..c8e75ca Binary files /dev/null and b/dataset/__pycache__/randaugment.cpython-38.pyc differ diff --git a/dataset/__pycache__/utils.cpython-38.pyc b/dataset/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000..aa2e740 Binary files /dev/null and b/dataset/__pycache__/utils.cpython-38.pyc differ diff --git a/dataset/__pycache__/ve_dataset.cpython-38.pyc b/dataset/__pycache__/ve_dataset.cpython-38.pyc new file mode 100644 index 0000000..1aa33b5 Binary files /dev/null and b/dataset/__pycache__/ve_dataset.cpython-38.pyc differ diff --git a/dataset/__pycache__/vqa_dataset.cpython-38.pyc b/dataset/__pycache__/vqa_dataset.cpython-38.pyc new file mode 100644 index 0000000..d1b91c8 Binary files /dev/null and b/dataset/__pycache__/vqa_dataset.cpython-38.pyc differ diff --git a/dataset/caption_dataset.py b/dataset/caption_dataset.py new file mode 100644 index 0000000..2d9a4cf --- /dev/null +++ b/dataset/caption_dataset.py @@ -0,0 +1,112 @@ +import json +import os +import random + +from torch.utils.data import Dataset + +from PIL import Image +from PIL import ImageFile +ImageFile.LOAD_TRUNCATED_IMAGES = True +Image.MAX_IMAGE_PIXELS = None + +from dataset.utils import pre_caption + + +class re_train_dataset(Dataset): + def __init__(self, ann_file, transform, image_root, max_words=30): + self.ann = [] + for f in ann_file: + self.ann += json.load(open(f,'r')) + self.transform = transform + self.image_root = image_root + self.max_words = max_words + self.img_ids = {} + + n = 0 + for ann in self.ann: + img_id = ann['image_id'] + if img_id not in self.img_ids.keys(): + self.img_ids[img_id] = n + n += 1 + + def __len__(self): + return len(self.ann) + + def __getitem__(self, index): + + ann = self.ann[index] + + image_path = os.path.join(self.image_root,ann['image']) + image = Image.open(image_path).convert('RGB') + image = self.transform(image) + + caption = pre_caption(ann['caption'], self.max_words) + + return image, caption, self.img_ids[ann['image_id']] + + + +class re_eval_dataset(Dataset): + def __init__(self, ann_file, transform, image_root, max_words=30): + self.ann = json.load(open(ann_file,'r')) + self.transform = transform + self.image_root = image_root + self.max_words = max_words + + self.text = [] + self.image = [] + self.txt2img = {} + self.img2txt = {} + + txt_id = 0 + for img_id, ann in enumerate(self.ann): + self.image.append(ann['image']) + self.img2txt[img_id] = [] + for i, caption in enumerate(ann['caption']): + self.text.append(pre_caption(caption,self.max_words)) + self.img2txt[img_id].append(txt_id) + self.txt2img[txt_id] = img_id + txt_id += 1 + + def __len__(self): + return len(self.image) + + def __getitem__(self, index): + + image_path = os.path.join(self.image_root, self.ann[index]['image']) + image = Image.open(image_path).convert('RGB') + image = self.transform(image) + + return image, index + + + +class pretrain_dataset(Dataset): + def __init__(self, ann_file, transform, max_words=30): + self.ann = [] + for f in ann_file: + self.ann += json.load(open(f,'r')) + self.transform = transform + self.max_words = max_words + + + def __len__(self): + return len(self.ann) + + + def __getitem__(self, index): + + ann = self.ann[index] + + if type(ann['caption']) == list: + caption = pre_caption(random.choice(ann['caption']), self.max_words) + else: + caption = pre_caption(ann['caption'], self.max_words) + + image = Image.open(ann['image']).convert('RGB') + image = self.transform(image) + + return image, caption + + + diff --git a/dataset/grounding_dataset.py b/dataset/grounding_dataset.py new file mode 100644 index 0000000..ac15d89 --- /dev/null +++ b/dataset/grounding_dataset.py @@ -0,0 +1,45 @@ +import json +import os +from torch.utils.data import Dataset +from PIL import Image +from dataset.utils import pre_caption + +class grounding_dataset(Dataset): + def __init__(self, ann_file, transform, image_root, max_words=30, mode='train'): + self.ann = [] + for f in ann_file: + self.ann += json.load(open(f,'r')) + self.transform = transform + self.image_root = image_root + self.max_words = max_words + self.mode = mode + + if self.mode == 'train': + self.img_ids = {} + n = 0 + for ann in self.ann: + img_id = ann['image'].split('/')[-1] + if img_id not in self.img_ids.keys(): + self.img_ids[img_id] = n + n += 1 + + + def __len__(self): + return len(self.ann) + + def __getitem__(self, index): + + ann = self.ann[index] + + image_path = os.path.join(self.image_root,ann['image']) + image = Image.open(image_path).convert('RGB') + image = self.transform(image) + + caption = pre_caption(ann['text'], self.max_words) + + if self.mode=='train': + img_id = ann['image'].split('/')[-1] + + return image, caption, self.img_ids[img_id] + else: + return image, caption, ann['ref_id'] \ No newline at end of file diff --git a/dataset/nlvr_dataset.py b/dataset/nlvr_dataset.py new file mode 100644 index 0000000..5375b61 --- /dev/null +++ b/dataset/nlvr_dataset.py @@ -0,0 +1,40 @@ +import json +import os +from torch.utils.data import Dataset +from PIL import Image +from dataset.utils import pre_caption + + +class nlvr_dataset(Dataset): + def __init__(self, ann_file, transform, image_root): + self.ann = [] + for f in ann_file: + self.ann += json.load(open(f,'r')) + self.transform = transform + self.image_root = image_root + self.max_words = 30 + + def __len__(self): + return len(self.ann) + + + def __getitem__(self, index): + + ann = self.ann[index] + + image0_path = os.path.join(self.image_root,ann['images'][0]) + image0 = Image.open(image0_path).convert('RGB') + image0 = self.transform(image0) + + image1_path = os.path.join(self.image_root,ann['images'][1]) + image1 = Image.open(image1_path).convert('RGB') + image1 = self.transform(image1) + + sentence = pre_caption(ann['sentence'], self.max_words) + + if ann['label']=='True': + label = 1 + else: + label = 0 + + return image0, image1, sentence, label \ No newline at end of file diff --git a/dataset/randaugment.py b/dataset/randaugment.py new file mode 100644 index 0000000..094d9f4 --- /dev/null +++ b/dataset/randaugment.py @@ -0,0 +1,340 @@ +import cv2 +import numpy as np + + +## aug functions +def identity_func(img): + return img + + +def autocontrast_func(img, cutoff=0): + ''' + same output as PIL.ImageOps.autocontrast + ''' + n_bins = 256 + + def tune_channel(ch): + n = ch.size + cut = cutoff * n // 100 + if cut == 0: + high, low = ch.max(), ch.min() + else: + hist = cv2.calcHist([ch], [0], None, [n_bins], [0, n_bins]) + low = np.argwhere(np.cumsum(hist) > cut) + low = 0 if low.shape[0] == 0 else low[0] + high = np.argwhere(np.cumsum(hist[::-1]) > cut) + high = n_bins - 1 if high.shape[0] == 0 else n_bins - 1 - high[0] + if high <= low: + table = np.arange(n_bins) + else: + scale = (n_bins - 1) / (high - low) + offset = -low * scale + table = np.arange(n_bins) * scale + offset + table[table < 0] = 0 + table[table > n_bins - 1] = n_bins - 1 + table = table.clip(0, 255).astype(np.uint8) + return table[ch] + + channels = [tune_channel(ch) for ch in cv2.split(img)] + out = cv2.merge(channels) + return out + + +def equalize_func(img): + ''' + same output as PIL.ImageOps.equalize + PIL's implementation is different from cv2.equalize + ''' + n_bins = 256 + + def tune_channel(ch): + hist = cv2.calcHist([ch], [0], None, [n_bins], [0, n_bins]) + non_zero_hist = hist[hist != 0].reshape(-1) + step = np.sum(non_zero_hist[:-1]) // (n_bins - 1) + if step == 0: return ch + n = np.empty_like(hist) + n[0] = step // 2 + n[1:] = hist[:-1] + table = (np.cumsum(n) // step).clip(0, 255).astype(np.uint8) + return table[ch] + + channels = [tune_channel(ch) for ch in cv2.split(img)] + out = cv2.merge(channels) + return out + + +def rotate_func(img, degree, fill=(0, 0, 0)): + ''' + like PIL, rotate by degree, not radians + ''' + H, W = img.shape[0], img.shape[1] + center = W / 2, H / 2 + M = cv2.getRotationMatrix2D(center, degree, 1) + out = cv2.warpAffine(img, M, (W, H), borderValue=fill) + return out + + +def solarize_func(img, thresh=128): + ''' + same output as PIL.ImageOps.posterize + ''' + table = np.array([el if el < thresh else 255 - el for el in range(256)]) + table = table.clip(0, 255).astype(np.uint8) + out = table[img] + return out + + +def color_func(img, factor): + ''' + same output as PIL.ImageEnhance.Color + ''' + ## implementation according to PIL definition, quite slow + # degenerate = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)[:, :, np.newaxis] + # out = blend(degenerate, img, factor) + # M = ( + # np.eye(3) * factor + # + np.float32([0.114, 0.587, 0.299]).reshape(3, 1) * (1. - factor) + # )[np.newaxis, np.newaxis, :] + M = ( + np.float32([ + [0.886, -0.114, -0.114], + [-0.587, 0.413, -0.587], + [-0.299, -0.299, 0.701]]) * factor + + np.float32([[0.114], [0.587], [0.299]]) + ) + out = np.matmul(img, M).clip(0, 255).astype(np.uint8) + return out + + +def contrast_func(img, factor): + """ + same output as PIL.ImageEnhance.Contrast + """ + mean = np.sum(np.mean(img, axis=(0, 1)) * np.array([0.114, 0.587, 0.299])) + table = np.array([( + el - mean) * factor + mean + for el in range(256) + ]).clip(0, 255).astype(np.uint8) + out = table[img] + return out + + +def brightness_func(img, factor): + ''' + same output as PIL.ImageEnhance.Contrast + ''' + table = (np.arange(256, dtype=np.float32) * factor).clip(0, 255).astype(np.uint8) + out = table[img] + return out + + +def sharpness_func(img, factor): + ''' + The differences the this result and PIL are all on the 4 boundaries, the center + areas are same + ''' + kernel = np.ones((3, 3), dtype=np.float32) + kernel[1][1] = 5 + kernel /= 13 + degenerate = cv2.filter2D(img, -1, kernel) + if factor == 0.0: + out = degenerate + elif factor == 1.0: + out = img + else: + out = img.astype(np.float32) + degenerate = degenerate.astype(np.float32)[1:-1, 1:-1, :] + out[1:-1, 1:-1, :] = degenerate + factor * (out[1:-1, 1:-1, :] - degenerate) + out = out.astype(np.uint8) + return out + + +def shear_x_func(img, factor, fill=(0, 0, 0)): + H, W = img.shape[0], img.shape[1] + M = np.float32([[1, factor, 0], [0, 1, 0]]) + out = cv2.warpAffine(img, M, (W, H), borderValue=fill, flags=cv2.INTER_LINEAR).astype(np.uint8) + return out + + +def translate_x_func(img, offset, fill=(0, 0, 0)): + ''' + same output as PIL.Image.transform + ''' + H, W = img.shape[0], img.shape[1] + M = np.float32([[1, 0, -offset], [0, 1, 0]]) + out = cv2.warpAffine(img, M, (W, H), borderValue=fill, flags=cv2.INTER_LINEAR).astype(np.uint8) + return out + + +def translate_y_func(img, offset, fill=(0, 0, 0)): + ''' + same output as PIL.Image.transform + ''' + H, W = img.shape[0], img.shape[1] + M = np.float32([[1, 0, 0], [0, 1, -offset]]) + out = cv2.warpAffine(img, M, (W, H), borderValue=fill, flags=cv2.INTER_LINEAR).astype(np.uint8) + return out + + +def posterize_func(img, bits): + ''' + same output as PIL.ImageOps.posterize + ''' + out = np.bitwise_and(img, np.uint8(255 << (8 - bits))) + return out + + +def shear_y_func(img, factor, fill=(0, 0, 0)): + H, W = img.shape[0], img.shape[1] + M = np.float32([[1, 0, 0], [factor, 1, 0]]) + out = cv2.warpAffine(img, M, (W, H), borderValue=fill, flags=cv2.INTER_LINEAR).astype(np.uint8) + return out + + +def cutout_func(img, pad_size, replace=(0, 0, 0)): + replace = np.array(replace, dtype=np.uint8) + H, W = img.shape[0], img.shape[1] + rh, rw = np.random.random(2) + pad_size = pad_size // 2 + ch, cw = int(rh * H), int(rw * W) + x1, x2 = max(ch - pad_size, 0), min(ch + pad_size, H) + y1, y2 = max(cw - pad_size, 0), min(cw + pad_size, W) + out = img.copy() + out[x1:x2, y1:y2, :] = replace + return out + + +### level to args +def enhance_level_to_args(MAX_LEVEL): + def level_to_args(level): + return ((level / MAX_LEVEL) * 1.8 + 0.1,) + return level_to_args + + +def shear_level_to_args(MAX_LEVEL, replace_value): + def level_to_args(level): + level = (level / MAX_LEVEL) * 0.3 + if np.random.random() > 0.5: level = -level + return (level, replace_value) + + return level_to_args + + +def translate_level_to_args(translate_const, MAX_LEVEL, replace_value): + def level_to_args(level): + level = (level / MAX_LEVEL) * float(translate_const) + if np.random.random() > 0.5: level = -level + return (level, replace_value) + + return level_to_args + + +def cutout_level_to_args(cutout_const, MAX_LEVEL, replace_value): + def level_to_args(level): + level = int((level / MAX_LEVEL) * cutout_const) + return (level, replace_value) + + return level_to_args + + +def solarize_level_to_args(MAX_LEVEL): + def level_to_args(level): + level = int((level / MAX_LEVEL) * 256) + return (level, ) + return level_to_args + + +def none_level_to_args(level): + return () + + +def posterize_level_to_args(MAX_LEVEL): + def level_to_args(level): + level = int((level / MAX_LEVEL) * 4) + return (level, ) + return level_to_args + + +def rotate_level_to_args(MAX_LEVEL, replace_value): + def level_to_args(level): + level = (level / MAX_LEVEL) * 30 + if np.random.random() < 0.5: + level = -level + return (level, replace_value) + + return level_to_args + + +func_dict = { + 'Identity': identity_func, + 'AutoContrast': autocontrast_func, + 'Equalize': equalize_func, + 'Rotate': rotate_func, + 'Solarize': solarize_func, + 'Color': color_func, + 'Contrast': contrast_func, + 'Brightness': brightness_func, + 'Sharpness': sharpness_func, + 'ShearX': shear_x_func, + 'TranslateX': translate_x_func, + 'TranslateY': translate_y_func, + 'Posterize': posterize_func, + 'ShearY': shear_y_func, +} + +translate_const = 10 +MAX_LEVEL = 10 +replace_value = (128, 128, 128) +arg_dict = { + 'Identity': none_level_to_args, + 'AutoContrast': none_level_to_args, + 'Equalize': none_level_to_args, + 'Rotate': rotate_level_to_args(MAX_LEVEL, replace_value), + 'Solarize': solarize_level_to_args(MAX_LEVEL), + 'Color': enhance_level_to_args(MAX_LEVEL), + 'Contrast': enhance_level_to_args(MAX_LEVEL), + 'Brightness': enhance_level_to_args(MAX_LEVEL), + 'Sharpness': enhance_level_to_args(MAX_LEVEL), + 'ShearX': shear_level_to_args(MAX_LEVEL, replace_value), + 'TranslateX': translate_level_to_args( + translate_const, MAX_LEVEL, replace_value + ), + 'TranslateY': translate_level_to_args( + translate_const, MAX_LEVEL, replace_value + ), + 'Posterize': posterize_level_to_args(MAX_LEVEL), + 'ShearY': shear_level_to_args(MAX_LEVEL, replace_value), +} + + +class RandomAugment(object): + + def __init__(self, N=2, M=10, isPIL=False, augs=[]): + self.N = N + self.M = M + self.isPIL = isPIL + if augs: + self.augs = augs + else: + self.augs = list(arg_dict.keys()) + + def get_random_ops(self): + sampled_ops = np.random.choice(self.augs, self.N) + return [(op, 0.5, self.M) for op in sampled_ops] + + def __call__(self, img): + if self.isPIL: + img = np.array(img) + ops = self.get_random_ops() + for name, prob, level in ops: + if np.random.random() > prob: + continue + args = arg_dict[name](level) + img = func_dict[name](img, *args) + return img + + +if __name__ == '__main__': + a = RandomAugment() + img = np.random.randn(32, 32, 3) + a(img) \ No newline at end of file diff --git a/dataset/utils.py b/dataset/utils.py new file mode 100644 index 0000000..44b9cb9 --- /dev/null +++ b/dataset/utils.py @@ -0,0 +1,215 @@ +import re + +def pre_question(question,max_ques_words): + question = re.sub( + r"([,.'!?\"()*#:;~])", + '', + question.lower(), + ).replace('-', ' ').replace('/', ' ') + question = question.rstrip(' ') + + #truncate question + question_words = question.split(' ') + if len(question_words)>max_ques_words: + question = ' '.join(question_words[:max_ques_words]) + + return question + + +def pre_caption(caption,max_words): + caption = re.sub( + r"([,.'!?\"()*#:;~])", + '', + caption.lower(), + ).replace('-', ' ').replace('/', ' ').replace('', 'person') + + caption = re.sub( + r"\s{2,}", + ' ', + caption, + ) + caption = caption.rstrip('\n') + caption = caption.strip(' ') + + #truncate caption + caption_words = caption.split(' ') + if len(caption_words)>max_words: + caption = ' '.join(caption_words[:max_words]) + + return caption + + +from vqaTools.vqaEval import VQAEval +from refTools.evaluation.refEvaluation import RefEvaluation + +import json +import os +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F + +import utils +from tqdm import tqdm + + +def vqa_eval(vqa, result_file, test_ques_path): + vqaRes = vqa.loadRes(result_file, test_ques_path) + # create vqaEval object by taking vqa and vqaRes + vqaEval = VQAEval(vqa, vqaRes, n=2) # n is precision of accuracy (number of places after decimal), default is 2 + # evaluate results + vqaEval.evaluate() + + # print accuracies + print("\n") + print("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) + print("Per Answer Type Accuracy is the following:") + for ansType in vqaEval.accuracy['perAnswerType']: + print("%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) + print("\n") + + return vqaEval + + + +def collect_result(result, result_dir, filename, is_json=True, is_list=True): + if is_json: + result_file = os.path.join(result_dir, '%s_rank%d.json'%(filename,utils.get_rank())) + final_result_file = os.path.join(result_dir, '%s.json'%filename) + json.dump(result,open(result_file,'w')) + else: + result_file = os.path.join(result_dir, '%s_rank%d.pth'%(filename,utils.get_rank())) + final_result_file = os.path.join(result_dir, '%s.pth'%filename) + torch.save(result,result_file) + + dist.barrier() + + result = None + if utils.is_main_process(): + # combine results from all processes + if is_list: + result = [] + else: + result = {} + for rank in range(utils.get_world_size()): + if is_json: + result_file = os.path.join(result_dir, '%s_rank%d.json'%(filename,rank)) + res = json.load(open(result_file,'r')) + else: + result_file = os.path.join(result_dir, '%s_rank%d.pth'%(filename,rank)) + res = torch.load(result_file) + if is_list: + result += res + else: + result.update(res) + + return result + + +def save_result(result, result_dir, filename, is_json=True, is_list=True): + if is_json: + result_file = os.path.join(result_dir, '%s_rank%d.json'%(filename,utils.get_rank())) + final_result_file = os.path.join(result_dir, '%s.json'%filename) + json.dump(result,open(result_file,'w')) + else: + result_file = os.path.join(result_dir, '%s_rank%d.pth'%(filename,utils.get_rank())) + final_result_file = os.path.join(result_dir, '%s.pth'%filename) + torch.save(result,result_file) + + dist.barrier() + + if utils.is_main_process(): + # combine results from all processes + if is_list: + result = [] + else: + result = {} + for rank in range(utils.get_world_size()): + if is_json: + result_file = os.path.join(result_dir, '%s_rank%d.json'%(filename,rank)) + res = json.load(open(result_file,'r')) + else: + result_file = os.path.join(result_dir, '%s_rank%d.pth'%(filename,rank)) + res = torch.load(result_file) + if is_list: + result += res + else: + result.update(res) + if is_json: + json.dump(result,open(final_result_file,'w')) + else: + torch.save(result,final_result_file) + + print('result file saved to %s'%final_result_file) + dist.barrier() + return final_result_file + + + +def grounding_eval(results,dets,cocos,refer,alpha,mask_size=24): + + correct_A_d, correct_B_d, correct_val_d = 0, 0, 0 + correct_A, correct_B, correct_val = 0, 0, 0 + num_A,num_B,num_val = 0,0,0 + + for res in tqdm(results): + + ref_id = res['ref_id'] + ref = refer.Refs[ref_id] + ref_box = refer.refToAnn[ref_id]['bbox'] + image = refer.Imgs[ref['image_id']] + + mask = res['pred'].cuda().view(1,1,mask_size,mask_size) + mask = F.interpolate(mask,size = (image['height'],image['width']), mode='bicubic').squeeze() + + # rank detection boxes + max_score = 0 + for det in dets[str(ref['image_id'])]: + score = mask[int(det[1]):int(det[1]+det[3]),int(det[0]):int(det[0]+det[2])] + area = det[2]*det[3] + score = score.sum() / area**alpha + if score>max_score: + pred_box = det[:4] + max_score = score + + IoU_det = computeIoU(ref_box, pred_box) + + if ref['split']=='testA': + num_A += 1 + if IoU_det >= 0.5: + correct_A_d += 1 + elif ref['split']=='testB': + num_B += 1 + if IoU_det >= 0.5: + correct_B_d += 1 + elif ref['split']=='val': + num_val += 1 + if IoU_det >= 0.5: + correct_val_d += 1 + + eval_result = {'val_d':correct_val_d/num_val,'testA_d':correct_A_d/num_A,'testB_d':correct_B_d/num_B} + + for metric, acc in eval_result.items(): + print(f'{metric}: {acc:.3f}') + + return eval_result + + + +# IoU function +def computeIoU(box1, box2): + # each box is of [x1, y1, w, h] + inter_x1 = max(box1[0], box2[0]) + inter_y1 = max(box1[1], box2[1]) + inter_x2 = min(box1[0]+box1[2]-1, box2[0]+box2[2]-1) + inter_y2 = min(box1[1]+box1[3]-1, box2[1]+box2[3]-1) + + if inter_x1 < inter_x2 and inter_y1 < inter_y2: + inter = (inter_x2-inter_x1+1)*(inter_y2-inter_y1+1) + else: + inter = 0 + union = box1[2]*box1[3] + box2[2]*box2[3] - inter + return float(inter)/union + + + \ No newline at end of file diff --git a/dataset/ve_dataset.py b/dataset/ve_dataset.py new file mode 100644 index 0000000..4028dbf --- /dev/null +++ b/dataset/ve_dataset.py @@ -0,0 +1,31 @@ +import json +import os +from torch.utils.data import Dataset +from PIL import Image +from dataset.utils import pre_caption + + +class ve_dataset(Dataset): + def __init__(self, ann_file, transform, image_root, max_words=30): + self.ann = json.load(open(ann_file,'r')) + self.transform = transform + self.image_root = image_root + self.max_words = max_words + self.labels = {'entailment':2,'neutral':1,'contradiction':0} + + def __len__(self): + return len(self.ann) + + + def __getitem__(self, index): + + ann = self.ann[index] + + image_path = os.path.join(self.image_root,'%s.jpg'%ann['image']) + image = Image.open(image_path).convert('RGB') + image = self.transform(image) + + sentence = pre_caption(ann['sentence'], self.max_words) + + return image, sentence, self.labels[ann['label']] + \ No newline at end of file diff --git a/dataset/vqa_dataset.py b/dataset/vqa_dataset.py new file mode 100644 index 0000000..e7bf9a4 --- /dev/null +++ b/dataset/vqa_dataset.py @@ -0,0 +1,70 @@ +import os +import json +import random +from PIL import Image +from torch.utils.data import Dataset +from dataset.utils import pre_question + + +class vqa_dataset(Dataset): + def __init__(self, ann_file, transform, vqa_root, vg_root, eos='[SEP]', split="train", max_ques_words=30, answer_list=''): + self.split = split + self.ann = [] + for f in ann_file: + self.ann += json.load(open(f,'r')) + + self.transform = transform + self.vqa_root = vqa_root + self.vg_root = vg_root + self.max_ques_words = max_ques_words + self.eos = eos + + if split=='test': + self.max_ques_words = 50 # do not limit question length during test + self.answer_list = json.load(open(answer_list,'r')) + + + def __len__(self): + return len(self.ann) + + def __getitem__(self, index): + + ann = self.ann[index] + + if ann['dataset']=='vqa': + image_path = os.path.join(self.vqa_root,ann['image']) + elif ann['dataset']=='vg': + image_path = os.path.join(self.vg_root,ann['image']) + + image = Image.open(image_path).convert('RGB') + image = self.transform(image) + + if self.split == 'test': + question = pre_question(ann['question'],self.max_ques_words) + question_id = ann['question_id'] + return image, question, question_id + + + elif self.split=='train': + + question = pre_question(ann['question'],self.max_ques_words) + + if ann['dataset']=='vqa': + + answer_weight = {} + for answer in ann['answer']: + if answer in answer_weight.keys(): + answer_weight[answer] += 1/len(ann['answer']) + else: + answer_weight[answer] = 1/len(ann['answer']) + + answers = list(answer_weight.keys()) + weights = list(answer_weight.values()) + + elif ann['dataset']=='vg': + answers = [ann['answer']] + weights = [0.5] + + answers = [answer+self.eos for answer in answers] + + return image, question, answers, weights \ No newline at end of file diff --git a/examples/image0.jpg b/examples/image0.jpg new file mode 100644 index 0000000..98bf635 Binary files /dev/null and b/examples/image0.jpg differ diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/__pycache__/__init__.cpython-38.pyc b/models/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..258d235 Binary files /dev/null and b/models/__pycache__/__init__.cpython-38.pyc differ diff --git a/models/__pycache__/deit.cpython-38.pyc b/models/__pycache__/deit.cpython-38.pyc new file mode 100644 index 0000000..5729c6b Binary files /dev/null and b/models/__pycache__/deit.cpython-38.pyc differ diff --git a/models/__pycache__/model_nlvr.cpython-38.pyc b/models/__pycache__/model_nlvr.cpython-38.pyc new file mode 100644 index 0000000..2ab00e7 Binary files /dev/null and b/models/__pycache__/model_nlvr.cpython-38.pyc differ diff --git a/models/__pycache__/model_pretrain.cpython-38.pyc b/models/__pycache__/model_pretrain.cpython-38.pyc new file mode 100644 index 0000000..3e891bb Binary files /dev/null and b/models/__pycache__/model_pretrain.cpython-38.pyc differ diff --git a/models/__pycache__/model_pretrain_nlvr.cpython-38.pyc b/models/__pycache__/model_pretrain_nlvr.cpython-38.pyc new file mode 100644 index 0000000..e6f82cf Binary files /dev/null and b/models/__pycache__/model_pretrain_nlvr.cpython-38.pyc differ diff --git a/models/__pycache__/model_pretrain_resnet.cpython-38.pyc b/models/__pycache__/model_pretrain_resnet.cpython-38.pyc new file mode 100644 index 0000000..9277f60 Binary files /dev/null and b/models/__pycache__/model_pretrain_resnet.cpython-38.pyc differ diff --git a/models/__pycache__/model_retrieval.cpython-38.pyc b/models/__pycache__/model_retrieval.cpython-38.pyc new file mode 100644 index 0000000..76c9acd Binary files /dev/null and b/models/__pycache__/model_retrieval.cpython-38.pyc differ diff --git a/models/__pycache__/model_ve.cpython-38.pyc b/models/__pycache__/model_ve.cpython-38.pyc new file mode 100644 index 0000000..c8061d2 Binary files /dev/null and b/models/__pycache__/model_ve.cpython-38.pyc differ diff --git a/models/__pycache__/model_ve_ensemble.cpython-38.pyc b/models/__pycache__/model_ve_ensemble.cpython-38.pyc new file mode 100644 index 0000000..04f3267 Binary files /dev/null and b/models/__pycache__/model_ve_ensemble.cpython-38.pyc differ diff --git a/models/__pycache__/model_vqa.cpython-38.pyc b/models/__pycache__/model_vqa.cpython-38.pyc new file mode 100644 index 0000000..a60bce8 Binary files /dev/null and b/models/__pycache__/model_vqa.cpython-38.pyc differ diff --git a/models/__pycache__/resnet.cpython-38.pyc b/models/__pycache__/resnet.cpython-38.pyc new file mode 100644 index 0000000..b2eda51 Binary files /dev/null and b/models/__pycache__/resnet.cpython-38.pyc differ diff --git a/models/__pycache__/tokenization_bert.cpython-38.pyc b/models/__pycache__/tokenization_bert.cpython-38.pyc new file mode 100644 index 0000000..1299d53 Binary files /dev/null and b/models/__pycache__/tokenization_bert.cpython-38.pyc differ diff --git a/models/__pycache__/vit.cpython-38.pyc b/models/__pycache__/vit.cpython-38.pyc new file mode 100644 index 0000000..5bbf517 Binary files /dev/null and b/models/__pycache__/vit.cpython-38.pyc differ diff --git a/models/__pycache__/xbert.cpython-38.pyc b/models/__pycache__/xbert.cpython-38.pyc new file mode 100644 index 0000000..cd9d789 Binary files /dev/null and b/models/__pycache__/xbert.cpython-38.pyc differ diff --git a/models/model_nlvr.py b/models/model_nlvr.py new file mode 100644 index 0000000..ce412f1 --- /dev/null +++ b/models/model_nlvr.py @@ -0,0 +1,128 @@ +from functools import partial +from models.vit import VisionTransformer +from models.xbert import BertConfig, BertModel + +import torch +from torch import nn +import torch.nn.functional as F + +class ALBEF(nn.Module): + def __init__(self, + text_encoder = None, + tokenizer = None, + config = None, + ): + super().__init__() + + self.tokenizer = tokenizer + self.distill = config['distill'] + + self.visual_encoder = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + + bert_config = BertConfig.from_json_file(config['bert_config']) + bert_config.num_hidden_layers = 18 + + self.text_encoder = BertModel.from_pretrained(text_encoder, config=bert_config, add_pooling_layer=False) + self.cls_head = nn.Sequential( + nn.Linear(self.text_encoder.config.hidden_size, self.text_encoder.config.hidden_size), + nn.ReLU(), + nn.Linear(self.text_encoder.config.hidden_size, 2) + ) + + self.share_cross_attention(self.text_encoder.encoder) + + if self.distill: + self.visual_encoder_m = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + self.text_encoder_m = BertModel.from_pretrained(text_encoder, config=bert_config, add_pooling_layer=False) + self.share_cross_attention(self.text_encoder_m.encoder) + + self.cls_head_m = nn.Sequential( + nn.Linear(self.text_encoder.config.hidden_size, self.text_encoder.config.hidden_size), + nn.ReLU(), + nn.Linear(self.text_encoder.config.hidden_size, 2) + ) + + self.model_pairs = [[self.visual_encoder,self.visual_encoder_m], + [self.text_encoder,self.text_encoder_m], + [self.cls_head,self.cls_head_m], + ] + self.copy_params() + self.momentum = 0.995 + + + def forward(self, image, text, targets, alpha=0, train=True): + + image_embeds = self.visual_encoder(image) + image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device) + + image0_embeds, image1_embeds = torch.split(image_embeds,targets.size(0)) + + output = self.text_encoder(text.input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = [image0_embeds,image1_embeds], + encoder_attention_mask = [image_atts[:image0_embeds.size(0)], + image_atts[image0_embeds.size(0):]], + return_dict = True, + ) + hidden_state = output.last_hidden_state[:,0,:] + prediction = self.cls_head(hidden_state) + + if train: + if self.distill: + with torch.no_grad(): + self._momentum_update() + image_embeds_m = self.visual_encoder_m(image) + image0_embeds_m, image1_embeds_m = torch.split(image_embeds_m,targets.size(0)) + output_m = self.text_encoder_m(text.input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = [image0_embeds_m,image1_embeds_m], + encoder_attention_mask = [image_atts[:image0_embeds.size(0)], + image_atts[image0_embeds.size(0):]], + return_dict = True, + ) + prediction_m = self.cls_head_m(output_m.last_hidden_state[:,0,:]) + + loss = (1-alpha)*F.cross_entropy(prediction, targets) - alpha*torch.sum( + F.log_softmax(prediction, dim=1)*F.softmax(prediction_m, dim=1),dim=1).mean() + else: + loss = F.cross_entropy(prediction, targets) + return loss + else: + return prediction + + + + @torch.no_grad() + def copy_params(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data.copy_(param.data) # initialize + param_m.requires_grad = False # not update by gradient + + + @torch.no_grad() + def _momentum_update(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data = param_m.data * self.momentum + param.data * (1. - self.momentum) + + + def share_cross_attention(self, model): + + for i in range(6): + layer_num = 6+i*2 + modules_0 = model.layer[layer_num].crossattention.self._modules + modules_1 = model.layer[layer_num+1].crossattention.self._modules + + for name in modules_0.keys(): + if 'key' in name or 'value' in name: + module_0 = modules_0[name] + module_1 = modules_1[name] + if hasattr(module_0, "weight"): + module_0.weight = module_1.weight + if hasattr(module_0, "bias"): + module_0.bias = module_1.bias \ No newline at end of file diff --git a/models/model_pretrain.py b/models/model_pretrain.py new file mode 100644 index 0000000..c4f7ad5 --- /dev/null +++ b/models/model_pretrain.py @@ -0,0 +1,284 @@ +from functools import partial +from models.vit import VisionTransformer, interpolate_pos_embed +from models.xbert import BertConfig, BertForMaskedLM + +import torch +import torch.nn.functional as F +from torch import nn + +import numpy as np +import random + + +class ALBEF(nn.Module): + def __init__(self, + text_encoder = None, + tokenizer = None, + config = None, + temp = 0.07, + init_deit = True + ): + super().__init__() + + self.tokenizer = tokenizer + self.mlm_probability = config['mlm_probability'] + embed_dim = config['embed_dim'] + + self.visual_encoder = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + + if init_deit: + checkpoint = torch.hub.load_state_dict_from_url( + url="https://dl.fbaipublicfiles.com/deit/deit_base_patch16_224-b5f2ef4d.pth", + map_location="cpu", check_hash=True) + state_dict = checkpoint["model"] + pos_embed_reshaped = interpolate_pos_embed(state_dict['pos_embed'], self.visual_encoder) + state_dict['pos_embed'] = pos_embed_reshaped + msg = self.visual_encoder.load_state_dict(state_dict,strict=False) + print(msg) + + vision_width = config['vision_width'] + bert_config = BertConfig.from_json_file(config['bert_config']) + + self.text_encoder = BertForMaskedLM.from_pretrained(text_encoder, config=bert_config) + + text_width = self.text_encoder.config.hidden_size + self.vision_proj = nn.Linear(vision_width, embed_dim) + self.text_proj = nn.Linear(text_width, embed_dim) + + self.temp = nn.Parameter(torch.ones([]) * config['temp']) + self.queue_size = config['queue_size'] + self.momentum = config['momentum'] + self.itm_head = nn.Linear(text_width, 2) + + # create momentum models + self.visual_encoder_m = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + self.vision_proj_m = nn.Linear(vision_width, embed_dim) + self.text_encoder_m = BertForMaskedLM.from_pretrained(text_encoder, config=bert_config) + self.text_proj_m = nn.Linear(text_width, embed_dim) + + self.model_pairs = [[self.visual_encoder,self.visual_encoder_m], + [self.vision_proj,self.vision_proj_m], + [self.text_encoder,self.text_encoder_m], + [self.text_proj,self.text_proj_m], + ] + + self.copy_params() + + # create the queue + self.register_buffer("image_queue", torch.randn(embed_dim, self.queue_size)) + self.register_buffer("text_queue", torch.randn(embed_dim, self.queue_size)) + self.register_buffer("queue_ptr", torch.zeros(1, dtype=torch.long)) + + self.image_queue = nn.functional.normalize(self.image_queue, dim=0) + self.text_queue = nn.functional.normalize(self.text_queue, dim=0) + + + + def forward(self, image, text, alpha=0): + with torch.no_grad(): + self.temp.clamp_(0.001,0.5) + + image_embeds = self.visual_encoder(image) + image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device) + + image_feat = F.normalize(self.vision_proj(image_embeds[:,0,:]),dim=-1) + + text_output = self.text_encoder.bert(text.input_ids, attention_mask = text.attention_mask, + return_dict = True, mode = 'text') + text_embeds = text_output.last_hidden_state + text_feat = F.normalize(self.text_proj(text_embeds[:,0,:]),dim=-1) + + # get momentum features + with torch.no_grad(): + self._momentum_update() + image_embeds_m = self.visual_encoder_m(image) + image_feat_m = F.normalize(self.vision_proj_m(image_embeds_m[:,0,:]),dim=-1) + image_feat_all = torch.cat([image_feat_m.t(),self.image_queue.clone().detach()],dim=1) + text_output_m = self.text_encoder_m.bert(text.input_ids, attention_mask = text.attention_mask, + return_dict = True, mode = 'text') + text_feat_m = F.normalize(self.text_proj_m(text_output_m.last_hidden_state[:,0,:]),dim=-1) + text_feat_all = torch.cat([text_feat_m.t(),self.text_queue.clone().detach()],dim=1) + + sim_i2t_m = image_feat_m @ text_feat_all / self.temp + sim_t2i_m = text_feat_m @ image_feat_all / self.temp + + sim_targets = torch.zeros(sim_i2t_m.size()).to(image.device) + sim_targets.fill_diagonal_(1) + + sim_i2t_targets = alpha * F.softmax(sim_i2t_m, dim=1) + (1 - alpha) * sim_targets + sim_t2i_targets = alpha * F.softmax(sim_t2i_m, dim=1) + (1 - alpha) * sim_targets + + sim_i2t = image_feat @ text_feat_all / self.temp + sim_t2i = text_feat @ image_feat_all / self.temp + + loss_i2t = -torch.sum(F.log_softmax(sim_i2t, dim=1)*sim_i2t_targets,dim=1).mean() + loss_t2i = -torch.sum(F.log_softmax(sim_t2i, dim=1)*sim_t2i_targets,dim=1).mean() + + loss_ita = (loss_i2t+loss_t2i)/2 + + self._dequeue_and_enqueue(image_feat_m, text_feat_m) + + ###=================================### + # forward the positve image-text pair + output_pos = self.text_encoder.bert(encoder_embeds = text_embeds, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True, + mode = 'fusion', + ) + with torch.no_grad(): + bs = image.size(0) + weights_i2t = F.softmax(sim_i2t[:,:bs],dim=1) + weights_t2i = F.softmax(sim_t2i[:,:bs],dim=1) + + weights_i2t.fill_diagonal_(0) + weights_t2i.fill_diagonal_(0) + + # select a negative image for each text + image_embeds_neg = [] + for b in range(bs): + neg_idx = torch.multinomial(weights_t2i[b], 1).item() + image_embeds_neg.append(image_embeds[neg_idx]) + image_embeds_neg = torch.stack(image_embeds_neg,dim=0) + + # select a negative text for each image + text_embeds_neg = [] + text_atts_neg = [] + for b in range(bs): + neg_idx = torch.multinomial(weights_i2t[b], 1).item() + text_embeds_neg.append(text_embeds[neg_idx]) + text_atts_neg.append(text.attention_mask[neg_idx]) + text_embeds_neg = torch.stack(text_embeds_neg,dim=0) + text_atts_neg = torch.stack(text_atts_neg,dim=0) + + text_embeds_all = torch.cat([text_embeds, text_embeds_neg],dim=0) + text_atts_all = torch.cat([text.attention_mask, text_atts_neg],dim=0) + + image_embeds_all = torch.cat([image_embeds_neg,image_embeds],dim=0) + image_atts_all = torch.cat([image_atts,image_atts],dim=0) + + output_neg = self.text_encoder.bert(encoder_embeds = text_embeds_all, + attention_mask = text_atts_all, + encoder_hidden_states = image_embeds_all, + encoder_attention_mask = image_atts_all, + return_dict = True, + mode = 'fusion', + ) + + vl_embeddings = torch.cat([output_pos.last_hidden_state[:,0,:], output_neg.last_hidden_state[:,0,:]],dim=0) + vl_output = self.itm_head(vl_embeddings) + + itm_labels = torch.cat([torch.ones(bs,dtype=torch.long),torch.zeros(2*bs,dtype=torch.long)], + dim=0).to(image.device) + loss_itm = F.cross_entropy(vl_output, itm_labels) + + ##================= MLM ========================## + input_ids = text.input_ids.clone() + labels = input_ids.clone() + + probability_matrix = torch.full(labels.shape, self.mlm_probability) + input_ids, labels = self.mask(input_ids, self.text_encoder.config.vocab_size, image.device, targets=labels, + probability_matrix = probability_matrix) + + with torch.no_grad(): + logits_m = self.text_encoder_m(input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds_m, + encoder_attention_mask = image_atts, + return_dict = True, + return_logits = True, + ) + mlm_output = self.text_encoder(input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True, + labels = labels, + soft_labels = F.softmax(logits_m,dim=-1), + alpha = alpha + ) + loss_mlm = mlm_output.loss + + return loss_mlm, loss_ita, loss_itm + + + + @torch.no_grad() + def copy_params(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data.copy_(param.data) # initialize + param_m.requires_grad = False # not update by gradient + + + @torch.no_grad() + def _momentum_update(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data = param_m.data * self.momentum + param.data * (1. - self.momentum) + + + + @torch.no_grad() + def _dequeue_and_enqueue(self, image_feat, text_feat): + # gather keys before updating queue + image_feats = concat_all_gather(image_feat) + text_feats = concat_all_gather(text_feat) + + batch_size = image_feats.shape[0] + + ptr = int(self.queue_ptr) + assert self.queue_size % batch_size == 0 # for simplicity + + # replace the keys at ptr (dequeue and enqueue) + self.image_queue[:, ptr:ptr + batch_size] = image_feats.T + self.text_queue[:, ptr:ptr + batch_size] = text_feats.T + ptr = (ptr + batch_size) % self.queue_size # move pointer + + self.queue_ptr[0] = ptr + + + def mask(self, input_ids, vocab_size, device, targets=None, masked_indices=None, probability_matrix=None): + if masked_indices is None: + masked_indices = torch.bernoulli(probability_matrix).bool() + + masked_indices[input_ids == self.tokenizer.pad_token_id] = False + masked_indices[input_ids == self.tokenizer.cls_token_id] = False + + if targets is not None: + targets[~masked_indices] = -100 # We only compute loss on masked tokens + + # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = torch.bernoulli(torch.full(input_ids.shape, 0.8)).bool() & masked_indices + input_ids[indices_replaced] = self.tokenizer.mask_token_id + + # 10% of the time, we replace masked input tokens with random word + indices_random = torch.bernoulli(torch.full(input_ids.shape, 0.5)).bool() & masked_indices & ~indices_replaced + random_words = torch.randint(vocab_size, input_ids.shape, dtype=torch.long).to(device) + input_ids[indices_random] = random_words[indices_random] + # The rest of the time (10% of the time) we keep the masked input tokens unchanged + + if targets is not None: + return input_ids, targets + else: + return input_ids + + +@torch.no_grad() +def concat_all_gather(tensor): + """ + Performs all_gather operation on the provided tensors. + *** Warning ***: torch.distributed.all_gather has no gradient. + """ + tensors_gather = [torch.ones_like(tensor) + for _ in range(torch.distributed.get_world_size())] + torch.distributed.all_gather(tensors_gather, tensor, async_op=False) + + output = torch.cat(tensors_gather, dim=0) + return output + diff --git a/models/model_pretrain_nlvr.py b/models/model_pretrain_nlvr.py new file mode 100644 index 0000000..dc57ec8 --- /dev/null +++ b/models/model_pretrain_nlvr.py @@ -0,0 +1,99 @@ +from functools import partial +from models.vit import VisionTransformer +from models.xbert import BertConfig, BertModel + +import torch +from torch import nn +import torch.nn.functional as F + +class ALBEF(nn.Module): + def __init__(self, + text_encoder = None, + tokenizer = None, + config = None, + ): + super().__init__() + + self.tokenizer = tokenizer + vision_width = config['vision_width'] + embed_dim = config['embed_dim'] + + self.visual_encoder = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + + bert_config = BertConfig.from_json_file(config['bert_config']) + bert_config.num_hidden_layers = 18 + self.text_encoder = BertModel.from_pretrained(text_encoder, config=bert_config, add_pooling_layer=False) + + #share the cross-attention layers for two images + self.share_cross_attention(self.text_encoder.encoder) + + text_width = self.text_encoder.config.hidden_size + self.vision_proj = nn.Linear(vision_width, embed_dim) + self.text_proj = nn.Linear(text_width, embed_dim) + self.temp = nn.Parameter(torch.ones([]) * 0.07) + self.ta_head = nn.Linear(self.text_encoder.config.hidden_size, 3) + + + def forward(self, image, text): + image_embeds = self.visual_encoder(image) + image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device) + with torch.no_grad(): + image_feat = F.normalize(self.vision_proj(image_embeds[:,0,:]),dim=-1) + sim = image_feat @ image_feat.t() / 0.07 + weights = F.softmax(sim,dim=1) + weights.fill_diagonal_(0) + + image_inputs = [[],[]] + labels = [] + for b in range(image.size(0)): + if torch.rand(1)>1/3: + idx = torch.multinomial(weights[b], 1).item() + if torch.rand(1)>0.5: + image_inputs[0].append(image_embeds[b]) + image_inputs[1].append(image_embeds[idx]) + labels.append(0) + else: + image_inputs[1].append(image_embeds[b]) + image_inputs[0].append(image_embeds[idx]) + labels.append(1) + else: + idx = torch.multinomial(weights[b], 2) + image_inputs[0].append(image_embeds[idx[0]]) + image_inputs[1].append(image_embeds[idx[1]]) + labels.append(2) + + image_inputs[0] = torch.stack(image_inputs[0],dim=0) + image_inputs[1] = torch.stack(image_inputs[1],dim=0) + labels = torch.LongTensor(labels).to(image.device) + + output = self.text_encoder(text.input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = image_inputs, + encoder_attention_mask = [image_atts,image_atts], + return_dict = True, + ) + + pred = self.ta_head(output.last_hidden_state[:,0,:]) + loss = F.cross_entropy(pred, labels) + + return loss + + + + def share_cross_attention(self, model): + + for i in range(6): + layer_num = 6+i*2 + modules_0 = model.layer[layer_num].crossattention.self._modules + modules_1 = model.layer[layer_num+1].crossattention.self._modules + + for name in modules_0.keys(): + if 'key' in name or 'value' in name: + module_0 = modules_0[name] + module_1 = modules_1[name] + if hasattr(module_0, "weight"): + module_0.weight = module_1.weight + if hasattr(module_0, "bias"): + module_0.bias = module_1.bias \ No newline at end of file diff --git a/models/model_pretrain_resnet.py b/models/model_pretrain_resnet.py new file mode 100644 index 0000000..d8f77e9 --- /dev/null +++ b/models/model_pretrain_resnet.py @@ -0,0 +1,274 @@ +from functools import partial +from models.resnet import resnet101 +from models.xbert import BertConfig, BertForMaskedLM + +import torch +import torch.nn.functional as F +from torch import nn + +import numpy as np +import random + + +class ALBEF(nn.Module): + def __init__(self, + text_encoder = None, + tokenizer = None, + config = None, + temp = 0.07, + ): + super().__init__() + + self.tokenizer = tokenizer + self.mlm_probability = config['mlm_probability'] + embed_dim = config['embed_dim'] + + self.visual_encoder = resnet101(pretrained=True) + self.vision_width = self.visual_encoder.width + self.visual_encoder.fc = nn.Sequential( + nn.Linear(self.vision_width,self.vision_width), + nn.ReLU(), + nn.Linear(self.vision_width,embed_dim), + ) + + bert_config = BertConfig.from_json_file(config['bert_config']) + bert_config.encoder_width = self.vision_width + + self.text_encoder = BertForMaskedLM.from_pretrained(text_encoder, config=bert_config) + + text_width = self.text_encoder.config.hidden_size + self.text_proj = nn.Linear(text_width, embed_dim) + + self.temp = nn.Parameter(torch.ones([]) * config['temp']) + self.queue_size = config['queue_size'] + self.momentum = config['momentum'] + self.itm_head = nn.Linear(text_width, 2) + + # create momentum models + self.visual_encoder_m = resnet101(pretrained=True) + self.visual_encoder_m.fc = nn.Sequential( + nn.Linear(self.vision_width,self.vision_width), + nn.ReLU(), + nn.Linear(self.vision_width,embed_dim), + ) + self.text_encoder_m = BertForMaskedLM.from_pretrained(text_encoder, config=bert_config) + self.text_proj_m = nn.Linear(text_width, embed_dim) + + self.model_pairs = [[self.visual_encoder,self.visual_encoder_m], + [self.text_encoder,self.text_encoder_m], + [self.text_proj,self.text_proj_m], + ] + + self.copy_params() + + # create the queue + self.register_buffer("image_queue", torch.randn(embed_dim, self.queue_size)) + self.register_buffer("text_queue", torch.randn(embed_dim, self.queue_size)) + self.register_buffer("queue_ptr", torch.zeros(1, dtype=torch.long)) + + self.image_queue = nn.functional.normalize(self.image_queue, dim=0) + self.text_queue = nn.functional.normalize(self.text_queue, dim=0) + + + + def forward(self, image, text, alpha=0): + with torch.no_grad(): + self.temp.clamp_(0.001,0.5) + + image_feat, image_embeds = self.visual_encoder(image) + image_feat = F.normalize(image_feat,dim=-1) + image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device) + + text_output = self.text_encoder.bert(text.input_ids, attention_mask = text.attention_mask, + return_dict = True, mode = 'text') + text_embeds = text_output.last_hidden_state + text_feat = F.normalize(self.text_proj(text_embeds[:,0,:]),dim=-1) + + # get momentum features + with torch.no_grad(): + self._momentum_update() + image_feat_m, image_embeds_m = self.visual_encoder_m(image) + + image_feat_m = F.normalize(image_feat_m,dim=-1) + image_feat_all = torch.cat([image_feat_m.t(),self.image_queue.clone().detach()],dim=1) + text_output_m = self.text_encoder_m.bert(text.input_ids, attention_mask = text.attention_mask, + return_dict = True, mode = 'text') + text_feat_m = F.normalize(self.text_proj_m(text_output_m.last_hidden_state[:,0,:]),dim=-1) + text_feat_all = torch.cat([text_feat_m.t(),self.text_queue.clone().detach()],dim=1) + + sim_i2t_m = image_feat_m @ text_feat_all / self.temp + sim_t2i_m = text_feat_m @ image_feat_all / self.temp + + sim_targets = torch.zeros(sim_i2t_m.size()).to(image.device) + sim_targets.fill_diagonal_(1) + + sim_i2t_targets = alpha * F.softmax(sim_i2t_m, dim=1) + (1 - alpha) * sim_targets + sim_t2i_targets = alpha * F.softmax(sim_t2i_m, dim=1) + (1 - alpha) * sim_targets + + sim_i2t = image_feat @ text_feat_all / self.temp + sim_t2i = text_feat @ image_feat_all / self.temp + + loss_i2t = -torch.sum(F.log_softmax(sim_i2t, dim=1)*sim_i2t_targets,dim=1).mean() + loss_t2i = -torch.sum(F.log_softmax(sim_t2i, dim=1)*sim_t2i_targets,dim=1).mean() + + loss_ita = (loss_i2t+loss_t2i)/2 + + self._dequeue_and_enqueue(image_feat_m, text_feat_m) + + ###=================================### + # forward the positve image-text pair + output_pos = self.text_encoder.bert(encoder_embeds = text_embeds, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True, + mode = 'fusion', + ) + with torch.no_grad(): + bs = image.size(0) + weights_i2t = F.softmax(sim_i2t[:,:bs],dim=1) + weights_t2i = F.softmax(sim_t2i[:,:bs],dim=1) + + weights_i2t.fill_diagonal_(0) + weights_t2i.fill_diagonal_(0) + + # select a negative image for each text + image_embeds_neg = [] + for b in range(bs): + neg_idx = torch.multinomial(weights_t2i[b], 1).item() + image_embeds_neg.append(image_embeds[neg_idx]) + image_embeds_neg = torch.stack(image_embeds_neg,dim=0) + + # select a negative text for each image + text_embeds_neg = [] + text_atts_neg = [] + for b in range(bs): + neg_idx = torch.multinomial(weights_i2t[b], 1).item() + text_embeds_neg.append(text_embeds[neg_idx]) + text_atts_neg.append(text.attention_mask[neg_idx]) + text_embeds_neg = torch.stack(text_embeds_neg,dim=0) + text_atts_neg = torch.stack(text_atts_neg,dim=0) + + text_embeds_all = torch.cat([text_embeds, text_embeds_neg],dim=0) + text_atts_all = torch.cat([text.attention_mask, text_atts_neg],dim=0) + + image_embeds_all = torch.cat([image_embeds_neg,image_embeds],dim=0) + image_atts_all = torch.cat([image_atts,image_atts],dim=0) + + output_neg = self.text_encoder.bert(encoder_embeds = text_embeds_all, + attention_mask = text_atts_all, + encoder_hidden_states = image_embeds_all, + encoder_attention_mask = image_atts_all, + return_dict = True, + mode = 'fusion', + ) + + vl_embeddings = torch.cat([output_pos.last_hidden_state[:,0,:], output_neg.last_hidden_state[:,0,:]],dim=0) + vl_output = self.itm_head(vl_embeddings) + + itm_labels = torch.cat([torch.ones(bs,dtype=torch.long),torch.zeros(2*bs,dtype=torch.long)], + dim=0).to(image.device) + loss_itm = F.cross_entropy(vl_output, itm_labels) + + ##================= MLM ========================## + input_ids = text.input_ids.clone() + labels = input_ids.clone() + + probability_matrix = torch.full(labels.shape, self.mlm_probability) + input_ids, labels = self.mask(input_ids, self.text_encoder.config.vocab_size, image.device, targets=labels, + probability_matrix = probability_matrix) + + with torch.no_grad(): + logits_m = self.text_encoder_m(input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds_m, + encoder_attention_mask = image_atts, + return_dict = True, + return_logits = True, + ) + mlm_output = self.text_encoder(input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True, + labels = labels, + soft_labels = F.softmax(logits_m,dim=-1), + alpha = alpha + ) + loss_mlm = mlm_output.loss + + return loss_mlm, loss_ita, loss_itm + + + @torch.no_grad() + def copy_params(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data.copy_(param.data) # initialize + param_m.requires_grad = False # not update by gradient + + + @torch.no_grad() + def _momentum_update(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data = param_m.data * self.momentum + param.data * (1. - self.momentum) + + @torch.no_grad() + def _dequeue_and_enqueue(self, image_feat, text_feat): + # gather keys before updating queue + image_feats = concat_all_gather(image_feat) + text_feats = concat_all_gather(text_feat) + + batch_size = image_feats.shape[0] + + ptr = int(self.queue_ptr) + assert self.queue_size % batch_size == 0 # for simplicity + + # replace the keys at ptr (dequeue and enqueue) + self.image_queue[:, ptr:ptr + batch_size] = image_feats.T + self.text_queue[:, ptr:ptr + batch_size] = text_feats.T + ptr = (ptr + batch_size) % self.queue_size # move pointer + + self.queue_ptr[0] = ptr + + + def mask(self, input_ids, vocab_size, device, targets=None, masked_indices=None, probability_matrix=None): + if masked_indices is None: + masked_indices = torch.bernoulli(probability_matrix).bool() + + masked_indices[input_ids == self.tokenizer.pad_token_id] = False + masked_indices[input_ids == self.tokenizer.cls_token_id] = False + + if targets is not None: + targets[~masked_indices] = -100 # We only compute loss on masked tokens + + # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = torch.bernoulli(torch.full(input_ids.shape, 0.8)).bool() & masked_indices + input_ids[indices_replaced] = self.tokenizer.mask_token_id + + # 10% of the time, we replace masked input tokens with random word + indices_random = torch.bernoulli(torch.full(input_ids.shape, 0.5)).bool() & masked_indices & ~indices_replaced + random_words = torch.randint(vocab_size, input_ids.shape, dtype=torch.long).to(device) + input_ids[indices_random] = random_words[indices_random] + # The rest of the time (10% of the time) we keep the masked input tokens unchanged + + if targets is not None: + return input_ids, targets + else: + return input_ids + + +@torch.no_grad() +def concat_all_gather(tensor): + """ + Performs all_gather operation on the provided tensors. + *** Warning ***: torch.distributed.all_gather has no gradient. + """ + tensors_gather = [torch.ones_like(tensor) + for _ in range(torch.distributed.get_world_size())] + torch.distributed.all_gather(tensors_gather, tensor, async_op=False) + + output = torch.cat(tensors_gather, dim=0) + return output + diff --git a/models/model_retrieval.py b/models/model_retrieval.py new file mode 100644 index 0000000..d00882a --- /dev/null +++ b/models/model_retrieval.py @@ -0,0 +1,217 @@ +from functools import partial +from models.vit import VisionTransformer +from models.xbert import BertConfig, BertModel + +import torch +from torch import nn +import torch.nn.functional as F + +class ALBEF(nn.Module): + def __init__(self, + text_encoder = None, + tokenizer = None, + config = None, + ): + super().__init__() + + self.tokenizer = tokenizer + self.distill = config['distill'] + embed_dim = config['embed_dim'] + vision_width = config['vision_width'] + self.visual_encoder = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + + bert_config = BertConfig.from_json_file(config['bert_config']) + self.text_encoder = BertModel.from_pretrained(text_encoder, config=bert_config, add_pooling_layer=False) + + text_width = self.text_encoder.config.hidden_size + self.vision_proj = nn.Linear(vision_width, embed_dim) + self.text_proj = nn.Linear(text_width, embed_dim) + + self.temp = nn.Parameter(torch.ones([]) * config['temp']) + self.queue_size = config['queue_size'] + self.momentum = config['momentum'] + self.itm_head = nn.Linear(text_width, 2) + + # create momentum models + self.visual_encoder_m = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + self.vision_proj_m = nn.Linear(vision_width, embed_dim) + self.text_encoder_m = BertModel.from_pretrained(text_encoder, config=bert_config, add_pooling_layer=False) + self.text_proj_m = nn.Linear(text_width, embed_dim) + + self.model_pairs = [[self.visual_encoder,self.visual_encoder_m], + [self.vision_proj,self.vision_proj_m], + [self.text_encoder,self.text_encoder_m], + [self.text_proj,self.text_proj_m], + ] + self.copy_params() + + # create the queue + self.register_buffer("image_queue", torch.randn(embed_dim, self.queue_size)) + self.register_buffer("text_queue", torch.randn(embed_dim, self.queue_size)) + self.register_buffer("idx_queue", torch.full((1,self.queue_size),-100)) + self.register_buffer("queue_ptr", torch.zeros(1, dtype=torch.long)) + + self.image_queue = nn.functional.normalize(self.image_queue, dim=0) + self.text_queue = nn.functional.normalize(self.text_queue, dim=0) + + + def forward(self, image, text, alpha, idx): + + image_embeds = self.visual_encoder(image) + image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device) + + image_feat = F.normalize(self.vision_proj(image_embeds[:,0,:]),dim=-1) + text_output = self.text_encoder(text.input_ids, attention_mask = text.attention_mask, + return_dict = True, mode = 'text') + text_embeds = text_output.last_hidden_state + text_feat = F.normalize(self.text_proj(text_embeds[:,0,:]),dim=-1) + + idx = idx.view(-1,1) + idx_all = torch.cat([idx.t(), self.idx_queue.clone().detach()],dim=1) + pos_idx = torch.eq(idx, idx_all).float() + sim_targets = pos_idx / pos_idx.sum(1,keepdim=True) + + with torch.no_grad(): + self._momentum_update() + image_embeds_m = self.visual_encoder_m(image) + image_feat_m = F.normalize(self.vision_proj_m(image_embeds_m[:,0,:]),dim=-1) + image_feat_all = torch.cat([image_feat_m.t(),self.image_queue.clone().detach()],dim=1) + text_output_m = self.text_encoder_m(text.input_ids, attention_mask = text.attention_mask, + return_dict = True, mode = 'text') + text_feat_m = F.normalize(self.text_proj_m(text_output_m.last_hidden_state[:,0,:]),dim=-1) + text_feat_all = torch.cat([text_feat_m.t(),self.text_queue.clone().detach()],dim=1) + + if self.distill: + sim_i2t_m = image_feat_m @ text_feat_all / self.temp + sim_t2i_m = text_feat_m @ image_feat_all / self.temp + + sim_i2t_targets = alpha * F.softmax(sim_i2t_m, dim=1) + (1 - alpha) * sim_targets + sim_t2i_targets = alpha * F.softmax(sim_t2i_m, dim=1) + (1 - alpha) * sim_targets + + sim_i2t = image_feat @ text_feat_all / self.temp + sim_t2i = text_feat @ image_feat_all / self.temp + + if self.distill: + loss_i2t = -torch.sum(F.log_softmax(sim_i2t, dim=1)*sim_i2t_targets,dim=1).mean() + loss_t2i = -torch.sum(F.log_softmax(sim_t2i, dim=1)*sim_t2i_targets,dim=1).mean() + else: + loss_i2t = -torch.sum(F.log_softmax(sim_i2t, dim=1)*sim_targets,dim=1).mean() + loss_t2i = -torch.sum(F.log_softmax(sim_t2i, dim=1)*sim_targets,dim=1).mean() + + loss_ita = (loss_i2t+loss_t2i)/2 + + self._dequeue_and_enqueue(image_feat_m, text_feat_m, idx) + + ###=================================### + # forward the positve image-text pair + output_pos = self.text_encoder(encoder_embeds = text_embeds, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True, + mode = 'fusion', + ) + with torch.no_grad(): + bs = image.size(0) + weights_i2t = F.softmax(sim_i2t[:,:bs],dim=1) + weights_t2i = F.softmax(sim_t2i[:,:bs],dim=1) + + mask = torch.eq(idx, idx.T) + weights_i2t.masked_fill_(mask, 0) + weights_t2i.masked_fill_(mask, 0) + + # select a negative image for each text + image_embeds_neg = [] + for b in range(bs): + neg_idx = torch.multinomial(weights_t2i[b], 1).item() + image_embeds_neg.append(image_embeds[neg_idx]) + image_embeds_neg = torch.stack(image_embeds_neg,dim=0) + + # select a negative text for each image + text_embeds_neg = [] + text_atts_neg = [] + for b in range(bs): + neg_idx = torch.multinomial(weights_i2t[b], 1).item() + text_embeds_neg.append(text_embeds[neg_idx]) + text_atts_neg.append(text.attention_mask[neg_idx]) + text_embeds_neg = torch.stack(text_embeds_neg,dim=0) + text_atts_neg = torch.stack(text_atts_neg,dim=0) + + text_embeds_all = torch.cat([text_embeds, text_embeds_neg],dim=0) + text_atts_all = torch.cat([text.attention_mask, text_atts_neg],dim=0) + + image_embeds_all = torch.cat([image_embeds_neg,image_embeds],dim=0) + image_atts_all = torch.cat([image_atts,image_atts],dim=0) + + output_neg = self.text_encoder(encoder_embeds = text_embeds_all, + attention_mask = text_atts_all, + encoder_hidden_states = image_embeds_all, + encoder_attention_mask = image_atts_all, + return_dict = True, + mode = 'fusion', + ) + + vl_embeddings = torch.cat([output_pos.last_hidden_state[:,0,:], output_neg.last_hidden_state[:,0,:]],dim=0) + vl_output = self.itm_head(vl_embeddings) + + itm_labels = torch.cat([torch.ones(bs,dtype=torch.long),torch.zeros(2*bs,dtype=torch.long)], + dim=0).to(image.device) + loss_itm = F.cross_entropy(vl_output, itm_labels) + + return loss_ita, loss_itm + + + + @torch.no_grad() + def copy_params(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data.copy_(param.data) # initialize + param_m.requires_grad = False # not update by gradient + + + @torch.no_grad() + def _momentum_update(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data = param_m.data * self.momentum + param.data * (1. - self.momentum) + + + @torch.no_grad() + def _dequeue_and_enqueue(self, image_feat, text_feat, idx): + # gather keys before updating queue + image_feats = concat_all_gather(image_feat) + text_feats = concat_all_gather(text_feat) + idxs = concat_all_gather(idx) + + batch_size = image_feats.shape[0] + + ptr = int(self.queue_ptr) + assert self.queue_size % batch_size == 0 # for simplicity + + # replace the keys at ptr (dequeue and enqueue) + self.image_queue[:, ptr:ptr + batch_size] = image_feats.T + self.text_queue[:, ptr:ptr + batch_size] = text_feats.T + self.idx_queue[:, ptr:ptr + batch_size] = idxs.T + ptr = (ptr + batch_size) % self.queue_size # move pointer + + self.queue_ptr[0] = ptr + + +@torch.no_grad() +def concat_all_gather(tensor): + """ + Performs all_gather operation on the provided tensors. + *** Warning ***: torch.distributed.all_gather has no gradient. + """ + tensors_gather = [torch.ones_like(tensor) + for _ in range(torch.distributed.get_world_size())] + torch.distributed.all_gather(tensors_gather, tensor, async_op=False) + + output = torch.cat(tensors_gather, dim=0) + return output + diff --git a/models/model_ve.py b/models/model_ve.py new file mode 100644 index 0000000..d659842 --- /dev/null +++ b/models/model_ve.py @@ -0,0 +1,110 @@ +from functools import partial +from models.vit import VisionTransformer +from models.xbert import BertConfig, BertModel + +import torch +from torch import nn +import torch.nn.functional as F + +class ALBEF(nn.Module): + def __init__(self, + text_encoder = None, + tokenizer = None, + config = None, + ): + super().__init__() + + self.tokenizer = tokenizer + self.distill = config['distill'] + + self.visual_encoder = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + + bert_config = BertConfig.from_json_file(config['bert_config']) + + self.text_encoder = BertModel.from_pretrained(text_encoder, config=bert_config, add_pooling_layer=False) + + self.cls_head = nn.Sequential( + nn.Linear(self.text_encoder.config.hidden_size, self.text_encoder.config.hidden_size), + nn.ReLU(), + nn.Linear(self.text_encoder.config.hidden_size, 3) + ) + + if self.distill: + self.visual_encoder_m = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + self.text_encoder_m = BertModel.from_pretrained(text_encoder, config=bert_config, add_pooling_layer=False) + self.cls_head_m = nn.Sequential( + nn.Linear(self.text_encoder.config.hidden_size, self.text_encoder.config.hidden_size), + nn.ReLU(), + nn.Linear(self.text_encoder.config.hidden_size, 3) + ) + + self.model_pairs = [[self.visual_encoder,self.visual_encoder_m], + [self.text_encoder,self.text_encoder_m], + [self.cls_head,self.cls_head_m], + ] + self.copy_params() + self.momentum = 0.995 + + + def forward(self, image, text, targets, alpha=0, train=True): + + image_embeds = self.visual_encoder(image) + image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device) + + if train: + output = self.text_encoder(text.input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True + ) + prediction = self.cls_head(output.last_hidden_state[:,0,:]) + if self.distill: + with torch.no_grad(): + self._momentum_update() + image_embeds_m = self.visual_encoder_m(image) + output_m = self.text_encoder_m(text.input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds_m, + encoder_attention_mask = image_atts, + return_dict = True + ) + prediction_m = self.cls_head_m(output_m.last_hidden_state[:,0,:]) + + loss = (1-alpha)*F.cross_entropy(prediction, targets) - alpha*torch.sum( + F.log_softmax(prediction, dim=1)*F.softmax(prediction_m, dim=1),dim=1).mean() + else: + loss = F.cross_entropy(prediction, targets) + return loss + + else: + output = self.text_encoder(text.input_ids, + attention_mask = text.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True + ) + prediction = self.cls_head(output.last_hidden_state[:,0,:]) + return prediction + + + + @torch.no_grad() + def copy_params(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data.copy_(param.data) # initialize + param_m.requires_grad = False # not update by gradient + + + @torch.no_grad() + def _momentum_update(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data = param_m.data * self.momentum + param.data * (1. - self.momentum) + + diff --git a/models/model_vqa.py b/models/model_vqa.py new file mode 100644 index 0000000..52711e9 --- /dev/null +++ b/models/model_vqa.py @@ -0,0 +1,213 @@ +from functools import partial +from models.vit import VisionTransformer +from models.xbert import BertConfig, BertModel, BertLMHeadModel + +import torch +from torch import nn +import torch.nn.functional as F + +import numpy as np + +class ALBEF(nn.Module): + def __init__(self, + text_encoder = None, + text_decoder = None, + tokenizer = None, + config = None, + ): + super().__init__() + + self.tokenizer = tokenizer + self.distill = config['distill'] + + self.visual_encoder = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + + config_encoder = BertConfig.from_json_file(config['bert_config']) + self.text_encoder = BertModel.from_pretrained(text_encoder, config=config_encoder, add_pooling_layer=False) + + config_decoder = BertConfig.from_json_file(config['bert_config']) + config_decoder.fusion_layer = 0 + config_decoder.num_hidden_layers = 6 + self.text_decoder = BertLMHeadModel.from_pretrained(text_decoder, config=config_decoder) + + if self.distill: + self.visual_encoder_m = VisionTransformer( + img_size=config['image_res'], patch_size=16, embed_dim=768, depth=12, num_heads=12, + mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) + self.text_encoder_m = BertModel.from_pretrained(text_encoder, config=config_encoder, add_pooling_layer=False) + self.text_decoder_m = BertLMHeadModel.from_pretrained(text_decoder, config=config_decoder) + self.model_pairs = [[self.visual_encoder,self.visual_encoder_m], + [self.text_encoder,self.text_encoder_m], + [self.text_decoder,self.text_decoder_m], + ] + self.copy_params() + self.momentum = 0.995 + + + def forward(self, image, quesiton, answer=None, alpha=0, k=None, weights=None, train=True): + + image_embeds = self.visual_encoder(image) + image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device) + + if train: + ''' + k: number of answers for each question + weights: weight for each answer + ''' + answer_targets = answer.input_ids.masked_fill(answer.input_ids == self.tokenizer.pad_token_id, -100) + + question_output = self.text_encoder(quesiton.input_ids, + attention_mask = quesiton.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True) + + question_states = [] + question_atts = [] + for b, n in enumerate(k): + question_states += [question_output.last_hidden_state[b]]*n + question_atts += [quesiton.attention_mask[b]]*n + question_states = torch.stack(question_states,0) + question_atts = torch.stack(question_atts,0) + + if self.distill: + with torch.no_grad(): + self._momentum_update() + image_embeds_m = self.visual_encoder_m(image) + question_output_m = self.text_encoder_m(quesiton.input_ids, + attention_mask = quesiton.attention_mask, + encoder_hidden_states = image_embeds_m, + encoder_attention_mask = image_atts, + return_dict = True) + + question_states_m = [] + for b, n in enumerate(k): + question_states_m += [question_output_m.last_hidden_state[b]]*n + question_states_m = torch.stack(question_states_m,0) + + logits_m = self.text_decoder_m(answer.input_ids, + attention_mask = answer.attention_mask, + encoder_hidden_states = question_states_m, + encoder_attention_mask = question_atts, + return_logits = True, + ) + + answer_output = self.text_decoder(answer.input_ids, + attention_mask = answer.attention_mask, + encoder_hidden_states = question_states, + encoder_attention_mask = question_atts, + labels = answer_targets, + return_dict = True, + soft_labels = F.softmax(logits_m,dim=-1), + reduction = 'none', + ) + else: + answer_output = self.text_decoder(answer.input_ids, + attention_mask = answer.attention_mask, + encoder_hidden_states = question_states, + encoder_attention_mask = question_atts, + labels = answer_targets, + return_dict = True, + reduction = 'none', + ) + loss = weights * answer_output.loss + loss = loss.sum()/image.size(0) + + return loss + + + else: + question_output = self.text_encoder(quesiton.input_ids, + attention_mask = quesiton.attention_mask, + encoder_hidden_states = image_embeds, + encoder_attention_mask = image_atts, + return_dict = True) + topk_ids, topk_probs = self.rank_answer(question_output.last_hidden_state, quesiton.attention_mask, + answer.input_ids, answer.attention_mask, k) + return topk_ids, topk_probs + + + + @torch.no_grad() + def copy_params(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data.copy_(param.data) # initialize + param_m.requires_grad = False # not update by gradient + + + @torch.no_grad() + def _momentum_update(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()): + param_m.data = param_m.data * self.momentum + param.data * (1. - self.momentum) + + + def rank_answer(self, question_states, question_atts, answer_ids, answer_atts, k): + + num_ques = question_states.size(0) + start_ids = answer_ids[0,0].repeat(num_ques,1) # bos token + + start_output = self.text_decoder(start_ids, + encoder_hidden_states = question_states, + encoder_attention_mask = question_atts, + return_dict = True, + reduction = 'none') + logits = start_output.logits[:,0,:] # first token's logit + + # topk_probs: top-k probability + # topk_ids: [num_question, k] + answer_first_token = answer_ids[:,1] + prob_first_token = F.softmax(logits,dim=1).index_select(dim=1, index=answer_first_token) + topk_probs, topk_ids = prob_first_token.topk(k,dim=1) + + # answer input: [num_question*k, answer_len] + input_ids = [] + input_atts = [] + for b, topk_id in enumerate(topk_ids): + input_ids.append(answer_ids.index_select(dim=0, index=topk_id)) + input_atts.append(answer_atts.index_select(dim=0, index=topk_id)) + input_ids = torch.cat(input_ids,dim=0) + input_atts = torch.cat(input_atts,dim=0) + + targets_ids = input_ids.masked_fill(input_ids == self.tokenizer.pad_token_id, -100) + + # repeat encoder's output for top-k answers + question_states = tile(question_states, 0, k) + question_atts = tile(question_atts, 0, k) + + output = self.text_decoder(input_ids, + attention_mask = input_atts, + encoder_hidden_states = question_states, + encoder_attention_mask = question_atts, + labels = targets_ids, + return_dict = True, + reduction = 'none') + + answer_loss = output.loss + answer_loss = answer_loss.view(input_ids.size(0),-1) + + # topk_prob: first token probability + topk_probs = topk_probs.view(-1,1) + log_probs = torch.cat([topk_probs.log(), -answer_loss],dim=1) + + # re-calculate log probabilities for the answer sequences using chain rule + log_probs_sum = log_probs.sum(1) + log_probs_sum = log_probs_sum.view(num_ques,k) + + topk_probs = F.softmax(log_probs_sum, dim=-1) + # get top-k after re-ranking + topk_probs, rerank_id = topk_probs.topk(k,dim=1) + topk_ids = torch.gather(topk_ids, 1, rerank_id) + + return topk_ids, topk_probs + +def tile(x, dim, n_tile): + init_dim = x.size(dim) + repeat_idx = [1] * x.dim() + repeat_idx[dim] = n_tile + x = x.repeat(*(repeat_idx)) + order_index = torch.LongTensor(np.concatenate([init_dim * np.arange(n_tile) + i for i in range(init_dim)])) + return torch.index_select(x, dim, order_index.to(x.device)) \ No newline at end of file diff --git a/models/resnet.py b/models/resnet.py new file mode 100644 index 0000000..58eaa06 --- /dev/null +++ b/models/resnet.py @@ -0,0 +1,384 @@ +import torch +from torch import Tensor +import torch.nn as nn +from torchvision.models.utils import load_state_dict_from_url +from typing import Type, Any, Callable, Union, List, Optional + + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', + 'wide_resnet50_2', 'wide_resnet101_2'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-f37072fd.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-b627a593.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-0676ba61.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-63fe2227.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-394f9c45.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d: + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + + +def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d: + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion: int = 1 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None + ) -> None: + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion: int = 4 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None + ) -> None: + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__( + self, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + num_classes: int = 1000, + zero_init_residual: bool = False, + groups: int = 1, + width_per_group: int = 64, + replace_stride_with_dilation: Optional[List[bool]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None + ) -> None: + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + self.width = 512 * block.expansion + #self.output_norm = nn.LayerNorm(self.width) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) # type: ignore[arg-type] + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) # type: ignore[arg-type] + + def _make_layer(self, block: Type[Union[BasicBlock, Bottleneck]], planes: int, blocks: int, + stride: int = 1, dilate: bool = False) -> nn.Sequential: + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x: Tensor) -> Tensor: + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + embeds = self.layer4(x) + + x = self.avgpool(embeds) + x = torch.flatten(x, 1) + x = self.fc(x) + + embeds = embeds.view(x.size(0),self.width,-1).permute(0,2,1) + #embeds = self.output_norm(embeds) + + return x, embeds + + def forward(self, x: Tensor) -> Tensor: + return self._forward_impl(x) + + +def _resnet( + arch: str, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + pretrained: bool, + progress: bool, + **kwargs: Any +) -> ResNet: + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + model.load_state_dict(state_dict,strict=False) + return model + + +def resnet18(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress, + **kwargs) + + +def resnet152(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress, + **kwargs) + + +def resnext50_32x4d(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def resnext101_32x8d(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) + + +def wide_resnet50_2(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_. + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def wide_resnet101_2(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_. + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) \ No newline at end of file diff --git a/models/tokenization_bert.py b/models/tokenization_bert.py new file mode 100644 index 0000000..96dea50 --- /dev/null +++ b/models/tokenization_bert.py @@ -0,0 +1,539 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes for Bert.""" + + +import collections +import os +import unicodedata +from typing import List, Optional, Tuple + +from transformers.tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + +VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"} + +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": { + "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt", + "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt", + "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt", + "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt", + "bert-base-multilingual-uncased": "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt", + "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt", + "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt", + "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt", + "bert-large-uncased-whole-word-masking": "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt", + "bert-large-cased-whole-word-masking": "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt", + "bert-large-uncased-whole-word-masking-finetuned-squad": "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt", + "bert-large-cased-whole-word-masking-finetuned-squad": "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt", + "bert-base-cased-finetuned-mrpc": "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt", + "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt", + "bert-base-german-dbmdz-uncased": "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt", + "TurkuNLP/bert-base-finnish-cased-v1": "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt", + "TurkuNLP/bert-base-finnish-uncased-v1": "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt", + "wietsedv/bert-base-dutch-cased": "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "bert-base-uncased": 512, + "bert-large-uncased": 512, + "bert-base-cased": 512, + "bert-large-cased": 512, + "bert-base-multilingual-uncased": 512, + "bert-base-multilingual-cased": 512, + "bert-base-chinese": 512, + "bert-base-german-cased": 512, + "bert-large-uncased-whole-word-masking": 512, + "bert-large-cased-whole-word-masking": 512, + "bert-large-uncased-whole-word-masking-finetuned-squad": 512, + "bert-large-cased-whole-word-masking-finetuned-squad": 512, + "bert-base-cased-finetuned-mrpc": 512, + "bert-base-german-dbmdz-cased": 512, + "bert-base-german-dbmdz-uncased": 512, + "TurkuNLP/bert-base-finnish-cased-v1": 512, + "TurkuNLP/bert-base-finnish-uncased-v1": 512, + "wietsedv/bert-base-dutch-cased": 512, +} + +PRETRAINED_INIT_CONFIGURATION = { + "bert-base-uncased": {"do_lower_case": True}, + "bert-large-uncased": {"do_lower_case": True}, + "bert-base-cased": {"do_lower_case": False}, + "bert-large-cased": {"do_lower_case": False}, + "bert-base-multilingual-uncased": {"do_lower_case": True}, + "bert-base-multilingual-cased": {"do_lower_case": False}, + "bert-base-chinese": {"do_lower_case": False}, + "bert-base-german-cased": {"do_lower_case": False}, + "bert-large-uncased-whole-word-masking": {"do_lower_case": True}, + "bert-large-cased-whole-word-masking": {"do_lower_case": False}, + "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True}, + "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False}, + "bert-base-cased-finetuned-mrpc": {"do_lower_case": False}, + "bert-base-german-dbmdz-cased": {"do_lower_case": False}, + "bert-base-german-dbmdz-uncased": {"do_lower_case": True}, + "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False}, + "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True}, + "wietsedv/bert-base-dutch-cased": {"do_lower_case": False}, +} + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + with open(vocab_file, "r", encoding="utf-8") as reader: + tokens = reader.readlines() + for index, token in enumerate(tokens): + token = token.rstrip("\n") + vocab[token] = index + return vocab + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class BertTokenizer(PreTrainedTokenizer): + r""" + Construct a BERT tokenizer. Based on WordPiece. + This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the main methods. + Users should refer to this superclass for more information regarding those methods. + Args: + vocab_file (:obj:`str`): + File containing the vocabulary. + do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to lowercase the input when tokenizing. + do_basic_tokenize (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to do basic tokenization before WordPiece. + never_split (:obj:`Iterable`, `optional`): + Collection of tokens which will never be split during tokenization. Only has an effect when + :obj:`do_basic_tokenize=True` + unk_token (:obj:`str`, `optional`, defaults to :obj:`"[UNK]"`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + sep_token (:obj:`str`, `optional`, defaults to :obj:`"[SEP]"`): + The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for + sequence classification or for a text and a question for question answering. It is also used as the last + token of a sequence built with special tokens. + pad_token (:obj:`str`, `optional`, defaults to :obj:`"[PAD]"`): + The token used for padding, for example when batching sequences of different lengths. + cls_token (:obj:`str`, `optional`, defaults to :obj:`"[CLS]"`): + The classifier token which is used when doing sequence classification (classification of the whole sequence + instead of per-token classification). It is the first token of the sequence when built with special tokens. + mask_token (:obj:`str`, `optional`, defaults to :obj:`"[MASK]"`): + The token used for masking values. This is the token used when training this model with masked language + modeling. This is the token which the model will try to predict. + tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to tokenize Chinese characters. + This should likely be deactivated for Japanese (see this `issue + `__). + strip_accents: (:obj:`bool`, `optional`): + Whether or not to strip all accents. If this option is not specified, then it will be determined by the + value for :obj:`lowercase` (as in the original BERT). + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__( + self, + vocab_file, + do_lower_case=True, + do_basic_tokenize=True, + never_split=None, + unk_token="[UNK]", + sep_token="[SEP]", + pad_token="[PAD]", + cls_token="[CLS]", + mask_token="[MASK]", + tokenize_chinese_chars=True, + strip_accents=None, + **kwargs + ): + super().__init__( + do_lower_case=do_lower_case, + do_basic_tokenize=do_basic_tokenize, + never_split=never_split, + unk_token=unk_token, + sep_token=sep_token, + pad_token=pad_token, + cls_token=cls_token, + mask_token=mask_token, + tokenize_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + **kwargs, + ) + + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " + "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file) + ) + self.vocab = load_vocab(vocab_file) + self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()]) + self.do_basic_tokenize = do_basic_tokenize + if do_basic_tokenize: + self.basic_tokenizer = BasicTokenizer( + do_lower_case=do_lower_case, + never_split=never_split, + tokenize_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + ) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=self.unk_token) + + @property + def do_lower_case(self): + return self.basic_tokenizer.do_lower_case + + @property + def vocab_size(self): + return len(self.vocab) + + def get_vocab(self): + return dict(self.vocab, **self.added_tokens_encoder) + + def _tokenize(self, text): + split_tokens = [] + if self.do_basic_tokenize: + for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens): + + # If the token is part of the never_split set + if token in self.basic_tokenizer.never_split: + split_tokens.append(token) + else: + split_tokens += self.wordpiece_tokenizer.tokenize(token) + else: + split_tokens = self.wordpiece_tokenizer.tokenize(text) + return split_tokens + + def _convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + return self.vocab.get(token, self.vocab.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.ids_to_tokens.get(index, self.unk_token) + + def convert_tokens_to_string(self, tokens): + """ Converts a sequence of tokens (string) in a single string. """ + out_string = " ".join(tokens).replace(" ##", "").strip() + return out_string + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A BERT sequence has the following format: + - single sequence: ``[CLS] X `` + - pair of sequences: ``[CLS] A [SEP] B [SEP]`` + Args: + token_ids_0 (:obj:`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (:obj:`List[int]`, `optional`): + Optional second list of IDs for sequence pairs. + Returns: + :obj:`List[int]`: List of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens. + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + token_ids_1 + sep + + def get_special_tokens_mask( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` method. + Args: + token_ids_0 (:obj:`List[int]`): + List of IDs. + token_ids_1 (:obj:`List[int]`, `optional`): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether or not the token list is already formatted with special tokens for the model. + Returns: + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError( + "You should not supply a second sequence if the provided sequence of " + "ids is already formatted with special tokens for the model." + ) + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is not None: + return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence + pair mask has the following format: + :: + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + If :obj:`token_ids_1` is :obj:`None`, this method only returns the first portion of the mask (0s). + Args: + token_ids_0 (:obj:`List[int]`): + List of IDs. + token_ids_1 (:obj:`List[int]`, `optional`): + Optional second list of IDs for sequence pairs. + Returns: + :obj:`List[int]`: List of `token type IDs <../glossary.html#token-type-ids>`_ according to the given + sequence(s). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + index = 0 + if os.path.isdir(save_directory): + vocab_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] + ) + else: + vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory + with open(vocab_file, "w", encoding="utf-8") as writer: + for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + "Saving vocabulary to {}: vocabulary indices are not consecutive." + " Please check that the vocabulary is not corrupted!".format(vocab_file) + ) + index = token_index + writer.write(token + "\n") + index += 1 + return (vocab_file,) + + +class BasicTokenizer(object): + """ + Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.). + Args: + do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to lowercase the input when tokenizing. + never_split (:obj:`Iterable`, `optional`): + Collection of tokens which will never be split during tokenization. Only has an effect when + :obj:`do_basic_tokenize=True` + tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to tokenize Chinese characters. + This should likely be deactivated for Japanese (see this `issue + `__). + strip_accents: (:obj:`bool`, `optional`): + Whether or not to strip all accents. If this option is not specified, then it will be determined by the + value for :obj:`lowercase` (as in the original BERT). + """ + + def __init__(self, do_lower_case=True, never_split=None, tokenize_chinese_chars=True, strip_accents=None): + if never_split is None: + never_split = [] + self.do_lower_case = do_lower_case + self.never_split = set(never_split) + self.tokenize_chinese_chars = tokenize_chinese_chars + self.strip_accents = strip_accents + + def tokenize(self, text, never_split=None): + """ + Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see + WordPieceTokenizer. + Args: + **never_split**: (`optional`) list of str + Kept for backward compatibility purposes. Now implemented directly at the base class level (see + :func:`PreTrainedTokenizer.tokenize`) List of token not to split. + """ + # union() returns a new set by concatenating the two sets. + never_split = self.never_split.union(set(never_split)) if never_split else self.never_split + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + if self.tokenize_chinese_chars: + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if token not in never_split: + if self.do_lower_case: + token = token.lower() + if self.strip_accents is not False: + token = self._run_strip_accents(token) + elif self.strip_accents: + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token, never_split)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text, never_split=None): + """Splits punctuation on a piece of text.""" + if never_split is not None and text in never_split: + return [text] + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xFFFD or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenization.""" + + def __init__(self, vocab, unk_token, max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """ + Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform + tokenization using the given vocabulary. + For example, :obj:`input = "unaffable"` wil return as output :obj:`["un", "##aff", "##able"]`. + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer`. + Returns: + A list of wordpiece tokens. + """ + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens \ No newline at end of file diff --git a/models/vit.py b/models/vit.py new file mode 100644 index 0000000..8253bc7 --- /dev/null +++ b/models/vit.py @@ -0,0 +1,204 @@ +# Copyright (c) 2015-present, Facebook, Inc. +# All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F +from functools import partial + +from timm.models.vision_transformer import _cfg, PatchEmbed +from timm.models.registry import register_model +from timm.models.layers import trunc_normal_, DropPath + + +class Mlp(nn.Module): + """ MLP as used in Vision Transformer, MLP-Mixer and related networks + """ + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim ** -0.5 + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + self.attn_gradients = None + self.attention_map = None + + def save_attn_gradients(self, attn_gradients): + self.attn_gradients = attn_gradients + + def get_attn_gradients(self): + return self.attn_gradients + + def save_attention_map(self, attention_map): + self.attention_map = attention_map + + def get_attention_map(self): + return self.attention_map + + def forward(self, x, register_hook=False): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + if register_hook: + self.save_attention_map(attn) + attn.register_hook(self.save_attn_gradients) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x, register_hook=False): + x = x + self.drop_path(self.attn(self.norm1(x), register_hook=register_hook)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class VisionTransformer(nn.Module): + """ Vision Transformer + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - + https://arxiv.org/abs/2010.11929 + """ + def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, + num_heads=12, mlp_ratio=4., qkv_bias=True, qk_scale=None, representation_size=None, + drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None): + """ + Args: + img_size (int, tuple): input image size + patch_size (int, tuple): patch size + in_chans (int): number of input channels + num_classes (int): number of classes for classification head + embed_dim (int): embedding dimension + depth (int): depth of transformer + num_heads (int): number of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + qk_scale (float): override default qk scale of head_dim ** -0.5 if set + representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set + drop_rate (float): dropout rate + attn_drop_rate (float): attention dropout rate + drop_path_rate (float): stochastic depth rate + norm_layer: (nn.Module): normalization layer + """ + super().__init__() + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + + self.patch_embed = PatchEmbed( + img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim)) + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + self.blocks = nn.ModuleList([ + Block( + dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer) + for i in range(depth)]) + self.norm = norm_layer(embed_dim) + + trunc_normal_(self.pos_embed, std=.02) + trunc_normal_(self.cls_token, std=.02) + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token'} + + def forward(self, x, register_blk=-1): + B = x.shape[0] + x = self.patch_embed(x) + + cls_tokens = self.cls_token.expand(B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + x = x + self.pos_embed[:,:x.size(1),:] + x = self.pos_drop(x) + + for i,blk in enumerate(self.blocks): + x = blk(x, register_blk==i) + x = self.norm(x) + + return x + + + +def interpolate_pos_embed(pos_embed_checkpoint, visual_encoder): + # interpolate position embedding + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = visual_encoder.patch_embed.num_patches + num_extra_tokens = visual_encoder.pos_embed.shape[-2] - num_patches + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches ** 0.5) + + if orig_size!=new_size: + # class_token and dist_token are kept unchanged + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + print('reshape position embedding from %d to %d'%(orig_size ** 2,new_size ** 2)) + + return new_pos_embed + else: + return pos_embed_checkpoint \ No newline at end of file diff --git a/models/xbert.py b/models/xbert.py new file mode 100644 index 0000000..4f3f716 --- /dev/null +++ b/models/xbert.py @@ -0,0 +1,1916 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model. """ + +import math +import os +import warnings +from dataclasses import dataclass +from typing import Optional, Tuple + +import torch +from torch import Tensor, device, dtype, nn +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss, MSELoss +import torch.nn.functional as F + +from transformers.activations import ACT2FN +from transformers.file_utils import ( + ModelOutput, + add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + replace_return_docstrings, +) +from transformers.modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithCrossAttentions, + MaskedLMOutput, + MultipleChoiceModelOutput, + NextSentencePredictorOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +) +from transformers.modeling_utils import ( + PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer, +) +from transformers.utils import logging +from transformers.models.bert.configuration_bert import BertConfig + +import transformers +transformers.logging.set_verbosity_error() + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "BertConfig" +_TOKENIZER_FOR_DOC = "BertTokenizer" + +BERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "bert-base-uncased", + "bert-large-uncased", + "bert-base-cased", + "bert-large-cased", + "bert-base-multilingual-uncased", + "bert-base-multilingual-cased", + "bert-base-chinese", + "bert-base-german-cased", + "bert-large-uncased-whole-word-masking", + "bert-large-cased-whole-word-masking", + "bert-large-uncased-whole-word-masking-finetuned-squad", + "bert-large-cased-whole-word-masking-finetuned-squad", + "bert-base-cased-finetuned-mrpc", + "bert-base-german-dbmdz-cased", + "bert-base-german-dbmdz-uncased", + "cl-tohoku/bert-base-japanese", + "cl-tohoku/bert-base-japanese-whole-word-masking", + "cl-tohoku/bert-base-japanese-char", + "cl-tohoku/bert-base-japanese-char-whole-word-masking", + "TurkuNLP/bert-base-finnish-cased-v1", + "TurkuNLP/bert-base-finnish-uncased-v1", + "wietsedv/bert-base-dutch-cased", + # See all BERT models at https://huggingface.co/models?filter=bert +] + + +def load_tf_weights_in_bert(model, config, tf_checkpoint_path): + """Load tf checkpoints in a pytorch model.""" + try: + import re + + import numpy as np + import tensorflow as tf + except ImportError: + logger.error( + "Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions." + ) + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + for name, array in zip(names, arrays): + name = name.split("/") + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any( + n in ["adam_v", "adam_m", "AdamWeightDecayOptimizer", "AdamWeightDecayOptimizer_1", "global_step"] + for n in name + ): + logger.info("Skipping {}".format("/".join(name))) + continue + pointer = model + for m_name in name: + if re.fullmatch(r"[A-Za-z]+_\d+", m_name): + scope_names = re.split(r"_(\d+)", m_name) + else: + scope_names = [m_name] + if scope_names[0] == "kernel" or scope_names[0] == "gamma": + pointer = getattr(pointer, "weight") + elif scope_names[0] == "output_bias" or scope_names[0] == "beta": + pointer = getattr(pointer, "bias") + elif scope_names[0] == "output_weights": + pointer = getattr(pointer, "weight") + elif scope_names[0] == "squad": + pointer = getattr(pointer, "classifier") + else: + try: + pointer = getattr(pointer, scope_names[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(scope_names) >= 2: + num = int(scope_names[1]) + pointer = pointer[num] + if m_name[-11:] == "_embeddings": + pointer = getattr(pointer, "weight") + elif m_name == "kernel": + array = np.transpose(array) + try: + assert ( + pointer.shape == array.shape + ), f"Pointer shape {pointer.shape} and array shape {array.shape} mismatched" + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info("Initialize PyTorch weight {}".format(name)) + pointer.data = torch.from_numpy(array) + return model + + +class BertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))) + self.position_embedding_type = getattr(config, "position_embedding_type", "absolute") + + self.config = config + + def forward( + self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0 + ): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length] + + if token_type_ids is None: + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == "absolute": + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertSelfAttention(nn.Module): + def __init__(self, config, is_cross_attention): + super().__init__() + self.config = config + if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"): + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads) + ) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + if is_cross_attention: + self.key = nn.Linear(config.encoder_width, self.all_head_size) + self.value = nn.Linear(config.encoder_width, self.all_head_size) + else: + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = getattr(config, "position_embedding_type", "absolute") + if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size) + self.save_attention = False + + def save_attn_gradients(self, attn_gradients): + self.attn_gradients = attn_gradients + + def get_attn_gradients(self): + return self.attn_gradients + + def save_attention_map(self, attention_map): + self.attention_map = attention_map + + def get_attention_map(self): + return self.attention_map + + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + + if is_cross_attention: + key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + + if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == "relative_key": + relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == "relative_key_query": + relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + if is_cross_attention and self.save_attention: + self.save_attention_map(attention_probs) + attention_probs.register_hook(self.save_attn_gradients) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs_dropped = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs_dropped = attention_probs_dropped * head_mask + + context_layer = torch.matmul(attention_probs_dropped, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + outputs = outputs + (past_key_value,) + return outputs + + +class BertSelfOutput(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertAttention(nn.Module): + def __init__(self, config, is_cross_attention=False): + super().__init__() + self.self = BertSelfAttention(config, is_cross_attention) + self.output = BertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads + ) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len(heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + return outputs + + +class BertIntermediate(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertLayer(nn.Module): + def __init__(self, config, layer_num): + super().__init__() + self.config = config + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = BertAttention(config) + + self.has_cross_attention = (layer_num >= config.fusion_layer) + if self.has_cross_attention: + self.layer_num = layer_num + self.crossattention = BertAttention(config, is_cross_attention=True) + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + + if self.has_cross_attention: + assert encoder_hidden_states is not None, "encoder_hidden_states must be given for cross-attention layers" + + if type(encoder_hidden_states) == list: + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states[(self.layer_num-self.config.fusion_layer)%len(encoder_hidden_states)], + encoder_attention_mask[(self.layer_num-self.config.fusion_layer)%len(encoder_hidden_states)], + output_attentions=output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[1:-1] + + else: + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions=output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[1:-1] # add cross attentions if we output attention weights + layer_output = apply_chunking_to_forward( + self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output + ) + outputs = (layer_output,) + outputs + + outputs = outputs + (present_key_value,) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class BertEncoder(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList([BertLayer(config,i) for i in range(config.num_hidden_layers)]) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + mode='multi_modal', + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + + + if mode=='text': + start_layer = 0 + output_layer = self.config.fusion_layer + + elif mode=='fusion': + start_layer = self.config.fusion_layer + output_layer = self.config.num_hidden_layers + + elif mode=='multi_modal': + start_layer = 0 + output_layer = self.config.num_hidden_layers + + for i in range(start_layer, output_layer): + layer_module = self.layer[i] + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[i] if past_key_values is not None else None + + if getattr(self.config, "gradient_checkpointing", False) and self.training: + + if use_cache: + logger.warn( + "`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting " + "`use_cache=False`..." + ) + use_cache = False + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs, past_key_value, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1],) + if output_attentions: + all_self_attentions = all_self_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] + if v is not None + ) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class BertPooler(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertPredictionHeadTransform(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class BertLMPredictionHead(nn.Module): + def __init__(self, config): + super().__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +class BertOnlyMLMHead(nn.Module): + def __init__(self, config): + super().__init__() + self.predictions = BertLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class BertOnlyNSPHead(nn.Module): + def __init__(self, config): + super().__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class BertPreTrainingHeads(nn.Module): + def __init__(self, config): + super().__init__() + self.predictions = BertLMPredictionHead(config) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +class BertPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = BertConfig + load_tf_weights = load_tf_weights_in_bert + base_model_prefix = "bert" + _keys_to_ignore_on_load_missing = [r"position_ids"] + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +@dataclass +class BertForPreTrainingOutput(ModelOutput): + """ + Output type of :class:`~transformers.BertForPreTraining`. + Args: + loss (`optional`, returned when ``labels`` is provided, ``torch.FloatTensor`` of shape :obj:`(1,)`): + Total loss as the sum of the masked language modeling loss and the next sequence prediction + (classification) loss. + prediction_logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + seq_relationship_logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, 2)`): + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation + before SoftMax). + hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``): + Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) + of shape :obj:`(batch_size, sequence_length, hidden_size)`. + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``): + Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, + sequence_length, sequence_length)`. + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + loss: Optional[torch.FloatTensor] = None + prediction_logits: torch.FloatTensor = None + seq_relationship_logits: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +BERT_START_DOCSTRING = r""" + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + Parameters: + config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. +""" + +BERT_INPUTS_DOCSTRING = r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`): + Indices of input sequence tokens in the vocabulary. + Indices can be obtained using :class:`~transformers.BertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for + details. + `What are input IDs? <../glossary.html#input-ids>`__ + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + `What are attention masks? <../glossary.html#attention-mask>`__ + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + `What are token type IDs? <../glossary.html#token-type-ids>`_ + position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, + config.max_position_embeddings - 1]``. + `What are position IDs? <../glossary.html#position-ids>`_ + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert :obj:`input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned + tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for + more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare Bert Model transformer outputting raw hidden-states without any specific head on top.", + BERT_START_DOCSTRING, +) +class BertModel(BertPreTrainedModel): + """ + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in `Attention is + all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an + input to the forward pass. + """ + + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + self.config = config + + self.embeddings = BertEmbeddings(config) + + self.encoder = BertEncoder(config) + + self.pooler = BertPooler(config) if add_pooling_layer else None + + self.init_weights() + + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + tokenizer_class=_TOKENIZER_FOR_DOC, + checkpoint="bert-base-uncased", + output_type=BaseModelOutputWithPoolingAndCrossAttentions, + config_class=_CONFIG_FOR_DOC, + ) + + + def get_extended_attention_mask(self, attention_mask: Tensor, input_shape: Tuple[int], device: device, is_decoder: bool) -> Tensor: + """ + Makes broadcastable attention and causal masks so that future and masked tokens are ignored. + + Arguments: + attention_mask (:obj:`torch.Tensor`): + Mask with ones indicating tokens to attend to, zeros for tokens to ignore. + input_shape (:obj:`Tuple[int]`): + The shape of the input to the model. + device: (:obj:`torch.device`): + The device of the input to the model. + + Returns: + :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`. + """ + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] + if is_decoder: + batch_size, seq_length = input_shape + seq_ids = torch.arange(seq_length, device=device) + causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] + # in case past_key_values are used we need to add a prefix ones mask to the causal mask + # causal and attention masks must have same type with pytorch version < 1.3 + causal_mask = causal_mask.to(attention_mask.dtype) + + if causal_mask.shape[1] < attention_mask.shape[1]: + prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1] + causal_mask = torch.cat( + [ + torch.ones( + (batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype + ), + causal_mask, + ], + axis=-1, + ) + + extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] + else: + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + "Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format( + input_shape, attention_mask.shape + ) + ) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + return extended_attention_mask + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=False, + mode='multi_modal', + ): + r""" + encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + batch_size, seq_length = input_shape + device = input_ids.device + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = inputs_embeds.device + elif encoder_embeds is not None: + input_shape = encoder_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = encoder_embeds.device + else: + raise ValueError("You have to specify either input_ids or inputs_embeds or encoder_embeds") + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length)), device=device) + if token_type_ids is None: + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, + device, is_decoder) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if encoder_hidden_states is not None: + if type(encoder_hidden_states) == list: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size() + else: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) + + if type(encoder_attention_mask) == list: + encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask] + elif encoder_attention_mask is None: + encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) + else: + encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) + + if encoder_embeds is None: + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + ) + else: + embedding_output = encoder_embeds + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + mode=mode, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + + +@add_start_docstrings( + """ + Bert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next + sentence prediction (classification)` head. + """, + BERT_START_DOCSTRING, +) +class BertForPreTraining(BertPreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config) + self.cls = BertPreTrainingHeads(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @replace_return_docstrings(output_type=BertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + next_sentence_label=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape ``(batch_size, sequence_length)``, `optional`): + Labels for computing the masked language modeling loss. Indices should be in ``[-100, 0, ..., + config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are ignored + (masked), the loss is only computed for the tokens with labels in ``[0, ..., config.vocab_size]`` + next_sentence_label (``torch.LongTensor`` of shape ``(batch_size,)``, `optional`): + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair + (see :obj:`input_ids` docstring) Indices should be in ``[0, 1]``: + - 0 indicates sequence B is a continuation of sequence A, + - 1 indicates sequence B is a random sequence. + kwargs (:obj:`Dict[str, any]`, optional, defaults to `{}`): + Used to hide legacy arguments that have been deprecated. + Returns: + Example:: + >>> from transformers import BertTokenizer, BertForPreTraining + >>> import torch + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + >>> model = BertForPreTraining.from_pretrained('bert-base-uncased') + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + >>> prediction_logits = outputs.prediction_logits + >>> seq_relationship_logits = outputs.seq_relationship_logits + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output, pooled_output = outputs[:2] + prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output) + + total_loss = None + if labels is not None and next_sentence_label is not None: + loss_fct = CrossEntropyLoss() + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) + next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + total_loss = masked_lm_loss + next_sentence_loss + + if not return_dict: + output = (prediction_scores, seq_relationship_score) + outputs[2:] + return ((total_loss,) + output) if total_loss is not None else output + + return BertForPreTrainingOutput( + loss=total_loss, + prediction_logits=prediction_scores, + seq_relationship_logits=seq_relationship_score, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """Bert Model with a `language modeling` head on top for CLM fine-tuning. """, BERT_START_DOCSTRING +) +class BertLMHeadModel(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r"pooler"] + _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"] + + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config, add_pooling_layer=False) + self.cls = BertOnlyMLMHead(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=True, + reduction='mean', + mode='multi_modal', + soft_labels=None, + alpha=0, + return_logits=False, + ): + r""" + encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in + ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are + ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]`` + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + Returns: + Example:: + >>> from transformers import BertTokenizer, BertLMHeadModel, BertConfig + >>> import torch + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-cased') + >>> config = BertConfig.from_pretrained("bert-base-cased") + >>> model = BertLMHeadModel.from_pretrained('bert-base-cased', config=config) + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + >>> prediction_logits = outputs.logits + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if labels is not None: + use_cache = False + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + is_decoder=is_decoder, + mode=mode, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + if return_logits: + return prediction_scores[:, :-1, :].contiguous() + + lm_loss = None + if labels is not None: + # we are doing next-token prediction; shift prediction scores and input ids by one + shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous() + labels = labels[:, 1:].contiguous() + loss_fct = CrossEntropyLoss(reduction=reduction) + lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) + lm_loss = lm_loss.view(prediction_scores.size(0),-1).sum(1) + + if soft_labels is not None: + loss_distill = -torch.sum(F.log_softmax(shifted_prediction_scores, dim=1)*soft_labels,dim=-1) + loss_distill = (loss_distill * (labels!=-100)).sum(1) + lm_loss = (1-alpha)*lm_loss + alpha*loss_distill + + if not return_dict: + output = (prediction_scores,) + outputs[2:] + return ((lm_loss,) + output) if lm_loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=lm_loss, + logits=prediction_scores, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + def prepare_inputs_for_generation(self, input_ids, past=None, attention_mask=None, **model_kwargs): + input_shape = input_ids.shape + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_shape) + + # cut decoder_input_ids if past is used + if past is not None: + input_ids = input_ids[:, -1:] + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "past_key_values": past, + "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None), + "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None), + "is_decoder": True, + } + + def _reorder_cache(self, past, beam_idx): + reordered_past = () + for layer_past in past: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past + + +@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, BERT_START_DOCSTRING) +class BertForMaskedLM(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r"pooler"] + _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"] + + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config, add_pooling_layer=False) + self.cls = BertOnlyMLMHead(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + tokenizer_class=_TOKENIZER_FOR_DOC, + checkpoint="bert-base-uncased", + output_type=MaskedLMOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=False, + mode='multi_modal', + soft_labels=None, + alpha=0, + return_logits=False, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the masked language modeling loss. Indices should be in ``[-100, 0, ..., + config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are ignored + (masked), the loss is only computed for the tokens with labels in ``[0, ..., config.vocab_size]`` + """ + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_embeds=encoder_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + is_decoder=is_decoder, + mode=mode, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + if return_logits: + return prediction_scores + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() # -100 index = padding token + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) + + if soft_labels is not None: + loss_distill = -torch.sum(F.log_softmax(prediction_scores, dim=1)*soft_labels,dim=-1) + loss_distill = loss_distill[labels!=-100].mean() + masked_lm_loss = (1-alpha)*masked_lm_loss + alpha*loss_distill + + if not return_dict: + output = (prediction_scores,) + outputs[2:] + return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output + + return MaskedLMOutput( + loss=masked_lm_loss, + logits=prediction_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **model_kwargs): + input_shape = input_ids.shape + effective_batch_size = input_shape[0] + + # add a dummy token + assert self.config.pad_token_id is not None, "The PAD token should be defined for generation" + attention_mask = torch.cat([attention_mask, attention_mask.new_zeros((attention_mask.shape[0], 1))], dim=-1) + dummy_token = torch.full( + (effective_batch_size, 1), self.config.pad_token_id, dtype=torch.long, device=input_ids.device + ) + input_ids = torch.cat([input_ids, dummy_token], dim=1) + + return {"input_ids": input_ids, "attention_mask": attention_mask} + + +@add_start_docstrings( + """Bert Model with a `next sentence prediction (classification)` head on top. """, + BERT_START_DOCSTRING, +) +class BertForNextSentencePrediction(BertPreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config) + self.cls = BertOnlyNSPHead(config) + + self.init_weights() + + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @replace_return_docstrings(output_type=NextSentencePredictorOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair + (see ``input_ids`` docstring). Indices should be in ``[0, 1]``: + - 0 indicates sequence B is a continuation of sequence A, + - 1 indicates sequence B is a random sequence. + Returns: + Example:: + >>> from transformers import BertTokenizer, BertForNextSentencePrediction + >>> import torch + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + >>> model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') + >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." + >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." + >>> encoding = tokenizer(prompt, next_sentence, return_tensors='pt') + >>> outputs = model(**encoding, labels=torch.LongTensor([1])) + >>> logits = outputs.logits + >>> assert logits[0, 0] < logits[0, 1] # next sentence was random + """ + + if "next_sentence_label" in kwargs: + warnings.warn( + "The `next_sentence_label` argument is deprecated and will be removed in a future version, use `labels` instead.", + FutureWarning, + ) + labels = kwargs.pop("next_sentence_label") + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = outputs[1] + + seq_relationship_scores = self.cls(pooled_output) + + next_sentence_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + next_sentence_loss = loss_fct(seq_relationship_scores.view(-1, 2), labels.view(-1)) + + if not return_dict: + output = (seq_relationship_scores,) + outputs[2:] + return ((next_sentence_loss,) + output) if next_sentence_loss is not None else output + + return NextSentencePredictorOutput( + loss=next_sentence_loss, + logits=seq_relationship_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled + output) e.g. for GLUE tasks. + """, + BERT_START_DOCSTRING, +) +class BertForSequenceClassification(BertPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + tokenizer_class=_TOKENIZER_FOR_DOC, + checkpoint="bert-base-uncased", + output_type=SequenceClassifierOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + loss = None + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + Bert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a + softmax) e.g. for RocStories/SWAG tasks. + """, + BERT_START_DOCSTRING, +) +class BertForMultipleChoice(BertPreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, 1) + + self.init_weights() + + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) + @add_code_sample_docstrings( + tokenizer_class=_TOKENIZER_FOR_DOC, + checkpoint="bert-base-uncased", + output_type=MultipleChoiceModelOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the multiple choice classification loss. Indices should be in ``[0, ..., + num_choices-1]`` where :obj:`num_choices` is the size of the second dimension of the input tensors. (See + :obj:`input_ids` above) + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] + + input_ids = input_ids.view(-1, input_ids.size(-1)) if input_ids is not None else None + attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None + token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None + position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None + inputs_embeds = ( + inputs_embeds.view(-1, inputs_embeds.size(-2), inputs_embeds.size(-1)) + if inputs_embeds is not None + else None + ) + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = logits.view(-1, num_choices) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels) + + if not return_dict: + output = (reshaped_logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return MultipleChoiceModelOutput( + loss=loss, + logits=reshaped_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + Bert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for + Named-Entity-Recognition (NER) tasks. + """, + BERT_START_DOCSTRING, +) +class BertForTokenClassification(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r"pooler"] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.bert = BertModel(config, add_pooling_layer=False) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + tokenizer_class=_TOKENIZER_FOR_DOC, + checkpoint="bert-base-uncased", + output_type=TokenClassifierOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the token classification loss. Indices should be in ``[0, ..., config.num_labels - + 1]``. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels) + active_labels = torch.where( + active_loss, labels.view(-1), torch.tensor(loss_fct.ignore_index).type_as(labels) + ) + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + Bert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear + layers on top of the hidden-states output to compute `span start logits` and `span end logits`). + """, + BERT_START_DOCSTRING, +) +class BertForQuestionAnswering(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r"pooler"] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.bert = BertModel(config, add_pooling_layer=False) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + tokenizer_class=_TOKENIZER_FOR_DOC, + checkpoint="bert-base-uncased", + output_type=QuestionAnsweringModelOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + start_positions=None, + end_positions=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + start_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (:obj:`sequence_length`). Position outside of the + sequence are not taken into account for computing the loss. + end_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (:obj:`sequence_length`). Position outside of the + sequence are not taken into account for computing the loss. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + total_loss = None + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + + if not return_dict: + output = (start_logits, end_logits) + outputs[2:] + return ((total_loss,) + output) if total_loss is not None else output + + return QuestionAnsweringModelOutput( + loss=total_loss, + start_logits=start_logits, + end_logits=end_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/optim/__init__.py b/optim/__init__.py new file mode 100644 index 0000000..33e4907 --- /dev/null +++ b/optim/__init__.py @@ -0,0 +1,13 @@ +from .adamp import AdamP +from .adamw import AdamW +from .adafactor import Adafactor +from .adahessian import Adahessian +from .lookahead import Lookahead +from .nadam import Nadam +from .novograd import NovoGrad +from .nvnovograd import NvNovoGrad +from .radam import RAdam +from .rmsprop_tf import RMSpropTF +from .sgdp import SGDP + +from .optim_factory import create_optimizer \ No newline at end of file diff --git a/optim/__pycache__/__init__.cpython-36.pyc b/optim/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..a458f82 Binary files /dev/null and b/optim/__pycache__/__init__.cpython-36.pyc differ diff --git a/optim/__pycache__/__init__.cpython-38.pyc b/optim/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..5358f8a Binary files /dev/null and b/optim/__pycache__/__init__.cpython-38.pyc differ diff --git a/optim/__pycache__/adafactor.cpython-36.pyc b/optim/__pycache__/adafactor.cpython-36.pyc new file mode 100644 index 0000000..08961ba Binary files /dev/null and b/optim/__pycache__/adafactor.cpython-36.pyc differ diff --git a/optim/__pycache__/adafactor.cpython-38.pyc b/optim/__pycache__/adafactor.cpython-38.pyc new file mode 100644 index 0000000..11cf014 Binary files /dev/null and b/optim/__pycache__/adafactor.cpython-38.pyc differ diff --git a/optim/__pycache__/adahessian.cpython-36.pyc b/optim/__pycache__/adahessian.cpython-36.pyc new file mode 100644 index 0000000..ffc3304 Binary files /dev/null and b/optim/__pycache__/adahessian.cpython-36.pyc differ diff --git a/optim/__pycache__/adahessian.cpython-38.pyc b/optim/__pycache__/adahessian.cpython-38.pyc new file mode 100644 index 0000000..45372f3 Binary files /dev/null and b/optim/__pycache__/adahessian.cpython-38.pyc differ diff --git a/optim/__pycache__/adamp.cpython-36.pyc b/optim/__pycache__/adamp.cpython-36.pyc new file mode 100644 index 0000000..a29f0bc Binary files /dev/null and b/optim/__pycache__/adamp.cpython-36.pyc differ diff --git a/optim/__pycache__/adamp.cpython-38.pyc b/optim/__pycache__/adamp.cpython-38.pyc new file mode 100644 index 0000000..66dff0d Binary files /dev/null and b/optim/__pycache__/adamp.cpython-38.pyc differ diff --git a/optim/__pycache__/adamw.cpython-36.pyc b/optim/__pycache__/adamw.cpython-36.pyc new file mode 100644 index 0000000..ff91214 Binary files /dev/null and b/optim/__pycache__/adamw.cpython-36.pyc differ diff --git a/optim/__pycache__/adamw.cpython-38.pyc b/optim/__pycache__/adamw.cpython-38.pyc new file mode 100644 index 0000000..2ce64e9 Binary files /dev/null and b/optim/__pycache__/adamw.cpython-38.pyc differ diff --git a/optim/__pycache__/lookahead.cpython-36.pyc b/optim/__pycache__/lookahead.cpython-36.pyc new file mode 100644 index 0000000..e34f801 Binary files /dev/null and b/optim/__pycache__/lookahead.cpython-36.pyc differ diff --git a/optim/__pycache__/lookahead.cpython-38.pyc b/optim/__pycache__/lookahead.cpython-38.pyc new file mode 100644 index 0000000..0171639 Binary files /dev/null and b/optim/__pycache__/lookahead.cpython-38.pyc differ diff --git a/optim/__pycache__/nadam.cpython-36.pyc b/optim/__pycache__/nadam.cpython-36.pyc new file mode 100644 index 0000000..782521e Binary files /dev/null and b/optim/__pycache__/nadam.cpython-36.pyc differ diff --git a/optim/__pycache__/nadam.cpython-38.pyc b/optim/__pycache__/nadam.cpython-38.pyc new file mode 100644 index 0000000..2c88887 Binary files /dev/null and b/optim/__pycache__/nadam.cpython-38.pyc differ diff --git a/optim/__pycache__/novograd.cpython-36.pyc b/optim/__pycache__/novograd.cpython-36.pyc new file mode 100644 index 0000000..d49f1a4 Binary files /dev/null and b/optim/__pycache__/novograd.cpython-36.pyc differ diff --git a/optim/__pycache__/novograd.cpython-38.pyc b/optim/__pycache__/novograd.cpython-38.pyc new file mode 100644 index 0000000..f8ed72d Binary files /dev/null and b/optim/__pycache__/novograd.cpython-38.pyc differ diff --git a/optim/__pycache__/nvnovograd.cpython-36.pyc b/optim/__pycache__/nvnovograd.cpython-36.pyc new file mode 100644 index 0000000..5896e21 Binary files /dev/null and b/optim/__pycache__/nvnovograd.cpython-36.pyc differ diff --git a/optim/__pycache__/nvnovograd.cpython-38.pyc b/optim/__pycache__/nvnovograd.cpython-38.pyc new file mode 100644 index 0000000..e52dd8b Binary files /dev/null and b/optim/__pycache__/nvnovograd.cpython-38.pyc differ diff --git a/optim/__pycache__/optim_factory.cpython-36.pyc b/optim/__pycache__/optim_factory.cpython-36.pyc new file mode 100644 index 0000000..dda9f07 Binary files /dev/null and b/optim/__pycache__/optim_factory.cpython-36.pyc differ diff --git a/optim/__pycache__/optim_factory.cpython-38.pyc b/optim/__pycache__/optim_factory.cpython-38.pyc new file mode 100644 index 0000000..b7cbb9a Binary files /dev/null and b/optim/__pycache__/optim_factory.cpython-38.pyc differ diff --git a/optim/__pycache__/radam.cpython-36.pyc b/optim/__pycache__/radam.cpython-36.pyc new file mode 100644 index 0000000..365ecc3 Binary files /dev/null and b/optim/__pycache__/radam.cpython-36.pyc differ diff --git a/optim/__pycache__/radam.cpython-38.pyc b/optim/__pycache__/radam.cpython-38.pyc new file mode 100644 index 0000000..7353ec2 Binary files /dev/null and b/optim/__pycache__/radam.cpython-38.pyc differ diff --git a/optim/__pycache__/rmsprop_tf.cpython-36.pyc b/optim/__pycache__/rmsprop_tf.cpython-36.pyc new file mode 100644 index 0000000..d3c8689 Binary files /dev/null and b/optim/__pycache__/rmsprop_tf.cpython-36.pyc differ diff --git a/optim/__pycache__/rmsprop_tf.cpython-38.pyc b/optim/__pycache__/rmsprop_tf.cpython-38.pyc new file mode 100644 index 0000000..f136466 Binary files /dev/null and b/optim/__pycache__/rmsprop_tf.cpython-38.pyc differ diff --git a/optim/__pycache__/sgdp.cpython-36.pyc b/optim/__pycache__/sgdp.cpython-36.pyc new file mode 100644 index 0000000..1fbf28c Binary files /dev/null and b/optim/__pycache__/sgdp.cpython-36.pyc differ diff --git a/optim/__pycache__/sgdp.cpython-38.pyc b/optim/__pycache__/sgdp.cpython-38.pyc new file mode 100644 index 0000000..b409b4e Binary files /dev/null and b/optim/__pycache__/sgdp.cpython-38.pyc differ diff --git a/optim/adafactor.py b/optim/adafactor.py new file mode 100644 index 0000000..088ce3a --- /dev/null +++ b/optim/adafactor.py @@ -0,0 +1,174 @@ +""" Adafactor Optimizer + +Lifted from https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py + +Original header/copyright below. + +""" +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import torch +import math + + +class Adafactor(torch.optim.Optimizer): + """Implements Adafactor algorithm. + This implementation is based on: `Adafactor: Adaptive Learning Rates with Sublinear Memory Cost` + (see https://arxiv.org/abs/1804.04235) + + Note that this optimizer internally adjusts the learning rate depending on the + *scale_parameter*, *relative_step* and *warmup_init* options. + + To use a manual (external) learning rate schedule you should set `scale_parameter=False` and + `relative_step=False`. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining parameter groups + lr (float, optional): external learning rate (default: None) + eps (tuple[float, float]): regularization constants for square gradient + and parameter scale respectively (default: (1e-30, 1e-3)) + clip_threshold (float): threshold of root mean square of final gradient update (default: 1.0) + decay_rate (float): coefficient used to compute running averages of square gradient (default: -0.8) + beta1 (float): coefficient used for computing running averages of gradient (default: None) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + scale_parameter (bool): if True, learning rate is scaled by root mean square of parameter (default: True) + relative_step (bool): if True, time-dependent learning rate is computed + instead of external learning rate (default: True) + warmup_init (bool): time-dependent learning rate computation depends on + whether warm-up initialization is being used (default: False) + """ + + def __init__(self, params, lr=None, eps=1e-30, eps_scale=1e-3, clip_threshold=1.0, + decay_rate=-0.8, betas=None, weight_decay=0.0, scale_parameter=True, warmup_init=False): + relative_step = lr is None + if warmup_init and not relative_step: + raise ValueError('warmup_init requires relative_step=True') + + beta1 = None if betas is None else betas[0] # make it compat with standard betas arg + defaults = dict(lr=lr, eps=eps, eps_scale=eps_scale, clip_threshold=clip_threshold, decay_rate=decay_rate, + beta1=beta1, weight_decay=weight_decay, scale_parameter=scale_parameter, + relative_step=relative_step, warmup_init=warmup_init) + super(Adafactor, self).__init__(params, defaults) + + @staticmethod + def _get_lr(param_group, param_state): + if param_group['relative_step']: + min_step = 1e-6 * param_state['step'] if param_group['warmup_init'] else 1e-2 + lr_t = min(min_step, 1.0 / math.sqrt(param_state['step'])) + param_scale = 1.0 + if param_group['scale_parameter']: + param_scale = max(param_group['eps_scale'], param_state['RMS']) + param_group['lr'] = lr_t * param_scale + return param_group['lr'] + + @staticmethod + def _get_options(param_group, param_shape): + factored = len(param_shape) >= 2 + use_first_moment = param_group['beta1'] is not None + return factored, use_first_moment + + @staticmethod + def _rms(tensor): + return tensor.norm(2) / (tensor.numel() ** 0.5) + + def _approx_sq_grad(self, exp_avg_sq_row, exp_avg_sq_col): + r_factor = (exp_avg_sq_row / exp_avg_sq_row.mean(dim=-1, keepdim=True)).rsqrt_().unsqueeze(-1) + c_factor = exp_avg_sq_col.unsqueeze(-2).rsqrt() + return torch.mul(r_factor, c_factor) + + def step(self, closure=None): + """Performs a single optimization step. + Arguments: + closure (callable, optional): A closure that reevaluates the model and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.dtype in {torch.float16, torch.bfloat16}: + grad = grad.float() + if grad.is_sparse: + raise RuntimeError('Adafactor does not support sparse gradients.') + + state = self.state[p] + grad_shape = grad.shape + + factored, use_first_moment = self._get_options(group, grad_shape) + # State Initialization + if len(state) == 0: + state['step'] = 0 + + if use_first_moment: + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(grad) + if factored: + state['exp_avg_sq_row'] = torch.zeros(grad_shape[:-1]).to(grad) + state['exp_avg_sq_col'] = torch.zeros(grad_shape[:-2] + grad_shape[-1:]).to(grad) + else: + state['exp_avg_sq'] = torch.zeros_like(grad) + + state['RMS'] = 0 + else: + if use_first_moment: + state['exp_avg'] = state['exp_avg'].to(grad) + if factored: + state['exp_avg_sq_row'] = state['exp_avg_sq_row'].to(grad) + state['exp_avg_sq_col'] = state['exp_avg_sq_col'].to(grad) + else: + state['exp_avg_sq'] = state['exp_avg_sq'].to(grad) + + p_data_fp32 = p.data + if p.data.dtype in {torch.float16, torch.bfloat16}: + p_data_fp32 = p_data_fp32.float() + + state['step'] += 1 + state['RMS'] = self._rms(p_data_fp32) + lr_t = self._get_lr(group, state) + + beta2t = 1.0 - math.pow(state['step'], group['decay_rate']) + update = grad ** 2 + group['eps'] + if factored: + exp_avg_sq_row = state['exp_avg_sq_row'] + exp_avg_sq_col = state['exp_avg_sq_col'] + + exp_avg_sq_row.mul_(beta2t).add_(1.0 - beta2t, update.mean(dim=-1)) + exp_avg_sq_col.mul_(beta2t).add_(1.0 - beta2t, update.mean(dim=-2)) + #exp_avg_sq_row.mul_(beta2t).add_(update.mean(dim=-1), alpha=1.0 - beta2t) # pytorch 1.6+ + #exp_avg_sq_col.mul_(beta2t).add_(update.mean(dim=-2), alpha=1.0 - beta2t) + + # Approximation of exponential moving average of square of gradient + update = self._approx_sq_grad(exp_avg_sq_row, exp_avg_sq_col) + update.mul_(grad) + else: + exp_avg_sq = state['exp_avg_sq'] + + exp_avg_sq.mul_(beta2t).add_(1.0 - beta2t, update) + #exp_avg_sq.mul_(beta2t).add_(update, alpha=1.0 - beta2t) # pytorch 1.6+ + update = exp_avg_sq.rsqrt().mul_(grad) + + update.div_((self._rms(update) / group['clip_threshold']).clamp_(min=1.0)) + update.mul_(lr_t) + + if use_first_moment: + exp_avg = state['exp_avg'] + exp_avg.mul_(group["beta1"]).add_(1 - group["beta1"], update) + #exp_avg.mul_(group['beta1']).add_(update, alpha=1 - group['beta1']) # pytorch 1.6+ + update = exp_avg + + if group['weight_decay'] != 0: + p_data_fp32.add_(-group["weight_decay"] * lr_t, p_data_fp32) + #p_data_fp32.add_(p_data_fp32, alpha=-group['weight_decay'] * lr_t) # pytorch 1.6+ + + p_data_fp32.add_(-update) + + if p.data.dtype in {torch.float16, torch.bfloat16}: + p.data.copy_(p_data_fp32) + + return loss \ No newline at end of file diff --git a/optim/adahessian.py b/optim/adahessian.py new file mode 100644 index 0000000..985c67c --- /dev/null +++ b/optim/adahessian.py @@ -0,0 +1,156 @@ +""" AdaHessian Optimizer + +Lifted from https://github.com/davda54/ada-hessian/blob/master/ada_hessian.py +Originally licensed MIT, Copyright 2020, David Samuel +""" +import torch + + +class Adahessian(torch.optim.Optimizer): + """ + Implements the AdaHessian algorithm from "ADAHESSIAN: An Adaptive Second OrderOptimizer for Machine Learning" + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining parameter groups + lr (float, optional): learning rate (default: 0.1) + betas ((float, float), optional): coefficients used for computing running averages of gradient and the + squared hessian trace (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0.0) + hessian_power (float, optional): exponent of the hessian trace (default: 1.0) + update_each (int, optional): compute the hessian trace approximation only after *this* number of steps + (to save time) (default: 1) + n_samples (int, optional): how many times to sample `z` for the approximation of the hessian trace (default: 1) + """ + + def __init__(self, params, lr=0.1, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.0, + hessian_power=1.0, update_each=1, n_samples=1, avg_conv_kernel=False): + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + if not 0.0 <= hessian_power <= 1.0: + raise ValueError(f"Invalid Hessian power value: {hessian_power}") + + self.n_samples = n_samples + self.update_each = update_each + self.avg_conv_kernel = avg_conv_kernel + + # use a separate generator that deterministically generates the same `z`s across all GPUs in case of distributed training + self.seed = 2147483647 + self.generator = torch.Generator().manual_seed(self.seed) + + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, hessian_power=hessian_power) + super(Adahessian, self).__init__(params, defaults) + + for p in self.get_params(): + p.hess = 0.0 + self.state[p]["hessian step"] = 0 + + @property + def is_second_order(self): + return True + + def get_params(self): + """ + Gets all parameters in all param_groups with gradients + """ + + return (p for group in self.param_groups for p in group['params'] if p.requires_grad) + + def zero_hessian(self): + """ + Zeros out the accumalated hessian traces. + """ + + for p in self.get_params(): + if not isinstance(p.hess, float) and self.state[p]["hessian step"] % self.update_each == 0: + p.hess.zero_() + + @torch.no_grad() + def set_hessian(self): + """ + Computes the Hutchinson approximation of the hessian trace and accumulates it for each trainable parameter. + """ + + params = [] + for p in filter(lambda p: p.grad is not None, self.get_params()): + if self.state[p]["hessian step"] % self.update_each == 0: # compute the trace only each `update_each` step + params.append(p) + self.state[p]["hessian step"] += 1 + + if len(params) == 0: + return + + if self.generator.device != params[0].device: # hackish way of casting the generator to the right device + self.generator = torch.Generator(params[0].device).manual_seed(self.seed) + + grads = [p.grad for p in params] + + for i in range(self.n_samples): + # Rademacher distribution {-1.0, 1.0} + zs = [torch.randint(0, 2, p.size(), generator=self.generator, device=p.device) * 2.0 - 1.0 for p in params] + h_zs = torch.autograd.grad( + grads, params, grad_outputs=zs, only_inputs=True, retain_graph=i < self.n_samples - 1) + for h_z, z, p in zip(h_zs, zs, params): + p.hess += h_z * z / self.n_samples # approximate the expected values of z*(H@z) + + @torch.no_grad() + def step(self, closure=None): + """ + Performs a single optimization step. + Arguments: + closure (callable, optional) -- a closure that reevaluates the model and returns the loss (default: None) + """ + + loss = None + if closure is not None: + loss = closure() + + self.zero_hessian() + self.set_hessian() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None or p.hess is None: + continue + + if self.avg_conv_kernel and p.dim() == 4: + p.hess = torch.abs(p.hess).mean(dim=[2, 3], keepdim=True).expand_as(p.hess).clone() + + # Perform correct stepweight decay as in AdamW + p.mul_(1 - group['lr'] * group['weight_decay']) + + state = self.state[p] + + # State initialization + if len(state) == 1: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p) + # Exponential moving average of Hessian diagonal square values + state['exp_hessian_diag_sq'] = torch.zeros_like(p) + + exp_avg, exp_hessian_diag_sq = state['exp_avg'], state['exp_hessian_diag_sq'] + beta1, beta2 = group['betas'] + state['step'] += 1 + + # Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(p.grad, alpha=1 - beta1) + exp_hessian_diag_sq.mul_(beta2).addcmul_(p.hess, p.hess, value=1 - beta2) + + bias_correction1 = 1 - beta1 ** state['step'] + bias_correction2 = 1 - beta2 ** state['step'] + + k = group['hessian_power'] + denom = (exp_hessian_diag_sq / bias_correction2).pow_(k / 2).add_(group['eps']) + + # make update + step_size = group['lr'] / bias_correction1 + p.addcdiv_(exp_avg, denom, value=-step_size) + + return loss diff --git a/optim/adamp.py b/optim/adamp.py new file mode 100644 index 0000000..468c3e8 --- /dev/null +++ b/optim/adamp.py @@ -0,0 +1,107 @@ +""" +AdamP Optimizer Implementation copied from https://github.com/clovaai/AdamP/blob/master/adamp/adamp.py + +Paper: `Slowing Down the Weight Norm Increase in Momentum-based Optimizers` - https://arxiv.org/abs/2006.08217 +Code: https://github.com/clovaai/AdamP + +Copyright (c) 2020-present NAVER Corp. +MIT license +""" + +import torch +import torch.nn as nn +from torch.optim.optimizer import Optimizer, required +import math + +class AdamP(Optimizer): + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, + weight_decay=0, delta=0.1, wd_ratio=0.1, nesterov=False): + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, + delta=delta, wd_ratio=wd_ratio, nesterov=nesterov) + super(AdamP, self).__init__(params, defaults) + + def _channel_view(self, x): + return x.view(x.size(0), -1) + + def _layer_view(self, x): + return x.view(1, -1) + + def _cosine_similarity(self, x, y, eps, view_func): + x = view_func(x) + y = view_func(y) + + x_norm = x.norm(dim=1).add_(eps) + y_norm = y.norm(dim=1).add_(eps) + dot = (x * y).sum(dim=1) + + return dot.abs() / x_norm / y_norm + + def _projection(self, p, grad, perturb, delta, wd_ratio, eps): + wd = 1 + expand_size = [-1] + [1] * (len(p.shape) - 1) + for view_func in [self._channel_view, self._layer_view]: + + cosine_sim = self._cosine_similarity(grad, p.data, eps, view_func) + + if cosine_sim.max() < delta / math.sqrt(view_func(p.data).size(1)): + p_n = p.data / view_func(p.data).norm(dim=1).view(expand_size).add_(eps) + perturb -= p_n * view_func(p_n * perturb).sum(dim=1).view(expand_size) + wd = wd_ratio + + return perturb, wd + + return perturb, wd + + def step(self, closure=None): + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + + grad = p.grad.data + beta1, beta2 = group['betas'] + nesterov = group['nesterov'] + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + state['exp_avg'] = torch.zeros_like(p.data) + state['exp_avg_sq'] = torch.zeros_like(p.data) + + # Adam + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + + state['step'] += 1 + bias_correction1 = 1 - beta1 ** state['step'] + bias_correction2 = 1 - beta2 ** state['step'] + + exp_avg.mul_(beta1).add_(1 - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + + denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps']) + step_size = group['lr'] / bias_correction1 + + if nesterov: + perturb = (beta1 * exp_avg + (1 - beta1) * grad) / denom + else: + perturb = exp_avg / denom + + # Projection + wd_ratio = 1 + if len(p.shape) > 1: + perturb, wd_ratio = self._projection(p, grad, perturb, group['delta'], group['wd_ratio'], group['eps']) + + # Weight decay + if group['weight_decay'] > 0: + p.data.mul_(1 - group['lr'] * group['weight_decay'] * wd_ratio) + + # Step + p.data.add_(-step_size, perturb) + + return loss diff --git a/optim/adamw.py b/optim/adamw.py new file mode 100644 index 0000000..66f9a95 --- /dev/null +++ b/optim/adamw.py @@ -0,0 +1,117 @@ +""" AdamW Optimizer +Impl copied from PyTorch master +""" +import math +import torch +from torch.optim.optimizer import Optimizer + + +class AdamW(Optimizer): + r"""Implements AdamW algorithm. + + The original Adam algorithm was proposed in `Adam: A Method for Stochastic Optimization`_. + The AdamW variant was proposed in `Decoupled Weight Decay Regularization`_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay coefficient (default: 1e-2) + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) + + .. _Adam\: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + """ + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, + weight_decay=1e-2, amsgrad=False): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + defaults = dict(lr=lr, betas=betas, eps=eps, + weight_decay=weight_decay, amsgrad=amsgrad) + super(AdamW, self).__init__(params, defaults) + + def __setstate__(self, state): + super(AdamW, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('amsgrad', False) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + + # Perform stepweight decay + p.data.mul_(1 - group['lr'] * group['weight_decay']) + + # Perform optimization step + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead') + amsgrad = group['amsgrad'] + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + if amsgrad: + # Maintains max of all exp. moving avg. of sq. grad. values + state['max_exp_avg_sq'] = torch.zeros_like(p.data) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + if amsgrad: + max_exp_avg_sq = state['max_exp_avg_sq'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + bias_correction1 = 1 - beta1 ** state['step'] + bias_correction2 = 1 - beta2 ** state['step'] + + # Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(1 - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + if amsgrad: + # Maintains the maximum of all 2nd moment running avg. till now + torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) + # Use the max. for normalizing running avg. of gradient + denom = (max_exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps']) + else: + denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps']) + + step_size = group['lr'] / bias_correction1 + + p.data.addcdiv_(-step_size, exp_avg, denom) + + return loss diff --git a/optim/lookahead.py b/optim/lookahead.py new file mode 100644 index 0000000..6b5b7f3 --- /dev/null +++ b/optim/lookahead.py @@ -0,0 +1,92 @@ +""" Lookahead Optimizer Wrapper. +Implementation modified from: https://github.com/alphadl/lookahead.pytorch +Paper: `Lookahead Optimizer: k steps forward, 1 step back` - https://arxiv.org/abs/1907.08610 + +Hacked together by / Copyright 2020 Ross Wightman +""" +import torch +from torch.optim.optimizer import Optimizer +from collections import defaultdict + + +class Lookahead(Optimizer): + def __init__(self, base_optimizer, alpha=0.5, k=6): + if not 0.0 <= alpha <= 1.0: + raise ValueError(f'Invalid slow update rate: {alpha}') + if not 1 <= k: + raise ValueError(f'Invalid lookahead steps: {k}') + defaults = dict(lookahead_alpha=alpha, lookahead_k=k, lookahead_step=0) + self.base_optimizer = base_optimizer + self.param_groups = self.base_optimizer.param_groups + self.defaults = base_optimizer.defaults + self.defaults.update(defaults) + self.state = defaultdict(dict) + # manually add our defaults to the param groups + for name, default in defaults.items(): + for group in self.param_groups: + group.setdefault(name, default) + + def update_slow(self, group): + for fast_p in group["params"]: + if fast_p.grad is None: + continue + param_state = self.state[fast_p] + if 'slow_buffer' not in param_state: + param_state['slow_buffer'] = torch.empty_like(fast_p.data) + param_state['slow_buffer'].copy_(fast_p.data) + slow = param_state['slow_buffer'] + slow.add_(group['lookahead_alpha'], fast_p.data - slow) + fast_p.data.copy_(slow) + + def sync_lookahead(self): + for group in self.param_groups: + self.update_slow(group) + + def step(self, closure=None): + #assert id(self.param_groups) == id(self.base_optimizer.param_groups) + loss = self.base_optimizer.step(closure) + for group in self.param_groups: + group['lookahead_step'] += 1 + if group['lookahead_step'] % group['lookahead_k'] == 0: + self.update_slow(group) + return loss + + def state_dict(self): + fast_state_dict = self.base_optimizer.state_dict() + slow_state = { + (id(k) if isinstance(k, torch.Tensor) else k): v + for k, v in self.state.items() + } + fast_state = fast_state_dict['state'] + param_groups = fast_state_dict['param_groups'] + return { + 'state': fast_state, + 'slow_state': slow_state, + 'param_groups': param_groups, + } + + def load_state_dict(self, state_dict): + fast_state_dict = { + 'state': state_dict['state'], + 'param_groups': state_dict['param_groups'], + } + self.base_optimizer.load_state_dict(fast_state_dict) + + # We want to restore the slow state, but share param_groups reference + # with base_optimizer. This is a bit redundant but least code + slow_state_new = False + if 'slow_state' not in state_dict: + print('Loading state_dict from optimizer without Lookahead applied.') + state_dict['slow_state'] = defaultdict(dict) + slow_state_new = True + slow_state_dict = { + 'state': state_dict['slow_state'], + 'param_groups': state_dict['param_groups'], # this is pointless but saves code + } + super(Lookahead, self).load_state_dict(slow_state_dict) + self.param_groups = self.base_optimizer.param_groups # make both ref same container + if slow_state_new: + # reapply defaults to catch missing lookahead specific ones + for name, default in self.defaults.items(): + for group in self.param_groups: + group.setdefault(name, default) diff --git a/optim/nadam.py b/optim/nadam.py new file mode 100644 index 0000000..d994d1b --- /dev/null +++ b/optim/nadam.py @@ -0,0 +1,88 @@ +import torch +from torch.optim import Optimizer + + +class Nadam(Optimizer): + """Implements Nadam algorithm (a variant of Adam based on Nesterov momentum). + + It has been proposed in `Incorporating Nesterov Momentum into Adam`__. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 2e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + schedule_decay (float, optional): momentum schedule decay (default: 4e-3) + + __ http://cs229.stanford.edu/proj2015/054_report.pdf + __ http://www.cs.toronto.edu/~fritz/absps/momentum.pdf + + Originally taken from: https://github.com/pytorch/pytorch/pull/1408 + NOTE: Has potential issues but does work well on some problems. + """ + + def __init__(self, params, lr=2e-3, betas=(0.9, 0.999), eps=1e-8, + weight_decay=0, schedule_decay=4e-3): + defaults = dict(lr=lr, betas=betas, eps=eps, + weight_decay=weight_decay, schedule_decay=schedule_decay) + super(Nadam, self).__init__(params, defaults) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + state['m_schedule'] = 1. + state['exp_avg'] = grad.new().resize_as_(grad).zero_() + state['exp_avg_sq'] = grad.new().resize_as_(grad).zero_() + + # Warming momentum schedule + m_schedule = state['m_schedule'] + schedule_decay = group['schedule_decay'] + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + eps = group['eps'] + state['step'] += 1 + t = state['step'] + + if group['weight_decay'] != 0: + grad = grad.add(group['weight_decay'], p.data) + + momentum_cache_t = beta1 * \ + (1. - 0.5 * (0.96 ** (t * schedule_decay))) + momentum_cache_t_1 = beta1 * \ + (1. - 0.5 * (0.96 ** ((t + 1) * schedule_decay))) + m_schedule_new = m_schedule * momentum_cache_t + m_schedule_next = m_schedule * momentum_cache_t * momentum_cache_t_1 + state['m_schedule'] = m_schedule_new + + # Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(1. - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1. - beta2, grad, grad) + exp_avg_sq_prime = exp_avg_sq / (1. - beta2 ** t) + denom = exp_avg_sq_prime.sqrt_().add_(eps) + + p.data.addcdiv_(-group['lr'] * (1. - momentum_cache_t) / (1. - m_schedule_new), grad, denom) + p.data.addcdiv_(-group['lr'] * momentum_cache_t_1 / (1. - m_schedule_next), exp_avg, denom) + + return loss diff --git a/optim/novograd.py b/optim/novograd.py new file mode 100644 index 0000000..4137c6a --- /dev/null +++ b/optim/novograd.py @@ -0,0 +1,77 @@ +"""NovoGrad Optimizer. +Original impl by Masashi Kimura (Convergence Lab): https://github.com/convergence-lab/novograd +Paper: `Stochastic Gradient Methods with Layer-wise Adaptive Moments for Training of Deep Networks` + - https://arxiv.org/abs/1905.11286 +""" + +import torch +from torch.optim.optimizer import Optimizer +import math + + +class NovoGrad(Optimizer): + def __init__(self, params, grad_averaging=False, lr=0.1, betas=(0.95, 0.98), eps=1e-8, weight_decay=0): + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) + super(NovoGrad, self).__init__(params, defaults) + self._lr = lr + self._beta1 = betas[0] + self._beta2 = betas[1] + self._eps = eps + self._wd = weight_decay + self._grad_averaging = grad_averaging + + self._momentum_initialized = False + + def step(self, closure=None): + loss = None + if closure is not None: + loss = closure() + + if not self._momentum_initialized: + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + state = self.state[p] + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('NovoGrad does not support sparse gradients') + + v = torch.norm(grad)**2 + m = grad/(torch.sqrt(v) + self._eps) + self._wd * p.data + state['step'] = 0 + state['v'] = v + state['m'] = m + state['grad_ema'] = None + self._momentum_initialized = True + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + state = self.state[p] + state['step'] += 1 + + step, v, m = state['step'], state['v'], state['m'] + grad_ema = state['grad_ema'] + + grad = p.grad.data + g2 = torch.norm(grad)**2 + grad_ema = g2 if grad_ema is None else grad_ema * \ + self._beta2 + g2 * (1. - self._beta2) + grad *= 1.0 / (torch.sqrt(grad_ema) + self._eps) + + if self._grad_averaging: + grad *= (1. - self._beta1) + + g2 = torch.norm(grad)**2 + v = self._beta2*v + (1. - self._beta2)*g2 + m = self._beta1*m + (grad / (torch.sqrt(v) + self._eps) + self._wd * p.data) + bias_correction1 = 1 - self._beta1 ** step + bias_correction2 = 1 - self._beta2 ** step + step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 + + state['v'], state['m'] = v, m + state['grad_ema'] = grad_ema + p.data.add_(-step_size, m) + return loss diff --git a/optim/nvnovograd.py b/optim/nvnovograd.py new file mode 100644 index 0000000..323312d --- /dev/null +++ b/optim/nvnovograd.py @@ -0,0 +1,118 @@ +""" Nvidia NovoGrad Optimizer. +Original impl by Nvidia from Jasper example: + - https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/SpeechRecognition/Jasper +Paper: `Stochastic Gradient Methods with Layer-wise Adaptive Moments for Training of Deep Networks` + - https://arxiv.org/abs/1905.11286 +""" + +import torch +from torch.optim.optimizer import Optimizer +import math + + +class NvNovoGrad(Optimizer): + """ + Implements Novograd algorithm. + + Args: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.95, 0.98)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + grad_averaging: gradient averaging + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) + """ + + def __init__(self, params, lr=1e-3, betas=(0.95, 0.98), eps=1e-8, + weight_decay=0, grad_averaging=False, amsgrad=False): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + defaults = dict(lr=lr, betas=betas, eps=eps, + weight_decay=weight_decay, + grad_averaging=grad_averaging, + amsgrad=amsgrad) + + super(NvNovoGrad, self).__init__(params, defaults) + + def __setstate__(self, state): + super(NvNovoGrad, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('amsgrad', False) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('Sparse gradients are not supported.') + amsgrad = group['amsgrad'] + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros([]).to(state['exp_avg'].device) + if amsgrad: + # Maintains max of all exp. moving avg. of sq. grad. values + state['max_exp_avg_sq'] = torch.zeros([]).to(state['exp_avg'].device) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + if amsgrad: + max_exp_avg_sq = state['max_exp_avg_sq'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + + norm = torch.sum(torch.pow(grad, 2)) + + if exp_avg_sq == 0: + exp_avg_sq.copy_(norm) + else: + exp_avg_sq.mul_(beta2).add_(1 - beta2, norm) + + if amsgrad: + # Maintains the maximum of all 2nd moment running avg. till now + torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) + # Use the max. for normalizing running avg. of gradient + denom = max_exp_avg_sq.sqrt().add_(group['eps']) + else: + denom = exp_avg_sq.sqrt().add_(group['eps']) + + grad.div_(denom) + if group['weight_decay'] != 0: + grad.add_(group['weight_decay'], p.data) + if group['grad_averaging']: + grad.mul_(1 - beta1) + exp_avg.mul_(beta1).add_(grad) + + p.data.add_(-group['lr'], exp_avg) + + return loss diff --git a/optim/optim_factory.py b/optim/optim_factory.py new file mode 100644 index 0000000..4c0aaca --- /dev/null +++ b/optim/optim_factory.py @@ -0,0 +1,122 @@ +""" Optimizer Factory w/ Custom Weight Decay +Hacked together by / Copyright 2020 Ross Wightman +""" +import torch +from torch import optim as optim + +from .adafactor import Adafactor +from .adahessian import Adahessian +from .adamp import AdamP +from .lookahead import Lookahead +from .nadam import Nadam +from .novograd import NovoGrad +from .nvnovograd import NvNovoGrad +from .radam import RAdam +from .rmsprop_tf import RMSpropTF +from .sgdp import SGDP + +try: + from apex.optimizers import FusedNovoGrad, FusedAdam, FusedLAMB, FusedSGD + has_apex = True +except ImportError: + has_apex = False + + +def add_weight_decay(model, weight_decay=1e-5, skip_list=()): + decay = [] + no_decay = [] + for name, param in model.named_parameters(): + if not param.requires_grad: + continue # frozen weights + if len(param.shape) == 1 or name.endswith(".bias") or name in skip_list: + no_decay.append(param) + else: + decay.append(param) + return [ + {'params': no_decay, 'weight_decay': 0.}, + {'params': decay, 'weight_decay': weight_decay}] + + +def create_optimizer(args, model, filter_bias_and_bn=True): + opt_lower = args.opt.lower() + weight_decay = args.weight_decay + if weight_decay and filter_bias_and_bn: + skip = {} + if hasattr(model, 'no_weight_decay'): + skip = model.no_weight_decay() + parameters = add_weight_decay(model, weight_decay, skip) + weight_decay = 0. + else: + parameters = model.parameters() + + if 'fused' in opt_lower: + assert has_apex and torch.cuda.is_available(), 'APEX and CUDA required for fused optimizers' + + opt_args = dict(lr=args.lr, weight_decay=weight_decay) + if hasattr(args, 'opt_eps') and args.opt_eps is not None: + opt_args['eps'] = args.opt_eps + if hasattr(args, 'opt_betas') and args.opt_betas is not None: + opt_args['betas'] = args.opt_betas + if hasattr(args, 'opt_args') and args.opt_args is not None: + opt_args.update(args.opt_args) + + opt_split = opt_lower.split('_') + opt_lower = opt_split[-1] + if opt_lower == 'sgd' or opt_lower == 'nesterov': + opt_args.pop('eps', None) + optimizer = optim.SGD(parameters, momentum=args.momentum, nesterov=True, **opt_args) + elif opt_lower == 'momentum': + opt_args.pop('eps', None) + optimizer = optim.SGD(parameters, momentum=args.momentum, nesterov=False, **opt_args) + elif opt_lower == 'adam': + optimizer = optim.Adam(parameters, **opt_args) + elif opt_lower == 'adamw': + optimizer = optim.AdamW(parameters, **opt_args) + elif opt_lower == 'nadam': + optimizer = Nadam(parameters, **opt_args) + elif opt_lower == 'radam': + optimizer = RAdam(parameters, **opt_args) + elif opt_lower == 'adamp': + optimizer = AdamP(parameters, wd_ratio=0.01, nesterov=True, **opt_args) + elif opt_lower == 'sgdp': + optimizer = SGDP(parameters, momentum=args.momentum, nesterov=True, **opt_args) + elif opt_lower == 'adadelta': + optimizer = optim.Adadelta(parameters, **opt_args) + elif opt_lower == 'adafactor': + if not args.lr: + opt_args['lr'] = None + optimizer = Adafactor(parameters, **opt_args) + elif opt_lower == 'adahessian': + optimizer = Adahessian(parameters, **opt_args) + elif opt_lower == 'rmsprop': + optimizer = optim.RMSprop(parameters, alpha=0.9, momentum=args.momentum, **opt_args) + elif opt_lower == 'rmsproptf': + optimizer = RMSpropTF(parameters, alpha=0.9, momentum=args.momentum, **opt_args) + elif opt_lower == 'novograd': + optimizer = NovoGrad(parameters, **opt_args) + elif opt_lower == 'nvnovograd': + optimizer = NvNovoGrad(parameters, **opt_args) + elif opt_lower == 'fusedsgd': + opt_args.pop('eps', None) + optimizer = FusedSGD(parameters, momentum=args.momentum, nesterov=True, **opt_args) + elif opt_lower == 'fusedmomentum': + opt_args.pop('eps', None) + optimizer = FusedSGD(parameters, momentum=args.momentum, nesterov=False, **opt_args) + elif opt_lower == 'fusedadam': + optimizer = FusedAdam(parameters, adam_w_mode=False, **opt_args) + elif opt_lower == 'fusedadamw': + optimizer = FusedAdam(parameters, adam_w_mode=True, **opt_args) + elif opt_lower == 'fusedlamb': + optimizer = FusedLAMB(parameters, **opt_args) + elif opt_lower == 'fusednovograd': + opt_args.setdefault('betas', (0.95, 0.98)) + optimizer = FusedNovoGrad(parameters, **opt_args) + else: + assert False and "Invalid optimizer" + raise ValueError + + if len(opt_split) > 1: + if opt_split[0] == 'lookahead': + optimizer = Lookahead(optimizer) + + return optimizer diff --git a/optim/radam.py b/optim/radam.py new file mode 100644 index 0000000..9987a33 --- /dev/null +++ b/optim/radam.py @@ -0,0 +1,152 @@ +"""RAdam Optimizer. +Implementation lifted from: https://github.com/LiyuanLucasLiu/RAdam +Paper: `On the Variance of the Adaptive Learning Rate and Beyond` - https://arxiv.org/abs/1908.03265 +""" +import math +import torch +from torch.optim.optimizer import Optimizer, required + + +class RAdam(Optimizer): + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) + self.buffer = [[None, None, None] for ind in range(10)] + super(RAdam, self).__init__(params, defaults) + + def __setstate__(self, state): + super(RAdam, self).__setstate__(state) + + def step(self, closure=None): + + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data.float() + if grad.is_sparse: + raise RuntimeError('RAdam does not support sparse gradients') + + p_data_fp32 = p.data.float() + + state = self.state[p] + + if len(state) == 0: + state['step'] = 0 + state['exp_avg'] = torch.zeros_like(p_data_fp32) + state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) + else: + state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) + state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + exp_avg.mul_(beta1).add_(1 - beta1, grad) + + state['step'] += 1 + buffered = self.buffer[int(state['step'] % 10)] + if state['step'] == buffered[0]: + N_sma, step_size = buffered[1], buffered[2] + else: + buffered[0] = state['step'] + beta2_t = beta2 ** state['step'] + N_sma_max = 2 / (1 - beta2) - 1 + N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) + buffered[1] = N_sma + + # more conservative since it's an approximated value + if N_sma >= 5: + step_size = group['lr'] * math.sqrt( + (1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / ( + N_sma_max - 2)) / (1 - beta1 ** state['step']) + else: + step_size = group['lr'] / (1 - beta1 ** state['step']) + buffered[2] = step_size + + if group['weight_decay'] != 0: + p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) + + # more conservative since it's an approximated value + if N_sma >= 5: + denom = exp_avg_sq.sqrt().add_(group['eps']) + p_data_fp32.addcdiv_(-step_size, exp_avg, denom) + else: + p_data_fp32.add_(-step_size, exp_avg) + + p.data.copy_(p_data_fp32) + + return loss + + +class PlainRAdam(Optimizer): + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) + + super(PlainRAdam, self).__init__(params, defaults) + + def __setstate__(self, state): + super(PlainRAdam, self).__setstate__(state) + + def step(self, closure=None): + + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data.float() + if grad.is_sparse: + raise RuntimeError('RAdam does not support sparse gradients') + + p_data_fp32 = p.data.float() + + state = self.state[p] + + if len(state) == 0: + state['step'] = 0 + state['exp_avg'] = torch.zeros_like(p_data_fp32) + state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) + else: + state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) + state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + exp_avg.mul_(beta1).add_(1 - beta1, grad) + + state['step'] += 1 + beta2_t = beta2 ** state['step'] + N_sma_max = 2 / (1 - beta2) - 1 + N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) + + if group['weight_decay'] != 0: + p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) + + # more conservative since it's an approximated value + if N_sma >= 5: + step_size = group['lr'] * math.sqrt( + (1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / ( + N_sma_max - 2)) / (1 - beta1 ** state['step']) + denom = exp_avg_sq.sqrt().add_(group['eps']) + p_data_fp32.addcdiv_(-step_size, exp_avg, denom) + else: + step_size = group['lr'] / (1 - beta1 ** state['step']) + p_data_fp32.add_(-step_size, exp_avg) + + p.data.copy_(p_data_fp32) + + return loss diff --git a/optim/rmsprop_tf.py b/optim/rmsprop_tf.py new file mode 100644 index 0000000..5115555 --- /dev/null +++ b/optim/rmsprop_tf.py @@ -0,0 +1,136 @@ +""" RMSProp modified to behave like Tensorflow impl + +Originally cut & paste from PyTorch RMSProp +https://github.com/pytorch/pytorch/blob/063946d2b3f3f1e953a2a3b54e0b34f1393de295/torch/optim/rmsprop.py +Licensed under BSD-Clause 3 (ish), https://github.com/pytorch/pytorch/blob/master/LICENSE + +Modifications Copyright 2020 Ross Wightman +""" + +import torch +from torch.optim import Optimizer + + +class RMSpropTF(Optimizer): + """Implements RMSprop algorithm (TensorFlow style epsilon) + + NOTE: This is a direct cut-and-paste of PyTorch RMSprop with eps applied before sqrt + and a few other modifications to closer match Tensorflow for matching hyper-params. + + Noteworthy changes include: + 1. Epsilon applied inside square-root + 2. square_avg initialized to ones + 3. LR scaling of update accumulated in momentum buffer + + Proposed by G. Hinton in his + `course `_. + + The centered version first appears in `Generating Sequences + With Recurrent Neural Networks `_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-2) + momentum (float, optional): momentum factor (default: 0) + alpha (float, optional): smoothing (decay) constant (default: 0.9) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-10) + centered (bool, optional) : if ``True``, compute the centered RMSProp, + the gradient is normalized by an estimation of its variance + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + decoupled_decay (bool, optional): decoupled weight decay as per https://arxiv.org/abs/1711.05101 + lr_in_momentum (bool, optional): learning rate scaling is included in the momentum buffer + update as per defaults in Tensorflow + + """ + + def __init__(self, params, lr=1e-2, alpha=0.9, eps=1e-10, weight_decay=0, momentum=0., centered=False, + decoupled_decay=False, lr_in_momentum=True): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= momentum: + raise ValueError("Invalid momentum value: {}".format(momentum)) + if not 0.0 <= weight_decay: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + if not 0.0 <= alpha: + raise ValueError("Invalid alpha value: {}".format(alpha)) + + defaults = dict(lr=lr, momentum=momentum, alpha=alpha, eps=eps, centered=centered, weight_decay=weight_decay, + decoupled_decay=decoupled_decay, lr_in_momentum=lr_in_momentum) + super(RMSpropTF, self).__init__(params, defaults) + + def __setstate__(self, state): + super(RMSpropTF, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('momentum', 0) + group.setdefault('centered', False) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('RMSprop does not support sparse gradients') + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + state['square_avg'] = torch.ones_like(p.data) # PyTorch inits to zero + if group['momentum'] > 0: + state['momentum_buffer'] = torch.zeros_like(p.data) + if group['centered']: + state['grad_avg'] = torch.zeros_like(p.data) + + square_avg = state['square_avg'] + one_minus_alpha = 1. - group['alpha'] + + state['step'] += 1 + + if group['weight_decay'] != 0: + if 'decoupled_decay' in group and group['decoupled_decay']: + p.data.add_(-group['weight_decay'], p.data) + else: + grad = grad.add(group['weight_decay'], p.data) + + # Tensorflow order of ops for updating squared avg + square_avg.add_(one_minus_alpha, grad.pow(2) - square_avg) + # square_avg.mul_(alpha).addcmul_(1 - alpha, grad, grad) # PyTorch original + + if group['centered']: + grad_avg = state['grad_avg'] + grad_avg.add_(one_minus_alpha, grad - grad_avg) + # grad_avg.mul_(alpha).add_(1 - alpha, grad) # PyTorch original + avg = square_avg.addcmul(-1, grad_avg, grad_avg).add(group['eps']).sqrt_() # eps moved in sqrt + else: + avg = square_avg.add(group['eps']).sqrt_() # eps moved in sqrt + + if group['momentum'] > 0: + buf = state['momentum_buffer'] + # Tensorflow accumulates the LR scaling in the momentum buffer + if 'lr_in_momentum' in group and group['lr_in_momentum']: + buf.mul_(group['momentum']).addcdiv_(group['lr'], grad, avg) + p.data.add_(-buf) + else: + # PyTorch scales the param update by LR + buf.mul_(group['momentum']).addcdiv_(grad, avg) + p.data.add_(-group['lr'], buf) + else: + p.data.addcdiv_(-group['lr'], grad, avg) + + return loss diff --git a/optim/sgdp.py b/optim/sgdp.py new file mode 100644 index 0000000..f4a94aa --- /dev/null +++ b/optim/sgdp.py @@ -0,0 +1,96 @@ +""" +SGDP Optimizer Implementation copied from https://github.com/clovaai/AdamP/blob/master/adamp/sgdp.py + +Paper: `Slowing Down the Weight Norm Increase in Momentum-based Optimizers` - https://arxiv.org/abs/2006.08217 +Code: https://github.com/clovaai/AdamP + +Copyright (c) 2020-present NAVER Corp. +MIT license +""" + +import torch +import torch.nn as nn +from torch.optim.optimizer import Optimizer, required +import math + +class SGDP(Optimizer): + def __init__(self, params, lr=required, momentum=0, dampening=0, + weight_decay=0, nesterov=False, eps=1e-8, delta=0.1, wd_ratio=0.1): + defaults = dict(lr=lr, momentum=momentum, dampening=dampening, weight_decay=weight_decay, + nesterov=nesterov, eps=eps, delta=delta, wd_ratio=wd_ratio) + super(SGDP, self).__init__(params, defaults) + + def _channel_view(self, x): + return x.view(x.size(0), -1) + + def _layer_view(self, x): + return x.view(1, -1) + + def _cosine_similarity(self, x, y, eps, view_func): + x = view_func(x) + y = view_func(y) + + x_norm = x.norm(dim=1).add_(eps) + y_norm = y.norm(dim=1).add_(eps) + dot = (x * y).sum(dim=1) + + return dot.abs() / x_norm / y_norm + + def _projection(self, p, grad, perturb, delta, wd_ratio, eps): + wd = 1 + expand_size = [-1] + [1] * (len(p.shape) - 1) + for view_func in [self._channel_view, self._layer_view]: + + cosine_sim = self._cosine_similarity(grad, p.data, eps, view_func) + + if cosine_sim.max() < delta / math.sqrt(view_func(p.data).size(1)): + p_n = p.data / view_func(p.data).norm(dim=1).view(expand_size).add_(eps) + perturb -= p_n * view_func(p_n * perturb).sum(dim=1).view(expand_size) + wd = wd_ratio + + return perturb, wd + + return perturb, wd + + def step(self, closure=None): + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + weight_decay = group['weight_decay'] + momentum = group['momentum'] + dampening = group['dampening'] + nesterov = group['nesterov'] + + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + state = self.state[p] + + # State initialization + if len(state) == 0: + state['momentum'] = torch.zeros_like(p.data) + + # SGD + buf = state['momentum'] + buf.mul_(momentum).add_(1 - dampening, grad) + if nesterov: + d_p = grad + momentum * buf + else: + d_p = buf + + # Projection + wd_ratio = 1 + if len(p.shape) > 1: + d_p, wd_ratio = self._projection(p, grad, d_p, group['delta'], group['wd_ratio'], group['eps']) + + # Weight decay + if weight_decay != 0: + p.data.mul_(1 - group['lr'] * group['weight_decay'] * wd_ratio / (1-momentum)) + + # Step + p.data.add_(-group['lr'], d_p) + + return loss diff --git a/refTools/__pycache__/refer_python3.cpython-36.pyc b/refTools/__pycache__/refer_python3.cpython-36.pyc new file mode 100644 index 0000000..2bd2037 Binary files /dev/null and b/refTools/__pycache__/refer_python3.cpython-36.pyc differ diff --git a/refTools/__pycache__/refer_python3.cpython-38.pyc b/refTools/__pycache__/refer_python3.cpython-38.pyc new file mode 100644 index 0000000..dc711bc Binary files /dev/null and b/refTools/__pycache__/refer_python3.cpython-38.pyc differ diff --git a/refTools/evaluation/__init__.py b/refTools/evaluation/__init__.py new file mode 100644 index 0000000..66e55e9 --- /dev/null +++ b/refTools/evaluation/__init__.py @@ -0,0 +1,3 @@ +__author__ = 'licheng' + + diff --git a/refTools/evaluation/__pycache__/__init__.cpython-36.pyc b/refTools/evaluation/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..7a2be60 Binary files /dev/null and b/refTools/evaluation/__pycache__/__init__.cpython-36.pyc differ diff --git a/refTools/evaluation/__pycache__/__init__.cpython-38.pyc b/refTools/evaluation/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..af0c07f Binary files /dev/null and b/refTools/evaluation/__pycache__/__init__.cpython-38.pyc differ diff --git a/refTools/evaluation/__pycache__/refEvaluation.cpython-36.pyc b/refTools/evaluation/__pycache__/refEvaluation.cpython-36.pyc new file mode 100644 index 0000000..7d3866f Binary files /dev/null and b/refTools/evaluation/__pycache__/refEvaluation.cpython-36.pyc differ diff --git a/refTools/evaluation/__pycache__/refEvaluation.cpython-38.pyc b/refTools/evaluation/__pycache__/refEvaluation.cpython-38.pyc new file mode 100644 index 0000000..7bbd25f Binary files /dev/null and b/refTools/evaluation/__pycache__/refEvaluation.cpython-38.pyc differ diff --git a/refTools/evaluation/bleu/LICENSE b/refTools/evaluation/bleu/LICENSE new file mode 100644 index 0000000..9ccf677 --- /dev/null +++ b/refTools/evaluation/bleu/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2015 Xinlei Chen, Hao Fang, Tsung-Yi Lin, and Ramakrishna Vedantam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/refTools/evaluation/bleu/__init__.py b/refTools/evaluation/bleu/__init__.py new file mode 100644 index 0000000..3f7d85b --- /dev/null +++ b/refTools/evaluation/bleu/__init__.py @@ -0,0 +1 @@ +__author__ = 'tylin' diff --git a/refTools/evaluation/bleu/__init__.pyc b/refTools/evaluation/bleu/__init__.pyc new file mode 100644 index 0000000..511902e Binary files /dev/null and b/refTools/evaluation/bleu/__init__.pyc differ diff --git a/refTools/evaluation/bleu/__pycache__/__init__.cpython-36.pyc b/refTools/evaluation/bleu/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..f9e7346 Binary files /dev/null and b/refTools/evaluation/bleu/__pycache__/__init__.cpython-36.pyc differ diff --git a/refTools/evaluation/bleu/__pycache__/__init__.cpython-38.pyc b/refTools/evaluation/bleu/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..4df280c Binary files /dev/null and b/refTools/evaluation/bleu/__pycache__/__init__.cpython-38.pyc differ diff --git a/refTools/evaluation/bleu/__pycache__/bleu.cpython-36.pyc b/refTools/evaluation/bleu/__pycache__/bleu.cpython-36.pyc new file mode 100644 index 0000000..6d754ec Binary files /dev/null and b/refTools/evaluation/bleu/__pycache__/bleu.cpython-36.pyc differ diff --git a/refTools/evaluation/bleu/__pycache__/bleu.cpython-38.pyc b/refTools/evaluation/bleu/__pycache__/bleu.cpython-38.pyc new file mode 100644 index 0000000..de1637c Binary files /dev/null and b/refTools/evaluation/bleu/__pycache__/bleu.cpython-38.pyc differ diff --git a/refTools/evaluation/bleu/__pycache__/bleu_scorer.cpython-36.pyc b/refTools/evaluation/bleu/__pycache__/bleu_scorer.cpython-36.pyc new file mode 100644 index 0000000..b863f71 Binary files /dev/null and b/refTools/evaluation/bleu/__pycache__/bleu_scorer.cpython-36.pyc differ diff --git a/refTools/evaluation/bleu/__pycache__/bleu_scorer.cpython-38.pyc b/refTools/evaluation/bleu/__pycache__/bleu_scorer.cpython-38.pyc new file mode 100644 index 0000000..9248a6c Binary files /dev/null and b/refTools/evaluation/bleu/__pycache__/bleu_scorer.cpython-38.pyc differ diff --git a/refTools/evaluation/bleu/bleu.py b/refTools/evaluation/bleu/bleu.py new file mode 100644 index 0000000..1dab5ad --- /dev/null +++ b/refTools/evaluation/bleu/bleu.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# +# File Name : bleu.py +# +# Description : Wrapper for BLEU scorer. +# +# Creation Date : 06-01-2015 +# Last Modified : Thu 19 Mar 2015 09:13:28 PM PDT +# Authors : Hao Fang and Tsung-Yi Lin + +from refTools.evaluation.bleu.bleu_scorer import BleuScorer + + +class Bleu: + def __init__(self, n=4): + # default compute Blue score up to 4 + self._n = n + self._hypo_for_image = {} + self.ref_for_image = {} + + def compute_score(self, gts, res): + + assert(gts.keys() == res.keys()) + imgIds = gts.keys() + + bleu_scorer = BleuScorer(n=self._n) + for id in imgIds: + hypo = res[id] + ref = gts[id] + + # Sanity check. + assert(type(hypo) is list) + assert(len(hypo) == 1) + assert(type(ref) is list) + assert(len(ref) >= 1) + + bleu_scorer += (hypo[0], ref) + + #score, scores = bleu_scorer.compute_score(option='shortest') + score, scores = bleu_scorer.compute_score(option='closest', verbose=1) + #score, scores = bleu_scorer.compute_score(option='average', verbose=1) + + # return (bleu, bleu_info) + return score, scores + + def method(self): + return "Bleu" diff --git a/refTools/evaluation/bleu/bleu.pyc b/refTools/evaluation/bleu/bleu.pyc new file mode 100644 index 0000000..8f15689 Binary files /dev/null and b/refTools/evaluation/bleu/bleu.pyc differ diff --git a/refTools/evaluation/bleu/bleu_scorer.py b/refTools/evaluation/bleu/bleu_scorer.py new file mode 100644 index 0000000..5419434 --- /dev/null +++ b/refTools/evaluation/bleu/bleu_scorer.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python + +# bleu_scorer.py +# David Chiang + +# Copyright (c) 2004-2006 University of Maryland. All rights +# reserved. Do not redistribute without permission from the +# author. Not for commercial use. + +# Modified by: +# Hao Fang +# Tsung-Yi Lin + +'''Provides: +cook_refs(refs, n=4): Transform a list of reference sentences as strings into a form usable by cook_test(). +cook_test(test, refs, n=4): Transform a test sentence as a string (together with the cooked reference sentences) into a form usable by score_cooked(). +''' + +import copy +import sys, math, re +from collections import defaultdict + +def precook(s, n=4, out=False): + """Takes a string as input and returns an object that can be given to + either cook_refs or cook_test. This is optional: cook_refs and cook_test + can take string arguments as well.""" + words = s.split() + counts = defaultdict(int) + for k in range(1,n+1): + for i in range(len(words)-k+1): + ngram = tuple(words[i:i+k]) + counts[ngram] += 1 + return (len(words), counts) + +def cook_refs(refs, eff=None, n=4): ## lhuang: oracle will call with "average" + '''Takes a list of reference sentences for a single segment + and returns an object that encapsulates everything that BLEU + needs to know about them.''' + + reflen = [] + maxcounts = {} + for ref in refs: + rl, counts = precook(ref, n) + reflen.append(rl) + for (ngram,count) in counts.items(): + maxcounts[ngram] = max(maxcounts.get(ngram,0), count) + + # Calculate effective reference sentence length. + if eff == "shortest": + reflen = min(reflen) + elif eff == "average": + reflen = float(sum(reflen))/len(reflen) + + ## lhuang: N.B.: leave reflen computaiton to the very end!! + + ## lhuang: N.B.: in case of "closest", keep a list of reflens!! (bad design) + + return (reflen, maxcounts) + +def cook_test(test, refparam, eff=None, n=4): + '''Takes a test sentence and returns an object that + encapsulates everything that BLEU needs to know about it.''' + + reflen, refmaxcounts = refparam[0], refparam[1] + testlen, counts = precook(test, n, True) + + result = {} + + # Calculate effective reference sentence length. + + if eff == "closest": + result["reflen"] = min((abs(l-testlen), l) for l in reflen)[1] + else: ## i.e., "average" or "shortest" or None + result["reflen"] = reflen + + result["testlen"] = testlen + + result["guess"] = [max(0,testlen-k+1) for k in range(1,n+1)] + + result['correct'] = [0]*n + for (ngram, count) in counts.items(): + result["correct"][len(ngram)-1] += min(refmaxcounts.get(ngram,0), count) + + return result + +class BleuScorer(object): + """Bleu scorer. + """ + + __slots__ = "n", "crefs", "ctest", "_score", "_ratio", "_testlen", "_reflen", "special_reflen" + # special_reflen is used in oracle (proportional effective ref len for a node). + + def copy(self): + ''' copy the refs.''' + new = BleuScorer(n=self.n) + new.ctest = copy.copy(self.ctest) + new.crefs = copy.copy(self.crefs) + new._score = None + return new + + def __init__(self, test=None, refs=None, n=4, special_reflen=None): + ''' singular instance ''' + + self.n = n + self.crefs = [] + self.ctest = [] + self.cook_append(test, refs) + self.special_reflen = special_reflen + + def cook_append(self, test, refs): + '''called by constructor and __iadd__ to avoid creating new instances.''' + + if refs is not None: + self.crefs.append(cook_refs(refs)) + if test is not None: + cooked_test = cook_test(test, self.crefs[-1]) + self.ctest.append(cooked_test) ## N.B.: -1 + else: + self.ctest.append(None) # lens of crefs and ctest have to match + + self._score = None ## need to recompute + + def ratio(self, option=None): + self.compute_score(option=option) + return self._ratio + + def score_ratio(self, option=None): + '''return (bleu, len_ratio) pair''' + return (self.fscore(option=option), self.ratio(option=option)) + + def score_ratio_str(self, option=None): + return "%.4f (%.2f)" % self.score_ratio(option) + + def reflen(self, option=None): + self.compute_score(option=option) + return self._reflen + + def testlen(self, option=None): + self.compute_score(option=option) + return self._testlen + + def retest(self, new_test): + if type(new_test) is str: + new_test = [new_test] + assert len(new_test) == len(self.crefs), new_test + self.ctest = [] + for t, rs in zip(new_test, self.crefs): + self.ctest.append(cook_test(t, rs)) + self._score = None + + return self + + def rescore(self, new_test): + ''' replace test(s) with new test(s), and returns the new score.''' + + return self.retest(new_test).compute_score() + + def size(self): + assert len(self.crefs) == len(self.ctest), "refs/test mismatch! %d<>%d" % (len(self.crefs), len(self.ctest)) + return len(self.crefs) + + def __iadd__(self, other): + '''add an instance (e.g., from another sentence).''' + + if type(other) is tuple: + ## avoid creating new BleuScorer instances + self.cook_append(other[0], other[1]) + else: + assert self.compatible(other), "incompatible BLEUs." + self.ctest.extend(other.ctest) + self.crefs.extend(other.crefs) + self._score = None ## need to recompute + + return self + + def compatible(self, other): + return isinstance(other, BleuScorer) and self.n == other.n + + def single_reflen(self, option="average"): + return self._single_reflen(self.crefs[0][0], option) + + def _single_reflen(self, reflens, option=None, testlen=None): + + if option == "shortest": + reflen = min(reflens) + elif option == "average": + reflen = float(sum(reflens))/len(reflens) + elif option == "closest": + reflen = min((abs(l-testlen), l) for l in reflens)[1] + else: + assert False, "unsupported reflen option %s" % option + + return reflen + + def recompute_score(self, option=None, verbose=0): + self._score = None + return self.compute_score(option, verbose) + + def compute_score(self, option=None, verbose=0): + n = self.n + small = 1e-9 + tiny = 1e-15 ## so that if guess is 0 still return 0 + bleu_list = [[] for _ in range(n)] + + if self._score is not None: + return self._score + + if option is None: + option = "average" if len(self.crefs) == 1 else "closest" + + self._testlen = 0 + self._reflen = 0 + totalcomps = {'testlen':0, 'reflen':0, 'guess':[0]*n, 'correct':[0]*n} + + # for each sentence + for comps in self.ctest: + testlen = comps['testlen'] + self._testlen += testlen + + if self.special_reflen is None: ## need computation + reflen = self._single_reflen(comps['reflen'], option, testlen) + else: + reflen = self.special_reflen + + self._reflen += reflen + + for key in ['guess','correct']: + for k in range(n): + totalcomps[key][k] += comps[key][k] + + # append per image bleu score + bleu = 1. + for k in range(n): + bleu *= (float(comps['correct'][k]) + tiny) \ + /(float(comps['guess'][k]) + small) + bleu_list[k].append(bleu ** (1./(k+1))) + ratio = (testlen + tiny) / (reflen + small) ## N.B.: avoid zero division + if ratio < 1: + for k in range(n): + bleu_list[k][-1] *= math.exp(1 - 1/ratio) + + if verbose > 1: + print(comps, reflen) + + totalcomps['reflen'] = self._reflen + totalcomps['testlen'] = self._testlen + + bleus = [] + bleu = 1. + for k in range(n): + bleu *= float(totalcomps['correct'][k] + tiny) \ + / (totalcomps['guess'][k] + small) + bleus.append(bleu ** (1./(k+1))) + ratio = (self._testlen + tiny) / (self._reflen + small) ## N.B.: avoid zero division + if ratio < 1: + for k in range(n): + bleus[k] *= math.exp(1 - 1/ratio) + + if verbose > 0: + print(totalcomps) + print("ratio:", ratio) + + self._score = bleus + return self._score, bleu_list diff --git a/refTools/evaluation/bleu/bleu_scorer.pyc b/refTools/evaluation/bleu/bleu_scorer.pyc new file mode 100644 index 0000000..8606fc7 Binary files /dev/null and b/refTools/evaluation/bleu/bleu_scorer.pyc differ diff --git a/refTools/evaluation/cider/__init__.py b/refTools/evaluation/cider/__init__.py new file mode 100644 index 0000000..3f7d85b --- /dev/null +++ b/refTools/evaluation/cider/__init__.py @@ -0,0 +1 @@ +__author__ = 'tylin' diff --git a/refTools/evaluation/cider/__init__.pyc b/refTools/evaluation/cider/__init__.pyc new file mode 100644 index 0000000..3e37f4c Binary files /dev/null and b/refTools/evaluation/cider/__init__.pyc differ diff --git a/refTools/evaluation/cider/__pycache__/__init__.cpython-36.pyc b/refTools/evaluation/cider/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..d71d8f8 Binary files /dev/null and b/refTools/evaluation/cider/__pycache__/__init__.cpython-36.pyc differ diff --git a/refTools/evaluation/cider/__pycache__/__init__.cpython-38.pyc b/refTools/evaluation/cider/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..33621b2 Binary files /dev/null and b/refTools/evaluation/cider/__pycache__/__init__.cpython-38.pyc differ diff --git a/refTools/evaluation/cider/__pycache__/cider.cpython-36.pyc b/refTools/evaluation/cider/__pycache__/cider.cpython-36.pyc new file mode 100644 index 0000000..d2354bc Binary files /dev/null and b/refTools/evaluation/cider/__pycache__/cider.cpython-36.pyc differ diff --git a/refTools/evaluation/cider/__pycache__/cider.cpython-38.pyc b/refTools/evaluation/cider/__pycache__/cider.cpython-38.pyc new file mode 100644 index 0000000..e251e5a Binary files /dev/null and b/refTools/evaluation/cider/__pycache__/cider.cpython-38.pyc differ diff --git a/refTools/evaluation/cider/__pycache__/cider_scorer.cpython-36.pyc b/refTools/evaluation/cider/__pycache__/cider_scorer.cpython-36.pyc new file mode 100644 index 0000000..33a7b1f Binary files /dev/null and b/refTools/evaluation/cider/__pycache__/cider_scorer.cpython-36.pyc differ diff --git a/refTools/evaluation/cider/__pycache__/cider_scorer.cpython-38.pyc b/refTools/evaluation/cider/__pycache__/cider_scorer.cpython-38.pyc new file mode 100644 index 0000000..61bb43b Binary files /dev/null and b/refTools/evaluation/cider/__pycache__/cider_scorer.cpython-38.pyc differ diff --git a/refTools/evaluation/cider/cider.py b/refTools/evaluation/cider/cider.py new file mode 100644 index 0000000..c96f4f4 --- /dev/null +++ b/refTools/evaluation/cider/cider.py @@ -0,0 +1,54 @@ +# Filename: cider.py +# +# Description: Describes the class to compute the CIDEr (Consensus-Based Image Description Evaluation) Metric +# by Vedantam, Zitnick, and Parikh (http://arxiv.org/abs/1411.5726) +# +# Creation Date: Sun Feb 8 14:16:54 2015 +# +# Authors: Ramakrishna Vedantam and Tsung-Yi Lin + +from refTools.evaluation.cider.cider_scorer import CiderScorer +import pdb + +class Cider: + """ + Main Class to compute the CIDEr metric + + """ + def __init__(self, test=None, refs=None, n=4, sigma=6.0): + # set cider to sum over 1 to 4-grams + self._n = n + # set the standard deviation parameter for gaussian penalty + self._sigma = sigma + + def compute_score(self, gts, res): + """ + Main function to compute CIDEr score + :param hypo_for_image (dict) : dictionary with key and value + ref_for_image (dict) : dictionary with key and value + :return: cider (float) : computed CIDEr score for the corpus + """ + + assert(gts.keys() == res.keys()) + imgIds = gts.keys() + + cider_scorer = CiderScorer(n=self._n, sigma=self._sigma) + + for id in imgIds: + hypo = res[id] + ref = gts[id] + + # Sanity check. + assert(type(hypo) is list) + assert(len(hypo) == 1) + assert(type(ref) is list) + assert(len(ref) > 0) + + cider_scorer += (hypo[0], ref) + + (score, scores) = cider_scorer.compute_score() + + return score, scores + + def method(self): + return "CIDEr" \ No newline at end of file diff --git a/refTools/evaluation/cider/cider.pyc b/refTools/evaluation/cider/cider.pyc new file mode 100644 index 0000000..1b3d645 Binary files /dev/null and b/refTools/evaluation/cider/cider.pyc differ diff --git a/refTools/evaluation/cider/cider_scorer.py b/refTools/evaluation/cider/cider_scorer.py new file mode 100644 index 0000000..a73405e --- /dev/null +++ b/refTools/evaluation/cider/cider_scorer.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +# Tsung-Yi Lin +# Ramakrishna Vedantam + +import copy +from collections import defaultdict +import numpy as np +import pdb +import math + +def precook(s, n=4, out=False): + """ + Takes a string as input and returns an object that can be given to + either cook_refs or cook_test. This is optional: cook_refs and cook_test + can take string arguments as well. + :param s: string : sentence to be converted into ngrams + :param n: int : number of ngrams for which representation is calculated + :return: term frequency vector for occuring ngrams + """ + words = s.split() + counts = defaultdict(int) + for k in xrange(1,n+1): + for i in xrange(len(words)-k+1): + ngram = tuple(words[i:i+k]) + counts[ngram] += 1 + return counts + +def cook_refs(refs, n=4): ## lhuang: oracle will call with "average" + '''Takes a list of reference sentences for a single segment + and returns an object that encapsulates everything that BLEU + needs to know about them. + :param refs: list of string : reference sentences for some image + :param n: int : number of ngrams for which (ngram) representation is calculated + :return: result (list of dict) + ''' + return [precook(ref, n) for ref in refs] + +def cook_test(test, n=4): + '''Takes a test sentence and returns an object that + encapsulates everything that BLEU needs to know about it. + :param test: list of string : hypothesis sentence for some image + :param n: int : number of ngrams for which (ngram) representation is calculated + :return: result (dict) + ''' + return precook(test, n, True) + +class CiderScorer(object): + """CIDEr scorer. + """ + + def copy(self): + ''' copy the refs.''' + new = CiderScorer(n=self.n) + new.ctest = copy.copy(self.ctest) + new.crefs = copy.copy(self.crefs) + return new + + def __init__(self, test=None, refs=None, n=4, sigma=6.0): + ''' singular instance ''' + self.n = n + self.sigma = sigma + self.crefs = [] + self.ctest = [] + self.document_frequency = defaultdict(float) + self.cook_append(test, refs) + self.ref_len = None + + def cook_append(self, test, refs): + '''called by constructor and __iadd__ to avoid creating new instances.''' + + if refs is not None: + self.crefs.append(cook_refs(refs)) + if test is not None: + self.ctest.append(cook_test(test)) ## N.B.: -1 + else: + self.ctest.append(None) # lens of crefs and ctest have to match + + def size(self): + assert len(self.crefs) == len(self.ctest), "refs/test mismatch! %d<>%d" % (len(self.crefs), len(self.ctest)) + return len(self.crefs) + + def __iadd__(self, other): + '''add an instance (e.g., from another sentence).''' + + if type(other) is tuple: + ## avoid creating new CiderScorer instances + self.cook_append(other[0], other[1]) + else: + self.ctest.extend(other.ctest) + self.crefs.extend(other.crefs) + + return self + def compute_doc_freq(self): + ''' + Compute term frequency for reference data. + This will be used to compute idf (inverse document frequency later) + The term frequency is stored in the object + :return: None + ''' + for refs in self.crefs: + # refs, k ref captions of one image + for ngram in set([ngram for ref in refs for (ngram,count) in ref.iteritems()]): + self.document_frequency[ngram] += 1 + # maxcounts[ngram] = max(maxcounts.get(ngram,0), count) + + def compute_cider(self): + def counts2vec(cnts): + """ + Function maps counts of ngram to vector of tfidf weights. + The function returns vec, an array of dictionary that store mapping of n-gram and tf-idf weights. + The n-th entry of array denotes length of n-grams. + :param cnts: + :return: vec (array of dict), norm (array of float), length (int) + """ + vec = [defaultdict(float) for _ in range(self.n)] + length = 0 + norm = [0.0 for _ in range(self.n)] + for (ngram,term_freq) in cnts.iteritems(): + # give word count 1 if it doesn't appear in reference corpus + df = np.log(max(1.0, self.document_frequency[ngram])) + # ngram index + n = len(ngram)-1 + # tf (term_freq) * idf (precomputed idf) for n-grams + vec[n][ngram] = float(term_freq)*(self.ref_len - df) + # compute norm for the vector. the norm will be used for computing similarity + norm[n] += pow(vec[n][ngram], 2) + + if n == 1: + length += term_freq + norm = [np.sqrt(n) for n in norm] + return vec, norm, length + + def sim(vec_hyp, vec_ref, norm_hyp, norm_ref, length_hyp, length_ref): + ''' + Compute the cosine similarity of two vectors. + :param vec_hyp: array of dictionary for vector corresponding to hypothesis + :param vec_ref: array of dictionary for vector corresponding to reference + :param norm_hyp: array of float for vector corresponding to hypothesis + :param norm_ref: array of float for vector corresponding to reference + :param length_hyp: int containing length of hypothesis + :param length_ref: int containing length of reference + :return: array of score for each n-grams cosine similarity + ''' + delta = float(length_hyp - length_ref) + # measure consine similarity + val = np.array([0.0 for _ in range(self.n)]) + for n in range(self.n): + # ngram + for (ngram,count) in vec_hyp[n].iteritems(): + # vrama91 : added clipping + val[n] += min(vec_hyp[n][ngram], vec_ref[n][ngram]) * vec_ref[n][ngram] + + if (norm_hyp[n] != 0) and (norm_ref[n] != 0): + val[n] /= (norm_hyp[n]*norm_ref[n]) + + assert(not math.isnan(val[n])) + # vrama91: added a length based gaussian penalty + val[n] *= np.e**(-(delta**2)/(2*self.sigma**2)) + return val + + # compute log reference length + self.ref_len = np.log(float(len(self.crefs))) + + scores = [] + for test, refs in zip(self.ctest, self.crefs): + # compute vector for test captions + vec, norm, length = counts2vec(test) + # compute vector for ref captions + score = np.array([0.0 for _ in range(self.n)]) + for ref in refs: + vec_ref, norm_ref, length_ref = counts2vec(ref) + score += sim(vec, vec_ref, norm, norm_ref, length, length_ref) + # change by vrama91 - mean of ngram scores, instead of sum + score_avg = np.mean(score) + # divide by number of references + score_avg /= len(refs) + # multiply score by 10 + score_avg *= 10.0 + # append score of an image to the score list + scores.append(score_avg) + return scores + + def compute_score(self, option=None, verbose=0): + # compute idf + self.compute_doc_freq() + # assert to check document frequency + assert(len(self.ctest) >= max(self.document_frequency.values())) + # compute cider score + score = self.compute_cider() + # debug + # print score + return np.mean(np.array(score)), np.array(score) \ No newline at end of file diff --git a/refTools/evaluation/cider/cider_scorer.pyc b/refTools/evaluation/cider/cider_scorer.pyc new file mode 100644 index 0000000..4c77a56 Binary files /dev/null and b/refTools/evaluation/cider/cider_scorer.pyc differ diff --git a/refTools/evaluation/meteor/__init__.py b/refTools/evaluation/meteor/__init__.py new file mode 100644 index 0000000..3f7d85b --- /dev/null +++ b/refTools/evaluation/meteor/__init__.py @@ -0,0 +1 @@ +__author__ = 'tylin' diff --git a/refTools/evaluation/meteor/__init__.pyc b/refTools/evaluation/meteor/__init__.pyc new file mode 100644 index 0000000..0170323 Binary files /dev/null and b/refTools/evaluation/meteor/__init__.pyc differ diff --git a/refTools/evaluation/meteor/__pycache__/__init__.cpython-36.pyc b/refTools/evaluation/meteor/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..168c166 Binary files /dev/null and b/refTools/evaluation/meteor/__pycache__/__init__.cpython-36.pyc differ diff --git a/refTools/evaluation/meteor/__pycache__/__init__.cpython-38.pyc b/refTools/evaluation/meteor/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..5997108 Binary files /dev/null and b/refTools/evaluation/meteor/__pycache__/__init__.cpython-38.pyc differ diff --git a/refTools/evaluation/meteor/__pycache__/meteor.cpython-36.pyc b/refTools/evaluation/meteor/__pycache__/meteor.cpython-36.pyc new file mode 100644 index 0000000..389ee88 Binary files /dev/null and b/refTools/evaluation/meteor/__pycache__/meteor.cpython-36.pyc differ diff --git a/refTools/evaluation/meteor/__pycache__/meteor.cpython-38.pyc b/refTools/evaluation/meteor/__pycache__/meteor.cpython-38.pyc new file mode 100644 index 0000000..9710c82 Binary files /dev/null and b/refTools/evaluation/meteor/__pycache__/meteor.cpython-38.pyc differ diff --git a/refTools/evaluation/meteor/data/paraphrase-en.gz b/refTools/evaluation/meteor/data/paraphrase-en.gz new file mode 100644 index 0000000..88033c8 Binary files /dev/null and b/refTools/evaluation/meteor/data/paraphrase-en.gz differ diff --git a/refTools/evaluation/meteor/meteor-1.5.jar b/refTools/evaluation/meteor/meteor-1.5.jar new file mode 100644 index 0000000..a833bc0 Binary files /dev/null and b/refTools/evaluation/meteor/meteor-1.5.jar differ diff --git a/refTools/evaluation/meteor/meteor.py b/refTools/evaluation/meteor/meteor.py new file mode 100644 index 0000000..a4915c2 --- /dev/null +++ b/refTools/evaluation/meteor/meteor.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# Python wrapper for METEOR implementation, by Xinlei Chen +# Acknowledge Michael Denkowski for the generous discussion and help + +import os +import sys +import subprocess +import threading + +# Assumes meteor-1.5.jar is in the same directory as meteor.py. Change as needed. +METEOR_JAR = 'meteor-1.5.jar' +# print METEOR_JAR + +class Meteor: + + def __init__(self): + self.meteor_cmd = ['java', '-jar', '-Xmx2G', METEOR_JAR, \ + '-', '-', '-stdio', '-l', 'en', '-norm'] + self.meteor_p = subprocess.Popen(self.meteor_cmd, \ + cwd=os.path.dirname(os.path.abspath(__file__)), \ + stdin=subprocess.PIPE, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + # Used to guarantee thread safety + self.lock = threading.Lock() + + def compute_score(self, gts, res): + assert(gts.keys() == res.keys()) + imgIds = gts.keys() + scores = [] + + eval_line = 'EVAL' + self.lock.acquire() + for i in imgIds: + assert(len(res[i]) == 1) + stat = self._stat(res[i][0], gts[i]) + eval_line += ' ||| {}'.format(stat) + + self.meteor_p.stdin.write('{}\n'.format(eval_line).encode()) + for i in range(0,len(imgIds)): + scores.append(float(self.meteor_p.stdout.readline().strip())) + score = float(self.meteor_p.stdout.readline().strip()) + self.lock.release() + + return score, scores + + def method(self): + return "METEOR" + + def _stat(self, hypothesis_str, reference_list): + # SCORE ||| reference 1 words ||| reference n words ||| hypothesis words + hypothesis_str = hypothesis_str.replace('|||','').replace(' ',' ') + score_line = ' ||| '.join(('SCORE', ' ||| '.join(reference_list), hypothesis_str)) + self.meteor_p.stdin.write('{}\n'.format(score_line).encode()) + return self.meteor_p.stdout.readline().decode().strip() + + def _score(self, hypothesis_str, reference_list): + self.lock.acquire() + # SCORE ||| reference 1 words ||| reference n words ||| hypothesis words + hypothesis_str = hypothesis_str.replace('|||','').replace(' ',' ') + score_line = ' ||| '.join(('SCORE', ' ||| '.join(reference_list), hypothesis_str)) + self.meteor_p.stdin.write('{}\n'.format(score_line)) + stats = self.meteor_p.stdout.readline().strip() + eval_line = 'EVAL ||| {}'.format(stats) + # EVAL ||| stats + self.meteor_p.stdin.write('{}\n'.format(eval_line)) + score = float(self.meteor_p.stdout.readline().strip()) + self.lock.release() + return score + + def __exit__(self): + self.lock.acquire() + self.meteor_p.stdin.close() + self.meteor_p.wait() + self.lock.release() diff --git a/refTools/evaluation/meteor/meteor.pyc b/refTools/evaluation/meteor/meteor.pyc new file mode 100644 index 0000000..fd280eb Binary files /dev/null and b/refTools/evaluation/meteor/meteor.pyc differ diff --git a/refTools/evaluation/readme.txt b/refTools/evaluation/readme.txt new file mode 100644 index 0000000..4491b5a --- /dev/null +++ b/refTools/evaluation/readme.txt @@ -0,0 +1,11 @@ +This folder contains modified coco-caption evaluation, which is downloaded from https://github.com/tylin/coco-caption.git +and refEvaluation which is to be called by the refer algorithm. + +More specifically, this folder contains: +1. bleu/ +2. cider/ +3. meteor/ +4. rouge/ +5. tokenizer/ +6. __init__.py +7. refEvaluation.py diff --git a/refTools/evaluation/refEvaluation.py b/refTools/evaluation/refEvaluation.py new file mode 100644 index 0000000..8045cb5 --- /dev/null +++ b/refTools/evaluation/refEvaluation.py @@ -0,0 +1,136 @@ +from refTools.evaluation.tokenizer.ptbtokenizer import PTBTokenizer +from refTools.evaluation.bleu.bleu import Bleu +from refTools.evaluation.meteor.meteor import Meteor +from refTools.evaluation.rouge.rouge import Rouge +from refTools.evaluation.cider.cider import Cider + +""" +Input: refer and Res = [{ref_id, sent}] + +Things of interest +evalRefs - list of ['ref_id', 'CIDEr', 'Bleu_1', 'Bleu_2', 'Bleu_3', 'Bleu_4', 'ROUGE_L', 'METEOR'] +eval - dict of {metric: score} +refToEval - dict of {ref_id: ['ref_id', 'CIDEr', 'Bleu_1', 'Bleu_2', 'Bleu_3', 'Bleu_4', 'ROUGE_L', 'METEOR']} +""" + +class RefEvaluation: + def __init__ (self, refer, Res): + """ + :param refer: refer class of current dataset + :param Res: [{'ref_id', 'sent'}] + """ + self.evalRefs = [] + self.eval = {} + self.refToEval = {} + self.refer = refer + self.Res = Res + + def evaluate(self): + + evalRefIds = [ann['ref_id'] for ann in self.Res] + + refToGts = {} + for ref_id in evalRefIds: + ref = self.refer.Refs[ref_id] + gt_sents = [sent['sent'].encode('ascii', 'ignore').decode('ascii') for sent in ref['sentences']] # up to 3 expressions + refToGts[ref_id] = gt_sents + refToRes = {ann['ref_id']: [ann['sent']] for ann in self.Res} + + print('tokenization...') + tokenizer = PTBTokenizer() + self.refToRes = tokenizer.tokenize(refToRes) + self.refToGts = tokenizer.tokenize(refToGts) + + # ================================================= + # Set up scorers + # ================================================= + print('setting up scorers...') + scorers = [ + (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]), + (Meteor(),"METEOR"), + (Rouge(), "ROUGE_L"), + (Cider(), "CIDEr") + ] + + # ================================================= + # Compute scores + # ================================================= + for scorer, method in scorers: + print('computing %s score...'%(scorer.method())) + score, scores = scorer.compute_score(self.refToGts, self.refToRes) + if type(method) == list: + for sc, scs, m in zip(score, scores, method): + self.setEval(sc, m) + self.setRefToEvalRefs(scs, self.refToGts.keys(), m) + print("%s: %0.3f"%(m, sc)) + else: + self.setEval(score, method) + self.setRefToEvalRefs(scores, self.refToGts.keys(), method) + print("%s: %0.3f"%(method, score)) + self.setEvalRefs() + + def setEval(self, score, method): + self.eval[method] = score + + def setRefToEvalRefs(self, scores, refIds, method): + for refId, score in zip(refIds, scores): + if not refId in self.refToEval: + self.refToEval[refId] = {} + self.refToEval[refId]["ref_id"] = refId + self.refToEval[refId][method] = score + + def setEvalRefs(self): + self.evalRefs = [eval for refId, eval in self.refToEval.items()] + + +if __name__ == '__main__': + + import os.path as osp + import sys + ROOT_DIR = osp.abspath(osp.join(osp.dirname(__file__), '..', '..')) + sys.path.insert(0, osp.join(ROOT_DIR, 'lib', 'datasets')) + from refer import REFER + + # load refer of dataset + dataset = 'refcoco' + refer = REFER(dataset, splitBy = 'google') + + # mimic some Res + val_refIds = refer.getRefIds(split='test') + ref_id = 49767 + print("GD: %s" % refer.Refs[ref_id]['sentences']) + Res = [{'ref_id': ref_id, 'sent': 'left bottle'}] + + # evaluate some refer expressions + refEval = RefEvaluation(refer, Res) + refEval.evaluate() + + # print output evaluation scores + for metric, score in refEval.eval.items(): + print('%s: %.3f'%(metric, score)) + + # demo how to use evalImgs to retrieve low score result + # evals = [eva for eva in refEval.evalRefs if eva['CIDEr']<30] + # print 'ground truth sents' + # refId = evals[0]['ref_id'] + # print 'refId: %s' % refId + # print [sent['sent'] for sent in refer.Refs[refId]['sentences']] + # + # print 'generated sent (CIDEr score %0.1f)' % (evals[0]['CIDEr']) + + # print refEval.refToEval[8] + + + + + + + + + + + + + + + diff --git a/refTools/evaluation/refEvaluation.pyc b/refTools/evaluation/refEvaluation.pyc new file mode 100644 index 0000000..a0c3b60 Binary files /dev/null and b/refTools/evaluation/refEvaluation.pyc differ diff --git a/refTools/evaluation/rouge/__init__.py b/refTools/evaluation/rouge/__init__.py new file mode 100644 index 0000000..43a773e --- /dev/null +++ b/refTools/evaluation/rouge/__init__.py @@ -0,0 +1 @@ +__author__ = 'vrama91' diff --git a/refTools/evaluation/rouge/__init__.pyc b/refTools/evaluation/rouge/__init__.pyc new file mode 100644 index 0000000..92f2e6b Binary files /dev/null and b/refTools/evaluation/rouge/__init__.pyc differ diff --git a/refTools/evaluation/rouge/__pycache__/__init__.cpython-36.pyc b/refTools/evaluation/rouge/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..f300eab Binary files /dev/null and b/refTools/evaluation/rouge/__pycache__/__init__.cpython-36.pyc differ diff --git a/refTools/evaluation/rouge/__pycache__/__init__.cpython-38.pyc b/refTools/evaluation/rouge/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..7b96e5c Binary files /dev/null and b/refTools/evaluation/rouge/__pycache__/__init__.cpython-38.pyc differ diff --git a/refTools/evaluation/rouge/__pycache__/rouge.cpython-36.pyc b/refTools/evaluation/rouge/__pycache__/rouge.cpython-36.pyc new file mode 100644 index 0000000..0219b69 Binary files /dev/null and b/refTools/evaluation/rouge/__pycache__/rouge.cpython-36.pyc differ diff --git a/refTools/evaluation/rouge/__pycache__/rouge.cpython-38.pyc b/refTools/evaluation/rouge/__pycache__/rouge.cpython-38.pyc new file mode 100644 index 0000000..7870dea Binary files /dev/null and b/refTools/evaluation/rouge/__pycache__/rouge.cpython-38.pyc differ diff --git a/refTools/evaluation/rouge/rouge.py b/refTools/evaluation/rouge/rouge.py new file mode 100644 index 0000000..3a10f5a --- /dev/null +++ b/refTools/evaluation/rouge/rouge.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# +# File Name : rouge.py +# +# Description : Computes ROUGE-L metric as described by Lin and Hovey (2004) +# +# Creation Date : 2015-01-07 06:03 +# Author : Ramakrishna Vedantam + +import numpy as np +import pdb + +def my_lcs(string, sub): + """ + Calculates longest common subsequence for a pair of tokenized strings + :param string : list of str : tokens from a string split using whitespace + :param sub : list of str : shorter string, also split using whitespace + :returns: length (list of int): length of the longest common subsequence between the two strings + + Note: my_lcs only gives length of the longest common subsequence, not the actual LCS + """ + if(len(string)< len(sub)): + sub, string = string, sub + + lengths = [[0 for i in range(0,len(sub)+1)] for j in range(0,len(string)+1)] + + for j in range(1,len(sub)+1): + for i in range(1,len(string)+1): + if(string[i-1] == sub[j-1]): + lengths[i][j] = lengths[i-1][j-1] + 1 + else: + lengths[i][j] = max(lengths[i-1][j] , lengths[i][j-1]) + + return lengths[len(string)][len(sub)] + +class Rouge(): + ''' + Class for computing ROUGE-L score for a set of candidate sentences for the MS COCO test set + + ''' + def __init__(self): + # vrama91: updated the value below based on discussion with Hovey + self.beta = 1.2 + + def calc_score(self, candidate, refs): + """ + Compute ROUGE-L score given one candidate and references for an image + :param candidate: str : candidate sentence to be evaluated + :param refs: list of str : COCO reference sentences for the particular image to be evaluated + :returns score: int (ROUGE-L score for the candidate evaluated against references) + """ + assert(len(candidate)==1) + assert(len(refs)>0) + prec = [] + rec = [] + + # split into tokens + token_c = candidate[0].split(" ") + + for reference in refs: + # split into tokens + token_r = reference.split(" ") + # compute the longest common subsequence + lcs = my_lcs(token_r, token_c) + prec.append(lcs/float(len(token_c))) + rec.append(lcs/float(len(token_r))) + + prec_max = max(prec) + rec_max = max(rec) + + if(prec_max!=0 and rec_max !=0): + score = ((1 + self.beta**2)*prec_max*rec_max)/float(rec_max + self.beta**2*prec_max) + else: + score = 0.0 + return score + + def compute_score(self, gts, res): + """ + Computes Rouge-L score given a set of reference and candidate sentences for the dataset + Invoked by evaluate_captions.py + :param hypo_for_image: dict : candidate / test sentences with "image name" key and "tokenized sentences" as values + :param ref_for_image: dict : reference MS-COCO sentences with "image name" key and "tokenized sentences" as values + :returns: average_score: float (mean ROUGE-L score computed by averaging scores for all the images) + """ + assert(gts.keys() == res.keys()) + imgIds = gts.keys() + + score = [] + for id in imgIds: + hypo = res[id] + ref = gts[id] + + score.append(self.calc_score(hypo, ref)) + + # Sanity check. + assert(type(hypo) is list) + assert(len(hypo) == 1) + assert(type(ref) is list) + assert(len(ref) > 0) + + average_score = np.mean(np.array(score)) + return average_score, np.array(score) + + def method(self): + return "Rouge" diff --git a/refTools/evaluation/rouge/rouge.pyc b/refTools/evaluation/rouge/rouge.pyc new file mode 100644 index 0000000..94d0c1c Binary files /dev/null and b/refTools/evaluation/rouge/rouge.pyc differ diff --git a/refTools/evaluation/tokenizer/__init__.py b/refTools/evaluation/tokenizer/__init__.py new file mode 100644 index 0000000..71357a4 --- /dev/null +++ b/refTools/evaluation/tokenizer/__init__.py @@ -0,0 +1 @@ +__author__ = 'hfang' diff --git a/refTools/evaluation/tokenizer/__init__.pyc b/refTools/evaluation/tokenizer/__init__.pyc new file mode 100644 index 0000000..eddf6e6 Binary files /dev/null and b/refTools/evaluation/tokenizer/__init__.pyc differ diff --git a/refTools/evaluation/tokenizer/__pycache__/__init__.cpython-36.pyc b/refTools/evaluation/tokenizer/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..9866bef Binary files /dev/null and b/refTools/evaluation/tokenizer/__pycache__/__init__.cpython-36.pyc differ diff --git a/refTools/evaluation/tokenizer/__pycache__/__init__.cpython-38.pyc b/refTools/evaluation/tokenizer/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..f45f5aa Binary files /dev/null and b/refTools/evaluation/tokenizer/__pycache__/__init__.cpython-38.pyc differ diff --git a/refTools/evaluation/tokenizer/__pycache__/ptbtokenizer.cpython-36.pyc b/refTools/evaluation/tokenizer/__pycache__/ptbtokenizer.cpython-36.pyc new file mode 100644 index 0000000..6cd5ef7 Binary files /dev/null and b/refTools/evaluation/tokenizer/__pycache__/ptbtokenizer.cpython-36.pyc differ diff --git a/refTools/evaluation/tokenizer/__pycache__/ptbtokenizer.cpython-38.pyc b/refTools/evaluation/tokenizer/__pycache__/ptbtokenizer.cpython-38.pyc new file mode 100644 index 0000000..127171d Binary files /dev/null and b/refTools/evaluation/tokenizer/__pycache__/ptbtokenizer.cpython-38.pyc differ diff --git a/refTools/evaluation/tokenizer/ptbtokenizer.py b/refTools/evaluation/tokenizer/ptbtokenizer.py new file mode 100644 index 0000000..52176fc --- /dev/null +++ b/refTools/evaluation/tokenizer/ptbtokenizer.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# +# File Name : ptbtokenizer.py +# +# Description : Do the PTB Tokenization and remove punctuations. +# +# Creation Date : 29-12-2014 +# Last Modified : Thu Mar 19 09:53:35 2015 +# Authors : Hao Fang and Tsung-Yi Lin + +import os +import sys +import subprocess +import tempfile +import itertools + +# path to the stanford corenlp jar +STANFORD_CORENLP_3_4_1_JAR = 'stanford-corenlp-3.4.1.jar' + +# punctuations to be removed from the sentences +PUNCTUATIONS = ["''", "'", "``", "`", "-LRB-", "-RRB-", "-LCB-", "-RCB-", \ + ".", "?", "!", ",", ":", "-", "--", "...", ";"] + +class PTBTokenizer: + """Python wrapper of Stanford PTBTokenizer""" + + def tokenize(self, captions_for_image): + cmd = ['java', '-cp', STANFORD_CORENLP_3_4_1_JAR, \ + 'edu.stanford.nlp.process.PTBTokenizer', \ + '-preserveLines', '-lowerCase'] + + # ====================================================== + # prepare data for PTB Tokenizer + # ====================================================== + final_tokenized_captions_for_image = {} + image_id = [k for k, v in captions_for_image.items() for _ in range(len(v))] + sentences = '\n'.join([c.replace('\n', ' ') for k, v in captions_for_image.items() for c in v]) + + # ====================================================== + # save sentences to temporary file + # ====================================================== + path_to_jar_dirname=os.path.dirname(os.path.abspath(__file__)) + tmp_file = tempfile.NamedTemporaryFile(delete=False, dir=path_to_jar_dirname) + tmp_file.write(sentences.encode()) + tmp_file.close() + + # ====================================================== + # tokenize sentence + # ====================================================== + cmd.append(os.path.basename(tmp_file.name)) + p_tokenizer = subprocess.Popen(cmd, cwd=path_to_jar_dirname, \ + stdout=subprocess.PIPE) + token_lines = p_tokenizer.communicate(input=sentences.rstrip())[0] + token_lines = token_lines.decode() + lines = token_lines.split('\n') + # remove temp file + os.remove(tmp_file.name) + + # ====================================================== + # create dictionary for tokenized captions + # ====================================================== + for k, line in zip(image_id, lines): + if not k in final_tokenized_captions_for_image: + final_tokenized_captions_for_image[k] = [] + tokenized_caption = ' '.join([w for w in line.rstrip().split(' ') \ + if w not in PUNCTUATIONS]) + final_tokenized_captions_for_image[k].append(tokenized_caption) + + return final_tokenized_captions_for_image diff --git a/refTools/evaluation/tokenizer/ptbtokenizer.pyc b/refTools/evaluation/tokenizer/ptbtokenizer.pyc new file mode 100644 index 0000000..62077b9 Binary files /dev/null and b/refTools/evaluation/tokenizer/ptbtokenizer.pyc differ diff --git a/refTools/evaluation/tokenizer/stanford-corenlp-3.4.1.jar b/refTools/evaluation/tokenizer/stanford-corenlp-3.4.1.jar new file mode 100644 index 0000000..3cfa0a0 Binary files /dev/null and b/refTools/evaluation/tokenizer/stanford-corenlp-3.4.1.jar differ diff --git a/refTools/evaluation/tokenizer/tmp37tp6xj8 b/refTools/evaluation/tokenizer/tmp37tp6xj8 new file mode 100644 index 0000000..13fc6e3 --- /dev/null +++ b/refTools/evaluation/tokenizer/tmp37tp6xj8 @@ -0,0 +1,7578 @@ +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the the trailer trailer trailer trailer trailer trailer trailer +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the fruit fruit fruit fruit fruit fruit fruit +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +of of course course course course course course course course +the the the snow snow snow snow snow snow snow +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the park park park park park park park +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the shop shop shop shop shop shop shop +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of buses buses buses buses buses buses buses buses +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the subway subway subway subway subway subway subway subway +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and groom groom groom groomedededed +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and chairs chairs chairs chairs chairs chairs chairs chairs +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +the the cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the tree tree tree tree tree tree tree +in in in in in in in in in in +the the the zoo zoo zoo zoo zoo zoo zoo +the the the game game game game game game game +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the team team team team team team team +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the wedding wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the snow snow snow snow snow snow snow snow +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the lake lake lake lake lake lake lake lake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the calculator calculator calculator calculator calculator calculator calculator +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the festival festival festival festival festival festival festival festival +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a bench bench bench bench bench bench bench bench +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the garden garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the the event event event event event event event +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +of of of people people people people people people people +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the dog dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the flag flag flag flag flag flag flag +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the boat boat boat boat boat boat boat +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the the birds birds birds birds birds birds birds +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +the the the tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the game game game game game game game +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pasture pasture pasture pasture pasture pasture pasture pasture +the the the temple temple temple temple temple temple temple +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the bed bed bed bed bed bed bed +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the summit summit summit summit summit summit summit summit +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the city city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of bananas bananas bananas bananas bananas bananas bananas bananas +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the office office office office office office office office +of of course course course course course course course course +and and and and and and and and and and +the the the field field field field field field field +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and white white white white white white white white +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the world world world world world world world +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +the the the shop shop shop shop shop shop shop +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of buses buses buses buses buses buses buses buses +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the table table table table table table table +the the glass glass glass glass glass glass glass glass +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +the the the sky sky sky sky sky sky sky +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the the zoo zoo zoo zoo zoo zoo zoo +the the the laptop laptop laptop laptop laptop laptop laptop +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the the street street street street street street street +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +the the dog dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the crowd crowd crowd crowd crowd crowd crowd crowd +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the movie movie movie movie movie movie movie +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the the pizza pizza pizza pizza pizza pizza pizza +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +the the the trailer trailer trailer trailer trailer trailer trailer +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the park park park park park park park +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the field field field field field field field +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the animals animals animals animals animals animals animals +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the subway subway subway subway subway subway subway subway +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +of of of of of of of of of of +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and groom groom groom groomedededed +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the tree tree tree tree tree tree tree +in in in in in in in in in in +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +the the wedding wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the team team team team team team team +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the lake lake lake lake lake lake lake lake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the team team team team team team team +and and and and and and and and and and +and and and and and and and and and and +the the the meeting meeting meeting meeting meeting meeting meeting +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the calculator calculator calculator calculator calculator calculator calculator +and and and and and and and and and and +the the fountain fountain fountain fountain fountain fountain fountain fountain +the the the party party party party party party party +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sunset sunset sunset sunset sunset sunset sunset sunset +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the fire fire fire fire fire fire fire fire +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the festival festival festival festival festival festival festival festival +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a bench bench bench bench bench bench bench bench +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +the the cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the garden garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of people people people people people people people +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the flag flag flag flag flag flag flag +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the table table table table table table table +and and and and and and and and and and +and and chairs chairs chairs chairs chairs chairs chairs chairs +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +the the tree tree tree tree tree tree tree tree +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +the the the birds birds birds birds birds birds birds +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the game game game game game game game +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +the the horse horse horse horse horse horse horse horse +and and and and and and and and and and +the the summit summit summit summit summit summit summit summit +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the city city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the team team team team team team team +in in in in in in in in in in +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the world world world world world world world +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the car car car car car car car +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the forest forest forest forest forest forest forest +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the toilet toilet toilet toilet toilet toilet toilet toilet +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +the the the table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the roof roof roof roof roof roof roof roof +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of buses buses buses buses buses buses buses buses +and and and and and and and and and and +and and and and and and and and and and +the the rocks rocks rocks rocks rocks rocks rocks rocks +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +in in in in in in in in in in +and and and and and and and and and and +the the the wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the glass glass glass glass glass glass glass glass +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the studio studio studio studio studio studio studio +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +the the the sky sky sky sky sky sky sky +and and and and and and and and and and +and and and and and and and and and and +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and person person person person person person person +the the beach beach beach beach beach beach beach beach +the the the zoo zoo zoo zoo zoo zoo zoo +the the the laptop laptop laptop laptop laptop laptop laptop +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the toilet toilet toilet toilet toilet toilet toilet toilet +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +the the toilet toilet toilet toilet toilet toilet toilet toilet +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the market market market market market market market market +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +the the the trailer trailer trailer trailer trailer trailer trailer +and and and and and and and and and and +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the movie movie movie movie movie movie movie +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the the pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +the the the bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +the the the park park park park park park park +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the field field field field field field field +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the subway subway subway subway subway subway subway subway +and and and and and and and and and and +and and and and and and and and and and +the the the animals animals animals animals animals animals animals +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +the the subway subway subway subway subway subway subway subway +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the show show show show show show show show +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +the the wedding wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the team team team team team team team +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the city city city city city city city city +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the snow snow snow snow snow snow snow snow +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the the team team team team team team team +and and and and and and and and and and +and and and and and and and and and and +the the the meeting meeting meeting meeting meeting meeting meeting +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +the the fountain fountain fountain fountain fountain fountain fountain fountain +the the the party party party party party party party +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sunset sunset sunset sunset sunset sunset sunset sunset +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the fire fire fire fire fire fire fire fire +the the the beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of apples apples apples apples apples apples apples apples +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +the the cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the garden garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of flowers flowers flowers flowers flowers flowers flowers flowers +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the flag flag flag flag flag flag flag +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the table table table table table table table +and and and and and and and and and and +and and chairs chairs chairs chairs chairs chairs chairs chairs +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the game game game game game game game +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the gym gym gym gym gym gym gym +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +of of birds birds birds birds birds birds birds birds +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the the bedroom bedroom bedroom bedroom bedroom bedroom bedroom +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the horse horse horse horse horse horse horse horse +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the wedding wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and carriage carriage carriage carriage carriage carriage carriage carriage +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +the the the team team team team team team team +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +the the the dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +the the rain rain rain rain rain rain rain rain +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the world world world world world world world +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the car car car car car car car +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the the beach beach beach beach beach beach beach +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the forest forest forest forest forest forest forest +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the wild wild wild wild wild wild wild +the the toilet toilet toilet toilet toilet toilet toilet toilet +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +the the the table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +of of of people people people people people people people +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the dog dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sign sign sign sign sign sign sign sign +and and and and and and and and and and +and and and and and and and and and and +the the rocks rocks rocks rocks rocks rocks rocks rocks +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +in in in in in in in in in in +and and and and and and and and and and +the the the wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the the pub pub pub pub pub pub pub +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the studio studio studio studio studio studio studio +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the toilet toilet toilet toilet toilet toilet toilet toilet +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the toilet toilet toilet toilet toilet toilet toilet toilet +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the market market market market market market market market +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +the the the trailer trailer trailer trailer trailer trailer trailer +the the the zoo zoo zoo zoo zoo zoo zoo +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the movie movie movie movie movie movie movie +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +the the the bus bus bus bus bus bus bus +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in love love love love love love love love +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bears bears bears bears bears bears bears bears +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the the animals animals animals animals animals animals animals +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +the the the city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and actor actor actor actor actor actor actor actor +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +in in in in in in in in in in +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the music music music music music music music +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the show show show show show show show show +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the team team team team team team team +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the sign sign sign sign sign sign sign sign +and and and and and and and and and and +and and and and and and and and and and +the the city city city city city city city city +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the snow snow snow snow snow snow snow snow +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the the team team team team team team team +and and and and and and and and and and +and and and and and and and and and and +the the the meeting meeting meeting meeting meeting meeting meeting +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +the the fountain fountain fountain fountain fountain fountain fountain fountain +the the the party party party party party party party +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sunset sunset sunset sunset sunset sunset sunset sunset +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the fire fire fire fire fire fire fire fire +the the the beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +the the street street street street street street street street +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of apples apples apples apples apples apples apples apples +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +the the cattle cattle cattle cattle cattle cattle cattle cattle +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and person person person person person person person person +and and and and and and and and and and +the the the baby baby baby baby baby baby baby +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the city city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of flowers flowers flowers flowers flowers flowers flowers flowers +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the flag flag flag flag flag flag flag +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the bedroom bedroom bedroom bedroom bedroom bedroom bedroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +in in in in in ski ski ski ski ski +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the game game game game game game game +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the gym gym gym gym gym gym gym +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of birds birds birds birds birds birds birds birds +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the the bedroom bedroom bedroom bedroom bedroom bedroom bedroom +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the forest forest forest forest forest forest forest +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the boat boat boat boat boat boat boat boat +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the wedding wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the ship ship ship ship ship ship ship +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and carriage carriage carriage carriage carriage carriage carriage carriage +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +of of glasses glasses glasses glasses glasses glasses glasses glasses +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +the the the team team team team team team team +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the protest protest protest protest protest protest protest protest +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the car car car car car car car +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +of of course course course course course course course course +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the the beach beach beach beach beach beach beach +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the field field field field field field field field +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +the the the wild wild wild wild wild wild wild +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of people people people people people people people +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a aaaaaaaa +the the dog dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sign sign sign sign sign sign sign sign +and and and and and and and and and and +the the snow snow snow snow snow snow snow snow +and and and and and and and and and and +the the the event event event event event event event +the the sheep sheep sheep sheep sheep sheep sheep sheep +the the the river river river river river river river +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the wedding wedding wedding wedding wedding wedding wedding +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +of of wine wine wine wine wine wine wine wine +in in in in in in in in in in +the the the pub pub pub pub pub pub pub +a a bench bench bench bench bench bench bench bench +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the oven oven oven oven oven oven oven oven +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the bedroom bedroom bedroom bedroom bedroom bedroom bedroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the market market market market market market market market +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +the the the trailer trailer trailer trailer trailer trailer trailer +the the the zoo zoo zoo zoo zoo zoo zoo +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +the the the park park park park park park park +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +the the summit summit summit summit summit summit summit summit +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the train train train train train train train train +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in love love love love love love love love +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bears bears bears bears bears bears bears bears +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a carriage carriage carriage carriage carriage carriage carriage carriage +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and actor actor actor actor actor actor actor actor +and and and and and and and and and and +the the the car car car car car car car +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +in in in in in in in in in in +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the music music music music music music music +the the street street street street street street street street +and and and and and and and and and and +and and and and and and and and and and +the the show show show show show show show show +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the market market market market market market market market +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the ranch ranch ranch ranch ranch ranch ranch +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +the the table table table table table table table table +the the the summit summit summit summit summit summit summit +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the sign sign sign sign sign sign sign sign +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the canal canal canal canal canal canal canal +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the the team team team team team team team +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +the the the party party party party party party party +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +of of course course course course course course course course +and and chairs chairs chairs chairs chairs chairs chairs chairs +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and person person person person person person person person +and and and and and and and and and and +the the the baby baby baby baby baby baby baby +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the city city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the event event event event event event event +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a boat boat boat boat boat boat boat boat +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the flowers flowers flowers flowers flowers flowers flowers flowers +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the bedroom bedroom bedroom bedroom bedroom bedroom bedroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in ski ski ski ski ski +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the the bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the farm farm farm farm farm farm farm farm +and and and and and and and and and and +the the barn barn barn barn barn barn barn barn +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the forest forest forest forest forest forest forest +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +the the the game game game game game game game +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the boat boat boat boat boat boat boat boat +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the wedding wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the ship ship ship ship ship ship ship +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of glasses glasses glasses glasses glasses glasses glasses glasses +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +the the the team team team team team team team +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a banana banana banana banana banana banana banana banana +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +of of course course course course course course course course +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the road road road road road road road +the the the ship ship ship ship ship ship ship +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the meeting meeting meeting meeting meeting meeting meeting meeting +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the field field field field field field field field +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the the field field field field field field field +the the the wild wild wild wild wild wild wild +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in love love love love love love love love +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of people people people people people people people +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +a a aaaaaaaa +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the event event event event event event event +the the sheep sheep sheep sheep sheep sheep sheep sheep +the the the river river river river river river river +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the wedding wedding wedding wedding wedding wedding wedding +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +of of wine wine wine wine wine wine wine wine +the the the zoo zoo zoo zoo zoo zoo zoo +the the the pub pub pub pub pub pub pub +a a bench bench bench bench bench bench bench bench +and and and and and and and and and and +of of flowers flowers flowers flowers flowers flowers flowers flowers +and and and and and and and and and and +the the oven oven oven oven oven oven oven oven +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the bedroom bedroom bedroom bedroom bedroom bedroom bedroom +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +the the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +the the summit summit summit summit summit summit summit summit +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the train train train train train train train train +the the bottle bottle bottle bottle bottle bottle bottle bottle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a carriage carriage carriage carriage carriage carriage carriage carriage +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom bathroom +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the car car car car car car car +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the market market market market market market market market +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +the the cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the ranch ranch ranch ranch ranch ranch ranch +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +the the table table table table table table table table +the the the summit summit summit summit summit summit summit +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the meeting meeting meeting meeting meeting meeting meeting meeting +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +the the the gym gym gym gym gym gym gym +and and and and and and and and and and +the the the canal canal canal canal canal canal canal +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +the the the team team team team team team team +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the night night night night night night night +the the the calculator calculator calculator calculator calculator calculator calculator +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and chairs chairs chairs chairs chairs chairs chairs chairs +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the city city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the event event event event event event event +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +a a boat boat boat boat boat boat boat boat +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the flowers flowers flowers flowers flowers flowers flowers flowers +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in ski ski ski ski ski +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the table table table table table table table table +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the the bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the farm farm farm farm farm farm farm farm +and and and and and and and and and and +the the barn barn barn barn barn barn barn barn +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +the the the game game game game game game game +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the wedding wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the ship ship ship ship ship ship ship +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the forest forest forest forest forest forest forest +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the park park park park park park park +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of glasses glasses glasses glasses glasses glasses glasses glasses +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the team team team team team team team +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the event event event event event event event +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the horse horse horse horse horse horse and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the road road road road road road road +the the the ship ship ship ship ship ship ship +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the meeting meeting meeting meeting meeting meeting meeting meeting +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the field field field field field field field field +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the shop shop shop shop shop shop shop +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the field field field field field field field +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in love love love love love love love love +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +a a aaaaaaaa +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the the event event event event event event event +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the table table table table table table table +of of wine wine wine wine wine wine wine wine +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of flowers flowers flowers flowers flowers flowers flowers flowers +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +the the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +the the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +the the summit summit summit summit summit summit summit summit +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the tree tree tree tree tree tree tree tree +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bottle bottle bottle bottle bottle bottle bottle bottle +the the beach beach beach beach beach beach beach beach +the the the game game game game game game game +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the shop shop shop shop shop shop shop +of of of course course course course course course course +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the farm farm farm farm farm farm farm +the the classroom classroom classroom classroom classroom classroom classroom classroom +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +in in in in in in in in in in +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and person person person person person person person +the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom bathroom +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the event event event event event event event +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +the the cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of bicycles bicycles bicycles bicycles bicycles bicycles bicycles bicycles +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the wedding wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the window window window window window window window window +the the meeting meeting meeting meeting meeting meeting meeting meeting +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +the the the gym gym gym gym gym gym gym +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +the the the night night night night night night night +the the the calculator calculator calculator calculator calculator calculator calculator +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the oven oven oven oven oven oven oven oven +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the deck deck deck deck deck deck deck deck +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the event event event event event event event +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the street street street street street street street street +of of of people people people people people people people +and and and person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the dog dog dog dog dog dog dog dog +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the flag flag flag flag flag flag flag +the the flowers flowers flowers flowers flowers flowers flowers flowers +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the show show show show show show show +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the oven oven oven oven oven oven oven oven +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the laptop laptop laptop laptop laptop laptop laptop laptop +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the bed bed bed bed bed bed bed +and and and and and and and and and and +and and and and and and and and and and +the the the game game game game game game game +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the ship ship ship ship ship ship ship +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the forest forest forest forest forest forest forest +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the park park park park park park park +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and helmet helmet helmet helmet helmet helmet helmet helmet +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the team team team team team team team +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +the the the event event event event event event event +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the horse horse horse horse horse horse and and +and and and and and and and and and and +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the cake cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the road road road road road road road +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the meeting meeting meeting meeting meeting meeting meeting meeting +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the shop shop shop shop shop shop shop +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the wild wild wild wild wild wild wild wild +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the table table table table table table table +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of flowers flowers flowers flowers flowers flowers flowers flowers +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +the the the oven oven oven oven oven oven oven +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the tree tree tree tree tree tree tree tree +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the beach beach beach beach beach beach beach beach +the the the game game game game game game game +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the the trailer trailer trailer trailer trailer trailer trailer +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the fruit fruit fruit fruit fruit fruit fruit +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +of of course course course course course course course course +the the the snow snow snow snow snow snow snow +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the shop shop shop shop shop shop shop +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of buses buses buses buses buses buses buses buses +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the farm farm farm farm farm farm farm +the the classroom classroom classroom classroom classroom classroom classroom classroom +and and and and and and and and and and +and and and and and and and and and and +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +in in in in in in in in in in +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +the the the clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and person person person person person person person +the the zoo zoo zoo zoo zoo zoo zoo zoo +of of sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the event event event event event event event +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and chairs chairs chairs chairs chairs chairs chairs chairs +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the rain rain rain rain rain rain rain rain +and and and and and and and and and and +the the cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the the game game game game game game game +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +the the the garden garden garden garden garden garden garden +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the wedding wedding wedding wedding wedding wedding wedding wedding +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the window window window window window window window window +the the meeting meeting meeting meeting meeting meeting meeting meeting +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the snow snow snow snow snow snow snow snow +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +the the the calculator calculator calculator calculator calculator calculator calculator +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +the the crowd crowd crowd crowd crowd crowd crowd crowd +the the kitchen kitchen kitchen kitchen kitchen kitchen kitchen kitchen +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +the the oven oven oven oven oven oven oven oven +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the deck deck deck deck deck deck deck deck +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the the event event event event event event event +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and person person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +the the street street street street street street street street +of of of people people people people people people people +and and and person person person person person person person +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the dog dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the flag flag flag flag flag flag flag +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the boat boat boat boat boat boat boat +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the grill grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +of of of of of of of of of of +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the show show show show show show show +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the the garden garden garden garden garden garden garden +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +the the the tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes tomatoes +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +the the oven oven oven oven oven oven oven oven +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the grill grill grill grill grill grill grill +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the laptop laptop laptop laptop laptop laptop laptop laptop +the the pasture pasture pasture pasture pasture pasture pasture pasture +the the the temple temple temple temple temple temple temple +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the bed bed bed bed bed bed bed +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bus bus bus bus bus bus bus bus +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the forest forest forest forest forest forest forest +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the farm farm farm farm farm farm farm +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of bananas bananas bananas bananas bananas bananas bananas bananas +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the sheep sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the office office office office office office office office +of of course course course course course course course course +and and and and and and and and and and +the the the field field field field field field field +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and white white white white white white white white +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +and and and and and and and and and and +and and and and and and and and and and +the the the event event event event event event event +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the world world world world world world world +the the table table table table table table table table +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +the the cake cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the city city city city city city city +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of cattle cattle cattle cattle cattle cattle cattle cattle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the meeting meeting meeting meeting meeting meeting meeting meeting +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +the the the shop shop shop shop shop shop shop +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the wild wild wild wild wild wild wild wild +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the bathroom bathroom bathroom bathroom bathroom bathroom bathroom bathroom +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of buses buses buses buses buses buses buses buses +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +in in in in in in in in in in +and and and and and and and and and and +the the the zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the table table table table table table table +in in in in in in in in in in +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of flowers flowers flowers flowers flowers flowers flowers flowers +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the zoo zoo zoo zoo zoo zoo zoo zoo +and and and and and and and and and and +and and and and and and and and and and +the the clock clock clock clock clock clock clock clock +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the pizza pizza pizza pizza pizza pizza pizza pizza +the the the truck truck truck truck truck truck truck +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the beach beach beach beach beach beach beach beach +the the the street street street street street street street +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of of course course course course course course course +and and and and and and and and and and +the the dog dog dog dog dog dog dog dog +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the crowd crowd crowd crowd crowd crowd crowd crowd +and and and and and and and and and and +and and and and and and and and and and +the the the cake cake cake cake cake cake cake +and and and and and and and and and and +and and and and and and and and and and +the the the motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle motorcycle +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the sheep sheep sheep sheep sheep sheep sheep +and and and and and and and and and and +the the the oven oven oven oven oven oven oven +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +of of course course course course course course course course +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +the the the pizza pizza pizza pizza pizza pizza pizza +the the pizza pizza pizza pizza pizza pizza pizza pizza +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and +and and and and and and and and and and \ No newline at end of file diff --git a/refTools/evaluation/tokenizer/tmp82iqkuu0 b/refTools/evaluation/tokenizer/tmp82iqkuu0 new file mode 100644 index 0000000..e69de29 diff --git a/refTools/evaluation/tokenizer/tmpn19wmqte b/refTools/evaluation/tokenizer/tmpn19wmqte new file mode 100644 index 0000000..e69de29 diff --git a/refTools/refer_python3.py b/refTools/refer_python3.py new file mode 100644 index 0000000..0f4f816 --- /dev/null +++ b/refTools/refer_python3.py @@ -0,0 +1,252 @@ +__author__ = 'licheng' + +""" +This interface provides access to four datasets: +1) refclef +2) refcoco +3) refcoco+ +4) refcocog +split by unc and google + +The following API functions are defined: +REFER - REFER api class +getRefIds - get ref ids that satisfy given filter conditions. +getAnnIds - get ann ids that satisfy given filter conditions. +getImgIds - get image ids that satisfy given filter conditions. +getCatIds - get category ids that satisfy given filter conditions. +loadRefs - load refs with the specified ref ids. +loadAnns - load anns with the specified ann ids. +loadImgs - load images with the specified image ids. +loadCats - load category names with the specified category ids. +getRefBox - get ref's bounding box [x, y, w, h] given the ref_id +""" + +import sys +import os.path as osp +import json +import _pickle as pickle +import time +import itertools +import skimage.io as io +import matplotlib.pyplot as plt +from matplotlib.collections import PatchCollection +from matplotlib.patches import Polygon, Rectangle +from pprint import pprint +import numpy as np +# import cv2 +# from skimage.measure import label, regionprops + +class REFER: + + def __init__(self, data_root, dataset='refcoco', splitBy='unc'): + # provide data_root folder which contains refclef, refcoco, refcoco+ and refcocog + # also provide dataset name and splitBy information + # e.g., dataset = 'refcoco', splitBy = 'unc' + print('loading dataset %s into memory...' % dataset) + self.ROOT_DIR = osp.abspath(osp.dirname(__file__)) + self.DATA_DIR = osp.join(data_root, dataset) + if dataset in ['refcoco', 'refcoco+', 'refcocog']: + self.IMAGE_DIR = osp.join(data_root, 'images/mscoco/images/train2014') + elif dataset == 'refclef': + self.IMAGE_DIR = osp.join(data_root, 'images/saiapr_tc-12') + else: + print('No refer dataset is called [%s]' % dataset) + sys.exit() + + # load refs from data/dataset/refs(dataset).json + tic = time.time() + ref_file = osp.join(self.DATA_DIR, 'refs('+splitBy+').p') + self.data = {} + self.data['dataset'] = dataset + self.data['refs'] = pickle.load(open(ref_file, 'rb')) + + # load annotations from data/dataset/instances.json + instances_file = osp.join(self.DATA_DIR, 'instances.json') + instances = json.load(open(instances_file, 'r')) + self.data['images'] = instances['images'] + self.data['annotations'] = instances['annotations'] + self.data['categories'] = instances['categories'] + + # create index + self.createIndex() + print('DONE (t=%.2fs)' % (time.time()-tic)) + + def createIndex(self): + # create sets of mapping + # 1) Refs: {ref_id: ref} + # 2) Anns: {ann_id: ann} + # 3) Imgs: {image_id: image} + # 4) Cats: {category_id: category_name} + # 5) Sents: {sent_id: sent} + # 6) imgToRefs: {image_id: refs} + # 7) imgToAnns: {image_id: anns} + # 8) refToAnn: {ref_id: ann} + # 9) annToRef: {ann_id: ref} + # 10) catToRefs: {category_id: refs} + # 11) sentToRef: {sent_id: ref} + # 12) sentToTokens: {sent_id: tokens} + print('creating index...') + # fetch info from instances + Anns, Imgs, Cats, imgToAnns = {}, {}, {}, {} + for ann in self.data['annotations']: + Anns[ann['id']] = ann + imgToAnns[ann['image_id']] = imgToAnns.get(ann['image_id'], []) + [ann] + for img in self.data['images']: + Imgs[img['id']] = img + for cat in self.data['categories']: + Cats[cat['id']] = cat['name'] + + # fetch info from refs + Refs, imgToRefs, refToAnn, annToRef, catToRefs = {}, {}, {}, {}, {} + Sents, sentToRef, sentToTokens = {}, {}, {} + for ref in self.data['refs']: + # ids + ref_id = ref['ref_id'] + ann_id = ref['ann_id'] + category_id = ref['category_id'] + image_id = ref['image_id'] + + # add mapping related to ref + Refs[ref_id] = ref + imgToRefs[image_id] = imgToRefs.get(image_id, []) + [ref] + catToRefs[category_id] = catToRefs.get(category_id, []) + [ref] + refToAnn[ref_id] = Anns[ann_id] + annToRef[ann_id] = ref + + # add mapping of sent + for sent in ref['sentences']: + Sents[sent['sent_id']] = sent + sentToRef[sent['sent_id']] = ref + sentToTokens[sent['sent_id']] = sent['tokens'] + + # create class members + self.Refs = Refs + self.Anns = Anns + self.Imgs = Imgs + self.Cats = Cats + self.Sents = Sents + self.imgToRefs = imgToRefs + self.imgToAnns = imgToAnns + self.refToAnn = refToAnn + self.annToRef = annToRef + self.catToRefs = catToRefs + self.sentToRef = sentToRef + self.sentToTokens = sentToTokens + print('index created.') + + def getRefIds(self, image_ids=[], cat_ids=[], ref_ids=[], split=''): + image_ids = image_ids if type(image_ids) == list else [image_ids] + cat_ids = cat_ids if type(cat_ids) == list else [cat_ids] + ref_ids = ref_ids if type(ref_ids) == list else [ref_ids] + + if len(image_ids)==len(cat_ids)==len(ref_ids)==len(split)==0: + refs = self.data['refs'] + else: + if not len(image_ids) == 0: + refs = [self.imgToRefs[image_id] for image_id in image_ids] + else: + refs = self.data['refs'] + if not len(cat_ids) == 0: + refs = [ref for ref in refs if ref['category_id'] in cat_ids] + if not len(ref_ids) == 0: + refs = [ref for ref in refs if ref['ref_id'] in ref_ids] + if not len(split) == 0: + if split in ['testA', 'testB', 'testC']: + refs = [ref for ref in refs if split[-1] in ref['split']] # we also consider testAB, testBC, ... + elif split in ['testAB', 'testBC', 'testAC']: + refs = [ref for ref in refs if ref['split'] == split] # rarely used I guess... + elif split == 'test': + refs = [ref for ref in refs if 'test' in ref['split']] + elif split == 'train' or split == 'val': + refs = [ref for ref in refs if ref['split'] == split] + else: + print('No such split [%s]' % split) + sys.exit() + ref_ids = [ref['ref_id'] for ref in refs] + return ref_ids + + def getAnnIds(self, image_ids=[], cat_ids=[], ref_ids=[]): + image_ids = image_ids if type(image_ids) == list else [image_ids] + cat_ids = cat_ids if type(cat_ids) == list else [cat_ids] + ref_ids = ref_ids if type(ref_ids) == list else [ref_ids] + + if len(image_ids) == len(cat_ids) == len(ref_ids) == 0: + ann_ids = [ann['id'] for ann in self.data['annotations']] + else: + if not len(image_ids) == 0: + lists = [self.imgToAnns[image_id] for image_id in image_ids if image_id in self.imgToAnns] # list of [anns] + anns = list(itertools.chain.from_iterable(lists)) + else: + anns = self.data['annotations'] + if not len(cat_ids) == 0: + anns = [ann for ann in anns if ann['category_id'] in cat_ids] + ann_ids = [ann['id'] for ann in anns] + if not len(ref_ids) == 0: + ids = set(ann_ids).intersection(set([self.Refs[ref_id]['ann_id'] for ref_id in ref_ids])) + return ann_ids + + def getImgIds(self, ref_ids=[]): + ref_ids = ref_ids if type(ref_ids) == list else [ref_ids] + + if not len(ref_ids) == 0: + image_ids = list(set([self.Refs[ref_id]['image_id'] for ref_id in ref_ids])) + else: + image_ids = self.Imgs.keys() + return image_ids + + def getCatIds(self): + return self.Cats.keys() + + def loadRefs(self, ref_ids=[]): + if type(ref_ids) == list: + return [self.Refs[ref_id] for ref_id in ref_ids] + elif type(ref_ids) == int: + return [self.Refs[ref_ids]] + + def loadAnns(self, ann_ids=[]): + if type(ann_ids) == list: + return [self.Anns[ann_id] for ann_id in ann_ids] + elif type(ann_ids) == int or type(ann_ids) == unicode: + return [self.Anns[ann_ids]] + + def loadImgs(self, image_ids=[]): + if type(image_ids) == list: + return [self.Imgs[image_id] for image_id in image_ids] + elif type(image_ids) == int: + return [self.Imgs[image_ids]] + + def loadCats(self, cat_ids=[]): + if type(cat_ids) == list: + return [self.Cats[cat_id] for cat_id in cat_ids] + elif type(cat_ids) == int: + return [self.Cats[cat_ids]] + + def getRefBox(self, ref_id): + ref = self.Refs[ref_id] + ann = self.refToAnn[ref_id] + return ann['bbox'] # [x, y, w, h] + + + +if __name__ == '__main__': + refer = REFER(dataset='refcocog', splitBy='google') + ref_ids = refer.getRefIds() + print(len(ref_ids)) + + print(len(refer.Imgs)) + print(len(refer.imgToRefs)) + + ref_ids = refer.getRefIds(split='train') + print('There are %s training referred objects.' % len(ref_ids)) + + for ref_id in ref_ids: + ref = refer.loadRefs(ref_id)[0] + if len(ref['sentences']) < 2: + continue + + pprint(ref) + print('The label is %s.' % refer.Cats[ref['category_id']]) + plt.figure() + refer.showRef(ref, seg_box='box') + plt.show() + diff --git a/scheduler/__init__.py b/scheduler/__init__.py new file mode 100644 index 0000000..6a77898 --- /dev/null +++ b/scheduler/__init__.py @@ -0,0 +1,5 @@ +from .cosine_lr import CosineLRScheduler +from .plateau_lr import PlateauLRScheduler +from .step_lr import StepLRScheduler +from .tanh_lr import TanhLRScheduler +from .scheduler_factory import create_scheduler diff --git a/scheduler/__pycache__/__init__.cpython-36.pyc b/scheduler/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..177818f Binary files /dev/null and b/scheduler/__pycache__/__init__.cpython-36.pyc differ diff --git a/scheduler/__pycache__/__init__.cpython-38.pyc b/scheduler/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..d7909a9 Binary files /dev/null and b/scheduler/__pycache__/__init__.cpython-38.pyc differ diff --git a/scheduler/__pycache__/cosine_lr.cpython-36.pyc b/scheduler/__pycache__/cosine_lr.cpython-36.pyc new file mode 100644 index 0000000..e18cd10 Binary files /dev/null and b/scheduler/__pycache__/cosine_lr.cpython-36.pyc differ diff --git a/scheduler/__pycache__/cosine_lr.cpython-38.pyc b/scheduler/__pycache__/cosine_lr.cpython-38.pyc new file mode 100644 index 0000000..10fe0ac Binary files /dev/null and b/scheduler/__pycache__/cosine_lr.cpython-38.pyc differ diff --git a/scheduler/__pycache__/plateau_lr.cpython-36.pyc b/scheduler/__pycache__/plateau_lr.cpython-36.pyc new file mode 100644 index 0000000..5668ad2 Binary files /dev/null and b/scheduler/__pycache__/plateau_lr.cpython-36.pyc differ diff --git a/scheduler/__pycache__/plateau_lr.cpython-38.pyc b/scheduler/__pycache__/plateau_lr.cpython-38.pyc new file mode 100644 index 0000000..d6c36bf Binary files /dev/null and b/scheduler/__pycache__/plateau_lr.cpython-38.pyc differ diff --git a/scheduler/__pycache__/scheduler.cpython-36.pyc b/scheduler/__pycache__/scheduler.cpython-36.pyc new file mode 100644 index 0000000..3b74bb6 Binary files /dev/null and b/scheduler/__pycache__/scheduler.cpython-36.pyc differ diff --git a/scheduler/__pycache__/scheduler.cpython-38.pyc b/scheduler/__pycache__/scheduler.cpython-38.pyc new file mode 100644 index 0000000..4d09802 Binary files /dev/null and b/scheduler/__pycache__/scheduler.cpython-38.pyc differ diff --git a/scheduler/__pycache__/scheduler_factory.cpython-36.pyc b/scheduler/__pycache__/scheduler_factory.cpython-36.pyc new file mode 100644 index 0000000..54fc9b7 Binary files /dev/null and b/scheduler/__pycache__/scheduler_factory.cpython-36.pyc differ diff --git a/scheduler/__pycache__/scheduler_factory.cpython-38.pyc b/scheduler/__pycache__/scheduler_factory.cpython-38.pyc new file mode 100644 index 0000000..c068186 Binary files /dev/null and b/scheduler/__pycache__/scheduler_factory.cpython-38.pyc differ diff --git a/scheduler/__pycache__/step_lr.cpython-36.pyc b/scheduler/__pycache__/step_lr.cpython-36.pyc new file mode 100644 index 0000000..739cd45 Binary files /dev/null and b/scheduler/__pycache__/step_lr.cpython-36.pyc differ diff --git a/scheduler/__pycache__/step_lr.cpython-38.pyc b/scheduler/__pycache__/step_lr.cpython-38.pyc new file mode 100644 index 0000000..df52be0 Binary files /dev/null and b/scheduler/__pycache__/step_lr.cpython-38.pyc differ diff --git a/scheduler/__pycache__/tanh_lr.cpython-36.pyc b/scheduler/__pycache__/tanh_lr.cpython-36.pyc new file mode 100644 index 0000000..db971a8 Binary files /dev/null and b/scheduler/__pycache__/tanh_lr.cpython-36.pyc differ diff --git a/scheduler/__pycache__/tanh_lr.cpython-38.pyc b/scheduler/__pycache__/tanh_lr.cpython-38.pyc new file mode 100644 index 0000000..91114bc Binary files /dev/null and b/scheduler/__pycache__/tanh_lr.cpython-38.pyc differ diff --git a/scheduler/cosine_lr.py b/scheduler/cosine_lr.py new file mode 100644 index 0000000..3349e73 --- /dev/null +++ b/scheduler/cosine_lr.py @@ -0,0 +1,117 @@ +""" Cosine Scheduler + +Cosine LR schedule with warmup, cycle/restarts, noise. + +Hacked together by / Copyright 2020 Ross Wightman +""" +import logging +import math +import numpy as np +import torch + +from .scheduler import Scheduler + +from pdb import set_trace as breakpoint + +_logger = logging.getLogger(__name__) + + +class CosineLRScheduler(Scheduler): + """ + Cosine decay with restarts. + This is described in the paper https://arxiv.org/abs/1608.03983. + + Inspiration from + https://github.com/allenai/allennlp/blob/master/allennlp/training/learning_rate_schedulers/cosine.py + """ + + def __init__(self, + optimizer: torch.optim.Optimizer, + t_initial: int, + t_mul: float = 1., + lr_min: float = 0., + decay_rate: float = 1., + warmup_t=0, + warmup_lr_init=0, + warmup_prefix=True, + cycle_limit=0, + t_in_epochs=True, + noise_range_t=None, + noise_pct=0.67, + noise_std=1.0, + noise_seed=42, + initialize=True) -> None: + super().__init__( + optimizer, param_group_field="lr", + noise_range_t=noise_range_t, noise_pct=noise_pct, noise_std=noise_std, noise_seed=noise_seed, + initialize=initialize) + + assert t_initial > 0 + assert lr_min >= 0 + if t_initial == 1 and t_mul == 1 and decay_rate == 1: + _logger.warning("Cosine annealing scheduler will have no effect on the learning " + "rate since t_initial = t_mul = eta_mul = 1.") + self.t_initial = t_initial + self.t_mul = t_mul + self.lr_min = lr_min + self.decay_rate = decay_rate + self.cycle_limit = cycle_limit + self.warmup_t = warmup_t + self.warmup_lr_init = warmup_lr_init + self.warmup_prefix = warmup_prefix + self.t_in_epochs = t_in_epochs + if self.warmup_t: + self.warmup_steps = [(v - warmup_lr_init) / self.warmup_t for v in self.base_values] + super().update_groups(self.warmup_lr_init) + else: + self.warmup_steps = [1 for _ in self.base_values] + + def _get_lr(self, t): + if t < self.warmup_t: + lrs = [self.warmup_lr_init + t * s for s in self.warmup_steps] + else: + if self.warmup_prefix: + t = t - self.warmup_t + + if self.t_mul != 1: + i = math.floor(math.log(1 - t / self.t_initial * (1 - self.t_mul), self.t_mul)) + t_i = self.t_mul ** i * self.t_initial + t_curr = t - (1 - self.t_mul ** i) / (1 - self.t_mul) * self.t_initial + else: + i = t // self.t_initial + t_i = self.t_initial + t_curr = t - (self.t_initial * i) + + gamma = self.decay_rate ** i + lr_min = self.lr_min * gamma + lr_max_values = [v * gamma for v in self.base_values] + + if self.cycle_limit == 0 or (self.cycle_limit > 0 and i < self.cycle_limit): + lrs = [ + lr_min + 0.5 * (lr_max - lr_min) * (1 + math.cos(math.pi * t_curr / t_i)) for lr_max in lr_max_values + ] + else: + lrs = [self.lr_min for _ in self.base_values] + + return lrs + + def get_epoch_values(self, epoch: int): + if self.t_in_epochs: + return self._get_lr(epoch) + else: + return None + + def get_update_values(self, num_updates: int): + if not self.t_in_epochs: + return self._get_lr(num_updates) + else: + return None + + def get_cycle_length(self, cycles=0): + if not cycles: + cycles = self.cycle_limit + cycles = max(1, cycles) + if self.t_mul == 1.0: + return self.t_initial * cycles + else: + return int(math.floor(-self.t_initial * (self.t_mul ** cycles - 1) / (1 - self.t_mul))) diff --git a/scheduler/plateau_lr.py b/scheduler/plateau_lr.py new file mode 100644 index 0000000..4f2cacb --- /dev/null +++ b/scheduler/plateau_lr.py @@ -0,0 +1,113 @@ +""" Plateau Scheduler + +Adapts PyTorch plateau scheduler and allows application of noise, warmup. + +Hacked together by / Copyright 2020 Ross Wightman +""" +import torch + +from .scheduler import Scheduler + + +class PlateauLRScheduler(Scheduler): + """Decay the LR by a factor every time the validation loss plateaus.""" + + def __init__(self, + optimizer, + decay_rate=0.1, + patience_t=10, + verbose=True, + threshold=1e-4, + cooldown_t=0, + warmup_t=0, + warmup_lr_init=0, + lr_min=0, + mode='max', + noise_range_t=None, + noise_type='normal', + noise_pct=0.67, + noise_std=1.0, + noise_seed=None, + initialize=True, + ): + super().__init__(optimizer, 'lr', initialize=initialize) + + self.lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( + self.optimizer, + patience=patience_t, + factor=decay_rate, + verbose=verbose, + threshold=threshold, + cooldown=cooldown_t, + mode=mode, + min_lr=lr_min + ) + + self.noise_range = noise_range_t + self.noise_pct = noise_pct + self.noise_type = noise_type + self.noise_std = noise_std + self.noise_seed = noise_seed if noise_seed is not None else 42 + self.warmup_t = warmup_t + self.warmup_lr_init = warmup_lr_init + if self.warmup_t: + self.warmup_steps = [(v - warmup_lr_init) / self.warmup_t for v in self.base_values] + super().update_groups(self.warmup_lr_init) + else: + self.warmup_steps = [1 for _ in self.base_values] + self.restore_lr = None + + def state_dict(self): + return { + 'best': self.lr_scheduler.best, + 'last_epoch': self.lr_scheduler.last_epoch, + } + + def load_state_dict(self, state_dict): + self.lr_scheduler.best = state_dict['best'] + if 'last_epoch' in state_dict: + self.lr_scheduler.last_epoch = state_dict['last_epoch'] + + # override the base class step fn completely + def step(self, epoch, metric=None): + if epoch <= self.warmup_t: + lrs = [self.warmup_lr_init + epoch * s for s in self.warmup_steps] + super().update_groups(lrs) + else: + if self.restore_lr is not None: + # restore actual LR from before our last noise perturbation before stepping base + for i, param_group in enumerate(self.optimizer.param_groups): + param_group['lr'] = self.restore_lr[i] + self.restore_lr = None + + self.lr_scheduler.step(metric, epoch) # step the base scheduler + + if self.noise_range is not None: + if isinstance(self.noise_range, (list, tuple)): + apply_noise = self.noise_range[0] <= epoch < self.noise_range[1] + else: + apply_noise = epoch >= self.noise_range + if apply_noise: + self._apply_noise(epoch) + + def _apply_noise(self, epoch): + g = torch.Generator() + g.manual_seed(self.noise_seed + epoch) + if self.noise_type == 'normal': + while True: + # resample if noise out of percent limit, brute force but shouldn't spin much + noise = torch.randn(1, generator=g).item() + if abs(noise) < self.noise_pct: + break + else: + noise = 2 * (torch.rand(1, generator=g).item() - 0.5) * self.noise_pct + + # apply the noise on top of previous LR, cache the old value so we can restore for normal + # stepping of base scheduler + restore_lr = [] + for i, param_group in enumerate(self.optimizer.param_groups): + old_lr = float(param_group['lr']) + restore_lr.append(old_lr) + new_lr = old_lr + old_lr * noise + param_group['lr'] = new_lr + self.restore_lr = restore_lr diff --git a/scheduler/scheduler.py b/scheduler/scheduler.py new file mode 100644 index 0000000..21d5150 --- /dev/null +++ b/scheduler/scheduler.py @@ -0,0 +1,105 @@ +from typing import Dict, Any + +import torch + + +class Scheduler: + """ Parameter Scheduler Base Class + A scheduler base class that can be used to schedule any optimizer parameter groups. + + Unlike the builtin PyTorch schedulers, this is intended to be consistently called + * At the END of each epoch, before incrementing the epoch count, to calculate next epoch's value + * At the END of each optimizer update, after incrementing the update count, to calculate next update's value + + The schedulers built on this should try to remain as stateless as possible (for simplicity). + + This family of schedulers is attempting to avoid the confusion of the meaning of 'last_epoch' + and -1 values for special behaviour. All epoch and update counts must be tracked in the training + code and explicitly passed in to the schedulers on the corresponding step or step_update call. + + Based on ideas from: + * https://github.com/pytorch/fairseq/tree/master/fairseq/optim/lr_scheduler + * https://github.com/allenai/allennlp/tree/master/allennlp/training/learning_rate_schedulers + """ + + def __init__(self, + optimizer: torch.optim.Optimizer, + param_group_field: str, + noise_range_t=None, + noise_type='normal', + noise_pct=0.67, + noise_std=1.0, + noise_seed=None, + initialize: bool = True) -> None: + self.optimizer = optimizer + self.param_group_field = param_group_field + self._initial_param_group_field = f"initial_{param_group_field}" + if initialize: + for i, group in enumerate(self.optimizer.param_groups): + if param_group_field not in group: + raise KeyError(f"{param_group_field} missing from param_groups[{i}]") + group.setdefault(self._initial_param_group_field, group[param_group_field]) + else: + for i, group in enumerate(self.optimizer.param_groups): + if self._initial_param_group_field not in group: + raise KeyError(f"{self._initial_param_group_field} missing from param_groups[{i}]") + self.base_values = [group[self._initial_param_group_field] for group in self.optimizer.param_groups] + self.metric = None # any point to having this for all? + self.noise_range_t = noise_range_t + self.noise_pct = noise_pct + self.noise_type = noise_type + self.noise_std = noise_std + self.noise_seed = noise_seed if noise_seed is not None else 42 + self.update_groups(self.base_values) + + def state_dict(self) -> Dict[str, Any]: + return {key: value for key, value in self.__dict__.items() if key != 'optimizer'} + + def load_state_dict(self, state_dict: Dict[str, Any]) -> None: + self.__dict__.update(state_dict) + + def get_epoch_values(self, epoch: int): + return None + + def get_update_values(self, num_updates: int): + return None + + def step(self, epoch: int, metric: float = None) -> None: + self.metric = metric + values = self.get_epoch_values(epoch) + if values is not None: + values = self._add_noise(values, epoch) + self.update_groups(values) + + def step_update(self, num_updates: int, metric: float = None): + self.metric = metric + values = self.get_update_values(num_updates) + if values is not None: + values = self._add_noise(values, num_updates) + self.update_groups(values) + + def update_groups(self, values): + if not isinstance(values, (list, tuple)): + values = [values] * len(self.optimizer.param_groups) + for param_group, value in zip(self.optimizer.param_groups, values): + param_group[self.param_group_field] = value + + def _add_noise(self, lrs, t): + if self.noise_range_t is not None: + if isinstance(self.noise_range_t, (list, tuple)): + apply_noise = self.noise_range_t[0] <= t < self.noise_range_t[1] + else: + apply_noise = t >= self.noise_range_t + if apply_noise: + g = torch.Generator() + g.manual_seed(self.noise_seed + t) + if self.noise_type == 'normal': + while True: + # resample if noise out of percent limit, brute force but shouldn't spin much + noise = torch.randn(1, generator=g).item() + if abs(noise) < self.noise_pct: + break + else: + noise = 2 * (torch.rand(1, generator=g).item() - 0.5) * self.noise_pct + lrs = [v + v * noise for v in lrs] + return lrs diff --git a/scheduler/scheduler_factory.py b/scheduler/scheduler_factory.py new file mode 100644 index 0000000..9f7748f --- /dev/null +++ b/scheduler/scheduler_factory.py @@ -0,0 +1,87 @@ +""" Scheduler Factory +Hacked together by / Copyright 2020 Ross Wightman +""" +from .cosine_lr import CosineLRScheduler +from .tanh_lr import TanhLRScheduler +from .step_lr import StepLRScheduler +from .plateau_lr import PlateauLRScheduler + + +def create_scheduler(args, optimizer): + num_epochs = args.epochs + + if getattr(args, 'lr_noise', None) is not None: + lr_noise = getattr(args, 'lr_noise') + if isinstance(lr_noise, (list, tuple)): + noise_range = [n * num_epochs for n in lr_noise] + if len(noise_range) == 1: + noise_range = noise_range[0] + else: + noise_range = lr_noise * num_epochs + else: + noise_range = None + + lr_scheduler = None + if args.sched == 'cosine': + lr_scheduler = CosineLRScheduler( + optimizer, + t_initial=num_epochs, + t_mul=getattr(args, 'lr_cycle_mul', 1.), + lr_min=args.min_lr, + decay_rate=args.decay_rate, + warmup_lr_init=args.warmup_lr, + warmup_t=args.warmup_epochs, + cycle_limit=getattr(args, 'lr_cycle_limit', 1), + t_in_epochs=True, + noise_range_t=noise_range, + noise_pct=getattr(args, 'lr_noise_pct', 0.67), + noise_std=getattr(args, 'lr_noise_std', 1.), + noise_seed=getattr(args, 'seed', 42), + ) + num_epochs = lr_scheduler.get_cycle_length() + args.cooldown_epochs + elif args.sched == 'tanh': + lr_scheduler = TanhLRScheduler( + optimizer, + t_initial=num_epochs, + t_mul=getattr(args, 'lr_cycle_mul', 1.), + lr_min=args.min_lr, + warmup_lr_init=args.warmup_lr, + warmup_t=args.warmup_epochs, + cycle_limit=getattr(args, 'lr_cycle_limit', 1), + t_in_epochs=True, + noise_range_t=noise_range, + noise_pct=getattr(args, 'lr_noise_pct', 0.67), + noise_std=getattr(args, 'lr_noise_std', 1.), + noise_seed=getattr(args, 'seed', 42), + ) + num_epochs = lr_scheduler.get_cycle_length() + args.cooldown_epochs + elif args.sched == 'step': + lr_scheduler = StepLRScheduler( + optimizer, + decay_t=args.decay_epochs, + decay_rate=args.decay_rate, + warmup_lr_init=args.warmup_lr, + warmup_t=args.warmup_epochs, + noise_range_t=noise_range, + noise_pct=getattr(args, 'lr_noise_pct', 0.67), + noise_std=getattr(args, 'lr_noise_std', 1.), + noise_seed=getattr(args, 'seed', 42), + ) + elif args.sched == 'plateau': + mode = 'min' if 'loss' in getattr(args, 'eval_metric', '') else 'max' + lr_scheduler = PlateauLRScheduler( + optimizer, + decay_rate=args.decay_rate, + patience_t=args.patience_epochs, + lr_min=args.min_lr, + mode=mode, + warmup_lr_init=args.warmup_lr, + warmup_t=args.warmup_epochs, + cooldown_t=0, + noise_range_t=noise_range, + noise_pct=getattr(args, 'lr_noise_pct', 0.67), + noise_std=getattr(args, 'lr_noise_std', 1.), + noise_seed=getattr(args, 'seed', 42), + ) + + return lr_scheduler, num_epochs diff --git a/scheduler/step_lr.py b/scheduler/step_lr.py new file mode 100644 index 0000000..f797e1a --- /dev/null +++ b/scheduler/step_lr.py @@ -0,0 +1,63 @@ +""" Step Scheduler + +Basic step LR schedule with warmup, noise. + +Hacked together by / Copyright 2020 Ross Wightman +""" +import math +import torch + +from .scheduler import Scheduler + + +class StepLRScheduler(Scheduler): + """ + """ + + def __init__(self, + optimizer: torch.optim.Optimizer, + decay_t: float, + decay_rate: float = 1., + warmup_t=0, + warmup_lr_init=0, + t_in_epochs=True, + noise_range_t=None, + noise_pct=0.67, + noise_std=1.0, + noise_seed=42, + initialize=True, + ) -> None: + super().__init__( + optimizer, param_group_field="lr", + noise_range_t=noise_range_t, noise_pct=noise_pct, noise_std=noise_std, noise_seed=noise_seed, + initialize=initialize) + + self.decay_t = decay_t + self.decay_rate = decay_rate + self.warmup_t = warmup_t + self.warmup_lr_init = warmup_lr_init + self.t_in_epochs = t_in_epochs + if self.warmup_t: + self.warmup_steps = [(v - warmup_lr_init) / self.warmup_t for v in self.base_values] + super().update_groups(self.warmup_lr_init) + else: + self.warmup_steps = [1 for _ in self.base_values] + + def _get_lr(self, t): + if t < self.warmup_t: + lrs = [self.warmup_lr_init + t * s for s in self.warmup_steps] + else: + lrs = [v * (self.decay_rate ** (t // self.decay_t)) for v in self.base_values] + return lrs + + def get_epoch_values(self, epoch: int): + if self.t_in_epochs: + return self._get_lr(epoch) + else: + return None + + def get_update_values(self, num_updates: int): + if not self.t_in_epochs: + return self._get_lr(num_updates) + else: + return None diff --git a/scheduler/tanh_lr.py b/scheduler/tanh_lr.py new file mode 100644 index 0000000..8cc338b --- /dev/null +++ b/scheduler/tanh_lr.py @@ -0,0 +1,120 @@ +""" TanH Scheduler + +TanH schedule with warmup, cycle/restarts, noise. + +Hacked together by / Copyright 2020 Ross Wightman +""" +import logging +import math +import numpy as np +import torch + +from .scheduler import Scheduler + + +_logger = logging.getLogger(__name__) + + +class TanhLRScheduler(Scheduler): + """ + Hyberbolic-Tangent decay with restarts. + This is described in the paper https://arxiv.org/abs/1806.01593 + """ + + def __init__(self, + optimizer: torch.optim.Optimizer, + t_initial: int, + lb: float = -6., + ub: float = 4., + t_mul: float = 1., + lr_min: float = 0., + decay_rate: float = 1., + warmup_t=0, + warmup_lr_init=0, + warmup_prefix=False, + cycle_limit=0, + t_in_epochs=True, + noise_range_t=None, + noise_pct=0.67, + noise_std=1.0, + noise_seed=42, + initialize=True) -> None: + super().__init__( + optimizer, param_group_field="lr", + noise_range_t=noise_range_t, noise_pct=noise_pct, noise_std=noise_std, noise_seed=noise_seed, + initialize=initialize) + + assert t_initial > 0 + assert lr_min >= 0 + assert lb < ub + assert cycle_limit >= 0 + assert warmup_t >= 0 + assert warmup_lr_init >= 0 + self.lb = lb + self.ub = ub + self.t_initial = t_initial + self.t_mul = t_mul + self.lr_min = lr_min + self.decay_rate = decay_rate + self.cycle_limit = cycle_limit + self.warmup_t = warmup_t + self.warmup_lr_init = warmup_lr_init + self.warmup_prefix = warmup_prefix + self.t_in_epochs = t_in_epochs + if self.warmup_t: + t_v = self.base_values if self.warmup_prefix else self._get_lr(self.warmup_t) + self.warmup_steps = [(v - warmup_lr_init) / self.warmup_t for v in t_v] + super().update_groups(self.warmup_lr_init) + else: + self.warmup_steps = [1 for _ in self.base_values] + + def _get_lr(self, t): + if t < self.warmup_t: + lrs = [self.warmup_lr_init + t * s for s in self.warmup_steps] + else: + if self.warmup_prefix: + t = t - self.warmup_t + + if self.t_mul != 1: + i = math.floor(math.log(1 - t / self.t_initial * (1 - self.t_mul), self.t_mul)) + t_i = self.t_mul ** i * self.t_initial + t_curr = t - (1 - self.t_mul ** i) / (1 - self.t_mul) * self.t_initial + else: + i = t // self.t_initial + t_i = self.t_initial + t_curr = t - (self.t_initial * i) + + if self.cycle_limit == 0 or (self.cycle_limit > 0 and i < self.cycle_limit): + gamma = self.decay_rate ** i + lr_min = self.lr_min * gamma + lr_max_values = [v * gamma for v in self.base_values] + + tr = t_curr / t_i + lrs = [ + lr_min + 0.5 * (lr_max - lr_min) * (1 - math.tanh(self.lb * (1. - tr) + self.ub * tr)) + for lr_max in lr_max_values + ] + else: + lrs = [self.lr_min * (self.decay_rate ** self.cycle_limit) for _ in self.base_values] + return lrs + + def get_epoch_values(self, epoch: int): + if self.t_in_epochs: + return self._get_lr(epoch) + else: + return None + + def get_update_values(self, num_updates: int): + if not self.t_in_epochs: + return self._get_lr(num_updates) + else: + return None + + def get_cycle_length(self, cycles=0): + if not cycles: + cycles = self.cycle_limit + cycles = max(1, cycles) + if self.t_mul == 1.0: + return self.t_initial * cycles + else: + return int(math.floor(-self.t_initial * (self.t_mul ** cycles - 1) / (1 - self.t_mul))) diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..8a93975 --- /dev/null +++ b/utils.py @@ -0,0 +1,258 @@ +import numpy as np +import io +import os +import time +from collections import defaultdict, deque +import datetime + +import torch +import torch.distributed as dist + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda') + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value) + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append( + "{}: {}".format(name, str(meter)) + ) + return self.delimiter.join(loss_str) + + def global_avg(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append( + "{}: {:.4f}".format(name, meter.global_avg) + ) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None): + i = 0 + if not header: + header = '' + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt='{avg:.4f}') + data_time = SmoothedValue(fmt='{avg:.4f}') + space_fmt = ':' + str(len(str(len(iterable)))) + 'd' + log_msg = [ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}' + ] + if torch.cuda.is_available(): + log_msg.append('max mem: {memory:.0f}') + log_msg = self.delimiter.join(log_msg) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB)) + else: + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time))) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('{} Total time: {} ({:.4f} s / it)'.format( + header, total_time_str, total_time / len(iterable))) + + + +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + +def compute_acc(logits, label, reduction='mean'): + ret = (torch.argmax(logits, dim=1) == label).float() + if reduction == 'none': + return ret.detach() + elif reduction == 'mean': + return ret.mean().item() + +def compute_n_params(model, return_str=True): + tot = 0 + for p in model.parameters(): + w = 1 + for x in p.shape: + w *= x + tot += w + if return_str: + if tot >= 1e6: + return '{:.1f}M'.format(tot / 1e6) + else: + return '{:.1f}K'.format(tot / 1e3) + else: + return tot + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ['WORLD_SIZE']) + args.gpu = int(os.environ['LOCAL_RANK']) + elif 'SLURM_PROCID' in os.environ: + args.rank = int(os.environ['SLURM_PROCID']) + args.gpu = args.rank % torch.cuda.device_count() + else: + print('Not using distributed mode') + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = 'nccl' + print('| distributed init (rank {}): {}'.format( + args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) \ No newline at end of file diff --git a/visualization.ipynb b/visualization.ipynb new file mode 100644 index 0000000..2d5ee44 --- /dev/null +++ b/visualization.ipynb @@ -0,0 +1,343 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "lovely-budapest", + "metadata": {}, + "source": [ + "# This is a notebook that shows how to produce Grad-CAM visualizations for ALBEF" + ] + }, + { + "cell_type": "markdown", + "id": "czech-surprise", + "metadata": {}, + "source": [ + "# 1. Set the paths for model checkpoint and configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "institutional-sarah", + "metadata": {}, + "outputs": [], + "source": [ + "model_path = '../VL/Example/refcoco.pth'\n", + "bert_config_path = 'configs/config_bert.json'\n", + "use_cuda = False" + ] + }, + { + "cell_type": "markdown", + "id": "lovely-passage", + "metadata": {}, + "source": [ + "# 2. Model defination" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "documented-symbol", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial\n", + "from models.vit import VisionTransformer\n", + "from models.xbert import BertConfig, BertModel\n", + "from models.tokenization_bert import BertTokenizer\n", + "\n", + "import torch\n", + "from torch import nn\n", + "from torchvision import transforms\n", + "\n", + "import json\n", + "\n", + "class VL_Transformer_ITM(nn.Module):\n", + " def __init__(self, \n", + " text_encoder = None,\n", + " config_bert = ''\n", + " ):\n", + " super().__init__()\n", + " \n", + " bert_config = BertConfig.from_json_file(config_bert)\n", + "\n", + " self.visual_encoder = VisionTransformer(\n", + " img_size=384, patch_size=16, embed_dim=768, depth=12, num_heads=12, \n", + " mlp_ratio=4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6)) \n", + "\n", + " self.text_encoder = BertModel.from_pretrained(text_encoder, config=bert_config, add_pooling_layer=False) \n", + " \n", + " self.itm_head = nn.Linear(768, 2) \n", + "\n", + " \n", + " def forward(self, image, text):\n", + " image_embeds = self.visual_encoder(image) \n", + "\n", + " image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)\n", + "\n", + " output = self.text_encoder(text.input_ids, \n", + " attention_mask = text.attention_mask,\n", + " encoder_hidden_states = image_embeds,\n", + " encoder_attention_mask = image_atts, \n", + " return_dict = True,\n", + " ) \n", + " \n", + " vl_embeddings = output.last_hidden_state[:,0,:]\n", + " vl_output = self.itm_head(vl_embeddings) \n", + " return vl_output" + ] + }, + { + "cell_type": "markdown", + "id": "renewable-eight", + "metadata": {}, + "source": [ + "# 3. Text Preprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "optional-brooklyn", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "\n", + "def pre_caption(caption,max_words=30):\n", + " caption = re.sub(\n", + " r\"([,.'!?\\\"()*#:;~])\",\n", + " '',\n", + " caption.lower(),\n", + " ).replace('-', ' ').replace('/', ' ')\n", + "\n", + " caption = re.sub(\n", + " r\"\\s{2,}\",\n", + " ' ',\n", + " caption,\n", + " )\n", + " caption = caption.rstrip('\\n') \n", + " caption = caption.strip(' ')\n", + "\n", + " #truncate caption\n", + " caption_words = caption.split(' ')\n", + " if len(caption_words)>max_words:\n", + " caption = ' '.join(caption_words[:max_words]) \n", + " return caption" + ] + }, + { + "cell_type": "markdown", + "id": "based-roads", + "metadata": {}, + "source": [ + "# 4. Image Preprocessing and Postpressing" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "subsequent-flesh", + "metadata": {}, + "outputs": [], + "source": [ + "from PIL import Image\n", + "\n", + "import cv2\n", + "import numpy as np\n", + "\n", + "from skimage import transform as skimage_transform\n", + "from scipy.ndimage import filters\n", + "from matplotlib import pyplot as plt\n", + "\n", + "def getAttMap(img, attMap, blur = True, overlap = True):\n", + " attMap -= attMap.min()\n", + " if attMap.max() > 0:\n", + " attMap /= attMap.max()\n", + " attMap = skimage_transform.resize(attMap, (img.shape[:2]), order = 3, mode = 'constant')\n", + " if blur:\n", + " attMap = filters.gaussian_filter(attMap, 0.02*max(img.shape[:2]))\n", + " attMap -= attMap.min()\n", + " attMap /= attMap.max()\n", + " cmap = plt.get_cmap('jet')\n", + " attMapV = cmap(attMap)\n", + " attMapV = np.delete(attMapV, 3, 2)\n", + " if overlap:\n", + " attMap = 1*(1-attMap**0.7).reshape(attMap.shape + (1,))*img + (attMap**0.7).reshape(attMap.shape+(1,)) * attMapV\n", + " return attMap\n", + "\n", + "\n", + "normalize = transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))\n", + "\n", + "transform = transforms.Compose([\n", + " transforms.Resize((384,384),interpolation=Image.BICUBIC),\n", + " transforms.ToTensor(),\n", + " normalize,\n", + "]) " + ] + }, + { + "cell_type": "markdown", + "id": "occasional-trace", + "metadata": {}, + "source": [ + "# 5. Load model and tokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "qualified-sleep", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['bert.pooler.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'bert.pooler.dense.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of BertModel were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['bert.encoder.layer.10.crossattention.self.key.weight', 'bert.encoder.layer.6.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.10.crossattention.output.dense.bias', 'bert.encoder.layer.8.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.9.crossattention.output.dense.bias', 'bert.encoder.layer.6.crossattention.output.dense.bias', 'bert.encoder.layer.9.crossattention.self.value.bias', 'bert.encoder.layer.9.crossattention.self.key.bias', 'bert.encoder.layer.8.crossattention.self.key.weight', 'bert.encoder.layer.6.crossattention.self.query.weight', 'bert.encoder.layer.8.crossattention.self.value.weight', 'bert.encoder.layer.9.crossattention.self.query.weight', 'bert.encoder.layer.9.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.11.crossattention.self.value.weight', 'bert.encoder.layer.7.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.8.crossattention.self.key.bias', 'bert.encoder.layer.7.crossattention.self.key.weight', 'bert.encoder.layer.7.crossattention.output.dense.weight', 'bert.encoder.layer.11.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.9.crossattention.self.key.weight', 'bert.encoder.layer.9.crossattention.self.value.weight', 'bert.encoder.layer.8.crossattention.self.query.bias', 'bert.encoder.layer.11.crossattention.self.key.bias', 'bert.encoder.layer.10.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.8.crossattention.output.dense.bias', 'bert.encoder.layer.7.crossattention.output.dense.bias', 'bert.encoder.layer.7.crossattention.self.value.weight', 'bert.encoder.layer.11.crossattention.self.query.bias', 'bert.encoder.layer.9.crossattention.self.query.bias', 'bert.encoder.layer.6.crossattention.output.dense.weight', 'bert.encoder.layer.10.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.6.crossattention.self.value.bias', 'bert.encoder.layer.8.crossattention.self.query.weight', 'bert.encoder.layer.11.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.11.crossattention.self.value.bias', 'bert.encoder.layer.6.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.9.crossattention.output.dense.weight', 'bert.encoder.layer.6.crossattention.self.query.bias', 'bert.encoder.layer.8.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.11.crossattention.self.key.weight', 'bert.encoder.layer.7.crossattention.self.key.bias', 'bert.encoder.layer.6.crossattention.self.value.weight', 'bert.encoder.layer.7.crossattention.self.query.bias', 'bert.encoder.layer.8.crossattention.self.value.bias', 'bert.encoder.layer.6.crossattention.self.key.bias', 'bert.encoder.layer.7.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.10.crossattention.self.value.weight', 'bert.encoder.layer.10.crossattention.self.query.weight', 'bert.encoder.layer.11.crossattention.output.dense.weight', 'bert.encoder.layer.11.crossattention.output.dense.bias', 'bert.encoder.layer.9.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.7.crossattention.self.query.weight', 'bert.encoder.layer.10.crossattention.output.dense.weight', 'bert.encoder.layer.6.crossattention.self.key.weight', 'bert.encoder.layer.10.crossattention.self.key.bias', 'bert.encoder.layer.8.crossattention.output.dense.weight', 'bert.encoder.layer.7.crossattention.self.value.bias', 'bert.encoder.layer.11.crossattention.self.query.weight', 'bert.encoder.layer.10.crossattention.self.value.bias', 'bert.encoder.layer.10.crossattention.self.query.bias']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + } + ], + "source": [ + "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n", + "\n", + "model = VL_Transformer_ITM(text_encoder='bert-base-uncased', config_bert=bert_config_path)\n", + "\n", + "checkpoint = torch.load(model_path, map_location='cpu') \n", + "msg = model.load_state_dict(checkpoint,strict=False)\n", + "model.eval()\n", + "\n", + "block_num = 8\n", + "\n", + "model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.save_attention = True\n", + "\n", + "if use_cuda:\n", + " model.cuda() " + ] + }, + { + "cell_type": "markdown", + "id": "apparent-captain", + "metadata": {}, + "source": [ + "# 6. Load Image and Text" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "finite-angle", + "metadata": {}, + "outputs": [], + "source": [ + "image_path = 'examples/image0.jpg'\n", + "image_pil = Image.open(image_path).convert('RGB') \n", + "image = transform(image_pil).unsqueeze(0) \n", + "\n", + "caption = 'the woman is working on her computer at the desk'\n", + "text = pre_caption(caption)\n", + "text_input = tokenizer(text, return_tensors=\"pt\")\n", + "\n", + "if use_cuda:\n", + " image = image.cuda()\n", + " text_input = text_input.to(image.device)" + ] + }, + { + "cell_type": "markdown", + "id": "gorgeous-matrix", + "metadata": {}, + "source": [ + "# 7. Compute GradCAM" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "driven-termination", + "metadata": {}, + "outputs": [], + "source": [ + "output = model(image, text_input)\n", + "loss = output[:,1].sum()\n", + "\n", + "model.zero_grad()\n", + "loss.backward() \n", + "\n", + "with torch.no_grad():\n", + " mask = text_input.attention_mask.view(text_input.attention_mask.size(0),1,-1,1,1)\n", + "\n", + " grads=model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.get_attn_gradients()\n", + " cams=model.text_encoder.base_model.base_model.encoder.layer[block_num].crossattention.self.get_attention_map()\n", + "\n", + " cams = cams[:, :, :, 1:].reshape(image.size(0), 12, -1, 24, 24) * mask\n", + " grads = grads[:, :, :, 1:].clamp(0).reshape(image.size(0), 12, -1, 24, 24) * mask\n", + "\n", + " gradcam = cams * grads\n", + " gradcam = gradcam[0].mean(0).cpu().detach()" + ] + }, + { + "cell_type": "markdown", + "id": "abroad-northern", + "metadata": {}, + "source": [ + "# 8. Visualize GradCam for each word" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fourth-cache", + "metadata": {}, + "outputs": [], + "source": [ + "num_image = len(text_input.input_ids[0]) \n", + "fig, ax = plt.subplots(num_image, 1, figsize=(15,5*num_image))\n", + "\n", + "rgb_image = cv2.imread(image_path)[:, :, ::-1]\n", + "rgb_image = np.float32(rgb_image) / 255\n", + "\n", + "ax[0].imshow(rgb_image)\n", + "ax[0].set_yticks([])\n", + "ax[0].set_xticks([])\n", + "ax[0].set_xlabel(\"Image\")\n", + " \n", + "for i,token_id in enumerate(text_input.input_ids[0][1:]):\n", + " word = tokenizer.decode([token_id])\n", + " gradcam_image = getAttMap(rgb_image, gradcam[i+1])\n", + " ax[i+1].imshow(gradcam_image)\n", + " ax[i+1].set_yticks([])\n", + " ax[i+1].set_xticks([])\n", + " ax[i+1].set_xlabel(word)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/vqaTools/__init__.py b/vqaTools/__init__.py new file mode 100644 index 0000000..072d8d9 --- /dev/null +++ b/vqaTools/__init__.py @@ -0,0 +1 @@ +__author__ = 'aagrawal' diff --git a/vqaTools/__pycache__/__init__.cpython-36.pyc b/vqaTools/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..516d3a3 Binary files /dev/null and b/vqaTools/__pycache__/__init__.cpython-36.pyc differ diff --git a/vqaTools/__pycache__/__init__.cpython-38.pyc b/vqaTools/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..9b08a9a Binary files /dev/null and b/vqaTools/__pycache__/__init__.cpython-38.pyc differ diff --git a/vqaTools/__pycache__/vqa.cpython-36.pyc b/vqaTools/__pycache__/vqa.cpython-36.pyc new file mode 100644 index 0000000..aa61d9c Binary files /dev/null and b/vqaTools/__pycache__/vqa.cpython-36.pyc differ diff --git a/vqaTools/__pycache__/vqa.cpython-38.pyc b/vqaTools/__pycache__/vqa.cpython-38.pyc new file mode 100644 index 0000000..543a65b Binary files /dev/null and b/vqaTools/__pycache__/vqa.cpython-38.pyc differ diff --git a/vqaTools/__pycache__/vqaEval.cpython-36.pyc b/vqaTools/__pycache__/vqaEval.cpython-36.pyc new file mode 100644 index 0000000..5b40010 Binary files /dev/null and b/vqaTools/__pycache__/vqaEval.cpython-36.pyc differ diff --git a/vqaTools/__pycache__/vqaEval.cpython-38.pyc b/vqaTools/__pycache__/vqaEval.cpython-38.pyc new file mode 100644 index 0000000..d416df9 Binary files /dev/null and b/vqaTools/__pycache__/vqaEval.cpython-38.pyc differ diff --git a/vqaTools/vqa.py b/vqaTools/vqa.py new file mode 100644 index 0000000..c0a22da --- /dev/null +++ b/vqaTools/vqa.py @@ -0,0 +1,177 @@ +__author__ = 'aagrawal' +__version__ = '0.9' + +# Interface for accessing the VQA dataset. + +# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: +# (https://github.com/pdollar/coco/blob/master/PythonAPI/pycocotools/coco.py). + +# The following functions are defined: +# VQA - VQA class that loads VQA annotation file and prepares data structures. +# getQuesIds - Get question ids that satisfy given filter conditions. +# getImgIds - Get image ids that satisfy given filter conditions. +# loadQA - Load questions and answers with the specified question ids. +# showQA - Display the specified questions and answers. +# loadRes - Load result file and create result object. + +# Help on each function can be accessed by: "help(COCO.function)" + +import json +import datetime +import copy + +class VQA: + def __init__(self, annotation_file=None, question_file=None): + """ + Constructor of VQA helper class for reading and visualizing questions and answers. + :param annotation_file (str): location of VQA annotation file + :return: + """ + # load dataset + self.dataset = {} + self.questions = {} + self.qa = {} + self.qqa = {} + self.imgToQA = {} + if not annotation_file == None and not question_file == None: + print('loading VQA annotations and questions into memory...') + time_t = datetime.datetime.utcnow() + dataset = json.load(open(annotation_file, 'r')) + questions = json.load(open(question_file, 'r')) + self.dataset = dataset + self.questions = questions + self.createIndex() + + def createIndex(self): + # create index + print('creating index...') + imgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']} + qa = {ann['question_id']: [] for ann in self.dataset['annotations']} + qqa = {ann['question_id']: [] for ann in self.dataset['annotations']} + for ann in self.dataset['annotations']: + imgToQA[ann['image_id']] += [ann] + qa[ann['question_id']] = ann + for ques in self.questions['questions']: + qqa[ques['question_id']] = ques + print('index created!') + + # create class members + self.qa = qa + self.qqa = qqa + self.imgToQA = imgToQA + + def info(self): + """ + Print information about the VQA annotation file. + :return: + """ + for key, value in self.datset['info'].items(): + print('%s: %s'%(key, value)) + + def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]): + """ + Get question ids that satisfy given filter conditions. default skips that filter + :param imgIds (int array) : get question ids for given imgs + quesTypes (str array) : get question ids for given question types + ansTypes (str array) : get question ids for given answer types + :return: ids (int array) : integer array of question ids + """ + imgIds = imgIds if type(imgIds) == list else [imgIds] + quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] + ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] + + if len(imgIds) == len(quesTypes) == len(ansTypes) == 0: + anns = self.dataset['annotations'] + else: + if not len(imgIds) == 0: + anns = sum([self.imgToQA[imgId] for imgId in imgIds if imgId in self.imgToQA],[]) + else: + anns = self.dataset['annotations'] + anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] + anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] + ids = [ann['question_id'] for ann in anns] + return ids + + def getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]): + """ + Get image ids that satisfy given filter conditions. default skips that filter + :param quesIds (int array) : get image ids for given question ids + quesTypes (str array) : get image ids for given question types + ansTypes (str array) : get image ids for given answer types + :return: ids (int array) : integer array of image ids + """ + quesIds = quesIds if type(quesIds) == list else [quesIds] + quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] + ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] + + if len(quesIds) == len(quesTypes) == len(ansTypes) == 0: + anns = self.dataset['annotations'] + else: + if not len(quesIds) == 0: + anns = sum([self.qa[quesId] for quesId in quesIds if quesId in self.qa],[]) + else: + anns = self.dataset['annotations'] + anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] + anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] + ids = [ann['image_id'] for ann in anns] + return ids + + def loadQA(self, ids=[]): + """ + Load questions and answers with the specified question ids. + :param ids (int array) : integer ids specifying question ids + :return: qa (object array) : loaded qa objects + """ + if type(ids) == list: + return [self.qa[id] for id in ids] + elif type(ids) == int: + return [self.qa[ids]] + + def showQA(self, anns): + """ + Display the specified annotations. + :param anns (array of object): annotations to display + :return: None + """ + if len(anns) == 0: + return 0 + for ann in anns: + quesId = ann['question_id'] + print("Question: %s" %(self.qqa[quesId]['question'])) + for ans in ann['answers']: + print("Answer %d: %s" %(ans['answer_id'], ans['answer'])) + + def loadRes(self, resFile, quesFile): + """ + Load result file and return a result object. + :param resFile (str) : file name of result file + :return: res (obj) : result api object + """ + res = VQA() + res.questions = json.load(open(quesFile)) + res.dataset['info'] = copy.deepcopy(self.questions['info']) + res.dataset['task_type'] = copy.deepcopy(self.questions['task_type']) + res.dataset['data_type'] = copy.deepcopy(self.questions['data_type']) + res.dataset['data_subtype'] = copy.deepcopy(self.questions['data_subtype']) + res.dataset['license'] = copy.deepcopy(self.questions['license']) + + print('Loading and preparing results... ') + time_t = datetime.datetime.utcnow() + anns = json.load(open(resFile)) + assert type(anns) == list, 'results is not an array of objects' + annsQuesIds = [ann['question_id'] for ann in anns] + assert set(annsQuesIds) == set(self.getQuesIds()), \ + 'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.' + for ann in anns: + quesId = ann['question_id'] + if res.dataset['task_type'] == 'Multiple Choice': + assert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices' + qaAnn = self.qa[quesId] + ann['image_id'] = qaAnn['image_id'] + ann['question_type'] = qaAnn['question_type'] + ann['answer_type'] = qaAnn['answer_type'] + print('DONE (t=%0.2fs)'%((datetime.datetime.utcnow() - time_t).total_seconds())) + + res.dataset['annotations'] = anns + res.createIndex() + return res diff --git a/vqaTools/vqaEval.py b/vqaTools/vqaEval.py new file mode 100644 index 0000000..6b66f98 --- /dev/null +++ b/vqaTools/vqaEval.py @@ -0,0 +1,184 @@ +# coding=utf-8 + +__author__='aagrawal' + +# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: +# (https://github.com/tylin/coco-caption/blob/master/pycocoevalcap/eval.py). +import sys +import re + +class VQAEval: + def __init__(self, vqa, vqaRes, n=2): + self.n = n + self.accuracy = {} + self.evalQA = {} + self.evalQuesType = {} + self.evalAnsType = {} + self.vqa = vqa + self.vqaRes = vqaRes + self.params = {'question_id': vqa.getQuesIds()} + self.contractions = {"aint": "ain't", "arent": "aren't", "cant": "can't", "couldve": "could've", "couldnt": "couldn't", + "couldn'tve": "couldn't've", "couldnt've": "couldn't've", "didnt": "didn't", "doesnt": "doesn't", "dont": "don't", "hadnt": "hadn't", + "hadnt've": "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent": "haven't", "hed": "he'd", "hed've": "he'd've", + "he'dve": "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll", "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", + "Im": "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've": "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's", + "maam": "ma'am", "mightnt": "mightn't", "mightnt've": "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've", + "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't", "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't", + "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat": "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve": "she'd've", + "she's": "she's", "shouldve": "should've", "shouldnt": "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve": "shouldn't've", + "somebody'd": "somebodyd", "somebodyd've": "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll": "somebody'll", + "somebodys": "somebody's", "someoned": "someone'd", "someoned've": "someone'd've", "someone'dve": "someone'd've", + "someonell": "someone'll", "someones": "someone's", "somethingd": "something'd", "somethingd've": "something'd've", + "something'dve": "something'd've", "somethingll": "something'll", "thats": "that's", "thered": "there'd", "thered've": "there'd've", + "there'dve": "there'd've", "therere": "there're", "theres": "there's", "theyd": "they'd", "theyd've": "they'd've", + "they'dve": "they'd've", "theyll": "they'll", "theyre": "they're", "theyve": "they've", "twas": "'twas", "wasnt": "wasn't", + "wed've": "we'd've", "we'dve": "we'd've", "weve": "we've", "werent": "weren't", "whatll": "what'll", "whatre": "what're", + "whats": "what's", "whatve": "what've", "whens": "when's", "whered": "where'd", "wheres": "where's", "whereve": "where've", + "whod": "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl": "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll", + "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve": "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've", + "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll": "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've", + "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd": "you'd", "youd've": "you'd've", "you'dve": "you'd've", + "youll": "you'll", "youre": "you're", "youve": "you've"} + self.manualMap = { 'none': '0', + 'zero': '0', + 'one': '1', + 'two': '2', + 'three': '3', + 'four': '4', + 'five': '5', + 'six': '6', + 'seven': '7', + 'eight': '8', + 'nine': '9', + 'ten': '10' + } + self.articles = ['a', + 'an', + 'the' + ] + + + self.periodStrip = re.compile("(?!<=\d)(\.)(?!\d)") + self.commaStrip = re.compile("(\d)(,)(\d)") + self.punct = [';', r"/", '[', ']', '"', '{', '}', + '(', ')', '=', '+', '\\', '_', '-', + '>', '<', '@', '`', ',', '?', '!'] + + + def evaluate(self, quesIds=None): + if quesIds == None: + quesIds = [quesId for quesId in self.params['question_id']] + gts = {} + res = {} + for quesId in quesIds: + gts[quesId] = self.vqa.qa[quesId] + res[quesId] = self.vqaRes.qa[quesId] + + # ================================================= + # Compute accuracy + # ================================================= + accQA = [] + accQuesType = {} + accAnsType = {} + print ("computing accuracy") + step = 0 + for quesId in quesIds: + resAns = res[quesId]['answer'] + resAns = resAns.replace('\n', ' ') + resAns = resAns.replace('\t', ' ') + resAns = resAns.strip() + resAns = self.processPunctuation(resAns) + resAns = self.processDigitArticle(resAns) + gtAcc = [] + gtAnswers = [ans['answer'] for ans in gts[quesId]['answers']] + if len(set(gtAnswers)) > 1: + for ansDic in gts[quesId]['answers']: + ansDic['answer'] = self.processPunctuation(ansDic['answer']) + for gtAnsDatum in gts[quesId]['answers']: + otherGTAns = [item for item in gts[quesId]['answers'] if item!=gtAnsDatum] + matchingAns = [item for item in otherGTAns if item['answer']==resAns] + acc = min(1, float(len(matchingAns))/3) + gtAcc.append(acc) + quesType = gts[quesId]['question_type'] + ansType = gts[quesId]['answer_type'] + avgGTAcc = float(sum(gtAcc))/len(gtAcc) + accQA.append(avgGTAcc) + if quesType not in accQuesType: + accQuesType[quesType] = [] + accQuesType[quesType].append(avgGTAcc) + if ansType not in accAnsType: + accAnsType[ansType] = [] + accAnsType[ansType].append(avgGTAcc) + self.setEvalQA(quesId, avgGTAcc) + self.setEvalQuesType(quesId, quesType, avgGTAcc) + self.setEvalAnsType(quesId, ansType, avgGTAcc) + if step%100 == 0: + self.updateProgress(step/float(len(quesIds))) + step = step + 1 + + self.setAccuracy(accQA, accQuesType, accAnsType) + print ("Done computing accuracy") + + def processPunctuation(self, inText): + outText = inText + for p in self.punct: + if (p + ' ' in inText or ' ' + p in inText) or (re.search(self.commaStrip, inText) != None): + outText = outText.replace(p, '') + else: + outText = outText.replace(p, ' ') + outText = self.periodStrip.sub("", + outText, + re.UNICODE) + return outText + + def processDigitArticle(self, inText): + outText = [] + tempText = inText.lower().split() + for word in tempText: + word = self.manualMap.setdefault(word, word) + if word not in self.articles: + outText.append(word) + else: + pass + for wordId, word in enumerate(outText): + if word in self.contractions: + outText[wordId] = self.contractions[word] + outText = ' '.join(outText) + return outText + + def setAccuracy(self, accQA, accQuesType, accAnsType): + self.accuracy['overall'] = round(100*float(sum(accQA))/len(accQA), self.n) + self.accuracy['perQuestionType'] = {quesType: round(100*float(sum(accQuesType[quesType]))/len(accQuesType[quesType]), self.n) for quesType in accQuesType} + self.accuracy['perAnswerType'] = {ansType: round(100*float(sum(accAnsType[ansType]))/len(accAnsType[ansType]), self.n) for ansType in accAnsType} + + def setEvalQA(self, quesId, acc): + self.evalQA[quesId] = round(100*acc, self.n) + + def setEvalQuesType(self, quesId, quesType, acc): + if quesType not in self.evalQuesType: + self.evalQuesType[quesType] = {} + self.evalQuesType[quesType][quesId] = round(100*acc, self.n) + + def setEvalAnsType(self, quesId, ansType, acc): + if ansType not in self.evalAnsType: + self.evalAnsType[ansType] = {} + self.evalAnsType[ansType][quesId] = round(100*acc, self.n) + + def updateProgress(self, progress): + barLength = 20 + status = "" + if isinstance(progress, int): + progress = float(progress) + if not isinstance(progress, float): + progress = 0 + status = "error: progress var must be float\r\n" + if progress < 0: + progress = 0 + status = "Halt...\r\n" + if progress >= 1: + progress = 1 + status = "Done...\r\n" + block = int(round(barLength*progress)) + text = "\rFinshed Percent: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), int(progress*100), status) + sys.stdout.write(text) + sys.stdout.flush() \ No newline at end of file