From c7bd6b602f32c1fa4ae9e9036c6b74fe10bf60d2 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 22 Mar 2019 02:53:39 -0400 Subject: [PATCH 01/40] genotypes.py random path --- cnn/genotypes.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index 66f733b..aed50b9 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -273,3 +273,28 @@ MULTI_CHANNEL_GREEDY_SCALAR_BOTTOM_UP = ['Source', 'Conv3x3_2', 'BatchNorm_2', 'layer_0_stride_1_c_in_128_c_out_256_op_type_SharpSepConv', 'layer_0_add_c_out_256_stride_1', 'layer_0_stride_2_c_in_256_c_out_32_op_type_ResizablePool', 'layer_0_add_c_out_32_stride_2', 'layer_1_stride_1_c_in_32_c_out_32_op_type_ResizablePool', 'layer_1_add_c_out_32_stride_1', 'layer_1_stride_2_c_in_32_c_out_32_op_type_ResizablePool', 'layer_1_add_c_out_32_stride_2', 'layer_2_stride_1_c_in_32_c_out_256_op_type_ResizablePool', 'layer_2_add_c_out_256_stride_1', 'layer_2_stride_2_c_in_256_c_out_256_op_type_ResizablePool', 'layer_2_add_c_out_256_stride_2', 'layer_3_stride_1_c_in_256_c_out_256_op_type_ResizablePool', 'layer_3_add_c_out_256_stride_1', 'layer_3_stride_2_c_in_256_c_out_32_op_type_SharpSepConv', 'layer_3_add_c_out_32_stride_2', 'SharpSepConv32', 'add-SharpSep', 'global_pooling', 'Linear'] MULTI_CHANNEL_GREEDY_MAX_W_TOP_DOWN = ['Source', 'Conv3x3_0', 'BatchNorm_0', 'layer_0_stride_1_c_in_32_c_out_32_op_type_ResizablePool', 'layer_0_add_c_out_32_stride_1', 'layer_0_stride_2_c_in_32_c_out_128_op_type_SharpSepConv', 'layer_0_add_c_out_128_stride_2', 'layer_1_stride_1_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_1_add_c_out_128_stride_1', 'layer_1_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_1_add_c_out_128_stride_2', 'layer_2_stride_1_c_in_128_c_out_64_op_type_SharpSepConv', 'layer_2_add_c_out_64_stride_1', 'layer_2_stride_2_c_in_64_c_out_64_op_type_SharpSepConv', 'layer_2_add_c_out_64_stride_2', 'layer_3_stride_1_c_in_64_c_out_128_op_type_SharpSepConv', 'layer_3_add_c_out_128_stride_1', 'layer_3_stride_2_c_in_128_c_out_128_op_type_ResizablePool', 'layer_3_add_c_out_128_stride_2', 'SharpSepConv128', 'add-SharpSep', 'global_pooling', 'Linear'] MULTI_CHANNEL_GREEDY_MAX_W_BOTTOM_UP = ['Source', 'Conv3x3_3', 'BatchNorm_3', 'layer_0_stride_1_c_in_256_c_out_128_op_type_ResizablePool', 'layer_0_add_c_out_128_stride_1', 'layer_0_stride_2_c_in_128_c_out_256_op_type_ResizablePool', 'layer_0_add_c_out_256_stride_2', 'layer_1_stride_1_c_in_256_c_out_128_op_type_ResizablePool', 'layer_1_add_c_out_128_stride_1', 'layer_1_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_1_add_c_out_128_stride_2', 'layer_2_stride_1_c_in_128_c_out_64_op_type_ResizablePool', 'layer_2_add_c_out_64_stride_1', 'layer_2_stride_2_c_in_64_c_out_64_op_type_ResizablePool', 'layer_2_add_c_out_64_stride_2', 'layer_3_stride_1_c_in_64_c_out_64_op_type_ResizablePool', 'layer_3_add_c_out_64_stride_1', 'layer_3_stride_2_c_in_64_c_out_64_op_type_ResizablePool', 'layer_3_add_c_out_64_stride_2', 'SharpSepConv64', 'add-SharpSep', 'global_pooling', 'Linear'] +MULTI_CHANNEL_RANDOM_PATH = ['Conv3x3_1', 'BatchNorm_1', 'layer_0_stride_1_c_in_64_c_out_128_op_type_ResizablePool', 'layer_0_add_c_out_128_stride_1', 'layer_0_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_0_add_c_out_128_stride_2', 'layer_1_stride_1_c_in_128_c_out_256_op_type_ResizablePool', 'layer_1_add_c_out_256_stride_1', 'layer_1_stride_2_c_in_256_c_out_256_op_type_SharpSepConv', 'layer_1_add_c_out_256_stride_2', 'layer_2_stride_1_c_in_256_c_out_128_op_type_SharpSepConv', 'layer_2_add_c_out_128_stride_1', 'layer_2_stride_2_c_in_128_c_out_32_op_type_SharpSepConv', 'layer_2_add_c_out_32_stride_2', 'layer_3_stride_1_c_in_32_c_out_64_op_type_ResizablePool', 'layer_3_add_c_out_64_stride_1', 'layer_3_stride_2_c_in_64_c_out_32_op_type_SharpSepConv', 'layer_3_add_c_out_32_stride_2', 'SharpSepConv32', 'add-SharpSep', 'global_pooling', 'Linear'] + +# costar@ubuntu|~/src/sharpDARTS/cnn on multi_channel_search!? +# ± export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --batch_size 48 --layers_of_cells 8 --layers_in_cells 4 --save max_w_SharpSepConvDARTS_SEARCH_`git rev-parse --short HEAD` --init_channels +# 16 --epochs 120 --cutout --autoaugment --seed 22 --weighting_algorithm max_w --primitives DARTS_PRIMITIVES +# Tensorflow is not installed. Skipping tf related imports +# Experiment dir : search-20190321-024555-max_w_SharpSepConvDARTS_SEARCH_5e49783-cifar10-DARTS_PRIMITIVES-OPS-0 +# 2019_03_21_19_11_44 epoch, 47, train_acc, 76.215997, valid_acc, 74.855997, train_loss, 0.687799, valid_loss, 0.734363, lr, 1.580315e-02, best_epoch, 47, best_valid_acc, 74.855997 +# 2019_03_21_19_11_44 genotype = +# 2019_03_21_19_11_44 alphas_normal = tensor([[0.1134, 0.0945, 0.0894, 0.1007, 0.1466, 0.1334, 0.1964, 0.1256], +# [0.1214, 0.0983, 0.1000, 0.1072, 0.1675, 0.1383, 0.1167, 0.1507], +# [0.1251, 0.1066, 0.1043, 0.1674, 0.1295, 0.1237, 0.1209, 0.1225], +# [0.1238, 0.1066, 0.1054, 0.1108, 0.1331, 0.1418, 0.1145, 0.1641], +# [0.1009, 0.0843, 0.0801, 0.0802, 0.2970, 0.1168, 0.1329, 0.1078], +# [0.1257, 0.1115, 0.1087, 0.1158, 0.1641, 0.1312, 0.1305, 0.1125], +# [0.1662, 0.1154, 0.1152, 0.1194, 0.1234, 0.1248, 0.1222, 0.1134], +# [0.1177, 0.0943, 0.1892, 0.0836, 0.1285, 0.1317, 0.1286, 0.1263], +# [0.3851, 0.0835, 0.0718, 0.0554, 0.1031, 0.1047, 0.1011, 0.0953], +# [0.1249, 0.1119, 0.1096, 0.1156, 0.1284, 0.1177, 0.1157, 0.1762], +# [0.1249, 0.1186, 0.1197, 0.1244, 0.1254, 0.1319, 0.1238, 0.1312], +# [0.1126, 0.0932, 0.2448, 0.0838, 0.1132, 0.1141, 0.1263, 0.1120], +# [0.0791, 0.4590, 0.0665, 0.0518, 0.0843, 0.0804, 0.0846, 0.0944], +# [0.0715, 0.0665, 0.4912, 0.0401, 0.0822, 0.0816, 0.0888, 0.0780]], +# device='cuda:0', grad_fn=) +SHARPSEPCONV_DARTS_MAX_W = Genotype(normal=[('dil_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 2), ('skip_connect', 0), ('avg_pool_3x3', 2), ('sep_conv_3x3', 0), ('avg_pool_3x3', 4), ('max_pool_3x3', 3)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('dil_conv_5x5', 2), ('sep_conv_5x5', 0), ('sep_conv_5x5', 0), ('sep_conv_3x3', 2), ('dil_conv_3x3', 0), ('dil_conv_5x5', 3)], reduce_concat=range(2, 6), layout='cell') \ No newline at end of file From 5c5c8664f8c0f2f1b7a5fd93dae3493d7d399211 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 22 Mar 2019 02:53:56 -0400 Subject: [PATCH 02/40] train.py add flops_shape --- cnn/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cnn/train.py b/cnn/train.py index 37692cd..7da376a 100644 --- a/cnn/train.py +++ b/cnn/train.py @@ -140,6 +140,7 @@ def main(): cnn_model = MultiChannelNetwork( args.init_channels, DATASET_CLASSES, layers=args.layers_of_cells, criterion=criterion, steps=args.layers_in_cells, weighting_algorithm=args.weighting_algorithm, genotype=genotype) + flops_shape = [1, 3, 32, 32] elif args.dataset == 'imagenet': cnn_model = NetworkImageNet(args.init_channels, DATASET_CLASSES, args.layers, args.auxiliary, genotype, op_dict=op_dict, C_mid=args.mid_channels) flops_shape = [1, 3, 224, 224] From 65a2a20ec0d04a81167a49035d490f00e7cb3f89 Mon Sep 17 00:00:00 2001 From: Priyanka Hubli Date: Sat, 6 Apr 2019 15:43:50 -0400 Subject: [PATCH 03/40] added new feature-mode:time_difference_images to classify time intervals between 2 frames --- cnn/dataset.py | 6 +- cnn/model.py | 112 +++++++++++++++++++++++++++++++++++ cnn/train_costar.py | 138 +++++++++++++++++++++++++++----------------- 3 files changed, 201 insertions(+), 55 deletions(-) diff --git a/cnn/dataset.py b/cnn/dataset.py index df9f6fd..2101000 100644 --- a/cnn/dataset.py +++ b/cnn/dataset.py @@ -83,13 +83,15 @@ costar_class_dict = {'translation_only': 3, 'rotation_only': 5, - 'all_features': 8} + 'all_features': 8, + 'time_difference_images': 6} costar_supercube_inp_channel_dict = {'translation_only': 52, 'rotation_only': 55, 'all_features': 57} costar_vec_size_dict = {'translation_only': 44, 'rotation_only': 49, - 'all_features': 49} + 'all_features': 49, + 'time_difference_images': 0} COSTAR_SET_NAMES = ['blocks_only', 'blocks_with_plush_toy'] COSTAR_SUBSET_NAMES = ['success_only', 'error_failure_only', 'task_failure_only', 'task_and_error_failure'] diff --git a/cnn/model.py b/cnn/model.py index 31df53c..413c4d5 100644 --- a/cnn/model.py +++ b/cnn/model.py @@ -509,3 +509,115 @@ def forward(self, input, log=False): def reset_noise(self): self.classifier.reset_noise() + +class residual_block(nn.Module): + """Implements a layer of ResNeXt. + + Based in https://blog.waya.ai/deep-residual-learning-9610bb62c355. + + Args: + input: Input for the block. + channels_in: Number of channels of data for the convolutional group. + channels_out: Number of chhannels generated as output. + cardinality: Number of convolution groups. Must be divisible by channels_in + is_training: Placeholder that indicates if the model is being trained + strides: Strides. + project_shortcut: Indicates whether the input should be projected to match the output's dimensions. + + Returns: + A tensor with shape (-1, channels_out, width, height). + + """ + def __init__(self, channels_in, channels_out, cardinality, strides=(1, 1), project_shortcut=False): + super(residual_block, self).__init__() + self.channels_in = channels_in + self.channels_out = channels_out + self.cardinality = cardinality + self.project_shortcut = project_shortcut + self.strides = strides + + self.conv1 = nn.Conv2d(64, self.channels_in, 1 , stride = 1) + self.norm1 = nn.BatchNorm2d(self.channels_in) + self.group_conv = nn.Conv2d(self.channels_in, self.channels_in, kernel_size=(3,3),stride=self.strides, groups=1,padding=1) + + self.norm2 = nn.BatchNorm2d(self.channels_in) + self.conv2 = nn.Conv2d(self.channels_in ,self.channels_out, 1 , stride = 1) + self.norm3 = nn.BatchNorm2d(self.channels_out) + + if self.project_shortcut or self.strides != (1,1): + self.conv3 = nn.Conv2d(64, self.channels_out, 1, stride = self.strides) + self.norm4 = nn.BatchNorm2d(self.channels_out) + + def forward(self, input): + shortcut = input + x = F.relu(self.norm1(self.conv1(input))) + x = self.group_conv(x) + x = F.relu(self.norm2(x)) + x = self.norm3(self.conv2(x)) + + if self.project_shortcut or self.strides!=(1,1): + # When the dimensions increase projection shortcut is used to match dimensions (done by 1×1 convolutions). + shortcut = self.norm4(self.conv3(shortcut)) + + output = F.relu(torch.add(shortcut, x)) + return output + +class TDCFeaturizer(nn.Module): + """Temporal Distance Classification featurizer + + Reference: "Playing hard exploration games by watching YouTube" + The task consists of presenting the network with 2 frames separated by n timesteps, + and making it classify the distance between the frames. + + We use the same network architecture as the paper: + 3 convolutional layers, followed by 3 residual blocks, + followed by 2 fully connected layers for the encoder. + + For the classifier, we do a multiplication between both feature vectors + followed by a fully connected layer. + + """ + def __init__(self): + super(TDCFeaturizer, self).__init__() + self.feature_vector_size = 1024 + + self.conv1 = nn.Conv2d(3, 32, 3, stride = 2, padding=1) + self.conv2 = nn.Conv2d(32, 64, 3, stride = 1,padding=1) + self.conv3 = nn.Conv2d(64, 64, 3, stride = 1,padding=1) + self.pool = nn.MaxPool2d(2, 2) + self.norm1 = nn.BatchNorm2d(32) + self.norm2 = nn.BatchNorm2d(64) + self.norm3 = nn.BatchNorm2d(64) + self.residual_block = residual_block(64, 64, 1) + self.fc1 = nn.Linear(3072, self.feature_vector_size) #dense layer + self.fc2 = nn.Linear(self.feature_vector_size, self.feature_vector_size) + self.fc3 = nn.Linear(1024, self.feature_vector_size) + self.fc4 = nn.Linear(self.feature_vector_size, 6) # size of each label + + + def forward(self, img_1, img_2): + logits_aux = None + x = torch.cat((img_1,img_2),0) + x = self.pool(self.norm1(F.relu(self.conv1(x)))) + x = self.pool(self.norm2(F.relu(self.conv2(x)))) + x = self.pool(self.norm3(F.relu(self.conv3(x)))) + + for i in range(3): + x = self.residual_block(x) + + x = x.view(-1, 3072) # flatten layer (3072 = 64*6*8) + x = F.relu(self.fc1(x)) + + feature_vector = self.fc2(x) + feature_vector = F.normalize(feature_vector, p=2, dim=1) + #if not self.training: + # return feature_vector + feature_vector_stack = feature_vector.view(-1, 2, self.feature_vector_size) + + combined_embeddings = torch.mul(feature_vector_stack[:, 0, :], feature_vector_stack[:, 1, :]) + + x = F.relu(self.fc3(combined_embeddings)) + prediction = F.relu(self.fc4(x)) + + return prediction, logits_aux + diff --git a/cnn/train_costar.py b/cnn/train_costar.py index bdc4aef..fb8ac60 100644 --- a/cnn/train_costar.py +++ b/cnn/train_costar.py @@ -33,6 +33,7 @@ import torchvision.datasets as datasets import torchvision.models as models import torchvision +from model import TDCFeaturizer import numpy as np import random @@ -70,6 +71,7 @@ # choices=model_names, help='model architecture: ' + ' | '.join(model_names) + + 'TDCFeaturizer for feature embeddings' ' (default: SHARP_DARTS)') parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', help='number of data loading workers (default: 4)') @@ -146,7 +148,7 @@ help='which subset to use in the CoSTAR BSD. Options are "success_only", ' '"error_failure_only", "task_failure_only", or "task_and_error_failure". Defaults to "success_only"') parser.add_argument('--feature_mode', type=str, default='all_features', - help='which feature mode to use. Options are "translation_only", "rotation_only", "stacking_reward", ' + help='which feature mode to use. Options are "translation_only", "rotation_only", "stacking_reward", "time_difference_images"' 'or the default "all_features"') parser.add_argument('--num_images_per_example', type=int, default=200, help='Number of times an example is visited per epoch. Default value is 200. Since the image for each visit to an ' @@ -168,7 +170,7 @@ best_combined_error = float('inf') args = parser.parse_args() logger = None - +model_input_shape = (224,224,3) # default input shape of images VECTOR_SIZE = dataset.costar_vec_size_dict[args.feature_mode] def fast_collate(batch): @@ -204,7 +206,7 @@ def main(): # note the gpu is used for directory creation and log files # which is needed when run as multiple processes args = utils.initialize_files_and_args(args) - logger = utils.logging_setup(args.log_file_path) + logger = utils.logging_setup(args.log_file_path) # # load the correct ops dictionary op_dict_to_load = "operations.%s" % args.ops @@ -222,13 +224,16 @@ def main(): if args.arch == 'NetworkResNetCOSTAR': # baseline model for comparison model = NetworkResNetCOSTAR(args.init_channels, classes, args.layers, args.auxiliary, None, vector_size=VECTOR_SIZE, op_dict=op_dict, C_mid=args.mid_channels) + model.drop_path_prob = 0.0 + elif args.arch == 'TDCFeaturizer': + model = TDCFeaturizer() else: # create model genotype = eval("genotypes.%s" % args.arch) # create the neural network model = NetworkCOSTAR(args.init_channels, classes, args.layers, args.auxiliary, genotype, vector_size=VECTOR_SIZE, op_dict=op_dict, C_mid=args.mid_channels) - model.drop_path_prob = 0.0 + model.drop_path_prob = 0.0 # if args.pretrained: # logger.info("=> using pre-trained model '{}'".format(args.arch)) # model = models.__dict__[args.arch](pretrained=True) @@ -248,19 +253,28 @@ def main(): # model = DDP(model) # delay_allreduce delays all communication to the end of the backward pass. model = DDP(model, delay_allreduce=True) - - # define loss function (criterion) and optimizer - criterion = nn.MSELoss().cuda() - # NOTE(rexxarchl): MSLE loss, indicated as better for rotation in costar_hyper/costar_block_stacking_train_regression.py - # is not available in PyTorch by default - + + # Scale learning rate based on global batch size args.learning_rate = args.learning_rate * float(args.batch_size * args.world_size)/256. init_lr = args.learning_rate / args.warmup_lr_divisor - optimizer = torch.optim.SGD(model.parameters(), init_lr, - momentum=args.momentum, - weight_decay=args.weight_decay) + # define loss function (criterion) and optimizer + if args.feature_mode == 'time_difference_images': + # 'time_difference_images' is a feature mode where we try to classify time intervals between two frames. + criterion = nn.CrossEntropyLoss().cuda() + optimizer = torch.optim.Adam(model.parameters(), lr=init_lr) + # Change collate function and model input shape for this feature mode + fast_collate = torch.utils.data.dataloader.default_collate + model_input_shape = (3,96,128) + else: + criterion = nn.MSELoss().cuda() + # NOTE(rexxarchl): MSLE loss, indicated as better for rotation in costar_hyper/costar_block_stacking_train_regression.py + # is not available in PyTorch by default + optimizer = torch.optim.SGD(model.parameters(), init_lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + # epoch_count = args.epochs - args.start_epoch # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(epoch_count)) # scheduler = warmup_scheduler.GradualWarmupScheduler( @@ -307,21 +321,20 @@ def resume(): num_workers=args.workers, costar_set_name=args.set_name, costar_subset_name=args.subset_name, costar_feature_mode=args.feature_mode, costar_version=args.version, costar_num_images_per_example=args.num_images_per_example, - costar_output_shape=(224, 224, 3), costar_random_augmentation=None, costar_one_hot_encoding=True, evaluate=evaluate) - + costar_output_shape=model_input_shape, costar_random_augmentation=None, costar_one_hot_encoding=True, evaluate=evaluate) if args.evaluate: # Load the test set test_loader = dataset.get_costar_test_queue( args.data, costar_set_name=args.set_name, costar_subset_name=args.subset_name, collate_fn=fast_collate, costar_feature_mode=args.feature_mode, costar_version=args.version, costar_num_images_per_example=args.num_images_per_example, - costar_output_shape=(224, 224, 3), costar_random_augmentation=None, costar_one_hot_encoding=True) - + costar_output_shape=model_input_shape, costar_random_augmentation=None, costar_one_hot_encoding=True) + # Evaluate on all splits, without any augmentation validate(train_loader, model, criterion, args, prefix='evaluate_train_') validate(val_loader, model, criterion, args, prefix='evaluate_val_') validate(test_loader, model, criterion, args, prefix='evaluate_test_') return - + lr_schedule = cosine_power_annealing( epochs=args.epochs, max_lr=args.learning_rate, min_lr=args.learning_rate_min, warmup_epochs=args.warmup_epochs, exponent_order=args.lr_power_annealing_exponent_order, @@ -345,7 +358,8 @@ def resume(): for param_group in optimizer.param_groups: param_group['lr'] = learning_rate # scheduler.step() - model.drop_path_prob = args.drop_path_prob * float(epoch) / float(args.epochs) + if args.feature_mode != 'time_difference_images': + model.drop_path_prob = args.drop_path_prob * float(epoch) / float(args.epochs) # train for one epoch train_stats = train(train_loader, model, criterion, optimizer, int(epoch), args) if args.prof: @@ -416,13 +430,13 @@ def resume(): num_workers=args.workers, costar_set_name=args.set_name, costar_subset_name=args.subset_name, costar_feature_mode=args.feature_mode, costar_version=args.version, costar_num_images_per_example=args.num_images_per_example, - costar_output_shape=(224, 224, 3), costar_random_augmentation=None, costar_one_hot_encoding=True, evaluate=evaluate) + costar_output_shape=model_input_shape, costar_random_augmentation=None, costar_one_hot_encoding=True, evaluate=evaluate) # Get the test set test_loader = dataset.get_costar_test_queue( args.data, costar_set_name=args.set_name, costar_subset_name=args.subset_name, collate_fn=fast_collate, costar_feature_mode=args.feature_mode, costar_version=args.version, costar_num_images_per_example=args.num_images_per_example, - costar_output_shape=(224, 224, 3), costar_random_augmentation=None, costar_one_hot_encoding=True) + costar_output_shape=model_input_shape, costar_random_augmentation=None, costar_one_hot_encoding=True) # Evaluate on all splits, without any augmentation validate(train_loader, model, criterion, args, prefix='best_final_train_') @@ -430,7 +444,7 @@ def resume(): validate(test_loader, model, criterion, args, prefix='best_final_test_') logger.info("Final evaluation complete! Save dir: ' + str(args.save)") - + class data_prefetcher(): def __init__(self, loader, cutout=False, cutout_length=112, cutout_cuts=1): self.loader = iter(loader) @@ -487,6 +501,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args): cart_error, angle_error = [], [] input_img, input_vec, target = prefetcher.next() + if args.feature_mode == 'time_difference_images': + target = target.type(torch.cuda.LongTensor) batch_size = input_img.size(0) i = -1 if args.local_rank == 0: @@ -507,7 +523,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args): # compute output # note here the term output is equivalent to logits output, logits_aux = model(input_img, input_vec) - output = sigmoid(output) + if args.feature_mode != 'time_difference_images': + output = sigmoid(output) loss = criterion(output, target) if logits_aux is not None and args.auxiliary: logits_aux = sigmoid(logits_aux) @@ -515,24 +532,29 @@ def train(train_loader, model, criterion, optimizer, epoch, args): loss += args.auxiliary_weight * loss_aux # measure accuracy and record loss - with torch.no_grad(): - output_np = output.cpu().detach().numpy() - target_np = target.cpu().detach().numpy() - batch_abs_cart_distance, batch_abs_angle_distance = accuracy(output_np, target_np) - abs_cart_f, abs_angle_f = np.mean(batch_abs_cart_distance), np.mean(batch_abs_angle_distance) - cart_error.extend(batch_abs_cart_distance) - angle_error.extend(batch_abs_angle_distance) + if args.feature_mode != 'time_difference_images': + with torch.no_grad(): + output_np = output.cpu().detach().numpy() + target_np = target.cpu().detach().numpy() + batch_abs_cart_distance, batch_abs_angle_distance = accuracy(output_np, target_np) + abs_cart_f, abs_angle_f = np.mean(batch_abs_cart_distance), np.mean(batch_abs_angle_distance) + cart_error.extend(batch_abs_cart_distance) + angle_error.extend(batch_abs_angle_distance) + + if args.distributed: + reduced_loss = reduce_tensor(loss.data) + abs_cart_f = reduce_tensor(abs_cart_f) + abs_angle_f = reduce_tensor(abs_angle_f) + else: + reduced_loss = loss.data - if args.distributed: - reduced_loss = reduce_tensor(loss.data) - abs_cart_f = reduce_tensor(abs_cart_f) - abs_angle_f = reduce_tensor(abs_angle_f) + losses.update(reduced_loss, batch_size) + abs_cart_m.update(abs_cart_f, batch_size) + abs_angle_m.update(abs_angle_f, batch_size) + else: - reduced_loss = loss.data - - losses.update(reduced_loss, batch_size) - abs_cart_m.update(abs_cart_f, batch_size) - abs_angle_m.update(abs_angle_f, batch_size) + reduced_loss = reduce_tensor(loss.data) if args.distributed else loss.data + losses.update(reduced_loss) # compute gradient and do SGD step optimizer.zero_grad() @@ -545,6 +567,8 @@ def train(train_loader, model, criterion, optimizer, epoch, args): end = time.time() input_img, input_vec, target = prefetcher.next() + if args.feature_mode == 'time_difference_images' and target is not None: + target = target.type(torch.cuda.LongTensor) if args.local_rank == 0: progbar.update() @@ -614,6 +638,8 @@ def validate(val_loader, model, criterion, args, prefix='val_'): cart_error, angle_error = [], [] prefetcher = data_prefetcher(val_loader) input_img, input_vec, target = prefetcher.next() + if args.feature_mode == 'time_difference_images': + target = target.type(torch.cuda.LongTensor) batch_size = input_img.size(0) i = -1 if args.local_rank == 0: @@ -634,21 +660,25 @@ def validate(val_loader, model, criterion, args, prefix='val_'): loss = criterion(output, target) # measure accuracy and record loss - batch_abs_cart_distance, batch_abs_angle_distance = accuracy(output.data.cpu().numpy(), target.data.cpu().numpy()) - abs_cart_f, abs_angle_f = np.mean(batch_abs_cart_distance), np.mean(batch_abs_angle_distance) - cart_error.extend(batch_abs_cart_distance) - angle_error.extend(batch_abs_angle_distance) - - if args.distributed: - reduced_loss = reduce_tensor(loss.data) - abs_cart_f = reduce_tensor(abs_cart_f) - abs_angle_f = reduce_tensor(abs_angle_f) - else: - reduced_loss = loss.data + if args.feature_mode != 'time_difference_images': + batch_abs_cart_distance, batch_abs_angle_distance = accuracy(output.data.cpu().numpy(), target.data.cpu().numpy()) + abs_cart_f, abs_angle_f = np.mean(batch_abs_cart_distance), np.mean(batch_abs_angle_distance) + cart_error.extend(batch_abs_cart_distance) + angle_error.extend(batch_abs_angle_distance) - losses.update(reduced_loss, batch_size) - abs_cart_m.update(abs_cart_f, batch_size) - abs_angle_m.update(abs_angle_f, batch_size) + if args.distributed: + reduced_loss = reduce_tensor(loss.data) + abs_cart_f = reduce_tensor(abs_cart_f) + abs_angle_f = reduce_tensor(abs_angle_f) + else: + reduced_loss = loss.data + + losses.update(reduced_loss, batch_size) + abs_cart_m.update(abs_cart_f, batch_size) + abs_angle_m.update(abs_angle_f, batch_size) + else: + reduced_loss = reduce_tensor(loss.data) if args.distributed else loss.data + losses.update(reduced_loss) # measure elapsed time batch_time.update(time.time() - end) @@ -673,6 +703,8 @@ def validate(val_loader, model, criterion, args, prefix='val_'): abs_cart=abs_cart_m, abs_angle=abs_angle_m)) input_img, input_vec, target = prefetcher.next() + if args.feature_mode == 'time_difference_images' and target is not None: + target = target.type(torch.cuda.LongTensor) # logger.info(' * combined_error {combined_error.avg:.3f} top5 {top5.avg:.3f}' # .format(combined_error=combined_error, top5=top5)) From a965ce06100c32f8df34842f6050dab43569806f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 9 Apr 2019 17:08:00 -0400 Subject: [PATCH 04/40] cnn/genotypes.py first try at sharper search space --- cnn/genotypes.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index 28daa8d..c59ba96 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -21,6 +21,26 @@ 'dil_choke_conv_3x3', ] +SHARPER_PRIMITIVES = [ + 'none', + 'max_pool_3x3', + 'avg_pool_3x3', + 'skip_connect', + 'sep_conv_3x3', + 'sep_conv_5x5', + 'sep_conv_7x7', + 'dil_conv_3x3', + 'dil_conv_5x5', + # 'nor_conv_3x3', + # 'nor_conv_5x5', + # 'nor_conv_7x7', + 'flood_conv_3x3', + 'flood_conv_5x5', + 'dil_flood_conv_3x3', + # 'choke_conv_3x3', + # 'dil_choke_conv_3x3', +] + # Primitives for the original darts search space DARTS_PRIMITIVES = [ 'none', From d9a54bf3a30fa31f423a55cadf661a34c4ff943a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 9 Apr 2019 17:11:35 -0400 Subject: [PATCH 05/40] cnn/visualize.py beter style params --- cnn/visualize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cnn/visualize.py b/cnn/visualize.py index d768aef..d599e6c 100644 --- a/cnn/visualize.py +++ b/cnn/visualize.py @@ -6,8 +6,8 @@ def plot(genotype, filename): g = Digraph( format='pdf', - edge_attr=dict(fontsize='20', fontname="times"), - node_attr=dict(style='filled', shape='rect', align='center', fontsize='20', height='0.5', width='0.5', penwidth='2', fontname="times"), + edge_attr=dict(fontsize='32', fontname="times"), + node_attr=dict(style='filled', shape='rect', align='center', fontsize='32', height='0.7', width='0.7', penwidth='2', fontname="times"), engine='dot') g.body.extend(['rankdir=LR']) From 284f92db9b9a79b12b71a387799b274086a6a22e Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 9 Apr 2019 17:13:10 -0400 Subject: [PATCH 06/40] WARNING UNCLEAR CODE STATE cnn/model_search.py merge graph search algorithms? not clear of code state --- cnn/model_search.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/cnn/model_search.py b/cnn/model_search.py index f074ddc..6176dc0 100644 --- a/cnn/model_search.py +++ b/cnn/model_search.py @@ -541,8 +541,47 @@ def forward(self, input_batch): # out = self.global_pooling(out) logits = self.classifier(out.view(out.size(0),-1)) # print('logits') + #print("Optimal_path_forward", nx.algorithms.dag.dag_longest_path(self.G)) + #print("Top down greedy", self.gen_greedy_path(self.G,"top_down")) + #print("Bottom up greedy",self.gen_greedy_path(self.G,"bottom_up")) return logits + def gen_greedy_path(self, G, strategy="top_down"): + if strategy == "top_down": + start_ = "Source" + current_node = "Source" + end_node = "Linear" + new_G = G + elif strategy == "bottom_up": + start_ = "Linear" + current_node = "Linear" + end_node = "Source" + new_G = G.reverse(copy=True) + wt = 0 + node_list = [] + while current_node != end_node: + neighbors = [n for n in new_G.neighbors(start_)] + for nodes in neighbors: + weight_ = new_G.get_edge_data(start_, nodes, "weight") + # print(weight_) + if len(weight_): + weight_ = weight_["weight"] + else: + weight_ = 0 + # print(weight_) + if weight_ > wt: + wt = weight_ + current_node = nodes + node_list.append(current_node) + # print("start",start_) + # print(node) + start_ = current_node + wt = -1 + # print(node_list) + if strategy == "bottom_up": + node_list = node_list[::-1] + node_list.append("Linear") + return node_list def arch_weights(self, stride_idx): # ops are stored as layer, stride, cin, cout, num_layer_types # while weights are ordered stride_index, layer, cin, cout, num_layer_types From 1354bea610dae70254c410da9c8e8cd580d2efa5 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 9 Apr 2019 17:14:16 -0400 Subject: [PATCH 07/40] WARNING UNCLEAR CODE STATE genotypes.py fixes to multichannel hyperparameter grid search path strings --- cnn/genotypes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index aed50b9..ab14f6e 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -273,7 +273,8 @@ MULTI_CHANNEL_GREEDY_SCALAR_BOTTOM_UP = ['Source', 'Conv3x3_2', 'BatchNorm_2', 'layer_0_stride_1_c_in_128_c_out_256_op_type_SharpSepConv', 'layer_0_add_c_out_256_stride_1', 'layer_0_stride_2_c_in_256_c_out_32_op_type_ResizablePool', 'layer_0_add_c_out_32_stride_2', 'layer_1_stride_1_c_in_32_c_out_32_op_type_ResizablePool', 'layer_1_add_c_out_32_stride_1', 'layer_1_stride_2_c_in_32_c_out_32_op_type_ResizablePool', 'layer_1_add_c_out_32_stride_2', 'layer_2_stride_1_c_in_32_c_out_256_op_type_ResizablePool', 'layer_2_add_c_out_256_stride_1', 'layer_2_stride_2_c_in_256_c_out_256_op_type_ResizablePool', 'layer_2_add_c_out_256_stride_2', 'layer_3_stride_1_c_in_256_c_out_256_op_type_ResizablePool', 'layer_3_add_c_out_256_stride_1', 'layer_3_stride_2_c_in_256_c_out_32_op_type_SharpSepConv', 'layer_3_add_c_out_32_stride_2', 'SharpSepConv32', 'add-SharpSep', 'global_pooling', 'Linear'] MULTI_CHANNEL_GREEDY_MAX_W_TOP_DOWN = ['Source', 'Conv3x3_0', 'BatchNorm_0', 'layer_0_stride_1_c_in_32_c_out_32_op_type_ResizablePool', 'layer_0_add_c_out_32_stride_1', 'layer_0_stride_2_c_in_32_c_out_128_op_type_SharpSepConv', 'layer_0_add_c_out_128_stride_2', 'layer_1_stride_1_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_1_add_c_out_128_stride_1', 'layer_1_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_1_add_c_out_128_stride_2', 'layer_2_stride_1_c_in_128_c_out_64_op_type_SharpSepConv', 'layer_2_add_c_out_64_stride_1', 'layer_2_stride_2_c_in_64_c_out_64_op_type_SharpSepConv', 'layer_2_add_c_out_64_stride_2', 'layer_3_stride_1_c_in_64_c_out_128_op_type_SharpSepConv', 'layer_3_add_c_out_128_stride_1', 'layer_3_stride_2_c_in_128_c_out_128_op_type_ResizablePool', 'layer_3_add_c_out_128_stride_2', 'SharpSepConv128', 'add-SharpSep', 'global_pooling', 'Linear'] MULTI_CHANNEL_GREEDY_MAX_W_BOTTOM_UP = ['Source', 'Conv3x3_3', 'BatchNorm_3', 'layer_0_stride_1_c_in_256_c_out_128_op_type_ResizablePool', 'layer_0_add_c_out_128_stride_1', 'layer_0_stride_2_c_in_128_c_out_256_op_type_ResizablePool', 'layer_0_add_c_out_256_stride_2', 'layer_1_stride_1_c_in_256_c_out_128_op_type_ResizablePool', 'layer_1_add_c_out_128_stride_1', 'layer_1_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_1_add_c_out_128_stride_2', 'layer_2_stride_1_c_in_128_c_out_64_op_type_ResizablePool', 'layer_2_add_c_out_64_stride_1', 'layer_2_stride_2_c_in_64_c_out_64_op_type_ResizablePool', 'layer_2_add_c_out_64_stride_2', 'layer_3_stride_1_c_in_64_c_out_64_op_type_ResizablePool', 'layer_3_add_c_out_64_stride_1', 'layer_3_stride_2_c_in_64_c_out_64_op_type_ResizablePool', 'layer_3_add_c_out_64_stride_2', 'SharpSepConv64', 'add-SharpSep', 'global_pooling', 'Linear'] -MULTI_CHANNEL_RANDOM_PATH = ['Conv3x3_1', 'BatchNorm_1', 'layer_0_stride_1_c_in_64_c_out_128_op_type_ResizablePool', 'layer_0_add_c_out_128_stride_1', 'layer_0_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_0_add_c_out_128_stride_2', 'layer_1_stride_1_c_in_128_c_out_256_op_type_ResizablePool', 'layer_1_add_c_out_256_stride_1', 'layer_1_stride_2_c_in_256_c_out_256_op_type_SharpSepConv', 'layer_1_add_c_out_256_stride_2', 'layer_2_stride_1_c_in_256_c_out_128_op_type_SharpSepConv', 'layer_2_add_c_out_128_stride_1', 'layer_2_stride_2_c_in_128_c_out_32_op_type_SharpSepConv', 'layer_2_add_c_out_32_stride_2', 'layer_3_stride_1_c_in_32_c_out_64_op_type_ResizablePool', 'layer_3_add_c_out_64_stride_1', 'layer_3_stride_2_c_in_64_c_out_32_op_type_SharpSepConv', 'layer_3_add_c_out_32_stride_2', 'SharpSepConv32', 'add-SharpSep', 'global_pooling', 'Linear'] +MULTI_CHANNEL_RANDOM_PATH = ['Source', 'Conv3x3_1', 'BatchNorm_1', 'layer_0_stride_1_c_in_64_c_out_128_op_type_ResizablePool', 'layer_0_add_c_out_128_stride_1', 'layer_0_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_0_add_c_out_128_stride_2', 'layer_1_stride_1_c_in_128_c_out_256_op_type_ResizablePool', 'layer_1_add_c_out_256_stride_1', 'layer_1_stride_2_c_in_256_c_out_256_op_type_SharpSepConv', 'layer_1_add_c_out_256_stride_2', 'layer_2_stride_1_c_in_256_c_out_128_op_type_SharpSepConv', 'layer_2_add_c_out_128_stride_1', 'layer_2_stride_2_c_in_128_c_out_32_op_type_SharpSepConv', 'layer_2_add_c_out_32_stride_2', 'layer_3_stride_1_c_in_32_c_out_64_op_type_ResizablePool', 'layer_3_add_c_out_64_stride_1', 'layer_3_stride_2_c_in_64_c_out_32_op_type_SharpSepConv', 'layer_3_add_c_out_32_stride_2', 'SharpSepConv32', 'add-SharpSep', 'global_pooling', 'Linear'] +MULTI_CHANNEL_RANDOM_OPTIMAL = ['Source', 'Conv3x3_1', 'BatchNorm_1', 'layer_0_stride_1_c_in_64_c_out_64_op_type_SharpSepConv', 'layer_0_add_c_out_64_stride_1', 'layer_0_stride_2_c_in_64_c_out_128_op_type_ResizablePool', 'layer_0_add_c_out_128_stride_2', 'layer_1_stride_1_c_in_128_c_out_32_op_type_ResizablePool', 'layer_1_add_c_out_32_stride_1', 'layer_1_stride_2_c_in_32_c_out_256_op_type_SharpSepConv', 'layer_1_add_c_out_256_stride_2', 'layer_2_stride_1_c_in_256_c_out_32_op_type_ResizablePool', 'layer_2_add_c_out_32_stride_1', 'layer_2_stride_2_c_in_32_c_out_256_op_type_ResizablePool', 'layer_2_add_c_out_256_stride_2', 'layer_3_stride_1_c_in_256_c_out_128_op_type_SharpSepConv', 'layer_3_add_c_out_128_stride_1', 'layer_3_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_3_add_c_out_128_stride_2', 'SharpSepConv128', 'add-SharpSep', 'global_pooling', 'Linear'] # costar@ubuntu|~/src/sharpDARTS/cnn on multi_channel_search!? # ± export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --batch_size 48 --layers_of_cells 8 --layers_in_cells 4 --save max_w_SharpSepConvDARTS_SEARCH_`git rev-parse --short HEAD` --init_channels @@ -297,4 +298,4 @@ # [0.0791, 0.4590, 0.0665, 0.0518, 0.0843, 0.0804, 0.0846, 0.0944], # [0.0715, 0.0665, 0.4912, 0.0401, 0.0822, 0.0816, 0.0888, 0.0780]], # device='cuda:0', grad_fn=) -SHARPSEPCONV_DARTS_MAX_W = Genotype(normal=[('dil_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 2), ('skip_connect', 0), ('avg_pool_3x3', 2), ('sep_conv_3x3', 0), ('avg_pool_3x3', 4), ('max_pool_3x3', 3)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('dil_conv_5x5', 2), ('sep_conv_5x5', 0), ('sep_conv_5x5', 0), ('sep_conv_3x3', 2), ('dil_conv_3x3', 0), ('dil_conv_5x5', 3)], reduce_concat=range(2, 6), layout='cell') \ No newline at end of file +SHARPSEPCONV_DARTS_MAX_W = Genotype(normal=[('dil_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 2), ('skip_connect', 0), ('avg_pool_3x3', 2), ('sep_conv_3x3', 0), ('avg_pool_3x3', 4), ('max_pool_3x3', 3)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('dil_conv_5x5', 2), ('sep_conv_5x5', 0), ('sep_conv_5x5', 0), ('sep_conv_3x3', 2), ('dil_conv_3x3', 0), ('dil_conv_5x5', 3)], reduce_concat=range(2, 6), layout='cell') From 6f0190ef60679fea25f63da4804ccdd0ad4c4597 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 9 Apr 2019 17:27:01 -0400 Subject: [PATCH 08/40] cnn/genotypes.py comments fix --- cnn/genotypes.py | 47 +++++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index 7ee94c5..f6833a7 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -276,31 +276,30 @@ MULTI_CHANNEL_RANDOM_PATH = ['Source', 'Conv3x3_1', 'BatchNorm_1', 'layer_0_stride_1_c_in_64_c_out_128_op_type_ResizablePool', 'layer_0_add_c_out_128_stride_1', 'layer_0_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_0_add_c_out_128_stride_2', 'layer_1_stride_1_c_in_128_c_out_256_op_type_ResizablePool', 'layer_1_add_c_out_256_stride_1', 'layer_1_stride_2_c_in_256_c_out_256_op_type_SharpSepConv', 'layer_1_add_c_out_256_stride_2', 'layer_2_stride_1_c_in_256_c_out_128_op_type_SharpSepConv', 'layer_2_add_c_out_128_stride_1', 'layer_2_stride_2_c_in_128_c_out_32_op_type_SharpSepConv', 'layer_2_add_c_out_32_stride_2', 'layer_3_stride_1_c_in_32_c_out_64_op_type_ResizablePool', 'layer_3_add_c_out_64_stride_1', 'layer_3_stride_2_c_in_64_c_out_32_op_type_SharpSepConv', 'layer_3_add_c_out_32_stride_2', 'SharpSepConv32', 'add-SharpSep', 'global_pooling', 'Linear'] MULTI_CHANNEL_RANDOM_OPTIMAL = ['Source', 'Conv3x3_1', 'BatchNorm_1', 'layer_0_stride_1_c_in_64_c_out_64_op_type_SharpSepConv', 'layer_0_add_c_out_64_stride_1', 'layer_0_stride_2_c_in_64_c_out_128_op_type_ResizablePool', 'layer_0_add_c_out_128_stride_2', 'layer_1_stride_1_c_in_128_c_out_32_op_type_ResizablePool', 'layer_1_add_c_out_32_stride_1', 'layer_1_stride_2_c_in_32_c_out_256_op_type_SharpSepConv', 'layer_1_add_c_out_256_stride_2', 'layer_2_stride_1_c_in_256_c_out_32_op_type_ResizablePool', 'layer_2_add_c_out_32_stride_1', 'layer_2_stride_2_c_in_32_c_out_256_op_type_ResizablePool', 'layer_2_add_c_out_256_stride_2', 'layer_3_stride_1_c_in_256_c_out_128_op_type_SharpSepConv', 'layer_3_add_c_out_128_stride_1', 'layer_3_stride_2_c_in_128_c_out_128_op_type_SharpSepConv', 'layer_3_add_c_out_128_stride_2', 'SharpSepConv128', 'add-SharpSep', 'global_pooling', 'Linear'] -# costar@ubuntu|~/src/sharpDARTS/cnn on multi_channel_search!? -# ± export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --batch_size 48 --layers_of_cells 8 --layers_in_cells 4 --save max_w_SharpSepConvDARTS_SEARCH_`git rev-parse --short HEAD` --init_channels -# 16 --epochs 120 --cutout --autoaugment --seed 22 --weighting_algorithm max_w --primitives DARTS_PRIMITIVES -# Tensorflow is not installed. Skipping tf related imports -# Experiment dir : search-20190321-024555-max_w_SharpSepConvDARTS_SEARCH_5e49783-cifar10-DARTS_PRIMITIVES-OPS-0 -# 2019_03_21_19_11_44 epoch, 47, train_acc, 76.215997, valid_acc, 74.855997, train_loss, 0.687799, valid_loss, 0.734363, lr, 1.580315e-02, best_epoch, 47, best_valid_acc, 74.855997 -# 2019_03_21_19_11_44 genotype = -# 2019_03_21_19_11_44 alphas_normal = tensor([[0.1134, 0.0945, 0.0894, 0.1007, 0.1466, 0.1334, 0.1964, 0.1256], -# [0.1214, 0.0983, 0.1000, 0.1072, 0.1675, 0.1383, 0.1167, 0.1507], -# [0.1251, 0.1066, 0.1043, 0.1674, 0.1295, 0.1237, 0.1209, 0.1225], -# [0.1238, 0.1066, 0.1054, 0.1108, 0.1331, 0.1418, 0.1145, 0.1641], -# [0.1009, 0.0843, 0.0801, 0.0802, 0.2970, 0.1168, 0.1329, 0.1078], -# [0.1257, 0.1115, 0.1087, 0.1158, 0.1641, 0.1312, 0.1305, 0.1125], -# [0.1662, 0.1154, 0.1152, 0.1194, 0.1234, 0.1248, 0.1222, 0.1134], -# [0.1177, 0.0943, 0.1892, 0.0836, 0.1285, 0.1317, 0.1286, 0.1263], -# [0.3851, 0.0835, 0.0718, 0.0554, 0.1031, 0.1047, 0.1011, 0.0953], -# [0.1249, 0.1119, 0.1096, 0.1156, 0.1284, 0.1177, 0.1157, 0.1762], -# [0.1249, 0.1186, 0.1197, 0.1244, 0.1254, 0.1319, 0.1238, 0.1312], -# [0.1126, 0.0932, 0.2448, 0.0838, 0.1132, 0.1141, 0.1263, 0.1120], -# [0.0791, 0.4590, 0.0665, 0.0518, 0.0843, 0.0804, 0.0846, 0.0944], -# [0.0715, 0.0665, 0.4912, 0.0401, 0.0822, 0.0816, 0.0888, 0.0780]], -# device='cuda:0', grad_fn=) -# SHARPSEPCONV_DARTS_MAX_W = Genotype(normal=[('dil_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 2), ('skip_connect', 0), ('avg_pool_3x3', 2), ('sep_conv_3x3', 0), ('avg_pool_3x3', 4), ('max_pool_3x3', 3)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('dil_conv_5x5', 2), ('sep_conv_5x5', 0), ('sep_conv_5x5', 0), ('sep_conv_3x3', 2), ('dil_conv_3x3', 0), ('dil_conv_5x5', 3)], reduce_concat=range(2, 6), layout='cell') - ''' +costar@ubuntu|~/src/sharpDARTS/cnn on multi_channel_search!? +± export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --batch_size 48 --layers_of_cells 8 --layers_in_cells 4 --save max_w_SharpSepConvDARTS_SEARCH_`git rev-parse --short HEAD` --init_channels +16 --epochs 120 --cutout --autoaugment --seed 22 --weighting_algorithm max_w --primitives DARTS_PRIMITIVES +Tensorflow is not installed. Skipping tf related imports +Experiment dir : search-20190321-024555-max_w_SharpSepConvDARTS_SEARCH_5e49783-cifar10-DARTS_PRIMITIVES-OPS-0 +2019_03_21_19_11_44 epoch, 47, train_acc, 76.215997, valid_acc, 74.855997, train_loss, 0.687799, valid_loss, 0.734363, lr, 1.580315e-02, best_epoch, 47, best_valid_acc, 74.855997 +2019_03_21_19_11_44 genotype = +2019_03_21_19_11_44 alphas_normal = tensor([[0.1134, 0.0945, 0.0894, 0.1007, 0.1466, 0.1334, 0.1964, 0.1256], + [0.1214, 0.0983, 0.1000, 0.1072, 0.1675, 0.1383, 0.1167, 0.1507], + [0.1251, 0.1066, 0.1043, 0.1674, 0.1295, 0.1237, 0.1209, 0.1225], + [0.1238, 0.1066, 0.1054, 0.1108, 0.1331, 0.1418, 0.1145, 0.1641], + [0.1009, 0.0843, 0.0801, 0.0802, 0.2970, 0.1168, 0.1329, 0.1078], + [0.1257, 0.1115, 0.1087, 0.1158, 0.1641, 0.1312, 0.1305, 0.1125], + [0.1662, 0.1154, 0.1152, 0.1194, 0.1234, 0.1248, 0.1222, 0.1134], + [0.1177, 0.0943, 0.1892, 0.0836, 0.1285, 0.1317, 0.1286, 0.1263], + [0.3851, 0.0835, 0.0718, 0.0554, 0.1031, 0.1047, 0.1011, 0.0953], + [0.1249, 0.1119, 0.1096, 0.1156, 0.1284, 0.1177, 0.1157, 0.1762], + [0.1249, 0.1186, 0.1197, 0.1244, 0.1254, 0.1319, 0.1238, 0.1312], + [0.1126, 0.0932, 0.2448, 0.0838, 0.1132, 0.1141, 0.1263, 0.1120], + [0.0791, 0.4590, 0.0665, 0.0518, 0.0843, 0.0804, 0.0846, 0.0944], + [0.0715, 0.0665, 0.4912, 0.0401, 0.0822, 0.0816, 0.0888, 0.0780]], + device='cuda:0', grad_fn=) +SHARPSEPCONV_DARTS_MAX_W = Genotype(normal=[('dil_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 2), ('skip_connect', 0), ('avg_pool_3x3', 2), ('sep_conv_3x3', 0), ('avg_pool_3x3', 4), ('max_pool_3x3', 3)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('dil_conv_5x5', 2), ('sep_conv_5x5', 0), ('sep_conv_5x5', 0), ('sep_conv_3x3', 2), ('dil_conv_3x3', 0), ('dil_conv_5x5', 3)], reduce_concat=range(2, 6), layout='cell') 2019_03_22_20_40_54 alphas_normal = tensor([[0.0232, 0.0151, 0.0171, 0.0194, 0.0332, 0.0312, 0.8298, 0.0309], [0.0286, 0.0168, 0.0197, 0.0224, 0.0504, 0.0883, 0.0418, 0.7321], [0.0877, 0.0575, 0.0670, 0.3865, 0.0951, 0.0988, 0.1257, 0.0818], From 0c41d9415cc4be4f98b73baf3484e20e59f0c413 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 9 Apr 2019 17:35:18 -0400 Subject: [PATCH 09/40] cnn/genotypes.py sharper add missing ops --- cnn/genotypes.py | 1 + cnn/operations.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index a3de208..3c05056 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -37,6 +37,7 @@ 'flood_conv_3x3', 'flood_conv_5x5', 'dil_flood_conv_3x3', + 'dil_flood_conv_5x5', # 'choke_conv_3x3', # 'dil_choke_conv_3x3', ] diff --git a/cnn/operations.py b/cnn/operations.py index d02cc52..502a7e8 100644 --- a/cnn/operations.py +++ b/cnn/operations.py @@ -9,6 +9,7 @@ 'skip_connect': lambda C_in, C_out, stride, affine, C_mid=None: Identity() if stride == 1 else FactorizedReduce(C_in, C_out, 1, stride, 0, affine=affine), 'sep_conv_3x3': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 3, stride, padding=1, affine=affine), 'sep_conv_5x5': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 5, stride, padding=2, affine=affine), + 'flood_conv_5x5': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 5, stride, padding=2, affine=affine, C_mid_mult=4), 'sep_conv_7x7': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 7, stride, padding=3, affine=affine), 'dil_conv_3x3': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 3, stride, padding=2, dilation=2, affine=affine), 'dil_conv_5x5': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 5, stride, padding=4, dilation=2, affine=affine), @@ -20,6 +21,7 @@ ), 'flood_conv_3x3': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 3, stride, padding=1, affine=affine, C_mid_mult=4), 'dil_flood_conv_3x3': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 3, stride, padding=2, dilation=2, affine=affine, C_mid_mult=4), + 'dil_flood_conv_5x5': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 5, stride, padding=2, dilation=2, affine=affine, C_mid_mult=4), 'choke_conv_3x3': lambda C_in, C_out, stride, affine, C_mid=32: SharpSepConv(C_in, C_out, 3, stride, padding=1, affine=affine, C_mid=C_mid), 'dil_choke_conv_3x3': lambda C_in, C_out, stride, affine, C_mid=32: SharpSepConv(C_in, C_out, 3, stride, padding=2, dilation=2, affine=affine, C_mid=C_mid), } From 6b5c16194563fd9a1ac0df53ae3f38fd03bf5958 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 9 Apr 2019 17:47:24 -0400 Subject: [PATCH 10/40] cnn/operations.py sharper flood_conv_3x3 added --- cnn/operations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cnn/operations.py b/cnn/operations.py index 502a7e8..992a72f 100644 --- a/cnn/operations.py +++ b/cnn/operations.py @@ -9,6 +9,7 @@ 'skip_connect': lambda C_in, C_out, stride, affine, C_mid=None: Identity() if stride == 1 else FactorizedReduce(C_in, C_out, 1, stride, 0, affine=affine), 'sep_conv_3x3': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 3, stride, padding=1, affine=affine), 'sep_conv_5x5': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 5, stride, padding=2, affine=affine), + 'flood_conv_3x3': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 3, stride, padding=2, affine=affine, C_mid_mult=4), 'flood_conv_5x5': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 5, stride, padding=2, affine=affine, C_mid_mult=4), 'sep_conv_7x7': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 7, stride, padding=3, affine=affine), 'dil_conv_3x3': lambda C_in, C_out, stride, affine, C_mid=None: SharpSepConv(C_in, C_out, 3, stride, padding=2, dilation=2, affine=affine), From 9342d1e7a53c1ef9db24fe654dd5214168ae2f9a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 9 Apr 2019 17:56:19 -0400 Subject: [PATCH 11/40] cnn/genotypes.py SHARPER_PRIMITIVES disable dil_flood_conv_5x5 due to bug --- cnn/genotypes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index 3c05056..d9f7648 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -37,7 +37,10 @@ 'flood_conv_3x3', 'flood_conv_5x5', 'dil_flood_conv_3x3', - 'dil_flood_conv_5x5', + # TODO(ahundt) sharpsepconv doesn't correctly support dil_flood_conv_5x5, padding is not sufficient + # w shape: torch.Size([]) op type: i: 12 self._primitives[i]: dil_flood_conv_5x5x size: torch.Size([16, 16, 32, 32]) stride: 1 + # op_out size: torch.Size([16, 16, 28, 28]) + # 'dil_flood_conv_5x5', # 'choke_conv_3x3', # 'dil_choke_conv_3x3', ] From efa1168b2e8d81bd56c5016d139d86ecb3ae96c9 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 9 Apr 2019 17:59:29 -0400 Subject: [PATCH 12/40] cnn/model_search.py fix debugging code for MixedOp() class --- cnn/model_search.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cnn/model_search.py b/cnn/model_search.py index d695814..dd63d9d 100644 --- a/cnn/model_search.py +++ b/cnn/model_search.py @@ -29,6 +29,7 @@ def __init__(self, C, stride, primitives=None, op_dict=None, weighting_algorithm self._stride = stride if primitives is None: primitives = PRIMITIVES + self._primitives = primitives if op_dict is None: op_dict = operations.OPS for primitive in primitives: @@ -44,7 +45,7 @@ def forward(self, x, weights): # print('-------------------- forward') # print('weights shape: ' + str(len(weights)) + ' ops shape: ' + str(len(self._ops))) # for i, (w, op) in enumerate(zip(weights, self._ops)): - # print('w shape: ' + str(w.shape) + ' op type: ' + str(type(op)) + ' i: ' + str(i) + ' PRIMITIVES[i]: ' + str(PRIMITIVES[i]) + 'x size: ' + str(x.size()) + ' stride: ' + str(self._stride)) + # print('w shape: ' + str(w.shape) + ' op type: ' + str(type(op)) + ' i: ' + str(i) + ' self._primitives[i]: ' + str(self._primitives[i]) + 'x size: ' + str(x.size()) + ' stride: ' + str(self._stride)) # op_out = op(x) # print('op_out size: ' + str(op_out.size())) # result += w * op_out From 17fbc78751fbdba8f381b3fbfdf9d90e7f556e2d Mon Sep 17 00:00:00 2001 From: Priyanka Hubli Date: Fri, 19 Apr 2019 20:25:35 -0400 Subject: [PATCH 13/40] modified TDCFeaturizer forward function to be able to choose different outputs --- cnn/model.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cnn/model.py b/cnn/model.py index 413c4d5..4b7c796 100644 --- a/cnn/model.py +++ b/cnn/model.py @@ -575,7 +575,9 @@ class TDCFeaturizer(nn.Module): For the classifier, we do a multiplication between both feature vectors followed by a fully connected layer. - + Set 'classify_frame_distance' as True if we want the predicted class from the model. + To just get the embeddings, set 'classify_frame_distance' as False and 'frame_embeddings' + as True. """ def __init__(self): super(TDCFeaturizer, self).__init__() @@ -593,7 +595,12 @@ def __init__(self): self.fc2 = nn.Linear(self.feature_vector_size, self.feature_vector_size) self.fc3 = nn.Linear(1024, self.feature_vector_size) self.fc4 = nn.Linear(self.feature_vector_size, 6) # size of each label + self.classify_frame_distance = True + self.frame_embeddings = False + def choose_outputs(self, classify_frame_distance = True, frame_embeddings = False): + self.classify_frame_distance = classify_frame_distance + self.frame_embeddings = frame_embeddings def forward(self, img_1, img_2): logits_aux = None @@ -609,9 +616,9 @@ def forward(self, img_1, img_2): x = F.relu(self.fc1(x)) feature_vector = self.fc2(x) - feature_vector = F.normalize(feature_vector, p=2, dim=1) - #if not self.training: - # return feature_vector + feature_vector = F.normalize(feature_vector, p=2, dim=1) + if self.frame_embeddings and not self.classify_frame_distance: + return feature_vector feature_vector_stack = feature_vector.view(-1, 2, self.feature_vector_size) combined_embeddings = torch.mul(feature_vector_stack[:, 0, :], feature_vector_stack[:, 1, :]) @@ -619,5 +626,8 @@ def forward(self, img_1, img_2): x = F.relu(self.fc3(combined_embeddings)) prediction = F.relu(self.fc4(x)) - return prediction, logits_aux + if self.classify_frame_distance and not self.frame_embeddings: + return prediction , logits_aux + if self.classify_frame_distance and self.frame_embeddings: + return (prediction, feature_vector, logits_aux) From 7e1636259f52812ffc2625b5a556fea910e92013 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 20 Apr 2019 12:49:45 -0400 Subject: [PATCH 14/40] cnn/genotypes.py add first "sharper" search space models --- cnn/genotypes.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index d9f7648..b29b59a 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -389,7 +389,6 @@ 2019_03_26_22_47_45 genotype = Genotype(normal=[('sep_conv_3x3', 0), ('choke_conv_3x3', 1), ('skip_connect', 0), ('dil_conv_3x3', 2), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 0), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('dil_choke_conv_3x3', 0), ('dil_flood_conv_3x3', 2), ('dil_conv_3x3', 0), ('skip_connect', 3), ('flood_conv_3x3', 0), ('skip_connect', 3)], reduce_concat=range(2, 6), layout='cell') 2019_03_26_22_47_45 alphas_normal = tensor([[0.0143, 0.0093, 0.0108, 0.8871, 0.0147, 0.0156, 0.0162, 0.0170, 0.0150], - [0.0372, 0.0194, 0.0285, 0.0438, 0.0300, 0.0522, 0.0522, 0.6729, 0.0637], [0.0251, 0.0151, 0.8027, 0.0271, 0.0251, 0.0254, 0.0278, 0.0270, 0.0247], [0.0345, 0.0207, 0.0290, 0.0423, 0.0373, 0.7211, 0.0346, 0.0405, 0.0400], @@ -437,3 +436,92 @@ Experiment dir : eval-20190327-141933-SHARP_DARTS_MAX_W_2k_c1059c7_cospower_min_1e-8-cifar10-SHARP_DARTS_MAX_W-0 """ SHARP_DARTS_MAX_W = Genotype(normal=[('sep_conv_3x3', 0), ('choke_conv_3x3', 1), ('skip_connect', 0), ('dil_conv_3x3', 2), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 0), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('dil_choke_conv_3x3', 0), ('dil_flood_conv_3x3', 2), ('dil_conv_3x3', 0), ('skip_connect', 3), ('flood_conv_3x3', 0), ('skip_connect', 3)], reduce_concat=range(2, 6), layout='cell') + +""" +ahundt@femur|~/src/darts/cnn on sharper? +± export CUDA_VISIBLE_DEVICES="2" && python3 train_search.py --dataset cifar10 --batch_size 16 --layers_of_cells 8 --layers_in_cells 4 --save SHARPER_SEARCH_`git rev-parse --short HEAD` --init_channels 16 --epochs 120 --cutout --autoaugment --seed 22 --primitives SHARPER_PRIMITIVES +2019_04_09_18_33_45 gpu device = 0 +2019_04_09_18_33_45 args = Namespace(arch='SHARPER_PRIMITIVES-OPS', arch_learning_rate=0.0003, arch_weight_decay=0.001, autoaugment=True, batch_size=16, cutout=True, cutout_length=16, data='../data', dataset='cifar10', drop_path_prob=0.3, epoch_stats_file='search-20190409-183345-SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0/eval-epoch-stats-20190409-183345.json', epochs=120, evaluate='', final_path=None, gpu=0, grad_clip=5, init_channels=16, layers_in_cells=4, layers_of_cells=8, learning_rate=0.025, learning_rate_min=0.0001, load='', load_args='', load_genotype=None, log_file_path='search-20190409-183345-SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0/log.txt', lr_power_annealing_exponent_order=2, mid_channels=32, model_path='saved_models', momentum=0.9, multi_channel=False, no_architect=False, ops='OPS', primitives='SHARPER_PRIMITIVES', random_eraser=False, report_freq=50, save='search-20190409-183345-SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0', seed=22, start_epoch=1, stats_file='search-20190409-183345-SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0/eval-stats-20190409-183345.json', train_portion=0.5, unrolled=False, warmup_epochs=5, weight_decay=0.0003, weighting_algorithm='scalar') +2019_04_09_18_33_45 loading op dict: operations.OPS +2019_04_09_18_33_45 loading primitives:genotypes.SHARPER_PRIMITIVES +2019_04_09_18_33_45 primitives: ['none', 'max_pool_3x3', 'avg_pool_3x3', 'skip_connect', 'sep_conv_3x3', 'sep_conv_5x5', 'sep_conv_7x7', 'dil_conv_3x3', 'dil_conv_5x5', 'flood_conv_3x3', 'flood_conv_5x5', 'dil_flood_conv_3x3'] +2019_04_09_18_33_49 param size = 9.707002MB +2019_04_19_14_33_25 alphas_normal = tensor([[0.1108, 0.0556, 0.0575, 0.2509, 0.1022, 0.0461, 0.0347, 0.0274, 0.0378, 0.2145, 0.0266, 0.0359], + [0.3534, 0.0249, 0.0230, 0.0314, 0.1636, 0.0603, 0.0392, 0.0490, 0.0627, 0.1044, 0.0464, 0.0417], + [0.5115, 0.0438, 0.0384, 0.0831, 0.0495, 0.0549, 0.0467, 0.0462, 0.0292, 0.0390, 0.0229, 0.0348], + [0.6162, 0.0238, 0.0217, 0.0320, 0.0882, 0.0679, 0.0205, 0.0213, 0.0237, 0.0291, 0.0289, 0.0267], + [0.7525, 0.0170, 0.0157, 0.0279, 0.0271, 0.0264, 0.0367, 0.0240, 0.0161, 0.0198, 0.0203, 0.0165], + [0.3173, 0.0881, 0.0614, 0.1120, 0.0474, 0.0473, 0.0461, 0.0410, 0.0378, 0.0895, 0.0414, 0.0707], + [0.3855, 0.0335, 0.0304, 0.0456, 0.0678, 0.0496, 0.0579, 0.0441, 0.0467, 0.1161, 0.0841, 0.0389], + [0.5562, 0.0272, 0.0226, 0.0429, 0.0706, 0.0511, 0.0392, 0.0321, 0.0275, 0.0596, 0.0366, 0.0344], + [0.1158, 0.0256, 0.0253, 0.0423, 0.1826, 0.0349, 0.0435, 0.0868, 0.0274, 0.0752, 0.1449, 0.1957], + [0.2988, 0.0673, 0.0460, 0.0676, 0.0678, 0.0567, 0.0483, 0.0704, 0.0604, 0.1230, 0.0485, 0.0452], + [0.3221, 0.0363, 0.0330, 0.0455, 0.0809, 0.0457, 0.0519, 0.0636, 0.0689, 0.1469, 0.0629, 0.0421], + [0.4835, 0.0269, 0.0227, 0.0398, 0.0528, 0.0671, 0.0407, 0.0762, 0.0554, 0.0495, 0.0554, 0.0300], + [0.0593, 0.0200, 0.0193, 0.0318, 0.0606, 0.0445, 0.0292, 0.0412, 0.0520, 0.1620, 0.0341, 0.4460], + [0.0821, 0.0228, 0.0230, 0.0340, 0.1011, 0.0903, 0.0396, 0.1702, 0.0370, 0.1469, 0.0921, 0.1609]], device='cuda:0', grad_fn=) +2019_04_19_14_33_25 alphas_reduce = tensor([[0.0628, 0.2237, 0.1198, 0.0752, 0.0712, 0.0598, 0.0775, 0.0537, 0.0651, 0.0707, 0.0613, 0.0594], + [0.0812, 0.1069, 0.1021, 0.1551, 0.0841, 0.0636, 0.0473, 0.0726, 0.0584, 0.0799, 0.0786, 0.0701], + [0.0625, 0.1197, 0.1379, 0.0985, 0.1186, 0.0799, 0.0425, 0.0679, 0.0458, 0.0692, 0.0832, 0.0743], + [0.0708, 0.0994, 0.1058, 0.1389, 0.0632, 0.0556, 0.0569, 0.0937, 0.0654, 0.1025, 0.0836, 0.0641], + [0.0874, 0.0765, 0.0767, 0.1159, 0.0823, 0.1001, 0.0772, 0.0783, 0.0534, 0.1009, 0.0804, 0.0708], + [0.0731, 0.0977, 0.1059, 0.1180, 0.0564, 0.1049, 0.0580, 0.0632, 0.0664, 0.0704, 0.0640, 0.1219], + [0.0816, 0.1009, 0.1261, 0.0929, 0.0817, 0.0604, 0.0824, 0.0925, 0.0606, 0.0622, 0.0848, 0.0740], + [0.0923, 0.0670, 0.0673, 0.0952, 0.1105, 0.0709, 0.0742, 0.0857, 0.1044, 0.0679, 0.0793, 0.0852], + [0.0977, 0.0673, 0.0777, 0.1163, 0.0792, 0.0727, 0.0850, 0.0836, 0.1078, 0.0856, 0.0502, 0.0769], + [0.0722, 0.1031, 0.1275, 0.0822, 0.0937, 0.0941, 0.0848, 0.0808, 0.0673, 0.0681, 0.0698, 0.0565], + [0.0762, 0.1123, 0.1090, 0.0942, 0.0699, 0.0770, 0.0775, 0.0765, 0.0812, 0.0897, 0.0716, 0.0650], + [0.0903, 0.0703, 0.0717, 0.1145, 0.0846, 0.0823, 0.0826, 0.0938, 0.0651, 0.0900, 0.0846, 0.0701], + [0.0935, 0.0614, 0.0651, 0.1099, 0.1085, 0.0799, 0.0833, 0.0786, 0.0622, 0.1417, 0.0515, 0.0644], + [0.1492, 0.0676, 0.0754, 0.1314, 0.0717, 0.1051, 0.0829, 0.0670, 0.0863, 0.0683, 0.0518, 0.0432]], device='cuda:0', grad_fn=) +Overview ***** best_epoch: 113 best_valid_acc: 86.48 ***** Progress: 99%|| 119/120 [235:59:34<1:59:19, 7159.36s/itTraceback (most recent call last):91.89, top 5: 98.82 progress: 5%|| 74/1563 [05:08<1:43:31, 4.17s/it] +2019_04_19_16_32_32 epoch, 120, train_acc, 91.084000, valid_acc, 86.732000, train_loss, 0.260623, valid_loss, 0.394877, lr, 1.000000e-04, best_epoch, 120, best_valid_acc, 86.732000 +Overview ***** best_epoch: 120 best_valid_acc: 86.73 ***** Progress: 100%|| 120/120 [237:58:43<00:00, 7156.24s/it] +2019_04_19_16_32_34 genotype = Genotype(normal=[('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 1), ('dil_flood_conv_3x3', 3), ('dil_conv_3x3', 4)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3), ('skip_connect', 4)], reduce_concat=range(2, 6), layout='cell') +2019_04_19_16_32_34 Search for Model Complete! Save dir: search-20190409-183345-SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0 +""" + +SHARPER_SCALAR = Genotype(normal=[('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 1), ('dil_flood_conv_3x3', 3), ('dil_conv_3x3', 4)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3), ('skip_connect', 4)], reduce_concat=range(2, 6), layout='cell') + +""" + +ahundt@femur|~/src/darts/cnn on sharper? +± export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --batch_size 16 --layers_of_cells 8 --layers_in_cells 4 --save max_w_SHARPER_SEARCH_`git rev-parse --short HEAD` --init_channels 16 --epochs 120 --cutout --autoaugment --seed 22 --weighting_algorithm max_w --primitives SHARPER_PRIMITIVES +2019_04_19_20_07_23 epoch, 119, train_acc, 88.696000, valid_acc, 84.868000, train_loss, 0.327119, valid_loss, 0.456210, lr, 1.032201e-04, best_epoch, 119, best_valid_acc, 84.868000 +2019_04_19_20_07_25 genotype = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') +2019_04_19_20_07_25 alphas_normal = tensor([[0.0064, 0.0045, 0.0048, 0.0053, 0.9306, 0.0070, 0.0069, 0.0070, 0.0064, 0.0066, 0.0069, 0.0076], + [0.0527, 0.0296, 0.0307, 0.0398, 0.0742, 0.0773, 0.0469, 0.1153, 0.2443, 0.1075, 0.0677, 0.1140], + [0.0392, 0.5563, 0.0226, 0.0266, 0.0424, 0.0451, 0.0452, 0.0414, 0.0485, 0.0419, 0.0433, 0.0476], + [0.0692, 0.0517, 0.0510, 0.0607, 0.0794, 0.2549, 0.0680, 0.0732, 0.0677, 0.0699, 0.0744, 0.0799], + [0.0454, 0.0421, 0.0343, 0.0414, 0.0426, 0.0447, 0.5136, 0.0453, 0.0458, 0.0522, 0.0464, 0.0460], + [0.0111, 0.0090, 0.0093, 0.0099, 0.0111, 0.0114, 0.0113, 0.0108, 0.0111, 0.8829, 0.0108, 0.0113], + [0.0610, 0.0434, 0.0440, 0.0507, 0.0652, 0.0654, 0.0673, 0.0664, 0.0790, 0.0715, 0.3231, 0.0629], + [0.0512, 0.0389, 0.0340, 0.4399, 0.0558, 0.0542, 0.0536, 0.0563, 0.0582, 0.0515, 0.0535, 0.0529], + [0.0081, 0.0071, 0.9128, 0.0058, 0.0081, 0.0083, 0.0082, 0.0083, 0.0082, 0.0085, 0.0083, 0.0082], + [0.0772, 0.0519, 0.0568, 0.0651, 0.0911, 0.1073, 0.1002, 0.1074, 0.0702, 0.0717, 0.0935, 0.1075], + [0.0779, 0.0605, 0.0629, 0.0704, 0.0858, 0.0788, 0.0745, 0.0790, 0.0771, 0.1685, 0.0820, 0.0824], + [0.0795, 0.0674, 0.0584, 0.1416, 0.0774, 0.0742, 0.0791, 0.0843, 0.0848, 0.0799, 0.0824, 0.0909], + [0.5488, 0.0359, 0.0321, 0.0295, 0.0464, 0.0443, 0.0413, 0.0444, 0.0459, 0.0438, 0.0430, 0.0445], + [0.6040, 0.0317, 0.0297, 0.0233, 0.0423, 0.0379, 0.0389, 0.0354, 0.0369, 0.0379, 0.0445, 0.0376]], device='cuda:0', grad_fn=) +2019_04_19_20_07_25 alphas_reduce = tensor([[0.0370, 0.0326, 0.0338, 0.0345, 0.0368, 0.0377, 0.0339, 0.0378, 0.0322, 0.0370, 0.6131, 0.0336], + [0.0800, 0.0891, 0.0890, 0.0891, 0.0888, 0.0890, 0.0672, 0.0886, 0.0890, 0.0880, 0.0530, 0.0892], + [0.0461, 0.0431, 0.0443, 0.0448, 0.5159, 0.0455, 0.0426, 0.0427, 0.0423, 0.0434, 0.0458, 0.0435], + [0.0825, 0.0883, 0.0882, 0.0886, 0.0820, 0.0883, 0.0698, 0.0882, 0.0715, 0.0776, 0.0869, 0.0882], + [0.0795, 0.0849, 0.0849, 0.1471, 0.0828, 0.0768, 0.0816, 0.0687, 0.0692, 0.0897, 0.0749, 0.0598], + [0.0694, 0.0658, 0.0676, 0.0679, 0.2674, 0.0675, 0.0653, 0.0684, 0.0669, 0.0629, 0.0648, 0.0661], + [0.0831, 0.0887, 0.0884, 0.0885, 0.0827, 0.0841, 0.0801, 0.0793, 0.0780, 0.0852, 0.0821, 0.0799], + [0.0795, 0.0818, 0.0828, 0.1563, 0.0836, 0.0785, 0.0753, 0.0717, 0.0760, 0.0711, 0.0712, 0.0721], + [0.0812, 0.0808, 0.0787, 0.1316, 0.0800, 0.0816, 0.0862, 0.0779, 0.0833, 0.0724, 0.0753, 0.0709], + [0.0801, 0.0770, 0.0786, 0.0791, 0.1442, 0.0784, 0.0740, 0.0758, 0.0773, 0.0812, 0.0776, 0.0767], + [0.0832, 0.0874, 0.0869, 0.0881, 0.0839, 0.0808, 0.0828, 0.0830, 0.0818, 0.0830, 0.0753, 0.0838], + [0.0823, 0.0844, 0.0854, 0.1030, 0.0826, 0.0882, 0.0793, 0.0819, 0.0845, 0.0774, 0.0792, 0.0719], + [0.0827, 0.0825, 0.0813, 0.1029, 0.0854, 0.0818, 0.0835, 0.0824, 0.0828, 0.0799, 0.0774, 0.0774], + [0.0799, 0.0759, 0.0747, 0.0774, 0.0808, 0.0749, 0.0827, 0.0802, 0.1554, 0.0747, 0.0713, 0.0722]], device='cuda:0', grad_fn=) +2019_04_19_22_09_50 epoch, 120, train_acc, 89.072000, valid_acc, 84.676000, train_loss, 0.310882, valid_loss, 0.459557, lr, 1.000000e-04, best_epoch, 119, best_valid_acc, 84.868000 +Overview ***** best_epoch: 119 best_valid_acc: 84.87 ***** Progress: 100%|| 120/120 [244:05:44<00:00, 7367.80s/it] +2019_04_19_22_09_52 genotype = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('max_pool_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5 +x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') +2019_04_19_22_09_52 Search for Model Complete! Save dir: search-20190409-180403-max_w_SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0 +""" + +SHARPER_MAX_W = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') From ddc4ff3a6501b4ffa99fb1012136fabc8ac05479 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 20 Apr 2019 13:00:19 -0400 Subject: [PATCH 15/40] cnn/genotypes.py sharper flops --- cnn/genotypes.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index b29b59a..ab1c4f7 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -479,6 +479,10 @@ Overview ***** best_epoch: 120 best_valid_acc: 86.73 ***** Progress: 100%|| 120/120 [237:58:43<00:00, 7156.24s/it] 2019_04_19_16_32_34 genotype = Genotype(normal=[('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 1), ('dil_flood_conv_3x3', 3), ('dil_conv_3x3', 4)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3), ('skip_connect', 4)], reduce_concat=range(2, 6), layout='cell') 2019_04_19_16_32_34 Search for Model Complete! Save dir: search-20190409-183345-SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0 + +2019_04_20_12_52_14 param size = 7.109470MB +2019_04_20_12_52_14 flops_shape = [1, 3, 32, 32] +2019_04_20_12_52_14 flops = 1.1GMac """ SHARPER_SCALAR = Genotype(normal=[('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 1), ('dil_flood_conv_3x3', 3), ('dil_conv_3x3', 4)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3), ('skip_connect', 4)], reduce_concat=range(2, 6), layout='cell') @@ -487,6 +491,12 @@ ahundt@femur|~/src/darts/cnn on sharper? ± export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --batch_size 16 --layers_of_cells 8 --layers_in_cells 4 --save max_w_SHARPER_SEARCH_`git rev-parse --short HEAD` --init_channels 16 --epochs 120 --cutout --autoaugment --seed 22 --weighting_algorithm max_w --primitives SHARPER_PRIMITIVES +2019_04_09_18_04_03 gpu device = 0 +2019_04_09_18_04_03 args = Namespace(arch='SHARPER_PRIMITIVES-OPS', arch_learning_rate=0.0003, arch_weight_decay=0.001, autoaugment=True, batch_size=16, cutout=True, cutout_length=16, data='../data', dataset='cifar10', drop_path_prob=0.3, epoch_stats_file='search-20190409-180403-max_w_SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0/eval-epoch-stats-20190409-180403.json', epochs=120, evaluate='', final_path=None, gpu=0, grad_clip=5, init_channels=16, layers_in_cells=4, layers_of_cells=8, learning_rate=0.025, learning_rate_min=0.0001, load='', load_args='', load_genotype=None, log_file_path='search-20190409-180403-max_w_SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0/log.txt', lr_power_annealing_exponent_order=2, mid_channels=32, model_path='saved_models', momentum=0.9, multi_channel=False, no_architect=False, ops='OPS', primitives='SHARPER_PRIMITIVES', random_eraser=False, report_freq=50, save='search-20190409-180403-max_w_SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0', seed=22, start_epoch=1, stats_file='search-20190409-180403-max_w_SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0/eval-stats-20190409-180403.json', train_portion=0.5, unrolled=False, warmup_epochs=5, weight_decay=0.0003, weighting_algorithm='max_w') +2019_04_09_18_04_03 loading op dict: operations.OPS +2019_04_09_18_04_03 loading primitives:genotypes.SHARPER_PRIMITIVES +2019_04_09_18_04_03 primitives: ['none', 'max_pool_3x3', 'avg_pool_3x3', 'skip_connect', 'sep_conv_3x3', 'sep_conv_5x5', 'sep_conv_7x7', 'dil_conv_3x3', 'dil_conv_5x5', 'flood_conv_3x3', 'flood_conv_5x5', 'dil_flood_conv_3x3'] +2019_04_09_18_04_07 param size = 9.707002MB 2019_04_19_20_07_23 epoch, 119, train_acc, 88.696000, valid_acc, 84.868000, train_loss, 0.327119, valid_loss, 0.456210, lr, 1.032201e-04, best_epoch, 119, best_valid_acc, 84.868000 2019_04_19_20_07_25 genotype = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') 2019_04_19_20_07_25 alphas_normal = tensor([[0.0064, 0.0045, 0.0048, 0.0053, 0.9306, 0.0070, 0.0069, 0.0070, 0.0064, 0.0066, 0.0069, 0.0076], @@ -522,6 +532,10 @@ 2019_04_19_22_09_52 genotype = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('max_pool_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5 x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') 2019_04_19_22_09_52 Search for Model Complete! Save dir: search-20190409-180403-max_w_SHARPER_SEARCH_efa1168-cifar10-SHARPER_PRIMITIVES-OPS-0 + +2019_04_20_12_51_18 param size = 6.087142MB +2019_04_20_12_51_18 flops_shape = [1, 3, 32, 32] +2019_04_20_12_51_18 flops = 950.22MMac """ SHARPER_MAX_W = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') From 8166e1c2d4f6ca63eadc75e756fc2bf19501dea8 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 20 Apr 2019 13:15:28 -0400 Subject: [PATCH 16/40] cnn/genotypes.py add cifar10 startup command to sharper models --- cnn/genotypes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index ab1c4f7..0e7025b 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -483,6 +483,8 @@ 2019_04_20_12_52_14 param size = 7.109470MB 2019_04_20_12_52_14 flops_shape = [1, 3, 32, 32] 2019_04_20_12_52_14 flops = 1.1GMac + +for i in {1..8}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --b 48 --save SHARPER_SCALAR_2k_`git rev-parse --short HEAD` --arch SHARPER_SCALAR --epochs 2000 --cutout --autoaugment --auxiliary ; done; """ SHARPER_SCALAR = Genotype(normal=[('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 1), ('dil_flood_conv_3x3', 3), ('dil_conv_3x3', 4)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3), ('skip_connect', 4)], reduce_concat=range(2, 6), layout='cell') @@ -536,6 +538,8 @@ 2019_04_20_12_51_18 param size = 6.087142MB 2019_04_20_12_51_18 flops_shape = [1, 3, 32, 32] 2019_04_20_12_51_18 flops = 950.22MMac + +for i in {1..8}; do export CUDA_VISIBLE_DEVICES="2" && python3 train.py --b 64 --save SHARPER_MAX_W_2k_`git rev-parse --short HEAD` --arch SHARPER_MAX_W --epochs 2000 --cutout --autoaugment --auxiliary ; done; """ SHARPER_MAX_W = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') From c4891da52248c982e2082aa81f872fe9ff8c291f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 12 May 2019 12:35:26 -0400 Subject: [PATCH 17/40] cnn/genotype_extractor.py added, can disable 'none' layer hack from DARTS. Also added hacked and unhacked versions of sharper search space. --- cnn/genotype_extractor.py | 137 ++++++++++++++++++++++++++++++++++++++ cnn/genotypes.py | 62 +++++++++++++++++ cnn/model_search.py | 67 +++++-------------- 3 files changed, 216 insertions(+), 50 deletions(-) create mode 100644 cnn/genotype_extractor.py diff --git a/cnn/genotype_extractor.py b/cnn/genotype_extractor.py new file mode 100644 index 0000000..eaa5b7e --- /dev/null +++ b/cnn/genotype_extractor.py @@ -0,0 +1,137 @@ +''' +This file includes code from the DARTS and sharpDARTS https://arxiv.org/abs/1903.09900 papers. +''' + +import numpy as np +import genotypes + + +def parse_cell(weights, primitives, steps=4, skip_primitive='none'): + """ Take a weight array and turn it into a list of pairs (primitive_string, node_index). + """ + gene = [] + n = 2 + start = 0 + for add_node_index in range(steps): + # Each step is a separate "add node" in the graph, so i is the integer index of the current node. + # A better name for i might be add_node_index. + end = start + n + # Only look at the weights relevant to this node. + # "Nodes" 0 and 1 will always be the output of the previous cells. + # + # All other nodes will be add nodes which need edges connecting back to the previous nodes: + # add node 0 will need 2: rows 0, 1 + # add node 1 will need 3: rows 2, 3, 4 + # add node 2 will need 4: rows 5, 6, 7, 8 + # add node 3 will need 5: rows 9, 10, 11, 12, 13 + # ...and so on if there are more than 4 nodes. + W = weights[start:end].copy() + # print('add_node_index: ' + str(add_node_index) + ' start: ' + str(start) + ' end: ' + str(end) + ' W shape: ' + str(W.shape)) + # Each row in the weights is a separate edge, and each column are the possible primitives that edge might use. + # The first "add node" can connect back to the two previous cells, which is why the edges are i + 2. + # The sorted function orders lists from lowest to highest, so we use -max in the lambda function to sort from highest to lowest. + # We currently say there will only be two edges connecting to each node, which is why there is [:2], to select the two highest score edges. + # Each later nodes can connect back to the previous cells or an internal node, so the range(i+2) of possible connections increases. + pre_edges = sorted(range(add_node_index + 2), + key=lambda x: -max(W[x][k] for k in range(len(W[x])) if skip_primitive is None or k != primitives.index(skip_primitive))) + edges = pre_edges[:2] + # print('edges: ' + str(edges)) + # We've now selected the two edges we will use for this node, so next let's select the layer primitives. + # Each edge needs a particular primitive, so go through all the edges and compare all the possible primitives. + for j in edges: + k_best = None + # note: This probably could be simpler via argmax... + # Loop through all the columns to find the highest score primitive for the chosen edge, excluding none. + for k in range(len(W[j])): + if skip_primitive is None or k != primitives.index(skip_primitive): + if k_best is None or W[j][k] > W[j][k_best]: + k_best = k + # Once the best primitive is chosen, create the new gene element, which is the + # string for the name of the primitive, and the index of the previous node to connect to, + gene.append((primitives[k_best], j)) + start = end + n += 1 + # Return the full list of (node, primitive) pairs for this set of weights. + return gene + +def genotype_cell(alphas_normal, alphas_reduce, primitives, steps=4, multiplier=4, skip_primitive='none'): + # skip_primitive = 'none' is a hack in original DARTS, which removes a no-op primitive. + # skip_primitive = None means no hack is applied + # note the printed weights from a call to Network::arch_weights() in model_search.py + # are already post-softmax, so we don't need to apply softmax again + # alphas_normal = torch.FloatTensor(genotypes.SHARPER_SCALAR_WEIGHTS.normal) + # alphas_reduce = torch.FloatTensor(genotypes.SHARPER_SCALAR_WEIGHTS.reduce) + # F.softmax(alphas_normal, dim=-1).data.cpu().numpy() + # F.softmax(alphas_reduce, dim=-1).data.cpu().numpy() + gene_normal = parse_cell(alphas_normal, primitives, steps, skip_primitive=skip_primitive) + gene_reduce = parse_cell(alphas_reduce, primitives, steps, skip_primitive=skip_primitive) + + concat = range(2+steps-multiplier, steps+2) + genotype = genotypes.Genotype( + normal=gene_normal, normal_concat=concat, + reduce=gene_reduce, reduce_concat=concat, + layout='cell', + ) + return genotype + + +def main(): + ''' + Parse raw weights with the hack that excludes the 'none' primitive, and without that hack. + Then print out the final genotypes. + ''' + # Set these variables to the ones you're using in this case from genotypes.py + skip_primitive = None + raw_weights_genotype = genotypes.SHARPER_SCALAR_WEIGHTS + primitives = genotypes.SHARPER_PRIMITIVES + # get the normal and reduce weights as a numpy array + alphas_normal = np.array(raw_weights_genotype.normal) + alphas_reduce = np.array(raw_weights_genotype.reduce) + + # for steps, see layers_in_cells in train_search.py + steps = 4 + # for multiplier, see multiplier for Network class in model_search.py + multiplier = 4 + # note the printed weights from a call to Network::arch_weights() in model_search.py + # are already post-softmax, so we don't need to apply softmax again + # alphas_normal = torch.FloatTensor(genotypes.SHARPER_SCALAR_WEIGHTS.normal) + # alphas_reduce = torch.FloatTensor(genotypes.SHARPER_SCALAR_WEIGHTS.reduce) + # F.softmax(alphas_normal, dim=-1).data.cpu().numpy() + # F.softmax(alphas_reduce, dim=-1).data.cpu().numpy() + + # skip_primitive = 'none' is a hack in original DARTS, which removes a no-op primitive. + # skip_primitive = None means no hack is applied + print('#################') + genotype = genotype_cell(alphas_normal, alphas_reduce, primitives, steps=4, multiplier=4, skip_primitive='none') + print('SHARPER_SCALAR_genotype_skip_none = ' + str(genotype)) + genotype = genotype_cell(alphas_normal, alphas_reduce, primitives, steps=4, multiplier=4, skip_primitive=None) + print('SHARPER_SCALAR_genotype_no_hack = ' + str(genotype)) + # Set these variables to the ones you're using in this case from genotypes.py + skip_primitive = None + raw_weights_genotype = genotypes.SHARPER_MAX_W_WEIGHTS + primitives = genotypes.SHARPER_PRIMITIVES + # get the normal and reduce weights as a numpy array + alphas_normal = np.array(raw_weights_genotype.normal) + alphas_reduce = np.array(raw_weights_genotype.reduce) + + # for steps, see layers_in_cells in train_search.py + steps = 4 + # for multiplier, see multiplier for Network class in model_search.py + multiplier = 4 + # note the printed weights from a call to Network::arch_weights() in model_search.py + # are already post-softmax, so we don't need to apply softmax again + # alphas_normal = torch.FloatTensor(genotypes.SHARPER_SCALAR_WEIGHTS.normal) + # alphas_reduce = torch.FloatTensor(genotypes.SHARPER_SCALAR_WEIGHTS.reduce) + # F.softmax(alphas_normal, dim=-1).data.cpu().numpy() + # F.softmax(alphas_reduce, dim=-1).data.cpu().numpy() + + # skip_primitive = 'none' is a hack in original DARTS, which removes a no-op primitive. + # skip_primitive = None means no hack is applied + print('#################') + genotype = genotype_cell(alphas_normal, alphas_reduce, primitives, steps=4, multiplier=4, skip_primitive='none') + print('SHARPER_MAX_W_genotype_skip_none = ' + str(genotype)) + genotype = genotype_cell(alphas_normal, alphas_reduce, primitives, steps=4, multiplier=4, skip_primitive=None) + print('SHARPER_MAX_W_genotype_no_hack = ' + str(genotype)) + +if __name__ == '__main__': + main() diff --git a/cnn/genotypes.py b/cnn/genotypes.py index 0e7025b..35c4a35 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -488,6 +488,37 @@ """ SHARPER_SCALAR = Genotype(normal=[('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 1), ('dil_flood_conv_3x3', 3), ('dil_conv_3x3', 4)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3), ('skip_connect', 4)], reduce_concat=range(2, 6), layout='cell') +SHARPER_SCALAR_WEIGHTS = Genotype(normal=[[0.1108, 0.0556, 0.0575, 0.2509, 0.1022, 0.0461, 0.0347, 0.0274, 0.0378, 0.2145, 0.0266, 0.0359], + [0.3534, 0.0249, 0.0230, 0.0314, 0.1636, 0.0603, 0.0392, 0.0490, 0.0627, 0.1044, 0.0464, 0.0417], + [0.5115, 0.0438, 0.0384, 0.0831, 0.0495, 0.0549, 0.0467, 0.0462, 0.0292, 0.0390, 0.0229, 0.0348], + [0.6162, 0.0238, 0.0217, 0.0320, 0.0882, 0.0679, 0.0205, 0.0213, 0.0237, 0.0291, 0.0289, 0.0267], + [0.7525, 0.0170, 0.0157, 0.0279, 0.0271, 0.0264, 0.0367, 0.0240, 0.0161, 0.0198, 0.0203, 0.0165], + [0.3173, 0.0881, 0.0614, 0.1120, 0.0474, 0.0473, 0.0461, 0.0410, 0.0378, 0.0895, 0.0414, 0.0707], + [0.3855, 0.0335, 0.0304, 0.0456, 0.0678, 0.0496, 0.0579, 0.0441, 0.0467, 0.1161, 0.0841, 0.0389], + [0.5562, 0.0272, 0.0226, 0.0429, 0.0706, 0.0511, 0.0392, 0.0321, 0.0275, 0.0596, 0.0366, 0.0344], + [0.1158, 0.0256, 0.0253, 0.0423, 0.1826, 0.0349, 0.0435, 0.0868, 0.0274, 0.0752, 0.1449, 0.1957], + [0.2988, 0.0673, 0.0460, 0.0676, 0.0678, 0.0567, 0.0483, 0.0704, 0.0604, 0.1230, 0.0485, 0.0452], + [0.3221, 0.0363, 0.0330, 0.0455, 0.0809, 0.0457, 0.0519, 0.0636, 0.0689, 0.1469, 0.0629, 0.0421], + [0.4835, 0.0269, 0.0227, 0.0398, 0.0528, 0.0671, 0.0407, 0.0762, 0.0554, 0.0495, 0.0554, 0.0300], + [0.0593, 0.0200, 0.0193, 0.0318, 0.0606, 0.0445, 0.0292, 0.0412, 0.0520, 0.1620, 0.0341, 0.4460], + [0.0821, 0.0228, 0.0230, 0.0340, 0.1011, 0.0903, 0.0396, 0.1702, 0.0370, 0.1469, 0.0921, 0.1609]], + reduce=[[0.0628, 0.2237, 0.1198, 0.0752, 0.0712, 0.0598, 0.0775, 0.0537, 0.0651, 0.0707, 0.0613, 0.0594], + [0.0812, 0.1069, 0.1021, 0.1551, 0.0841, 0.0636, 0.0473, 0.0726, 0.0584, 0.0799, 0.0786, 0.0701], + [0.0625, 0.1197, 0.1379, 0.0985, 0.1186, 0.0799, 0.0425, 0.0679, 0.0458, 0.0692, 0.0832, 0.0743], + [0.0708, 0.0994, 0.1058, 0.1389, 0.0632, 0.0556, 0.0569, 0.0937, 0.0654, 0.1025, 0.0836, 0.0641], + [0.0874, 0.0765, 0.0767, 0.1159, 0.0823, 0.1001, 0.0772, 0.0783, 0.0534, 0.1009, 0.0804, 0.0708], + [0.0731, 0.0977, 0.1059, 0.1180, 0.0564, 0.1049, 0.0580, 0.0632, 0.0664, 0.0704, 0.0640, 0.1219], + [0.0816, 0.1009, 0.1261, 0.0929, 0.0817, 0.0604, 0.0824, 0.0925, 0.0606, 0.0622, 0.0848, 0.0740], + [0.0923, 0.0670, 0.0673, 0.0952, 0.1105, 0.0709, 0.0742, 0.0857, 0.1044, 0.0679, 0.0793, 0.0852], + [0.0977, 0.0673, 0.0777, 0.1163, 0.0792, 0.0727, 0.0850, 0.0836, 0.1078, 0.0856, 0.0502, 0.0769], + [0.0722, 0.1031, 0.1275, 0.0822, 0.0937, 0.0941, 0.0848, 0.0808, 0.0673, 0.0681, 0.0698, 0.0565], + [0.0762, 0.1123, 0.1090, 0.0942, 0.0699, 0.0770, 0.0775, 0.0765, 0.0812, 0.0897, 0.0716, 0.0650], + [0.0903, 0.0703, 0.0717, 0.1145, 0.0846, 0.0823, 0.0826, 0.0938, 0.0651, 0.0900, 0.0846, 0.0701], + [0.0935, 0.0614, 0.0651, 0.1099, 0.1085, 0.0799, 0.0833, 0.0786, 0.0622, 0.1417, 0.0515, 0.0644], + [0.1492, 0.0676, 0.0754, 0.1314, 0.0717, 0.1051, 0.0829, 0.0670, 0.0863, 0.0683, 0.0518, 0.0432]], normal_concat=[], reduce_concat=[], layout='raw_weights') +# Retrieved from SHARPER_SCALAR_WEIGHTS by running genotype_extractor.py +SHARPER_SCALAR_genotype_skip_none = Genotype(normal=[('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 1), ('dil_flood_conv_3x3', 3), ('dil_conv_3x3', 4)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3), ('skip_connect', 4)], reduce_concat=range(2, 6), layout='cell') +SHARPER_SCALAR_genotype_no_hack = Genotype(normal=[('none', 1), ('skip_connect', 0), ('none', 2), ('none', 1), ('none', 2), ('none', 1), ('none', 2), ('dil_flood_conv_3x3', 3)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('none', 4), ('flood_conv_3x3', 3)], reduce_concat=range(2, 6), layout='cell') """ @@ -543,3 +574,34 @@ """ SHARPER_MAX_W = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') +SHARPER_MAX_W_WEIGHTS = Genotype(normal=[[0.0064, 0.0045, 0.0048, 0.0053, 0.9306, 0.0070, 0.0069, 0.0070, 0.0064, 0.0066, 0.0069, 0.0076], + [0.0527, 0.0296, 0.0307, 0.0398, 0.0742, 0.0773, 0.0469, 0.1153, 0.2443, 0.1075, 0.0677, 0.1140], + [0.0392, 0.5563, 0.0226, 0.0266, 0.0424, 0.0451, 0.0452, 0.0414, 0.0485, 0.0419, 0.0433, 0.0476], + [0.0692, 0.0517, 0.0510, 0.0607, 0.0794, 0.2549, 0.0680, 0.0732, 0.0677, 0.0699, 0.0744, 0.0799], + [0.0454, 0.0421, 0.0343, 0.0414, 0.0426, 0.0447, 0.5136, 0.0453, 0.0458, 0.0522, 0.0464, 0.0460], + [0.0111, 0.0090, 0.0093, 0.0099, 0.0111, 0.0114, 0.0113, 0.0108, 0.0111, 0.8829, 0.0108, 0.0113], + [0.0610, 0.0434, 0.0440, 0.0507, 0.0652, 0.0654, 0.0673, 0.0664, 0.0790, 0.0715, 0.3231, 0.0629], + [0.0512, 0.0389, 0.0340, 0.4399, 0.0558, 0.0542, 0.0536, 0.0563, 0.0582, 0.0515, 0.0535, 0.0529], + [0.0081, 0.0071, 0.9128, 0.0058, 0.0081, 0.0083, 0.0082, 0.0083, 0.0082, 0.0085, 0.0083, 0.0082], + [0.0772, 0.0519, 0.0568, 0.0651, 0.0911, 0.1073, 0.1002, 0.1074, 0.0702, 0.0717, 0.0935, 0.1075], + [0.0779, 0.0605, 0.0629, 0.0704, 0.0858, 0.0788, 0.0745, 0.0790, 0.0771, 0.1685, 0.0820, 0.0824], + [0.0795, 0.0674, 0.0584, 0.1416, 0.0774, 0.0742, 0.0791, 0.0843, 0.0848, 0.0799, 0.0824, 0.0909], + [0.5488, 0.0359, 0.0321, 0.0295, 0.0464, 0.0443, 0.0413, 0.0444, 0.0459, 0.0438, 0.0430, 0.0445], + [0.6040, 0.0317, 0.0297, 0.0233, 0.0423, 0.0379, 0.0389, 0.0354, 0.0369, 0.0379, 0.0445, 0.0376]], + reduce=[[0.0370, 0.0326, 0.0338, 0.0345, 0.0368, 0.0377, 0.0339, 0.0378, 0.0322, 0.0370, 0.6131, 0.0336], + [0.0800, 0.0891, 0.0890, 0.0891, 0.0888, 0.0890, 0.0672, 0.0886, 0.0890, 0.0880, 0.0530, 0.0892], + [0.0461, 0.0431, 0.0443, 0.0448, 0.5159, 0.0455, 0.0426, 0.0427, 0.0423, 0.0434, 0.0458, 0.0435], + [0.0825, 0.0883, 0.0882, 0.0886, 0.0820, 0.0883, 0.0698, 0.0882, 0.0715, 0.0776, 0.0869, 0.0882], + [0.0795, 0.0849, 0.0849, 0.1471, 0.0828, 0.0768, 0.0816, 0.0687, 0.0692, 0.0897, 0.0749, 0.0598], + [0.0694, 0.0658, 0.0676, 0.0679, 0.2674, 0.0675, 0.0653, 0.0684, 0.0669, 0.0629, 0.0648, 0.0661], + [0.0831, 0.0887, 0.0884, 0.0885, 0.0827, 0.0841, 0.0801, 0.0793, 0.0780, 0.0852, 0.0821, 0.0799], + [0.0795, 0.0818, 0.0828, 0.1563, 0.0836, 0.0785, 0.0753, 0.0717, 0.0760, 0.0711, 0.0712, 0.0721], + [0.0812, 0.0808, 0.0787, 0.1316, 0.0800, 0.0816, 0.0862, 0.0779, 0.0833, 0.0724, 0.0753, 0.0709], + [0.0801, 0.0770, 0.0786, 0.0791, 0.1442, 0.0784, 0.0740, 0.0758, 0.0773, 0.0812, 0.0776, 0.0767], + [0.0832, 0.0874, 0.0869, 0.0881, 0.0839, 0.0808, 0.0828, 0.0830, 0.0818, 0.0830, 0.0753, 0.0838], + [0.0823, 0.0844, 0.0854, 0.1030, 0.0826, 0.0882, 0.0793, 0.0819, 0.0845, 0.0774, 0.0792, 0.0719], + [0.0827, 0.0825, 0.0813, 0.1029, 0.0854, 0.0818, 0.0835, 0.0824, 0.0828, 0.0799, 0.0774, 0.0774], + [0.0799, 0.0759, 0.0747, 0.0774, 0.0808, 0.0749, 0.0827, 0.0802, 0.1554, 0.0747, 0.0713, 0.0722]], normal_concat=[], reduce_concat=[], layout='raw_weights') +# Retrieved from SHARPER_MAX_W_WEIGHTS by running genotype_extractor.py +SHARPER_MAX_W_genotype_skip_none = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') +SHARPER_MAX_W_genotype_no_hack = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('none', 4), ('none', 3)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') \ No newline at end of file diff --git a/cnn/model_search.py b/cnn/model_search.py index dd63d9d..6b28fc2 100644 --- a/cnn/model_search.py +++ b/cnn/model_search.py @@ -11,6 +11,7 @@ from networkx.readwrite import json_graph import json import time +import genotype_extractor class MixedOp(nn.Module): @@ -203,54 +204,20 @@ def arch_weights(self, stride_idx): weights = F.softmax(weights_softmax_view, dim=-1) return weights - def genotype(self): - - def _parse(weights): - """ Take a weight array and turn it into a list of pairs (primitive_string, node_index). - """ - gene = [] - n = 2 - start = 0 - for i in range(self._steps): - # Each step is a separate "add node" in the graph, so i is the integer index of the current node. - # A better name for i might be add_node_index. - end = start + n - # Only look at the weights relevant to this node. - # "Nodes" 0 and 1 will always be the output of the previous cells. - # - # All other nodes will be add nodes which need edges connecting back to the previous nodes: - # add node 0 will need 2: rows 0, 1 - # add node 1 will need 3: rows 2, 3, 4 - # add node 2 will need 4: rows 5, 6, 7, 8 - # add node 3 will need 5: rows 9, 10, 11, 12, 13 - # ...and so on if there are more than 4 nodes. - W = weights[start:end].copy() - # Each row in the weights is a separate edge, and each column are the possible primitives that edge might use. - # The first "add node" can connect back to the two previous cells, which is why the edges are i + 2. - # The sorted function orders lists from lowest to highest, so we use -max in the lambda function to sort from highest to lowest. - # We currently say there will only be two edges connecting to each node, which is why there is [:2], to select the two highest score edges. - # Each later nodes can connect back to the previous cells or an internal node, so the range(i+2) of possible connections increases. - edges = sorted(range(i + 2), key=lambda x: -max(W[x][k] for k in range(len(W[x])) if k != self.primitives.index('none')))[:2] - # We've now selected the two edges we will use for this node, so next let's select the layer primitives. - # Each edge needs a particular primitive, so go through all the edges and compare all the possible primitives. - for j in edges: - k_best = None - # note: This probably could be simpler via argmax... - # Loop through all the columns to find the highest score primitive for the chosen edge, excluding none. - for k in range(len(W[j])): - if k != self.primitives.index('none'): - if k_best is None or W[j][k] > W[j][k_best]: - k_best = k - # Once the best primitive is chosen, create the new gene element, which is the - # string for the name of the primitive, and the index of the previous node to connect to, - gene.append((self.primitives[k_best], j)) - start = end - n += 1 - # Return the full list of (node, primitive) pairs for this set of weights. - return gene - - gene_normal = _parse(F.softmax(self.alphas_normal, dim=-1).data.cpu().numpy()) - gene_reduce = _parse(F.softmax(self.alphas_reduce, dim=-1).data.cpu().numpy()) + def genotype(self, skip_primitive='none'): + ''' + Extract the genotype, or specific connections within a cell, as encoded by the weights. + # Arguments + skip_primitives: hack was added by DARTS to temporarily workaround the + 'strong gradient' problem identified in the sharpDARTS paper https://arxiv.org/abs/1903.09900, + set skip_primitive=None to not skip any primitives. + ''' + gene_normal = genotype_extractor.parse_cell( + F.softmax(self.alphas_normal, dim=-1).data.cpu().numpy(), + primitives=self.primitives, steps=self._steps, skip_primitive=skip_primitive) + gene_reduce = genotype_extractor.parse_cell( + F.softmax(self.alphas_reduce, dim=-1).data.cpu().numpy(), + primitives=self.primitives, steps=self._steps, skip_primitive=skip_primitive) concat = range(2+self._steps-self._multiplier, self._steps+2) genotype = Genotype( @@ -569,7 +536,7 @@ def forward(self, input_batch): # print('logits') #print("Optimal_path_forward", nx.algorithms.dag.dag_longest_path(self.G)) #print("Top down greedy", self.gen_greedy_path(self.G,"top_down")) - #print("Bottom up greedy",self.gen_greedy_path(self.G,"bottom_up")) + #print("Bottom up greedy",self.gen_greedy_path(self.G,"bottom_up")) return logits def gen_greedy_path(self, G, strategy="top_down"): @@ -626,7 +593,7 @@ def _loss(self, input_batch, target): return self._criterion(logits, target) def _initialize_alphas(self, genotype=None): - + if genotype is None or genotype[-1] == 'longest_path': init_alpha = 1e-3*torch.randn(self.arch_weights_shape) else: From d7791b5de9a0179ede303912bdeb6bb17520c48d Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 12 May 2019 12:40:00 -0400 Subject: [PATCH 18/40] cnn/visualize.py different genotypes get different filenames --- cnn/visualize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cnn/visualize.py b/cnn/visualize.py index d599e6c..c372e34 100644 --- a/cnn/visualize.py +++ b/cnn/visualize.py @@ -47,9 +47,9 @@ def plot(genotype, filename): try: genotype = eval('genotypes.{}'.format(genotype_name)) except AttributeError: - print("{} is not specified in genotypes.py".format(genotype_name)) + print("{} is not specified in genotypes.py".format(genotype_name)) sys.exit(1) - plot(genotype.normal, "normal") - plot(genotype.reduce, "reduction") + plot(genotype.normal, genotype_name + "_normal") + plot(genotype.reduce, genotype_name + "_reduction") From f0f88d65ff5979406c2139a3acbb7db9e7eb93ea Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 15 May 2019 13:41:01 -0400 Subject: [PATCH 19/40] README.md first sharpDARTS readme --- README.md | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f711a0..67582e8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,89 @@ -# Differentiable Architecture Search +# sharpDARTS: Faster, More Accurate Differentiable Architecture Search + +Please cite sharpDARTS if you use this code as part of your research! By using any part of the code you are agreeing to comply with our permissive Apache 2.0 license. + +``` +@article{DBLP:journals/corr/abs-1903-09900, + author = {Andrew Hundt and + Varun Jain and + Gregory D. Hager}, + title = {sharpDARTS: Faster and More Accurate Differentiable Architecture Search}, + journal = {CoRR}, + volume = {abs/1903.09900}, + year = {2019}, + url = {http://arxiv.org/abs/1903.09900}, + archivePrefix = {arXiv}, + eprint = {1903.09900}, + biburl = {https://dblp.org/rec/bib/journals/corr/abs-1903-09900}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +``` + +For algorithm details see: +[sharpDARTS: Faster, More Accurate Differentiable Architecture Search](http://arxiv.org/abs/1903.09900) + +Requires pytorch 1.0. + + +## Searching for a model + +To see the configuration options, run `python3 train_search.py --help` from the directory `cnn`. The code is also commented. + +Run `cnn/train_search.py` with your configuration. Place the best genotype into `genotypes.py`, preferably with a record of the raw weights as well and other command execution data so similar results can be reproduced in the future. + +## Training a final Model + +To see the configuration options, run `python3 train.py --help` from the directory `cnn`. The code is also commented. + +### CIFAR-10 Training + +Best Results with SharpSepConvDARTS: +``` +export CUDA_VISIBLE_DEVICES="0" && python3 train.py --autoaugment --auxiliary --cutout --batch_size 64 --epochs 2000 --save SHARPSEPCONV_DARTS_2k_`git rev-parse --short HEAD`_cospower_min_1e-8 --arch SHARPSEPCONV_DARTS --learning_rate 0.025 --learning_rate_min 1e-8 --cutout_length 16 --init_channels 36 --dataset cifar10 +``` + +Collecting multiple data points: + +``` +for i in {1..8}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --autoaugment --auxiliary --cutout --batch_size 64 --epochs 2000 --save SHARPSEPCONV_DARTS_MAX_W_2k_`git rev-parse --short HEAD`_cospower_min_1e-8 --learning_rate 0.025 --learning_rate_min 1e-8 --cutout_length 16 --init_channels 36 --dataset cifar10 --arch SHARPSEPCONV_DARTS_MAX_W ; done; +``` + + +### ImageNet Training + +We train with a modified version the fp16 optimizer from [APEX](https://github.com/NVIDIA/apex), we suspect this means results will be slightly below the ideal possible accuracy, but training time is substantially reduced. +These configurations are designed for an NVIDIA RTX 1080Ti, `--fp16` should be removed for older GPUs. + +You'll need to first [acquire the imagenet files](http://image-net.org/download) and preprocess them. + +Best SharpSepConvDARTS Results: + +``` +python3 -m torch.distributed.launch --nproc_per_node=2 main_fp16_optimizer.py --fp16 --b 224 --save `git rev-parse --short HEAD`_DARTS_PRIMITIVES_DIL_IS_SEPCONV --epochs 300 --dynamic-loss-scale --workers 20 --autoaugment --auxiliary --cutout --data /home/costar/datasets/imagenet/ --learning_rate 0.05 --learning_rate_min 0.00075 --arch DARTS_PRIMITIVES_DIL_IS_SEPCONV +``` + +Resuming Training (you may need to do this if your results don't match the paper on the first run): + +You'll need to make sure to specify the correct checkpoint directory, which will change every run. +``` +python3 -m torch.distributed.launch --nproc_per_node=2 main_fp16_optimizer.py --fp16 --b 224 --save `git rev-parse --short HEAD`_DARTS_PRIMITIVES_DIL_IS_SEPCONV --epochs 100 --dynamic-loss-scale --workers 20 --autoaugment --auxiliary --cutout --data /home/costar/datasets/imagenet/ --learning_rate 0.0005 --learning_rate_min 0.0000075 --arch DARTS_PRIMITIVES_DIL_IS_SEPCONV --resume eval-20190213-182625-975c657_DARTS_PRIMITIVES_DIL_IS_SEPCONV-imagenet-DARTS_PRIMITIVES_DIL_IS_SEPCONV-0/checkpoint.pth.tar +``` + +Training SHARP_DARTS model on ImageNet: + +``` +export CUDA_VISIBLE_DEVICES="0" && python3 main_fp16_optimizer.py --fp16 --b 256 --save `git rev-parse --short HEAD` --epochs 400 --dynamic-loss-scale --workers 20 --autoaugment --auxiliary --cutout --data /home/costar/datasets/imagenet/ --learning_rate_min 1e-8 --mid_channels 96 --lr 1e-4 --load eval-20190222-175718-4f5892f-imagenet-SHARP_DARTS-0/model_best.pth.tar +``` + +### Differentiable Hyperparameter Search + +``` +export CUDA_VISIBLE_DEVICES="1" && python2 train_search.py --dataset cifar10 --batch_size 64 --save MULTI_CHANNEL_SEARCH_`git rev-parse --short HEAD`_search_weighting_max_w --init_channels 36 --epochs 120 --cutout --autoaugment --seed 10 --weighting_algorithm max_w --multi_channel +``` + + +# sharpDARTS Repository based on Differentiable Architecture Search (DARTS) + Code accompanying the paper > [DARTS: Differentiable Architecture Search](https://arxiv.org/abs/1806.09055)\ > Hanxiao Liu, Karen Simonyan, Yiming Yang.\ From b374f37768bdcbefb8eef9c0afcc867600d0e6cb Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 15 May 2019 14:01:22 -0400 Subject: [PATCH 20/40] train.py add SHARPER_PRIMITIVES option to help --- cnn/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cnn/train.py b/cnn/train.py index 7da376a..8d5151a 100644 --- a/cnn/train.py +++ b/cnn/train.py @@ -67,7 +67,7 @@ def main(): parser.add_argument('--ops', type=str, default='OPS', help='which operations to use, options are OPS and DARTS_OPS') parser.add_argument('--primitives', type=str, default='PRIMITIVES', help='which primitive layers to use inside a cell search space,' - ' options are PRIMITIVES and DARTS_PRIMITIVES') + ' options are PRIMITIVES, SHARPER_PRIMITIVES, and DARTS_PRIMITIVES') parser.add_argument('--optimizer', type=str, default='sgd', help='which optimizer to use, options are padam and sgd') parser.add_argument('--load', type=str, default='', metavar='PATH', help='load weights at specified location') parser.add_argument('--grad_clip', type=float, default=5, help='gradient clipping') From fc7909153c9c07d4789c71bf2b7d995364478c35 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 15 May 2019 14:28:31 -0400 Subject: [PATCH 21/40] genotypes.py update sharper no hacks run command --- cnn/genotypes.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cnn/genotypes.py b/cnn/genotypes.py index 35c4a35..b1801e0 100644 --- a/cnn/genotypes.py +++ b/cnn/genotypes.py @@ -518,6 +518,19 @@ [0.1492, 0.0676, 0.0754, 0.1314, 0.0717, 0.1051, 0.0829, 0.0670, 0.0863, 0.0683, 0.0518, 0.0432]], normal_concat=[], reduce_concat=[], layout='raw_weights') # Retrieved from SHARPER_SCALAR_WEIGHTS by running genotype_extractor.py SHARPER_SCALAR_genotype_skip_none = Genotype(normal=[('skip_connect', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('dil_flood_conv_3x3', 3), ('flood_conv_3x3', 1), ('dil_flood_conv_3x3', 3), ('dil_conv_3x3', 4)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('flood_conv_3x3', 3), ('skip_connect', 4)], reduce_concat=range(2, 6), layout='cell') +""" +costar@ubuntu|/media/costar/7d094c19-d61f-48fe-93cb-0f7287e05292/datasets/sharpDARTS/cnn on sharper!? +± for i in {1..8}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --autoaugment --auxiliary --cutout --batch_size 64 --epochs 2000 --save SHARPER_SCALAR_genotype_no_hack_2k_`git rev-parse --short HEAD`_cospower_min_1e-8 --learning_rate 0.025 --learning_rate_min 1e-8 --cutout_length 16 --init_channels 36 --dataset cifar10 --arch SHARPER_SCALAR_genotype_no_hack --primitives SHARPER_PRIMITIVES ; done; +Tensorflow is not installed. Skipping tf related imports +Experiment dir : eval-20190515-140449-SHARPER_SCALAR_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_SCALAR_genotype_no_hack-0 +2019_05_15_14_04_49 gpu device = 0 +2019_05_15_14_04_49 args = Namespace(arch='SHARPER_SCALAR_genotype_no_hack', autoaugment=True, auxiliary=True, auxiliary_weight=0.4, batch_size=64, cutout=True, cutout_length=16, data='../data', dataset='cifar10', drop_path_prob=0.2, epoch_stats_file='eval-20190515-140449-SHARPER_SCALAR_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_SCALAR_genotype_no_hack-0/eval-epoch-stats-20190515-140449.json', epochs=2000, evaluate='', flops=False, gpu=0, grad_clip=5, init_channels=36, layers=20, layers_in_cells=4, layers_of_cells=8, learning_rate=0.025, learning_rate_min=1e-08, load='', load_args='', load_genotype=None, log_file_path='eval-20190515-140449-SHARPER_SCALAR_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_SCALAR_genotype_no_hack-0/log.txt', lr_power_annealing_exponent_order=2, mid_channels=32, mixed_auxiliary=False, model_path='saved_models', momentum=0.9, multi_channel=False, ops='OPS', optimizer='sgd', partial=0.125, primitives='SHARPER_PRIMITIVES', random_eraser=False, report_freq=50, save='eval-20190515-140449-SHARPER_SCALAR_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_SCALAR_genotype_no_hack-0', seed=0, start_epoch=1, stats_file='eval-20190515-140449-SHARPER_SCALAR_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_SCALAR_genotype_no_hack-0/eval-stats-20190515-140449.json', warm_restarts=20, warmup_epochs=5, weight_decay=0.0003, weighting_algorithm='scalar') +2019_05_15_14_04_49 output channels: 10 +2019_05_15_14_04_49 loading op dict: operations.OPS +2019_05_15_14_04_49 loading primitives:genotypes.SHARPER_PRIMITIVES +2019_05_15_14_04_49 primitives: ['none', 'max_pool_3x3', 'avg_pool_3x3', 'skip_connect', 'sep_conv_3x3', 'sep_conv_5x5', 'sep_conv_7x7', 'dil_conv_3x3', 'dil_conv_5x5', 'flood_conv_3x3', 'flood_conv_5x5', 'dil_flood_conv_3x3'] +2019_05_15_14_04_51 param size = 3.250846MB +""" SHARPER_SCALAR_genotype_no_hack = Genotype(normal=[('none', 1), ('skip_connect', 0), ('none', 2), ('none', 1), ('none', 2), ('none', 1), ('none', 2), ('dil_flood_conv_3x3', 3)], normal_concat=range(2, 6), reduce=[('max_pool_3x3', 0), ('skip_connect', 1), ('skip_connect', 1), ('avg_pool_3x3', 0), ('avg_pool_3x3', 1), ('dil_flood_conv_3x3', 0), ('none', 4), ('flood_conv_3x3', 3)], reduce_concat=range(2, 6), layout='cell') """ @@ -604,4 +617,19 @@ [0.0799, 0.0759, 0.0747, 0.0774, 0.0808, 0.0749, 0.0827, 0.0802, 0.1554, 0.0747, 0.0713, 0.0722]], normal_concat=[], reduce_concat=[], layout='raw_weights') # Retrieved from SHARPER_MAX_W_WEIGHTS by running genotype_extractor.py SHARPER_MAX_W_genotype_skip_none = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('flood_conv_3x3', 1), ('skip_connect', 2)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') +""" +costar@ubuntu|/media/costar/7d094c19-d61f-48fe-93cb-0f7287e05292/datasets/sharpDARTS/cnn on sharper!? +± for i in {1..8}; do export CUDA_VISIBLE_DEVICES="1" && python3 train.py --autoaugment --auxiliary --cutout --batch_size 48 --epochs 2000 --save SHARPER_MAX_W_genotype_no_hack_2k_`git rev-parse --short HEAD`_cospower_min_1e-8 --learning_rate 0.025 --learning_rate_min 1e-8 --cutout_length 16 --init_channels 36 --dataset cifar10 --arch SHARPER_MAX_W_genotype_no_hack --primitives SHARPER_PRIMITIVES ; done; +Tensorflow is not installed. Skipping tf related imports +Experiment dir : eval-20190515-142614-SHARPER_MAX_W_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_MAX_W_genotype_no_hack-0 +2019_05_15_14_26_14 gpu device = 0 +2019_05_15_14_26_14 args = Namespace(arch='SHARPER_MAX_W_genotype_no_hack', autoaugment=True, auxiliary=True, auxiliary_weight=0.4, batch_size=48, cutout=True, cutout_length=16, data='../data', dataset='cifar10', drop_path_prob=0.2, epoch_stats_file='eval-20190515-142614-SHARPER_MAX_W_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_MAX_W_genotype_no_hack-0/eval-epoch-stats-20190515-142614.json', epochs=2000, evaluate='', flops=False, gpu=0, grad_clip=5, init_channels=36, layers=20, layers_in_cells=4, layers_of_cells=8, learning_rate=0.025, learning_rate_min=1e-08, load='', load_args='', load_genotype=None, log_file_path='eval-20190515-142614-SHARPER_MAX_W_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_MAX_W_genotype_no_hack-0/log.txt', lr_power_annealing_exponent_order=2, mid_channels=32, mixed_auxiliary=False, model_path='saved_models', momentum=0.9, multi_channel=False, ops='OPS', optimizer='sgd', partial=0.125, primitives='SHARPER_PRIMITIVES', random_eraser=False, report_freq=50, save='eval-20190515-142614-SHARPER_MAX_W_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_MAX_W_genotype_no_hack-0', seed=0, start_epoch=1, stats_file='eval-20190515-142614-SHARPER_MAX_W_genotype_no_hack_2k_b374f37_cospower_min_1e-8-cifar10-SHARPER_MAX_W_genotype_no_hack-0/eval-stats-20190515-142614.json', warm_restarts=20, warmup_epochs=5, weight_decay=0.0003, weighting_algorithm='scalar') +2019_05_15_14_26_14 output channels: 10 +2019_05_15_14_26_14 loading op dict: operations.OPS +2019_05_15_14_26_14 loading primitives:genotypes.SHARPER_PRIMITIVES +2019_05_15_14_26_14 primitives: ['none', 'max_pool_3x3', 'avg_pool_3x3', 'skip_connect', 'sep_conv_3x3', 'sep_conv_5x5', 'sep_conv_7x7', 'dil_conv_3x3', 'dil_conv_5x5', 'flood_conv_3x3', 'flood_conv_5x5', 'dil_flood_conv_3x3'] +2019_05_15_14_26_17 param size = 4.697614MB + + +""" SHARPER_MAX_W_genotype_no_hack = Genotype(normal=[('sep_conv_3x3', 0), ('dil_conv_5x5', 1), ('max_pool_3x3', 0), ('sep_conv_7x7', 2), ('avg_pool_3x3', 3), ('flood_conv_3x3', 0), ('none', 4), ('none', 3)], normal_concat=range(2, 6), reduce=[('flood_conv_5x5', 0), ('dil_flood_conv_3x3', 1), ('sep_conv_3x3', 0), ('skip_connect', 2), ('sep_conv_3x3', 0), ('skip_connect', 2), ('dil_conv_5x5', 4), ('sep_conv_3x3', 0)], reduce_concat=range(2, 6), layout='cell') \ No newline at end of file From 72601bdbf71a8c1aa12dadf486d4960b48656052 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 15 May 2019 14:57:55 -0400 Subject: [PATCH 22/40] README.md ack-grep command --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 67582e8..7b1e902 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,10 @@ Collecting multiple data points: for i in {1..8}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --autoaugment --auxiliary --cutout --batch_size 64 --epochs 2000 --save SHARPSEPCONV_DARTS_MAX_W_2k_`git rev-parse --short HEAD`_cospower_min_1e-8 --learning_rate 0.025 --learning_rate_min 1e-8 --cutout_length 16 --init_channels 36 --dataset cifar10 --arch SHARPSEPCONV_DARTS_MAX_W ; done; ``` +Collecting results from multiple runs into a file from `sharperDARTS/cnn` directory with [ack-grep](https://beyondgrep.com/) (on some machiens it is just `ack`): +``` +ack-grep --match "cifar10.1" */*.txt > cifar10.1_results_femur.txt +``` ### ImageNet Training @@ -64,7 +68,7 @@ python3 -m torch.distributed.launch --nproc_per_node=2 main_fp16_optimizer.py -- Resuming Training (you may need to do this if your results don't match the paper on the first run): -You'll need to make sure to specify the correct checkpoint directory, which will change every run. +You'll need to make sure to specify the correct checkpoint directory, which will change every run. ``` python3 -m torch.distributed.launch --nproc_per_node=2 main_fp16_optimizer.py --fp16 --b 224 --save `git rev-parse --short HEAD`_DARTS_PRIMITIVES_DIL_IS_SEPCONV --epochs 100 --dynamic-loss-scale --workers 20 --autoaugment --auxiliary --cutout --data /home/costar/datasets/imagenet/ --learning_rate 0.0005 --learning_rate_min 0.0000075 --arch DARTS_PRIMITIVES_DIL_IS_SEPCONV --resume eval-20190213-182625-975c657_DARTS_PRIMITIVES_DIL_IS_SEPCONV-imagenet-DARTS_PRIMITIVES_DIL_IS_SEPCONV-0/checkpoint.pth.tar ``` From ee54e5913e7b5cc58ff5ef03333ce21df13d6dfb Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 15 May 2019 15:02:59 -0400 Subject: [PATCH 23/40] README.md add differentiable hyperparameter search --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b1e902..57e1290 100644 --- a/README.md +++ b/README.md @@ -79,12 +79,19 @@ Training SHARP_DARTS model on ImageNet: export CUDA_VISIBLE_DEVICES="0" && python3 main_fp16_optimizer.py --fp16 --b 256 --save `git rev-parse --short HEAD` --epochs 400 --dynamic-loss-scale --workers 20 --autoaugment --auxiliary --cutout --data /home/costar/datasets/imagenet/ --learning_rate_min 1e-8 --mid_channels 96 --lr 1e-4 --load eval-20190222-175718-4f5892f-imagenet-SHARP_DARTS-0/model_best.pth.tar ``` -### Differentiable Hyperparameter Search +## Differentiable Hyperparameter Search on CIFAR-10 +Searching for a Model: ``` export CUDA_VISIBLE_DEVICES="1" && python2 train_search.py --dataset cifar10 --batch_size 64 --save MULTI_CHANNEL_SEARCH_`git rev-parse --short HEAD`_search_weighting_max_w --init_channels 36 --epochs 120 --cutout --autoaugment --seed 10 --weighting_algorithm max_w --multi_channel ``` +Add the best path printed during the search process to `genotypes.py`, you can see examples in that file. + +Training a final model, this one will reproduce the best results from Max-W search: +``` +for i in {1..5}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --b 512 --save MULTI_CHANNEL_MAX_W_PATH_FULL_2k_`git rev-parse --short HEAD`_LR_0.1_to_1e-8 --arch MULTI_CHANNEL_MAX_W_PATH --epochs 2000 --multi_channel --cutout --autoaugment --learning_rate 0.1 ; done; +``` # sharpDARTS Repository based on Differentiable Architecture Search (DARTS) From 6904ca277abcb86bda1c8068ee280c6577606395 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 15 May 2019 15:08:28 -0400 Subject: [PATCH 24/40] README.md explain cosine power annealing --- README.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/README.md b/README.md index 57e1290..242e30b 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,67 @@ Training a final model, this one will reproduce the best results from Max-W sear for i in {1..5}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --b 512 --save MULTI_CHANNEL_MAX_W_PATH_FULL_2k_`git rev-parse --short HEAD`_LR_0.1_to_1e-8 --arch MULTI_CHANNEL_MAX_W_PATH --epochs 2000 --multi_channel --cutout --autoaugment --learning_rate 0.1 ; done; ``` +## Cosine Power Annealing + +Cosine Power annealing is designed to be a learning rate schedule which improves on cosine annealing. +See `consine_power_annealing.py` for the code, and `cnn/train.py` for an example of using the API. + +```python +def cosine_power_annealing( + epochs=None, max_lr=0.1, min_lr=1e-4, exponent_order=10, + max_epoch=None, warmup_epochs=None, return_intermediates=False, + start_epoch=1, restart_lr=True): + """ Cosine Power annealing is designed to be an improvement on cosine annealing. + + Often the cosine annealing schedule decreases too slowly at the beginning + and to quickly at the end. A simple exponential annealing curve does the + opposite, decreasing too quickly at the beginning and too slowly at the end. + Power cosine annealing strikes a configurable balance between the two. + The larger the exponent order, the faster exponential decay occurs. + The smaller the exponent order, the more like cosine annealing the curve becomes. + + # Arguments + + epochs: An integer indicating the number of epochs to train. + If you are resuming from epoch 100 and want to run until epoch 300, + specify 200. + max_lr: The maximum learning rate which is also the initial learning rate. + min_lr: The minimum learning rate which is also the final learning rate. + exponent_order: Determines how fast the learning rate decays. + A value of 1 will perform standard cosine annealing, while + 10 will decay with an exponential base of 10. + max_epoch: The maximum epoch number that will be encountered. + This is usually specified for when you are getting a single learning rate + value at the current epoch, or for resuming training runs. + return_intermediates: True changes the return value to be + [cos_power_annealing, cos_power_proportions, cos_proportions] + which is useful for comparing, understanding, and plotting several possible + learning rate curves. False returns only the cos_power_annealing + learning rate values., + start_epoch: The epoch number to start training from which will be at index 0 + of the returned numpy array. + restart_lr: If True the training curve will be returned as if starting from + epoch 1, even if you are resuming from a later epoch. Otherwise we will + return with the learning rate as if you have already trained up to + the value specified by start_epoch. + + # Returns + + A 1d numpy array cos_power_annealing, which will contain the learning + rate you should use at each of the specified epochs. + """ +``` + +Here is the key line with the settings you'll want to use: + +```python + lr_schedule = cosine_power_annealing( + epochs=args.epochs, max_lr=args.learning_rate, min_lr=args.learning_rate_min, + warmup_epochs=args.warmup_epochs, exponent_order=args.lr_power_annealing_exponent_order) +``` + +lr_schedule is a numpy array containing the learning rate at each epoch. + # sharpDARTS Repository based on Differentiable Architecture Search (DARTS) Code accompanying the paper From 9cd0551af62abffbcae65bd6f5ffd5cfd9aeba62 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 30 May 2019 13:26:20 -0400 Subject: [PATCH 25/40] cnn/visualize.py improve viz to make sense --- cnn/visualize.py | 14 +++++++++----- cnn/visualize_all.sh | 5 +++++ 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 cnn/visualize_all.sh diff --git a/cnn/visualize.py b/cnn/visualize.py index c372e34..7afa430 100644 --- a/cnn/visualize.py +++ b/cnn/visualize.py @@ -11,13 +11,14 @@ def plot(genotype, filename): engine='dot') g.body.extend(['rankdir=LR']) - g.node("c_{k-2}", fillcolor='darkseagreen2') - g.node("c_{k-1}", fillcolor='darkseagreen2') + g.node(str(0), label="c_{k-2}", fillcolor='darkseagreen2') + g.node(str(1), label="c_{k-1}", fillcolor='darkseagreen2') assert len(genotype) % 2 == 0 steps = len(genotype) // 2 for i in range(steps): - g.node(str(i), fillcolor='lightblue') + g.node(str(i), label='+', fillcolor='lightblue') + # g.node(str(i), label='+_{' + str(i) + '}', fillcolor='lightblue') for i in range(steps): for k in [2*i, 2*i + 1]: @@ -29,11 +30,14 @@ def plot(genotype, filename): else: u = str(j-2) v = str(i) - g.edge(u, v, label=op, fillcolor="gray") + name_ijk = str(i) + '_' + str(j) + '_' + str(k) + op + g.node(name_ijk, label=op, fillcolor='darkseagreen2') + g.edge(u, name_ijk, fillcolor="black") + g.edge(name_ijk, v, fillcolor="black") g.node("c_{k}", fillcolor='palegoldenrod') for i in range(steps): - g.edge(str(i), "c_{k}", fillcolor="gray") + g.edge(str(i), "c_{k}", fillcolor="black") g.render(filename, view=True) diff --git a/cnn/visualize_all.sh b/cnn/visualize_all.sh new file mode 100644 index 0000000..ffcbbad --- /dev/null +++ b/cnn/visualize_all.sh @@ -0,0 +1,5 @@ + +python3 visualize.py SHARPER_SCALAR_genotype_skip_none +python3 visualize.py SHARPER_SCALAR_genotype_no_hack +python3 visualize.py SHARPER_MAX_W_genotype_no_hack +python3 visualize.py SHARPER_MAX_W_genotype_skip_none \ No newline at end of file From 690f581e2cabbafe15dfcd42d34a74945680041c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 30 May 2019 15:21:07 -0400 Subject: [PATCH 26/40] LICENSE copyright update --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index c440a85..c95e646 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Copyright (c) 2018, Hanxiao Liu. + Copyright (c) 2019 Andrew Hundt, Varun Jain, and the Code Authors. Copyright (c) 2018, Hanxiao Liu. All rights reserved. Apache License From 4415de83b072a76fdb4539cb8d92b146e40cb33b Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 30 May 2019 16:32:12 -0400 Subject: [PATCH 27/40] dataset.py reference source for some lines in this file --- cnn/dataset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cnn/dataset.py b/cnn/dataset.py index df9f6fd..58240cd 100644 --- a/cnn/dataset.py +++ b/cnn/dataset.py @@ -1,3 +1,6 @@ +# Code to load various datasets for training. +# +# Some data loading code is from https://github.com/DRealArun/darts/ with the same license as DARTS. import os import sys import time From 45a1bd2bd8d2b1a30e0ff52acd82f8ca6f9ec2d5 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 30 May 2019 13:26:20 -0400 Subject: [PATCH 28/40] cnn/visualize.py improve viz to make sense --- cnn/visualize.py | 14 +++++++++----- cnn/visualize_all.sh | 5 +++++ 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 cnn/visualize_all.sh diff --git a/cnn/visualize.py b/cnn/visualize.py index c372e34..7afa430 100644 --- a/cnn/visualize.py +++ b/cnn/visualize.py @@ -11,13 +11,14 @@ def plot(genotype, filename): engine='dot') g.body.extend(['rankdir=LR']) - g.node("c_{k-2}", fillcolor='darkseagreen2') - g.node("c_{k-1}", fillcolor='darkseagreen2') + g.node(str(0), label="c_{k-2}", fillcolor='darkseagreen2') + g.node(str(1), label="c_{k-1}", fillcolor='darkseagreen2') assert len(genotype) % 2 == 0 steps = len(genotype) // 2 for i in range(steps): - g.node(str(i), fillcolor='lightblue') + g.node(str(i), label='+', fillcolor='lightblue') + # g.node(str(i), label='+_{' + str(i) + '}', fillcolor='lightblue') for i in range(steps): for k in [2*i, 2*i + 1]: @@ -29,11 +30,14 @@ def plot(genotype, filename): else: u = str(j-2) v = str(i) - g.edge(u, v, label=op, fillcolor="gray") + name_ijk = str(i) + '_' + str(j) + '_' + str(k) + op + g.node(name_ijk, label=op, fillcolor='darkseagreen2') + g.edge(u, name_ijk, fillcolor="black") + g.edge(name_ijk, v, fillcolor="black") g.node("c_{k}", fillcolor='palegoldenrod') for i in range(steps): - g.edge(str(i), "c_{k}", fillcolor="gray") + g.edge(str(i), "c_{k}", fillcolor="black") g.render(filename, view=True) diff --git a/cnn/visualize_all.sh b/cnn/visualize_all.sh new file mode 100644 index 0000000..ffcbbad --- /dev/null +++ b/cnn/visualize_all.sh @@ -0,0 +1,5 @@ + +python3 visualize.py SHARPER_SCALAR_genotype_skip_none +python3 visualize.py SHARPER_SCALAR_genotype_no_hack +python3 visualize.py SHARPER_MAX_W_genotype_no_hack +python3 visualize.py SHARPER_MAX_W_genotype_skip_none \ No newline at end of file From a4e9ac033ab3557a26fb050daa4f7cc9ae97b4a5 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 30 May 2019 16:51:46 -0400 Subject: [PATCH 29/40] README.md updated with new details --- README.md | 96 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 242e30b..fb37f03 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # sharpDARTS: Faster, More Accurate Differentiable Architecture Search -Please cite sharpDARTS if you use this code as part of your research! By using any part of the code you are agreeing to comply with our permissive Apache 2.0 license. +Please cite [sharpDARTS: Faster, More Accurate Differentiable Architecture Search](http://arxiv.org/abs/1903.09900) if you use this code as part of your research! By using any part of the code you are agreeing to comply with our permissive Apache 2.0 license. ``` -@article{DBLP:journals/corr/abs-1903-09900, +@article{hundt2019sharpdarts, author = {Andrew Hundt and Varun Jain and Gregory D. Hager}, @@ -19,25 +19,45 @@ Please cite sharpDARTS if you use this code as part of your research! By using a } ``` -For algorithm details see: -[sharpDARTS: Faster, More Accurate Differentiable Architecture Search](http://arxiv.org/abs/1903.09900) - -Requires pytorch 1.0. - +Requires pytorch 1.0. References and licenses for external code sources are included inline in the source code itself. ## Searching for a model To see the configuration options, run `python3 train_search.py --help` from the directory `cnn`. The code is also commented. +### Scalar Search + +Here is how to do a search configured to find a model like SharpSepConvDARTS: + +``` +export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --batch_size 48 --layers_of_cells 8 --layers_in_cells 4 --save scalar_SharpSepConvDARTS_SEARCH_`git rev-parse --short HEAD` --init_channels 16 --epochs 120 --cutout --autoaugment --seed 22 --weighting_algorithm scalar --primitives DARTS_PRIMITIVES +``` + +### Max-W Regularization Search + + +export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --batch_size 48 --layers_of_cells 8 --layers_in_cells 4 --save max_w_SharpSepConvDARTS_SEARCH_`git rev-parse --short HEAD` --init_channels 16 --epochs 120 --cutout --autoaugment --seed 22 --weighting_algorithm max_w --primitives DARTS_PRIMITIVES Run `cnn/train_search.py` with your configuration. Place the best genotype into `genotypes.py`, preferably with a record of the raw weights as well and other command execution data so similar results can be reproduced in the future. +### Visualization + +Here are examples of how to visualize the cell structures from a genotype definition found int `genotypes.py`: +``` +cd cnn + +python3 visualize.py SHARPSEPCONV_DARTS +python3 visualize.py SHARP_DARTS +python3 visualize.py DARTS +``` + ## Training a final Model To see the configuration options, run `python3 train.py --help` from the directory `cnn`. The code is also commented. ### CIFAR-10 Training -Best Results with SharpSepConvDARTS: +To reproduce the SharpSepConvDARTS results with 1.93% (1.98+/-0.07) validation error on CIFAR-10 and 5.5% error (5.8+/-0.3) on the CIFAR-10.1 test set: + ``` export CUDA_VISIBLE_DEVICES="0" && python3 train.py --autoaugment --auxiliary --cutout --batch_size 64 --epochs 2000 --save SHARPSEPCONV_DARTS_2k_`git rev-parse --short HEAD`_cospower_min_1e-8 --arch SHARPSEPCONV_DARTS --learning_rate 0.025 --learning_rate_min 1e-8 --cutout_length 16 --init_channels 36 --dataset cifar10 ``` @@ -48,7 +68,7 @@ Collecting multiple data points: for i in {1..8}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --autoaugment --auxiliary --cutout --batch_size 64 --epochs 2000 --save SHARPSEPCONV_DARTS_MAX_W_2k_`git rev-parse --short HEAD`_cospower_min_1e-8 --learning_rate 0.025 --learning_rate_min 1e-8 --cutout_length 16 --init_channels 36 --dataset cifar10 --arch SHARPSEPCONV_DARTS_MAX_W ; done; ``` -Collecting results from multiple runs into a file from `sharperDARTS/cnn` directory with [ack-grep](https://beyondgrep.com/) (on some machiens it is just `ack`): +Collecting results from multiple runs into a file from `sharperDARTS/cnn` directory with [ack-grep](https://beyondgrep.com/) (on some machines the command is just `ack`): ``` ack-grep --match "cifar10.1" */*.txt > cifar10.1_results_femur.txt ``` @@ -60,7 +80,7 @@ These configurations are designed for an NVIDIA RTX 1080Ti, `--fp16` should be r You'll need to first [acquire the imagenet files](http://image-net.org/download) and preprocess them. -Best SharpSepConvDARTS Results: +Training a SharpSepConvDARTS model on ImageNet where we expect approximately 25.1% top-1 (7.8% top-5) error: ``` python3 -m torch.distributed.launch --nproc_per_node=2 main_fp16_optimizer.py --fp16 --b 224 --save `git rev-parse --short HEAD`_DARTS_PRIMITIVES_DIL_IS_SEPCONV --epochs 300 --dynamic-loss-scale --workers 20 --autoaugment --auxiliary --cutout --data /home/costar/datasets/imagenet/ --learning_rate 0.05 --learning_rate_min 0.00075 --arch DARTS_PRIMITIVES_DIL_IS_SEPCONV @@ -154,9 +174,10 @@ Here is the key line with the settings you'll want to use: lr_schedule is a numpy array containing the learning rate at each epoch. -# sharpDARTS Repository based on Differentiable Architecture Search (DARTS) +## The sharpDARTS Repository is based on Differentiable Architecture Search (DARTS) + +The following is a lightly edited and updated reproduction of the README.md from the original [DARTS Code](https://github.com/quark0/darts/tree/f276dd346a09ae3160f8e3aca5c7b193fda1da37) accompanying the paper [DARTS: Differentiable Architecture Search](https://arxiv.org/abs/1806.09055): -Code accompanying the paper > [DARTS: Differentiable Architecture Search](https://arxiv.org/abs/1806.09055)\ > Hanxiao Liu, Karen Simonyan, Yiming Yang.\ > _arXiv:1806.09055_. @@ -164,18 +185,18 @@ Code accompanying the paper

darts

+ The algorithm is based on continuous relaxation and gradient descent in the architecture space. It is able to efficiently design high-performance convolutional architectures for image classification (on CIFAR-10 and ImageNet) and recurrent architectures for language modeling (on Penn Treebank and WikiText-2). Only a single GPU is required. -## Requirements +### Requirements ``` Python >= 3.5.5, PyTorch == 0.3.1, torchvision == 0.2.0 ``` -NOTE: PyTorch 0.4 is not supported at this moment and would lead to OOM. -## Datasets +### Datasets Instructions for acquiring PTB and WT2 can be found [here](https://github.com/salesforce/awd-lstm-lm). While CIFAR-10 can be automatically downloaded by torchvision, ImageNet needs to be manually downloaded (preferably to a SSD) following the instructions [here](https://github.com/pytorch/examples/tree/master/imagenet). -## Pretrained models +### Pretrained models The easist way to get started is to evaluate our pretrained DARTS models. **CIFAR-10** ([cifar10_model.pt](https://drive.google.com/file/d/1Y13i4zKGKgjtWBdC0HWLavjO7wvEiGOc/view?usp=sharing)) @@ -196,7 +217,7 @@ cd cnn && python test_imagenet.py --auxiliary --model_path imagenet_model.pt ``` * Expected result: 26.7% top-1 error and 8.7% top-5 error with 4.7M model params. -## Architecture search (using small proxy models) +### Architecture search (using small proxy models) To carry out architecture search using 2nd-order approximation, run ``` cd cnn && python train_search.py --unrolled # for conv cells on CIFAR-10 @@ -215,7 +236,7 @@ Also be aware that different runs would end up with different local minimum. To Figure: Snapshots of the most likely normal conv, reduction conv, and recurrent cells over time.

-## Architecture evaluation (using full-sized models) +### Architecture evaluation (using full-sized models) To evaluate our best cells by training from scratch, run ``` cd cnn && python train.py --auxiliary --cutout # CIFAR-10 @@ -237,20 +258,43 @@ The CIFAR-10 result at the end of training is subject to variance due to the non Figure: Expected learning curves on CIFAR-10 (4 runs), ImageNet and PTB.

-## Visualization +### Visualization Package [graphviz](https://graphviz.readthedocs.io/en/stable/index.html) is required to visualize the learned cells ``` python visualize.py DARTS ``` where `DARTS` can be replaced by any customized architectures in `genotypes.py`. -## Citation -If you use any part of this code in your research, please cite our [paper](https://arxiv.org/abs/1806.09055): +## Citations + +Please cite sharpDARTS if you use the sharpDARTS code as part of your research! By using any part of the code you are agreeing to comply with our permissive Apache 2.0 license. + +``` +@article{hundt2019sharpdarts, + author = {Andrew Hundt and + Varun Jain and + Gregory D. Hager}, + title = {sharpDARTS: Faster and More Accurate Differentiable Architecture Search}, + journal = {CoRR}, + volume = {abs/1903.09900}, + year = {2019}, + url = {http://arxiv.org/abs/1903.09900}, + archivePrefix = {arXiv}, + eprint = {1903.09900}, + biburl = {https://dblp.org/rec/bib/journals/corr/abs-1903-09900}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +``` + +If you use any part of the DARTS code in your research, please cite [DARTS: Differentiable Architecture Search](https://arxiv.org/abs/1806.09055): + ``` -@article{liu2018darts, - title={DARTS: Differentiable Architecture Search}, - author={Liu, Hanxiao and Simonyan, Karen and Yang, Yiming}, - journal={arXiv preprint arXiv:1806.09055}, - year={2018} +@inproceedings{ + liu2018darts, + title={{DARTS}: {D}ifferentiable {A}rchitecture {S}earch}, + author={Hanxiao Liu and Karen Simonyan and Yiming Yang}, + booktitle={International Conference on Learning Representations}, + year={2019}, + url={https://openreview.net/forum?id=S1eYHoC5FX}, } ``` From dc4058042f0601879e4cfe6e44f24ccdac039fd8 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 30 May 2019 17:21:28 -0400 Subject: [PATCH 30/40] README.md add images --- README.md | 6 + img/cos_power_annealing_imagenet.png | Bin 0 -> 211605 bytes img/multi_channel_net_2x2_x2x2x2.svg | 743 +++++++++++++++++++++++++++ 3 files changed, 749 insertions(+) create mode 100644 img/cos_power_annealing_imagenet.png create mode 100644 img/multi_channel_net_2x2_x2x2x2.svg diff --git a/README.md b/README.md index fb37f03..19d2610 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # sharpDARTS: Faster, More Accurate Differentiable Architecture Search +![Differentiable Hyperparameter Search Example Graph](img/multi_channel_net_2x2_x2x2x2.svg) + Please cite [sharpDARTS: Faster, More Accurate Differentiable Architecture Search](http://arxiv.org/abs/1903.09900) if you use this code as part of your research! By using any part of the code you are agreeing to comply with our permissive Apache 2.0 license. ``` @@ -101,6 +103,8 @@ export CUDA_VISIBLE_DEVICES="0" && python3 main_fp16_optimizer.py --fp16 --b 256 ## Differentiable Hyperparameter Search on CIFAR-10 +![Differentiable Hyperparameter Search Example Graph](img/multi_channel_net_2x2_x2x2x2.svg) + Searching for a Model: ``` export CUDA_VISIBLE_DEVICES="1" && python2 train_search.py --dataset cifar10 --batch_size 64 --save MULTI_CHANNEL_SEARCH_`git rev-parse --short HEAD`_search_weighting_max_w --init_channels 36 --epochs 120 --cutout --autoaugment --seed 10 --weighting_algorithm max_w --multi_channel @@ -115,6 +119,8 @@ for i in {1..5}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --b 512 ## Cosine Power Annealing +![Cosine Power Annealing Example Curve](img/cos_power_annealing_imagenet.png) + Cosine Power annealing is designed to be a learning rate schedule which improves on cosine annealing. See `consine_power_annealing.py` for the code, and `cnn/train.py` for an example of using the API. diff --git a/img/cos_power_annealing_imagenet.png b/img/cos_power_annealing_imagenet.png new file mode 100644 index 0000000000000000000000000000000000000000..b964ef41a616675e89aecd0bb24a5de3804afba4 GIT binary patch literal 211605 zcmeFZWn7f)w>^$xAgN-KiiC7`sfdV_bc29^ba$x;s3<5Yty0q6F$e`&;H7XD4$dtc z>4*1KTw|9<-Acn%N5zkK1#NGHxm_`S`c&oe`75RlGy&E7KLnGv1j1*PBT_W`a%8a) zed^4tq|C~hQX6(Gth~y$wWsjTDB_(rQH&ztJ}-b0cO8hkb6#NN`^zf5{`%P=tln<9 zxJQD+&g|G)RUcDEc6K%~Tt4zcp5^Iq{oh@j+@i~+r~lo*LTh{W-@Ut!Cy<4LpQoQ{ zuKv6C_0}uqfA{X)Kdt`n-pki#vj5$~IeS6p)W5s<|M!B(Lj2zv{tbu!3!A^o@PC=| zcNuX04;=ok!~cQ9|7&nCFD)a$;j>)mN_LwKR}Z0G87@(%^FMa{Tf_!7Gd1#PxJ}z- zL$+2&h2^4o23xQG3{Wt~)qbU}=SGKJL!HdA;r{&$i&g`<1`ER_Yw!y($A~KkpMamc zzX{a`@C$9mt9z*UEH#9j77K4?{C)A4Vh6gw~WFvbbH zjD6_+`}7+B-a|`rv33?l!oPq2e_4omM@A4{;HiEpuBJOtisT!!nVBrTy}g=6=Bn_w2;-Ir<3Hae1-CjSy1(A{ZPTT({M`EQo%%eN%J2K0V`OF? zZbjnFQn$Ie`N=o)#77^w9OhbO<3$AFrCVg!s|uqsd|4wA4!AdPsTkJeO2OQbitjn1 zF1S<{t-`z9Cau3`nnDYnH-)t6P+CU)HzCYShSf3~F1DKMNYK|ZyrB{EUqteyP?6;z z7qeRC@9&apN|C>xM_9RJ%hDAo2!g@wr@pt;W1#odsQ~zu8nbospRaJZE*v=O} z{`;ht?xlS?QMEov)6WQ{}x_lu;$9d3Lv)XfC%6mmRl-@Ws&iP4*<5C|hMAN}< zi@S?{*YzHs93OGpP5si^RY(vM?M_wb_;g#se$=UdHoIcA`P6xe+^F<3IM2NPt)1ty z0+Y5O7K5ce%dPo8f^%Pto^We0GcnosC`u(mfJF&8nV%z}ioMFLHj`B{*38*R5&BIc zz$937XP{LT7iZo3-{Q#4yuB0j*vV;oj?JiG5wh)IXCla~X4UL&v#r88?TG_?#S-O}%GEF4lP1s$_H|+3K_RZpHX)8~?a$NTKzp5L~B5 zRxYo!^N3xe$iaM~*=V^_$n)Aa7s^(Ohs73^PT>sT{F55g8YR}FU3o^0ZMqF=)BUJp zr?jIs8*grSUER1NfYdO^%FK2Q4$c4<9wI*mzN@II&8edfROJOG-z=9-2buL{YLf9= z$F7u&i%C&?Ic8TMEFvF*oWd0TWxq+mJEV?8TqI7&>B)4%$L?(1isgb1v9iO>7R-=E zPH#oZ#&jcEd;CC+SY*AX(4wF1BN^`#*xD{w@#TuOYQeQijC~`e%k$NdGU}OcksrwU z*-(4+?3;rf-h36S(ADW=Cg`k9j!nU=$L=!cM6^~Rzro1N5m8!%XwuBDKYtw zsO}%R-8N?kZ*bf)jdwHI888Yigp|B`^(vCD?z^jTCkHtv&4KLIrGK`wkPB3Gw?jv)XE2;VMH0$2kwBIgSI8$OJDPo%AJ_ zmkaHVdDygzU|Jyy?jX1cY0(9-80*xhS+-V<`wlb!0(B&21t( z=Dw6Ar$zwi9ZN(F@G)RGV&B5Lzv_b0HnkNrYWUFlh4YT@d6ITjb=`0X2_D!zow(@{ z=V7a|tu7gEpW{QV-Hv!s;R4f+_7-+jtp4HD_5Gp#>WHNrlo$~ikA|t7x~`L|_f{gc z@oaOr1!at&gNaFdY}DsFQYI5MuNKBpKF3(piN_skC#%#1Z}-)z-7%qH_yo-G37W-^ zH-bl8r$3hMO$G96oM4G4g?LP3?H;8lB7?Krk=Yc0O9dUsRM{&^Ob zK_Nc2DU<>IEr6=|9Yg#Vux90)c*xu0u|3I#AQ z9VxYQI+rgg8^uK@-IZ%lx3$>IT(O)lgYX`pK{K$~Ey#L#9c-zntE2tNt$*hmx7Y)d z>9<%0ESnP^-A=_zLGU^<~YoP{pKnK6dXRnSrMLkCkS>`^$zEZns`>j7xrto;_5U7k> z3*8J=?%PkGq$6(x8O5!c`|SRhH+IEyQ8}SGF<316POwOR`vww2z>UJf!;SlOUD}{J zMHgEQr+oiTgCxTb#l$BbFUGufN7FJhTlJ4OBe&b%9CGM*ZTId^lK6a~9RxDfUE#8( zqM|Z08>!EBHkFh?zEzqbE(}s5#h|WSygf?P!#PN3@rTmgNLo?%t=~}mKEcKytLnCp z9Jz@WKM_PH_#6kE)QFvi`0%Jc=*=O9qNcJGZq9ruOgq6V!zLrDZD!maf^R*K4& z^IMb@74!O>&=5=FC;J^m4`RLtiERExID{bJa5qq*q9An$8+M<4nd)^v_U90GH#wf& zYZqF|q;Q}AMoxi^89y{p%hdRsaI|c?f6$N8V=j#WN+1Sx+jMy#521h6*wMAVXulFR z@xw`y0?WaCPwen`J1mRl5*kVTSmzeV~}0E)qMh>Ih* zOyxH7c?5XD{vlc%^|P@m4FO){f`Fz6k2v+~4gy!OphN-8ccXkxR8&<#i0L)yx{MaJ zb8LjjurZ#s2j$fS94Gqt2;0BN4G0qJeXxL}X%_@J3HrX*8)AWpmlU=&r=obwlo)NT zAjT0=MigJ+xxX0;8=s6CMiQ-Ecuf}NePq7v0iO?fyHcpYn`z*GVhJ5^un=|}cN zsY1KNdRr*fqCY!%XK!PgK%X4vrCbLUVsu)nA(Ek_dONFYg@C=1;{+pqqHH=4O$)_U z%DQr+fx@Ac(}EJ)?ozEBZRQTd9j)MVpbTtBQ*?U!Vnd5V`l#E4a2#ZGUS2Yv@ zDZoUeLnMd=Y1MV?>VV0FLgMr5U1I?$`t*|B1moQm*mE`FZACsxfhgGYsOI%x7$1Sys!JvMnoC zilx=qJA^C;a)lcA_d9&_qKjNMn;4LIYtb^W2q}E=hX&ym?Skl7+6>$lV7|%ioa_Lg zu^y{sON82?2xhg6pB(QZfGd5J&?0BNzd36L?||f4QEeZs;7Vb){YL%uNNW-@o*(UY zkl*SSf=d8nUI_2o88l50+4x0j2I3@wRlE2C)z7{q*q_$jhPYhA>QWo*jZM>9_1)5<&QcKhYtaO@89b`kMpeiujzbJtd^DDzU zm@z$`u4=L7wr{Y%V&kKYFI?fY6hUrX`mIv{cwP&4hlp~ySGOF0GN{(2WuG?Q{;=1j zAcBy%gbV~iG6aqQyGnP!!b@D@->VgC@w9HRP)knoYhzWIWOgsJxW-*zPFpL({7s8d zg?Ao(7?}71&*^|5E^D6#+-N=77e7gLtjMk@0=>&~ClAVZ*=&Til3w+3FiUpXT(pva zC-7$L-1hCijJxGB3AI>dU&nFc`skXsOO=OfpU$z%-uk4|4#JI}tN?EkNWhMu`vmoX z8QVkEyZU2mdX^uxFPC_}bAL1Q=0^#C!lp>n;cVANG`D2{;mwc>r1YK>UgZ3JZ!VB> z8UA`(xjz>ZCGO)5aT2k(=;JqOu_F}+B?JmpdAsIM3k1l>E9amhDC0}^FGp60F~r(F zFms+iXs85c(I+_f_R^hAcxO84AhCma$N5eJ-~LhfPzA%*6~<6>ZUVd`3KgASYk`#X zI)9=T0EZx{&p{A$8*PT~+@C}#G<^EuXGAg0{@!emNZJTVr;~|zzOy`t)Kg&k zdKn!^`kD8eO3!ro9Aox>l6XJ;F&6B5mf%{?!Grr=JB3G6*HIeyS^*@~@sfep!cA;! zGNV17=Bx$Y=I!;g(#d}={cY(q&idrV()`}^wGiQ$+7LP!dr-A=pQWhMoKC_RmF*#B zOgg{bFWYQpKFJ!-JdJZtk-ch5DXq#R!r&5Uc%~u}vAqfRIEUZ21Hw-CL-Fks>Pwc1 zq;Q>hnKQ`RNV?^>II|G zkPkPxQwk4VfimaAPFkwZbrEJA9R%u$xwNw&AE9p-rcmU z^z?(r4W6J5WmY(i8hY?amMlO`T8z4Y&YeX>R+Vob?;=LWnrYkZIAmKq0E6zxxjJNZ zw?;P+>d{H@D~q=NKsMd-u!vN>gw4%O$AvDs>BeA%&7ZILP1>TvhqmLgv~ zqMsW}q)b+sa-4I~$)CmFg6+{$egxutwnhKgp3(B?7lYVma+^v{By}OEwY8L7+fY!z zQXQ?m;-NYzi}kOJ--EVNYgh+>wB!I^DqXJ{ba45pTyT_G^0J!-eL_Ph9DYdkyTn@Gt$|hbDaGA7zIoymyv8ecj=(vnJCGP+SO@Q}xs(Ljd zB}j^?Q{nz~03qvx5^=MyYLDVFev(!NNp`iG4+keEzUFG55pZv$@P*PI9c;^> z+vAp&cM)#`KFnjtVJgq{`@o)S)V^0%vRK9Nj*S=}YFj{M2XNHVVsZd2J7`_8g8#7> zoF}CU6OKGFFans0KwOoKRfBWUJ{dr*7p!;Cl?|cF(VEgbkg428Tzy0z_7ZZ zoPEp`(=4+$jtBNfn-R%Y1+0i)0Gn4G>>$$*no6l+j-{H!u+5{4cK=VXci~trZ<^11 z5uG_$iA+pPPh^M`V)(N9#_P&uWB69;*sKCoG5b+)h3Od?{Cm5I(3Oo7DsuFY$ho;x z0NOfoXnQoHY)-73w*$xprXYzUvk^p{40fVNCw=LNjMqF2q+zb3yw{E(z=fJH;MbEC z(6HC3U?X=Ji$KEcyT>UP=qHGI@h!Ros}Wo+o!$iV<2h8uyQ3$dLkMQ)^Val0P#ZSw>oz$3O*%u%n`rEF|o+WA%?sPZLVv+N-zuxYkMaSs^T{N&)IAaJfnib*S)XL5jx-AekfrRP7X~W)bhrTZBR$fh@0KDk{BeEzFq{7u>~$IRZi88R$A4!S@%8)eNZ)7 z{P!)%GSbzJo8HsK%=p4IK`~1swU1}oQN{QX8-P7Ck9-@BVb2dma>CAQ?0T_~nHB9A z{zt&RfFn1CS@*k$H|#RApY&^Q@p60jn}YS~mt8<71f{488V0l(rsUU%vMkc~J=yds zU8}ro05`IZ0}G@m>>j3h{iRJOJ}0|AK5u(Hx1xdloA;wl2P@q$%O&I9$;053jZ6q4 zFfFrkv^(yTxZ~}G(sdrB10~EfS`9efH;kIw0O6R5fP4-SPE88XB8}ka7UnDq!Aah7 z#|0&G1&E41^3;!I^16Ima{WH%J1Vz3JXgxPm34I^@)sML(i<2RLc`Sg<$ZbN>Ub0+ zqf-?%19{kNKAR8?C#fb9oAs9(Y4MvynaDA9WlGoaWZZ4p*&T^6Q4o*MhyKS{?)_#a zCZ#_3gPd1>zj!#MQnLWAx!7!=+=;BvEXLnzIP=&ByOTChtxQ&IlaRl=f+oPZ$9Y}H z_Wto2C<@I|TLXIVUS!zHlAeAkh1{JDO*njxaB@124KI_Y@O6eB}_$G+r6h~|=0sCPMug@&A{@;&I zTL$it!Q_rMgu44&&qs8-o(JkU8gf+o6mb*nUycWe{)x4;|a@)z;kL8G& z+K06Pn|!>qR@|Q{GUr$`EL@dgRwmRRQ;jW%1Nx)KPh?>>wb@d!s(evrp4)`t2QV?=`Q@M<@m3 zO+mCtL%8_wkFv`o{K#yIXY!g39L9Tvk-cPfg)Ath>g&;Zn|k2o7c`x4{|dD zF&r^_)4@4T%TPU8SxFDT>YfWzmS^-!w?Nt>FYv9W8q8P> z!VI#+6vXzQBmOfe=4Tl`Y#!T-s!KAsf0YUmS)m#VP%Tv-<&6jhGXCSeWrI<02f%D$ zlUD1nJwg`ar^CVSn!eUGSCFa*E#rRv`%}?@D5y}u1N$ZyjEJ*P&PhE`Or>q z?T?53ibaS?uzRS(Sv@Ot|3TOpFhzS8s+xJ{9v=U?5?dAS@H0n5LL^!~@n# zBhE93*znqpXpdXjc8KS|Aigh+c%HNwyUHs=MHUFgBb*R1FQD_p1tPjj5OBf%Z#P1- zKgOmy3T#sm;4*Fq$wSHpkMB10b((;hXy88x={?o{^E|cAp4%P*B1nr!6O;znl^}qz^*{{uR;l#^ z@q%~Ea&+8dO7G&e3{t~llpoqzN?7o-(+fIabq-(Ucr`Po&O}>eOXnZh*n!hO2=10e zn`4%sav|)%O`aD|iNz1sTVn;ep(`Yh*qqDY+$WO(MV$RWC0gz%A3#irINwC=@mayp z6+*gYNPo5faOVRVFB9S;BRX{%lqNz?5PMgPrVNB)*~u}sJNH{K^ty_9j!B6666dHm zpb4*UGaikV1uNb$FRBg*f2f1KFB80N}w!&T@i!W&&+48EDss z^nO!poA{9eh@k@-v24w9jbcyYws2V&0!h=6o6l5sE}`Nw_(omR{kM;aMW zQCX1|(Lyx{PsFjOq@q@mS%^F;i=MYb6FxG%L>1qkP?8%kD zBhgT#TF}s6L3X!kvGHmD^gJNoD*U=Dvq#PVSXDF9ssVq~6sR&hKOM834uqr(R7pW2 zm4{`%1ujnR0-lb6DB>@e+q{UHkPJM&if8>|e{y%Y^Ej6_bxVjwxa1(ARlk^nAmf_x zyOF{ROoMED58Guc~z7 zRq1YH>F!zS(AlQqmh^5zyDmdTO)VD7u!z9sF6YTj9<8E{G^@e)=_NXy0&#wrlY z!yfiw4%pHReOX_`54jjVq#MQMygWci#&cIebX$d9KK2W|W-1U75TeZ>6l2A_4w9hO z?DwRKsV=qT^HPEn=C<9dj`WabK+Cj)I18r*Cqa1!tN~;la*DRr8}B1sC!~dj7-|sg z(A16D$DmEYM`}U3?ocuL5kv+H(!8zRx;MSoB-6e71?U%laEg$$1m9R6@!nzjCWlAD zQ5OMRBoC^wt1U=L0G%2hL4$-nVu{DPFQy?){l!E3hVxD!vW)>4{`OFSaxsw7%l?Mi z)(y=aq?92|N~Eju@i-G1FOk>Wj?P)ERkMcbGH8|>7m_)afiyO$U0Y+U%Ng~tL;WGN z;4nQy^2lkY3gH!?#kramMU(k@16ZZGKuja`rfG37_w>*~nwDm`%XL)D%|MgqL$A%E zbOZ${*IY$B#KrP@&`}CuvneMpp007Wq-S)0&sWq!l}HS=n@18C#5@n!C+If*+3Kjb zrx&e$Ra@R&wV5kVx1_Q_J0VkO;;ZA3&DwIcM8EiXI^ zGEHY~U#w$p{4+ulc_q5Rrq3x^i=GC5M`w+HWq)3U=f`98TwHgV8+vOr+a_dYv@y~| z{`hL?YQd4WpiOWUw?X@x4{@S=n8-H$wJ*KK#!=(>Oc-tX<|JKcPDZP(ecGg};v_yV+WdjW(DD%H=nzG8wp?(_`{dEhUn6vG zrU#ikd+3aXt(}-bulAW`F;n|u-8{!CqhihCTa9v55!3V}Ewz!+Gs!FNvo^xZQ5HhW z5sUBNZ!HgqcsQ9wi3*oA$H!@>m^RL(y_mqg46{6S0J$-1UTYr7!=`ageI}XEyDNra z7mZ*rW6<-7!R-iwfNMnq&4>(#ec$s`uD4*Tr9(o2K_;?=eq2AUDaxEg>78q7Ztg0Q z?02;K!<(>h97QWVyV?A~-9^P8#Vt}db~D1xp&`rVa5thdh9PAerVzDy zeocj^&0%;g>Zk6(%xZ~C^H&4qP}+~=5oprklHX&xTBayUZBvwWR`)l_QLF}xu~6$H z0|D~FD}|nO+zDw8Hp4J&GWVK*4VknGrA2z(KtK~ctEb&3BYejrxU{1JXakOS^&A@0;t^YCLHvn+BeZfJ_mEQgE^-%gc`&q zVv>c-q>3MzX_Z*(&I@_+q^nno+m4n-{w$Uoo_j;ZFax*es@v%wq6F!EQwiHcmE&8( z&MNo0jge!ji~5xS6P6bvH|_vFLqykEHaUix=IRW9y^1Ozo&{@QO-z z<(o+C*LvvB@4kppupH+iI+Ke`L>2q?DhQbROZf>JVo~ZmMh5F=u^zXb7l0SOTR! zXqd8MTRlaQ8;0B}6p0q6KGUNqoq!1*$T8a<}m>OM}V*`Ur z?An>^rzF4<+K>Shq)~WN!1mdrV5&Gb%v`cjV~`-wcBCaT`_-3tXvA2FdTqSU30DTc zUmeC0oUN-IkQ>{`QthF`syOENFR9IeUyWQwilZ`TP!)TlhX=FzX#PZ>&^r}WQO6Z9 zD@g2R?iYWRQaGgzjZH;viz_AECpkWHvl1l6gG)9&f+j7qfeu4U3bO?{;&GA1OW^?? z6w#5xE3rM7lGrrLX?%QEMh}mQTtaDAqInO-af)?`N6U1}i$_gJMYy3@NI`jbZiaM< zM*7GwoLh6!G3F&DX;)-`Kicppy>Os{&hn1!;if5=7yI; zH4J}_i%e%S@wJH2_>J{imCGFB#B!LnS98IyVV zP}Tq8#WUY3h3`K+OF%&2o19FCbBl-P3OGNEXew2`Z_^C71{+q?hp;ltV1jt-KjsHo1) z&U=`-ww4NE_$B;=%eSM`TU%w%oIQIA=QJ*^!HXAf-M1GFz@t1%)v;EP#$@Qnd`Xqj zE#L}U7JmQXV>%^)9_N=5R_8s})d`8^nGsw-n?puNmLydvU{` zj{;v;PIq5ShPKmsAa}aVYcpL5l6g1Mn-%q-YLV*gBkh4UQxof+X;0!5;i(+;qNRu z3@^ohQhPSuZ9G`ICqQRjaD&d_aAJW?>Rig*e@w4J-SRRF8D$BRmH`4ig+7gD_$#4b1dT{UP?c$ zUW=#B(6}4x5tsvKl+Xojgk^ke|-JKP=SVy7n$!s7-2ZI zZ*{Ry?1@~p&|uU(g$m0I|95ZS;@95xgFoQsiN6Fs=~YT*op(Q;DE_czoxbfykeHJe zd`a$6O;H|mr9@TU^d#{}V0miis$3uWGX99)QSk0GcSQ$B%HWMJYTT-1%5o(_gu1$l z-Suai#I9eeFy`)BJ9! zR@cF$ey3NJAX%cCHyFj;OK2B1m#-GphZ=G9hxY5R1=pN1uH#<2%0gW@}TBVrBwcp&iuh!>?rub~`*Tz?IQ1IgWTi^LpDNn?cql_C` zK1g$$7gs&L;q8Ti`Tm7{v7ir`C;n9N4g=I#ndV<@3WnxKGx=D|=fe&k{AZIdsd8Ms z(|zyZR`&Bb>Cy-XzYMN6*=U0(4A&XDGX@1GCb4)zr=o7@N5#a42gRKdwwq`=g(SA3 z+@t&I%KYPBgjlt{1~uX*iF^2eaa#K9cqjmC*H14@ca}3l?Y1{s{>Df0favVHTfJQm zKAK0*_m5B{$7`AU6?2;6WJ>@^R;M&W{qE;x0}~~usz`PkcFqrf{;aw_ zlA*P|RA;zB(<5$2aSr$VX>)UYtNhECFBZMH^!i{+a%yAxw`gz4Lz?@f4!EA9ZP%nw ziyxK>DyGm1(Nhw_q%VF*-?^m}#RzY+QOe?JTjYs25B zU%74Lm*HNYVcXWC(@)_b)cW(?D~9!`QPZc9cj@&=yGV2Q+6}kFd#D*+%m1d64*!#> zeSHpf^!|V$NCM^KbMN~XmNr5Xsb7T)uAEyANn>S+Jn~k!l!R-~KCUDVPp^(pY4B!V zJy*yR5byl+-E!;0)d2G4w7eR^_by!MU~4ad?Zi>KWt!QLb6H8SPyA+l?M*T0QT)^)D*$(hCQ zdwb&@8w%9Y%RMr2b*;|ZD&+0VDHBxl9K-kNSUA<8xWbWSMR{ZXiswfg)Z510fWtr5 zKLtMwd!J`mlbVt3DzfPLgv{%P!zbd$3>jS?|RyQerPpywS`s2I-^k zN9*ebJKNrpm$*w@$f+DpE4qJ3D<*cx$#-DI;^?i8(`okSbf2*E z*Y@=A&cb4~0rsTSJ0`tMXSNptlekUsDa&u{vG13=t`E;Fsa*}p$r=Bhi*~*%Hg1a> zcir8nX+a7vNRDw!=YIjj8ri=KP$^AUug34M)QT?t)u*SOH{vG9vdwaJ2i4?`8|*=~ zw>6(yO6FbE>3aFy8TZkOy15)9ju`Iq=I74gTXEf?xhQ7F^`J3W>XF8mX0KkA$XY@5 zO{cnbzWL!yKAv98iNlr8d*?FPMEtr9sXePFg^0(9B(%;2P|*L+#(aPO>Vi5=y3(dv z>6GZ>($hAfNu!zNlfPfy`7-7#w8gV^W#|mUS);?UjW6~F_ouMwFC1@S%l^mtQz z?$J|ucjm3~3tfmJ*i=WiVH-jN%csT?uUv4Cyu=@M1oL+pi;$GaH$^V1IE+>3zgH z-T$l(4HJ=U>1D;<&&gkO!hWFiFZ5IJ7FiftHxhnw;Mei6M2(Y$-v+an+Rt%>}=P(8shK^TXG-6=LtnsjJ`paKeoD zcD#)A{*m~j&G1@uAnS*!GBNM@@iY9WCI8+!t-7Xm+VR@~-1+%-dkJ`+#j8T@Ii(*c z`}HTb$?MUv|GAbQJe>bFDU{JNw35N0^x8wF&xT)D?op7t68c)6XI$>MoS)n4!|YE% ze){BwPq<l~T$&l*!=e>sX`Q~j@B&zeH$Y#kjRLW|;C zRu++kg#`>!pBHvszLye0ZY)zbTH%rh(~>Yg#gJQ6M0EIwGN(!0gaowDe zj*6||l988hfF{i3^702bvT{#Gh5*E!zKCYr}!Jo=SK?x zMBGr9F8#qa|8EZdzHX89lH&e8G#okHw_ZGyl0pxsz}dZlDi8kU<>e={v3h!XU${+) zVJ1NFh4#CLufZ`!cB=EwAI5MD%5~%P_F^wCj)t~&Mo;kjse|n$TPLS_4ElWnrNh9$ zz%y8PTU*;7SJ1(4-`2oL2P{>b4Y{taZl+GzWov6|rux?3ziVLlEZ2Uv*)z4|^JCgC zB5qvZ{huNF_*!bn-p=kP3=I&(GMPdHglRYU?GFKaF?C}T6Pmks@o+BQxPd1jA<>he zPJ(-m@b0>+cY>ViAFTe6U zNv#FNRdhE=>4H!n|AWSZM|`-U64el<(v3>{%gcGJiA7B?(SA$^YHgJiLj}M#* zT$~9QHpaupM=qn1oMFkFI^O1n~X-{vDmzLlvjrVXc!|XZ^3JJyMn< zCDoFx8T0n|Gw_r%?SS_9fv5`ILj;tdhMMzY5 z-oStcA*GOtY_aoFAJNaBKLy;k?(*4;5mHlAk5ssjsHv&pjH%TV!W6Iy7CO1q&tUBG zZD3$sTMU1m$F6mzK8gwkB(B%i)os9ey0=zdo}MsBBp$Z7&m0PR75WyBWu1#({5c&= zE#3hBPOe!uJ+v&y6yinh!?7%!O1bC6@EA{T=EZ%~!6Gx~pgBleK$dCP-+_E%0%8h* zm$|tdrRC-CS4KxiHvsZZ8(9y+d6(iR$x1BR#f0!uuOcI_S5{U+Pn;C_jZ&J*H?R9i zN{ldtbDE!@KU1%Y491f$GuH#G{}+oTyx*jAPc^TWmw!0Acztonk6qjQg3~+CO-=gW zxeYEvI>8lF(v{{8H>H!5dJaABS_VU609#JX4^ z_fP*zR(8e_-1II5eo@t%Onro!d^;zU{?%0K9M6xOVmE`|WPUPu_m`Ocn1}N=Z{NH* zgL4lc)ea6e+(1VSB|}?PzD(8#=RXh?H-bd=|`q=r-hC{Lk*;H*5n zN^+t^su+IzZpCoXdj_3T_axF^ZLl^Nxu8aKGtvYUlPh zwA~F9hAs9Q_8BG|{aBhkPPpLj8mE^-{8^5aD(ajpq$WINO)z@|4a*?t&%a;FP|so8 z*l!G?Bw%1*2p>9zGbf!rJ!jVW2A#pNy_|sJYW5mURY(YeGgORL>J`pbFs3KBV0U%p z5W0qvJ1G96o%uhwFuA2PB+nFX2;+rQ2QUJ0k(4yCa0RA6E&-O{aBy(cjFdTGL{DHz zzvkqS;NTGxH$dYCV`X7uL!zrY>bVPPs9673!`#Hbnr&|W>2Vl|u=Z76F3UHfUc1k9 zFm_Kve>APMw6_21LPZ)*mA!i_t$b|2&HhdWzm8qUz+vyAqgJuXK!W?xxp<$=3wqeS zut&R?SHG^Mr3C;h1&PDqbL=VNy8g9OfRmGR4|;o(Gc#|1K}Y~~ zXlQ77(VKo1X7ygd2zmNQYE8{O7&suM6moo-m&XYV1mP;m%E~Z{eGW#!YoWz=k(ydR zB!CSc53dFeE5XE2!gpvEc6N9B`um@gm6h!*w$g?PXnv+$6dZZ_^q3a@Ti@UH$tf)r zV9_bP2odHBukGmQ_{_-27ec{pjRbzm^qb_#f1d}n2Sj5!bd^YdkojyLqD3q3#euchsXK$|`3?eV~=LExQ8NkV>WTA01H1B8i zPW-A;?ALZN7>TD(pZW=-Zs7rr|0orM^GT+OU=D;Smjsel<-&!BDwBTx{`fd&a2~vP@xsj|wd=_?h7`Uug7)}K z;S0?%-Spg5sqyRQD9CQ{Y+I(UPZu^yBf{FG&8#Ukft8OSvqr}O^EmbJnmiu&$=22QO82AogGUb^(oHWdF+;*gt#5C$ zu4O;v%+jq;`=MEovUg&0UmVVJw(_?_RXB@_>w7v_U0InF!KL5N^G{>9pyc40o0~Ij zH}&=w{px=PJd-I-hUW8+9v{f?jW2Lk9#1`6=MoDG3w|XVIiKZSjyagVsNaF3RadD* zTyMeQ`qG-?c4%e)>ZcDVDk{3M3*GQqU~6ZstgQSTpoBeGEp4{7I(z?OenElXLU$^G zQX0>=#bJ&Xv+hT(JIk7R z*yv%nn)*^_XJ;l#BMS>=sscE^gYR-I6-?Enn*){Yfl+xy#r1<~J;1BHHo$iLj|;Q2 zbh}t>m9p{OV(U?@ndx_L-_xKSdf|%Cva_@QY-y4CBJ7f#3#Y9ofqVM>^O8dna)aN_ zi_$-DCLv{3L~B3ujwa);A3OHQJ4onV-`ahHbARMImj2pwM^Gb>?+xP&_HVaYJvB_; zl?%zzdwn#gq`1iGaP2v}e)CH3cr|WX$y&`+FkNSU9^-anXym*2YxFK9^>5g=?WVkG zi}k#S$8#V0Ip5SA9PnVc;<5JTyN8kT2W+m`P;XXK^9$feT=3Nyw7`dzbYR;hU^wQP zg#`iZ3$XEN7;L_j(LPqGC`MKfQ&lr_E!|yRuR}vg;lKm{O)zZzEjBh{=y)Im?7@Qv zxjrY}$v)HSnHsNPivyPc!4Rfq4pITwJ8a&WbLVJo-NJ!qB->NM6L4@~q$jteBp@q` z6*B4+FE8&yNy*8bm0>&*l7N52uB*E{&!mkk+z^Y!8o~J+;D!EaX^e2{1^33yo3}+o zD8|Od9?Qr`sHt6R2&N8(q6{fW?AP$&?c00s9Uqr&-h2nqG64sw?!jaS6cSwEqO|nY zN8(E`4e`3FN)#EyKMYL#1t0hwjHR8%X>Du!lcvmYqZ3aPVf4d2cP$JLt5X1y-C0C$YY88NYvW3*NXs&+zYzF`5<{3_w?xrs2=n4 zFYL6T=h>NWEc2N5GQu@?*T&D|971&gjr$9@{xj250XbR4E8F|cW|0n~_SE`{?Fk;* zusTm6z{3rhgvF$QdC=Us6WG%~Vg(fRb66NHF}EzBSU7b7(L)vE8A>IM+I?+;q~sL8 zG^>>Fp>7PV)gt77%uc*IuittGhVO{C6M3MR&KUw-f^b7Mn{ldbV#R64JaRUhqW+f~8AY8b9& zZf$M-Wd*AU3CRWn>V!A~uIsm8C;fpoH`j+WJ>P>P%)hqBy|F|vPT=bwa&l*SYRY!z z_ocF#$996k!bWfof4(;(vWtEC?EM;jpCdmgjkkn^$bhrd!<11ce2K^dBlyM-O0FnE zpTmvwz#z|T>+s+9_xJw}zh2+nm2!6GgZUj{R}YWtWEcp%2k!1d5UYN`_nJi6A;xZq zxaQtK!?bN33~oNzKKTJ>R*9pdqcw`mufhxw^L1fSQQd4SxBu}U&i2iga_s2=U*nxe)Z~Ay_kz4rGPC?jy~!f4#bfoP=kRy3JQve ziHVoj+4bIDPzUh~Uxbt1X~fONH8C|sQ=RZ7I=U8s@Aii0TqEA%oy{hfQw)U9_ldKF1qN`QVX%slsC! ziJ4u8PwrS^^P@f7q@khY1C7l)pC5Nt$KM|6=l<3G@7^+HEnD7V&0cp_{Ty`rma5pAn^|CR@O9=Zr=G$5p9W+~ z^$JW8y@#2LyQBObJC=D}J%9dO0a!|+VxgDLI7Xi&5fC%~=67u@% z*IOgyP6Win#C|)fst}~YE~_e9@6wjj69LXkIAL8gJWMVmB!s_GSX8tD{NUcjGsuTyWMtHUv*oW)!G29q z7S~>md?Y!WrkW8^8e8(DEHf!7=^3E+)WO70UvSP2~(k zU^j<=J%>qg%mu6M+R!90$9Z*izIK-r(WEFy9E7IYV@XM<8)6!RpElOvM#Ci@Ofx?I z|EPKocr4pKZv3K@Rg{E;5QXeygd~-fvQ=cYh^>2qimNQBqMhWv`4-5~XBDR(AGY z|LpIWtIDW_P_g&`(ZKDAx_5bvLU{NjptZ?$MbQ9dT?}eYG zrKM%Tylz%Sg@zr4`TZ=eg>Ub9hvnI3<1DBX-oJmphmVg6-(&@B*hlO}jq^?4jVDjFI)BXw0&`>i9F7v9W3uX&O&UUwv|Dzg~CXF#?v!kP~Stw-caMw10Mds$Q#<5GdgMy zEhV#${d8NNuRrVA9Y;i*i`S&lu(@rF%$FZ$Tn1JFDkaxxJdxoMDk>@x(BVnuf8|9- zI+ts8*5n~MDOo>bD-#ow-Tq!QMjv<#9q6vOqBTInng4#8duJ0=hjs(?Y_cwMw0?em z%BN4G!Rc9xUDPx;H^)?aZZI|sSv5j4u{eh-(DafNVukbetS-;ryT!>JhDl0QJwJK- z+x_w|h?%kJZ_7vv4i3%-5WK8tY=hJA_$!SkwlF=owkw@D(Su$Fey;ZR7l1x|=CseB zXN67uJz`rzqoOE1JUkA<_WY&y(MP#v`=gQyeG$uYdjwBSdQ3FJA;<6J?EGK^bJsDX zf7PxO*A~04?!r3S){KtWXiyXa7)FM4c=b-Gs#4>!FQiUK&QI&^91ylpzhvjnAa2v) zIN1hv=(*!Pf)*0`NNY?_(zNc{^?M%!}_v1abR6VC-iVr)! zFT6+P=8ZGUH5{5H$4)A_&$GSJ)~<3p>*q(wCb^Y2Y+w1+4k_i%ubI9Dy3JZOgY~l_ z#V;g+7EhSW`n+=5`Q-T*Az98X4s^@r{fqmU=i|2+vZcnq`1oD`V>rLP{tQlXro;8< zs@`7$a33B7>26q*E&e^dLtafy4VG~-$?F{oXnkqVS^_=7qyo;9yW_XlQD?=pCPX8yiO+S8|7=E|jd!_HO#>+hg1D%4ZvnF;Z{IuGM>f4%s zc63mu>IhwPcX#i}9K?CgZ#0j#H(eR>t>bld_3Cw?H=$~{AuUb^wrG2&o zdxteI)wmt)+RXL89LtGLPM@>g=UfXP7cQ*c?fUw)nA^Q%QK0eS`p@W?N1@(S1H8XR z^o-4|-e!d;>%?^FtiQPM<&E{7%#6|#)zbs(WAq!9gpED3m)AtrS0!KfKh_^JYCIFx z^XK@qDgISYde=<4lGSTXrQk7aXSXkuQ}KBi=AhwKYsj(^HuOix)BY`1c)Hn&f&hm ziOcpu={b5T?Z(>gb=aGt-~tK_{=|~_X3wTgn|yM=ewg*Jzuajd-Kkmke1ENN1LJDj zZ}%-s5x+ibz6#-!0u27XAI+0Fcmw{b&w}#ur5|egF#TgSC+p4hUw@U7a(>i->32_+ zKeNn@*HOPlyWSlE254}3K>K)S^Bw-USHM0G2nuck_OGz$5}j;sJU-=ATHCY26PR@1 zx}Di?>+=yA)_5c5`M*^4u@80UyKD2x9-@>kye!{$^?Kv(F2jXQNBxfW$di%F{<;0@ z=;(%(#v!Ib%jCJES}E>s-&Y+txNwe-ydCJ9MrF2^}^KamT}#R3uznTo#s=%;5ZO0iz^Um#WTNi6}f<6siM8g*z~V zFF;K%3ZcB&JO`=%shmJ0F}85SRpW!QUEbKZ9qZEmnAWYKMhdir8}BF%Om2KO_1X5g z=1x(6#U}nNjP?%ie09^F35`1P4oIBe{zmo%1*1=B=(gw2pOZ+0yJsj0gVBRezP^{H zFFX>zcU^-#74TF`o(;$MPm2y_cAYg-xw`$EJXiYCD};gvQV!G`H%4KI*Jd`8vdLMB zk+h*^J0~Y0q5G$RFQxeqfxpvTX1x!d^R?}ziW~$@wwUO58AUl<+FI}PVV_FFz1Wjy z%;+Ny3*QU$M7^^tEXIm!@c|Y{glWD zP9ze#GD)wV%;5?KXpU082o2wq`OV>S|0Yvhx7xra=C=Cn=Z1+9MVDEN@66BqVi(PX zp?;PUE@|n47M13XnuM48HOQW+|G9O0`C%naVs>_RgJLI9^v9k!b$f9tf*nbgI$-7G zB-QcrXZOIsW;(iHvsn%jMwbIZ!R&Io&~@^Qgz7(hud1ufl5YK{+14^W=@7M6%QZPi zu0JrDR1;WibAg#Lp*(sc95a6D^p%sLV6vaSj|}O=fSUt^7i9?d0C=Q~^JKn&wz=f4BS|(!TFy zBG*bx(*1K;-O9YRXodxAjvG3fy#wI5@+R?-f*L(a$Rh9%3_C#SzLZ12b@2O|<5#FzR6QbCk; z%W)%%j{L0r)Y{?#G^7}$^_^WUoMr~M!I0t!M1y2A&?$F&>Z0GnM+toM%LWDp%o~sH zpNSeBwet4$eK6uv63O`Tbpz9$BkMO+{G9!2A0LXD z*;yJnUt$cRBgUZN58|{DN_QF46I8fDO;DK)+N}j?v-<`zR6nYh?}+1@$RF`cwQ6Dr zY+_^694~XfDrxaNQ~1`HjQ8mn-%&{$)sFeb#8yI#^dZtYYOLH-Fnfx*J zxM5_!ZFzm`F}+*e#?L^6myBbydw$z$R9u48Qy;XxmNrW2ioOdQ~2nANyr z4?Av!ES52k=psL^-PDvVGa5>m9(|B5m5%20KfNq*7C# zJYhr4IiGl}Xy?t>LpUM)r;kFzd`@UFmc!WCc=EO`P%FTD4?4ek^t+d`%Q<_0`0#|U!lFuCXz^1?zv)^OaEUNaRN2&Gy+oMcv<)g(y z=G`s0J1ej2!ofp2#WYU6tASBBOt))^>8?LKSy@5;qo~+KWKi43E#vf4#fkf&A;(Wf zzHUfJ3%q%1l)3lLks3#DEaU|13Wi3J)zikV*(=*M7T@U%)!)P$&FhXEYEytX*@l_0 z>Ot5t&8ULAiq`+M0I-6UH#B6F%_SuzwYD~12?6}rK6%Mhc%PN3Oyya-r%K+7fm2`<@s|(6DA!@ zogiV=RaGmHUO;0tJ~YH9G>;@E<>{!Mdx4`NoRdMo3pdK{Plt2?^{+P}E>S&o#c?X( z7+Q~ko~)YfXy!gr5cdrLUF6_n9N-n5ofizdKq`2SIgHx?xU#a+iBbJ+^wgLu^cM=z z&SsjTL?Bf)Xnx6xPnlo?8Cw*=qM>BiS>$NrIWA~nVSzdy*Jw@*ZmVU4=8VNl(I@4y z^labimH2C(V|!`}dQ1F?`r{ zR{Z*9f;6B613Q3(-gHM+&KRx_r({s8+ro>!SyNM!)VbAK)~ZFMV0ChG@`Iio_1%uj z+dK8-qB+3QiHe3tMeWz$65iaDq91K5=v8ZPO<)2z8p%i-kCwrOMqMmA4jMyqNJ8e? zj{|nM!BOz*p8CCiNaQ`hK3hjerse?zjM$jfc6AvGvCILAiWuyW#@YBK?YyMj2YUwx zhu9+?>=6+W3ESW8k+A$u&#?CO>sNjrqjdzU_goIw%s;`$H!DoHb`oLf@ zsY}{*^G&~%9QWPp%tAFrwnxs*3Ol3z(?Z11UwAWQZP~|a>)DHYz3vtC%=*wIG|o5v zhQ3*@SH>)|XajKRr0hVOtK`1cg?_?^etO$ntvLLL6I9+}^4Yh%;CAs z1m=TRL!xv(qcit}rOzC0u%28rkVn;*eoNW+ zQ9@jjdGX`+f`SVUYNt=vKnpOch+4!TEncLEXr2!I&Hm{O&_-Eq~7N zyLqEag}DV`hj52$z;O_uXfv<0Gz;dTDr~9vgaqBIb%@VVO#KB`NkYc)PXOqEfy2c% zT3Xw|qJK$HV{%2@Adk~Q8#l;?AaB@2BI*>ghzZHe+`PYSVR13uV|~pmF80ZjulQnJ z%>isEa)sVMvbY*lT~qS{WHfR5+{v4cc`E0!6E813*{kIf5wX+C+8RiTt(@zEZF~(u z>Q|Cqxp?szrnCHKeQ)c5P=3LV`>vr%tE`LRaM91>45myIyh9q`cEPO zs9Zp8pAH13cWt@you1<)c@=|9s0z zmZwre1=MPJ@-sXf2fQ#JRS$IsRG+B^*{%DO+pYs^tAQ$TX7cM4+^R#M#cB4e-@|h zrDSA!0Fu? z>kP=yX=TCAv+40n9Eytz+73jXmRDB7D{Nk~b+^1LGd_rjxcD(0ogMffcV52S%e-rs z88S|zByHG9i}7I&;D{>$6MR8tIzb13kY+yOE{iSi2V|$Bq5_g;%lpB>HRuEB>FM|I zPQu!)(^GTt;ze(qKF{ER4f%Lc3IrR7pD!gYju?M`VPoYxO+GjHe9hF<6B9$kLUCEU zr6vw#yAmV>LKzgOT7%nYhW*L&rxJmM6A*flJ1w(!4-NU@4HUSWuQYaqW8FL^6koBi zvGi+AwJ#{5~gQJHz4Fe_BQXJ&Xk+8v$L@5lf8~?JoWj~%{4%3-9Ql$c}0dj25lZKfHNQ?+yK@<9pS>6 z+HG1g$-v}-yTp0e-P?N?V9B1thgqTWz{=f>L=+MUK@*#>=Xi5DBrJD8GNNfx2Ncx= z(-3t#H!)cPTr&m!V!w=|(x2#(m}Ah))ovv;x|jCvH9XtBRG%$+?XHpAt!}k(;Yr!a zf_ynS0k&_#8YgYFL^@&$QeyWuS$-kigypV9-_J*+H@CstjYvjJL|V=7q*uxWlYpk5Y5eXpcF(m zkKXAXmIj#WZs?|PExw9`qz-XNe1V9Hz41NuECU2C$5E-kxY@y^<8uqkVgYC21k!%? zuO3l?a$&olk~+DMxA&gCdzFTy6SVlixMfUGIBB-Fx0?e$Wt`~V;Vx45|W(DLiRz%4{&hXzCI zu;bY$SpOVIT`m77H{3$Xzdg-Mka^7-xO*7&n1QCNy8X>E zwb8+n4c~+`DnBbP(q#*br@i-o&qS~zSCf+tWj`#8=S-E&cy_ZdnH&58KOa7v- zPlZ_(_3NRb@HX@Pr?iqba^#7e98b*29M{+{TYscH<>ifnLuXsbi?93snkcaia7+%) z;%HGgsVx|HpD3C4a9s5wD&!9IAR2M@`yxn>d?*2Ki-LoYpwUY11#1P~#>-DCjpm z!NSa3hH8xRQjKs&>HysxP+H!3_wEP+pC2H_1IutD7-T|B1Mm~F5nf4s!6Zy_#L@pIc8Obb5f>be`R=FPOIf{#{SLAxX?@&z@VW&Oi*~Gf$^zavNMXpbPZ4x0>>y^WtUG_8caI z-f3Dti`sWiqTiY;vY&1deSMSdwTeJz$VX_~Jmcaxur;yxJyEnG9UW0aV^dNDa5Php zFc{GhbbY-XFt+ElLIV!EmoM{rd3ljYAT31>9#n#K3}L8u$b$#wTXh)u;yUg^aPcUw zf^oN2zTs7636YFSS#_JkY}-HV+cJLQXB9w-K3_JW03|NZ$%V;GgHqb4df^j?So|341)jY~=gt#8AWh4y5p>5m z+c?y0*kfi}s-xAt$5~k&b9EE88|fU5KQ{i_6V9}OZA;x?6Q#U(p~;yMnKOsP41RrT zX3py0>1$K!R~D%gvQ;7vjGw=}7#iwf_`mt9zv) zr&#*Dw7JXpm`KWH-aI)@IWqHxfU1N{wc`(7eJhq8{?kdZ<=W!ntv5Zr+hiNJe2f&1 z%x@T1g_&qY^d-gGQlYqvq=;VI@2N26nX`5*4qLlU8md>(1m549T0|4&WKZTCJ0>pYv)W3Y zIemh%@IdLE&-zcf#RZCXJF5_vpTGws2$S&L@Nhaby1Br|7^%$XL1EC68#e=bv zLA(9XU__V){}VbM`X7kcA5y{St0KW*HI~Du?UvYM`Ciw~B_hP9K!1RXS8$X0;~-C^ z)`ddV*Q_k%mhFeUPe1m4bgg-_=Wm)%!BrnXBQw;gYCJuyO*5q1w8_=F4-1 zopbnRp<&ERNwJlc=3`DjX{$Tj=WMTEJ*QCuwu$R`BNrt(R8)6xWbOZ%k)9re@@o!J zI20k^sQdQO)-{*O{N9c z1!tD|s{PLuZ`6Dz>>XlMkbJc1<#t+&D^zhTMj{DCg4q%~)&n*le`WIhN4ssiGtKE2 z4?o365^toVg~z{?K@Q*j=g(F&P(&fcYyvy&9h5$qTch{U5pqM^gJ644o!SiSTB7-3 z_^TG7d;@eSW`d_CE;7#dC~+4~NM$81m9ak;DBYbbz)dih}x$H|tLA3tcE*0;Y{b>+gb&Fn&H zwkC=v4`!G@%QTJFGm z{!KsiUOaY|C>Zm!^JKnhxr4A?Y02VPHYvCzxK~ETzFj-r=-B|q{mAyGw+3vy1K97TdV6o5FOVIwOfvs{GSJpN znZda&hHsQ$o^MazCJ+Fqar-5`Mcl0b-NeEyd7@mb_vk_SERIFRw}2a7vWJ zg8cSxV&O{AmHqeg-0R;>ytgBLr7k(f^jb#RRfD(j9dr4TcFb%tHr~A0$08KYm>wSC89{bvrsGcTW$XUjSG8KA-5nbZsorV5bb_>^%z0qz zRZxL>MsrgpyeLroGkpF+=<1qPsa!dGDan~WcgU&)l-nyNqLpVPP-D3wM z@ZXOvJ7KX4kC(b9faUmhcvn+a;_|K=ZwB|~5i^#ti$~ye`GEh9(=YS{4aPgj1P>oB z7&|K}O35-bcR{z1+~yw3)0PCXdiP%M^gwmSxmTawWcAgV&rIeA#2n4)|8neq-xs=$ zzg|bN9oy4~v8`{(49iT|&M7#i{IPGp)GDT4B(>$$g;;(q*_-C(1~y%r5Xj!&d;BE# z#h0vN7mD{=L<;OytF&y9eIxjzFxNM_G@@edmF7q0!tPC%SBCr-9tHgW_qAqK^#A){0hDP zfpV)5R^u+5I-Q>_dpmUYiN%L0db4BZ~x7Y!RuQsOyvN8%78J9T6+}N^3 z5bwq%XT1f81;g&eH{y@aGVdP^R;e#*E*dX)TV_+@O$Xln_o&Cx$m`?smX~ZQFUFM_ym& z^Dy(TiO44t$2?pHE&0}-#?stZ3{Hs5>^&j!=Iz-r)&H|3by@#i8%s2GyFvHT8sYf* z>3&GwW}%?tR`j0A0Do~1R<5BvNi57|meagfTO}icnYQ_bY~v1LSzF^Z!f6>WtV1`D zC=-9LB)Vd)?*_|n_PLrbTbd^i@)zuOCQzk+KRTj+xAk3h(QazJ{L)CYeWeZyTaU4B z4R04>xESSbCsTiKhv?9`7n@QK+4$|srh^?mgNr9EsH?MryG|ewm`J+5f?bGvhn|}$oc=SD>~ISP1XKkIy|l)tRtz!)aRqOj?<5GRcU9O zWzajHr;uWLtY9qK)YK*{Jv5Yf6JBV|@@`wl#teIxTMzFaZTL`I{ZKp@Yzkuso!Wu_ zc34vV=g(UpkHTM1pFUM>C-foK9UUE=7VAX8d_84K*w;5EIOL0UFI@;zeYsFtInQh* zX>h$rJ;b!F$|9fMK3;k9{mT@vFmrWZ&TK3yIJ%R~2w&a}iDV12$zMBLq^rpyd}XMY zHyD>z@4J@X6D%GaBJ_#>zfYiPp?vC8)p~2ucxGa{ufGVR5K^XEI-EJv;zrKQyCrI?MRp1(Gtc;?KP=l73m8d7rYarr>~adp+S;4{zN zJ$uN<2W~q(?dx#piZy(w_Q9pgfdDPg2%7bM2(`U+Yg1gqm9Q@$qSHn=OC!a^#4>M< zor4aN5sxbIJhR^$

4*0PehnOPc8}gCZs8OVelG0-P^3Lt<i)a6B^rQz1hm!D+dJa?8ZS$k zMDfU$$NM1l!35jhs)5O@{7+^DD$Qr(-`WZ;o(u0uijFE1+E2MFsN>}|6<68qK*sWa z-OQ;m=Nk-vwDO3WOKiJ2c02hN5AnL3FXdI#)b1Ln$91RlKtl+0 zp1yJY$)YkeNw7>2vjNCdGWPM$;mL%i2!enKfcS;Gg@~)QB;zrp;tV9`PCnCJ+v~ZH z{`+T>4~O-i-e4O&81?M}FJEO{NOXSqtrXSC@9L+%-e|b~TGG1KLO7_vpUKlRGtgOdkx6V}5b!wbRIrE1gLr(+x3Cnt{?S`GH8Y>4b#|O~099*4W zuK}9-3%1^rPn>^O#m+Mo;q;CBby95;hP+uhuqt?rB23OqAB~+evlA_)TqB@t@8u5;Qa#E~XdvKD<)@@VxHNGZwEWMc+Eo;0nC1F}OvKOMa9N~{P3WzxVui~jueNPI~oGFk7&dPGLrK>-e?5DLuNA!Guc z$K$~^4X@W{-&~x!WSDQ;{UkV?_{wtbZBO?Osg6%otO!qJxc;a%EPXX*b|CxR&XVdr zwXn(k%bMXm7A*!ZhMxYUXJ)Z$qfu0xurF^m+56BkxIo@6r?DnU;dI^bPsQswBe`Gp zO?2%JYPoo>{qKu=hs=k=MG(Nh5`3(W!B45Bz+s<@2awY*KYm2C=K;u zMj9@i93Fdaw%Uu(YLAIF?;9qUzdUoRVn{(a`^`h+F&Wzt2`g%MZ!x=GRb_lhnno8c zuoM1FXyX759|K$mWIFTB8%3-~!o|^1q`x@2gN22~pu|~%VE>`NW5mWMZ8&Ke?97h0lqfiOLfwl>t&{a#zik22;!g&Q- z&^8W^j1`g&^ZCUaS<=$dgyQEbV15{@kDWTTMV37{&CrDvZnX+fWfKU#tSB!hgX)Kv z{UBQ)tPVD%i<=;7@mv^NmHY9}f>+|=QAm$Z;g(Fgll8ucGxR?oabxv!=Qw=%z z_*Qi&RwM@Fk&S`@)+4y{QT2G+$(Xkzj*JZgeRWJ9_l!XcjcKke(bo z_-L$mZ0vI`_!t(@QGC!n?AbvLhnW{(U&6^ruWQO1`}{c<@~z2m+KrA^K#h#&iJr8z z6=^@+dF@5%S$ITok6m5YmXD0#A#Sk__lq6Ok8_1mB5t`HS*WI0u55<g~ZH-R5IlH&b7xdPh6CbA6L}Eiv$_AymHZ(@Lan z(7X))1HWtSLDMotsDPC9o^u8`QvM2htaonZR(xqd+2VMwv*Y)Zz z-jq{UR3-ulDq+Ea8&UM8+>WUT*jMXev&o9{2wz=+D4Du5FQ`YjFh^+61dz-p+MWu&AsR{jWyH2 zC+fiWJu0ov$%*{b3DCKC&Bz<@1&0NY5kSePte-4=Qdz%_RYbrZn_K>;w~6bQzXg0z7YVIrU{4? z0XcW5*e_vIhgUyFxBzYk|8(1H!N!b8*jBz@9mU^Y&m0yyB3aA8!g6*r2t+$7ykJDppbJpQ{YNN_jBz>lN{8?}#iv3a;X6Z&QDEWB^VE~l)6TQq-OEcmS) zFW=_%_8q>9Zbl-7{&EY9iJZLN^oS3JtC*Y| z3F!EUlH}z@j)y92ARPK<&$0jz^@FWru7z)UVbYv%_7)Tr5K0@6SEiRP`9MNuSmK=1 zxXQ?vVc7KjyAlGgzZBngKpDJa%8Y_Q1Ry}|bVqq_1umR|v@rEaSoeV&(s&_gp)kv0 z#z}e{eG3fI!T7+O2;YP7Z9hMM1>%Rj!1^DAm$xwrL-#pPd#a zIEfSC{(Va5cD07GbhNea0=)+>WP)?RQ{fI23Eg<8`U!h`F=pR#T!W^TmKPpHk#yt0 zHN+5yWjc;um{@eZ@u!1DNQm$#!`|{9^_Pr_V{=giI&&*T76D-#gm{od!b3?Y;jO}f4!vgTx!xdue{&l==R3Y}4dfG{fWXYq zvI_l{QJN^xJ@v_H)=(KM-lraX@ND=K&DP&KZ1TfRMd|mHIvH;B+^+lb?bs!Lqs>ot zz3Cx}TQhdHPbMSD9zWuYZq1x1Q#j<(ZVeY8D(TYNLUe1?f z�T1uEF;gOZ^fuDsL5a^+)aVAT3StP}zFugf3jh^LCo07-P}5nuCg!c6;FOAC1js z$8H0Gmspk(*ETRhwR3oQBqLFc9MvIptRrkT$vQeG3M+IGl229!ivNRKIy&3SP$jmm zL4i))n|}7K@3bqZ&jxs`ZI`eDMM_IcV^zO?_@EqZfYN8hfi|ZgI1glfZZJ^cW!XO6 zCwMMLhE*3r^FIdWAwwrO7@i{sJj%Er%YA*-QKM-9O?H2ESh_71iaALHIqei-aSc!0 zUSwxK+U8$nZ?1EB_@!Uy)1RFioPf~E;0Oc z*I_rhA8FlWG+VmN+J$DC+S*~;C^y!j_hAlIem@k6Ys1s2ACHan_P!9g3=f0(_3QO> zY;w*Ar>7D|X~iJGtAk*n#T_9r)B@VX6|dll$LiOwr+18P_w$y9a%%b9hB9v-AHLn@ z);jsbyOW~O!id7hC)V4ZZ^u0#=hR=z=+lsZWnpy$$Leo1T4Jlv*RbFw`PTG`Lj5(1 zY&FBGWK&(x@^%-dQ7hGTbRMa{gX~BYosiWL507=r;5_7=TghHxTyK0Skc)*qZ5ix| z!DxeS{&uaK*}lZQX`;l)Z)mYae@a+xi`+`bz!j-owgP%}VS=(%i0fspxT6OK4RHHZQ?~03MiM*xJRmhplAOJ@T`fKKGb~Z875RP4v z0tzb*&OHnb;FwW9NJP2$Xe9a8RqhNvOs?u7ad8IR!r+QhlGKP~gqpH)89F$!V(I$s z?ry?@+3}zc><|+A8Aew@A?q!*K2J@hMT~kGtQXHxQa-gNrl)U}UmrunWUe(U5CA%~ zdIjwiG(Y{U@16WLJ{}BSJkz?dX=+?iMTH7w7oMO?bE}+krH~_;;PH%B?U)q$ ztED~GWY3@dJTOgYJ)k@LI5X1vb{_w83`=#R`UQavU=72eCTbr+?|QL@ID3>ma{VR5GL z3>Cd0E7ROJF%o_&*H`Lr3G0tompji!sA8=bt1d;Yy3h)Bs8uGHUl(co8p1(<1!pu= zwmR?rquE^PBy7ZTi=tYw*Kx`KL0%?dPe8l3K=*RK%Fo}w*Y$SkrB6GfUp~)On`(23>4ej_D`>lQJfjG*w^&~NUTX!oQ zJxX>*lFAdhTY-RANlAWqat;(ql5zUz7A7XTk4YUq?67p(#ibX2*op)4x%#=Oy_Vn| zzh)R8LFz`x7y^x1HM&AV=ruYXxUWo}JmgpEG-+0#y;W^`a+2V~>tO>ymjhS6z>{I^ z{X2S=$KUDidV2`r#v3t-?ccwjun+3A4Zx16gp7B_>*G)W;0pTsqOWh9R$scP!f{op; zQ#_mKdGO%nvdZ{r%tRqef&BlWK_RvY)cRVU1-a6B;56NeIa$BW_~gmWXhR~4j*#jj=Quezp&au_vC}1F zECOR-N15ktRSl5tXwS;v=?|)5ccH;0 z%zz|_hqS@ge~^usITuMtJ7nhU8&w7{k|e<4>SlWJ;!CZLyqjn*Mt>GDp$)DCYJ>E# z4{Lb-!QHgotLTUJgz8){l1C*-XumDX-j-|qsZ-c^K3_(Z%(hc|%Iren4ZpP9-Lg(1 z-ioaa&WtzFT*=7TTvkXLb)+Cc*bMO%iSW$VJvqz-4}9$Hk_|sy*u#hXetHhx zit*~?h0kk)I7m=Q`IN}PMxx50A1=S;tfAp<^ciHuD!)?f>tg{qxSTJnShH5n;7K1P z=dft>=tUQqJiG80rt&!B$He*4?zNOpgv^t*`v37G0>;wOrjpeIf?@e4C zdk-I&GdSQlEOT7_m4=TL_lDNZYcWFVT4(N{Dw}#t&>yr)y>??|Ht>prc5F;5+o$@D z%E|VRvDO(=Y7?#LG0fvlw9_lGoH|V3zwQSFENc$!*k$o*Gsl5n&T( z8m>?`=gA56e_Pj+6(UJ?JL9+)p{qX$bG-CSlu z(hfD$)m>!xiPPU8*D6}IH}dK4{5LOSijz)WtHqp|W#6wF9ySXs0nGvIk&8dPzw_q1VaGZ^bD_lgb6zk^7^+_1{T;?Rd}YjRK`Fpgq!=*Zl$U~7)t$W zx3GTeB!US~I4^lHu_aHfjek#KaFV_7!YJa{((JF%(K_ks2g>?eUL_{>po9VOa!#pX zqW0N;;J`*w_^Vq&7-?-%&M~SH`){7((r!x5KreNllwD{)79X=5tPGO3Nn{eDUzIHR z_I2TfNRwC<(^%;l>7BtfcdI13o*j_eu*f9*W~^t1cJHYuQqhLx_{%HT6D?hhS*|9T zukIWBHDc+Rar!;ih{U|w1EcN++cCRMhql-R@7(Ep^PnnrWe%UPaIkhacSeP#ZlfPH zG*IDlS-_cWA~Tx#tpRTh;^4xGg@m3v*r`M+#+iR%CWKg(kW`}wpr^E)BPW8j0GzDN z<2#^Lj1Ni6Ry}=sV6Wk|-h4QB%Bre-j0O-d7jdd3e4nO|BEm&vNyRE(Fx&0IcnXc7 zsle(>hV-dKhKLLMva_>yz<8l+vC@d#n(*rxfkV$(e9Lh{yx&2Bv!1caYheRX}ycL#<9Le`zAaK~o0&N-`4)lzSK0v=J+uC-4X6aYgJ!_9(!*3syr0Zmu4Pe|-Ne>$?npCSm)BX?VqQOz1@42aPR)_dAWeOqp>zfzvzJ2CscJp24h2UPE(d*54U2Lh!So`!IA|PTYQ$F_g zAYl~SYJ*dl(i8OYPi1dcXmkraQNYM92n}TjN)S4ix~Foi4RJ@VG~497eR~YR>qYis zGM>*fGK{&YN*qlw6^xEyjxhy-rMmp?lWJYt-qq!E@7(2(fucJWb!25?gNqB$5h9A^ z5wGOPbk-yJdnIGb@3L;kOuJW=CfDtKcK_{L@t&R@(w&m4&&B$$8E&}a@BiuikwXLn z(kZ-B$W9_^y_hz&w!Q!BXL&|8uHP0}HCQ)nqN&*V>qPAq3YU-jzBJz(6FwZzD^bEh zDlWBWMnH(~v*Cmp@c_hhAz#MlUr98g_dBs_^tK>SC2!k)J+& zO3~V$oW7Tbr~B(WrdAf{`r*`7T^{J~*OeKwbK_iHewg0la^5T}Dow9{dJ$1Ex(VSg zlS`|+PBxC`R8%O2Utpr3pa{$@WOOQ;ZpTpZ=&5+hb9@n_*1A8Kgaic4n`_eZj}Uio zSW-2AyW%d2#MGW$={nu2I@5mB`xc6S4#XY`Rq|%x6sM15(N8cK-nsMY(kBf{{?zeD zE>n9OC9P8D3|O~cQIm^ZsbJ_WxhP=t<^uBcGBgeB`FB=v*|7x#(JLyS8tEtycUcCp zXT$c&_3=4UK3wst%gXa3(9edmpEfdLBQhz_#eP9%SMDli3*84VC@V$s?Osgz7X#~T zX-DxBySDAe4?9sw$tz(LD9OaKjZuP)-WP)2S01;?58(jE5l`Gf5<%0ZrCEp97)QQr z7QJa_*PYtVciPFx>2AS-#!?9!jyaY!pV0SP48`!sBlF1=1Ag{8Al3eKX+vW~$l5;i zNG@J~@%;I&x{=MVm%i6E4&;H}*&`s3(&+ZmI0l|9n-X_{vXV@QzP#K@_^03zdCL?8 zm>Pb&{xpsl0uCBy`gdbnsrB&>O{XlxDJo;{*|lBpzhFL`kfdCzmaKK))X9@AE7@$a zoQ|p;b!>-2a%?)cWP6yJaw4#0uj6_$QW(86GK5`>n{KY?Ot#y+(`_3- zXwW5_Pv zEi*Sy7PsD|FPN;;7jgPi2R#niqhnqE=TUTm03BIYg~>5eDYqje~ z;PW%K-a7DvL3K55Z}Nmku`DJfZ@@u!BpJUZ?+(VBW#xrG>C!<^@0PnB!mmGn`c#S3 zh448#YHBl^dCd5#BlX*H)qSfcsPq+;d(YTy(SNKJr2{Oul6;I>kq0Y@0k&LIMSU(e7iBW<|{(+;Uq-CEY{J!SnI z4srw}-7c!#Z=AP7AwkzkmzbdvbgyGZKRItzRpCR2cEY2k+cFQ+0a6i;bT)mld;-)N9({^mr+kp7`J(k^kN_B< zkf%@UEyP7dO~cNjyR=d?eu_gt;=cC?=HixiVwT0MQBg|ZtY}5MsxmvHryMp;T(%yq zMgNPz(XZs4>!N+aG7f@WXgDv8pN1PdSM5-pXPqf2B9IB~(QU%wm-?(KCD zRi;qekFF9nGkDf3anAS6y#sYhxUq;-7OV3>J!4{lO;y6wQpc4x@P zxB?IHo&AF+`oVxi0hZ#^*>5`M0HXlA@&d8`#=4QQs3{)!4*-66Md>jYKWueJh#=y4 z@0Sd|#`i$vgsC6F+=lR{`z7yS@xVVQUmpIHoqdp&S>(>CiPuRgb%3YJCZ8z zkuUiRI9#ke_zZNSRrr-no(g42!ztrLFb@c75(#yH21eI{@g{rW3&mx~9YV_=fH~G7 zcXH8nX1pt|goB>`4k(-2_e22c*LQf#c2$qFv1kj>iCacj5qkhiykLACGwYnRekyN9 z!(+Grkr7C;5h~NXbZQiym$)nKV&&~*FQNdwu-XjjnDRWeigU^(@=35Jb{%6 zr6NrXe&xn}AOUrbl?ONG1BlP1bM`OJVVv_{#iXm1!bYTM>X=7daklVz#%JXfcN#J?)o>&gE#iqHeTQ~|FknVkdI%n@jKtuTMHuJQ?eq? z9DCn8De~Mw+ttLz+hX(cNfE)gx2wTLeD`mM-aQqUdvZDP#J?%SW&{k+NrdyAkDmj& z(u>Xz(Ju_=vaf=_zYn4Y;LF>ONSk_uI#q?CPSmDTmSpoo8%!o23?_sV4VA##+uOF3 zmIE3WG&Verh9G;&5mGCHUnW>C{IaQl0StLy7e1a9N5)*}Aef85-3(ymv4|*Pmr?>& zKm@ho@l;!BD+zxH*#CF_*}wt406XyqNFas=Y{c7%*@&0h@!Qe%_f1<-0 zSbC{z_O<3nX$9ca@y;X8`rFs9N8ys`n}Z>$oG2sK;XH<(l2nsKfD?cP+Da1;1TDq4 z#ffZ17#QN?!@~zh2^4M5U%cr4{d*H|T;Lbao;@S_ z7eJU~6TC+YU`~WMrdXr;+&2&|wg4Hi*f@au5)2WhBNJ=uEm)XQ(a}fuZpUD@0X!7q zIHMb#h%O2koTiQrE@mUn>t`4ahz&`QGTXM5S4T1ZPt-XOn8#66u}UHB-jQ^ zxm<-kWxxRm=OD?Zo(|2(aR4E${uTj&fs<(F-Y%oObmKP$n=?45X=o;a>=B~|z7gO5 zLl}Kzc?V_770d2n5mC`{;K+nrg20a<2vWk!eT4-VSKN)u%eU{RcCib{$+>wqhWatl zC;Zj8Y*(-1^J$8*=I#8S7GQt3YG=?>U-sPj&LB;XAVvFf4{co^o_nVKf#C*yS@)kP zYA_}myKdFg+HQaPiCO#mAr|GwvJ|&A(;04jZUYqpF^}MEHGJpJ24qA(Kls(F zSE>#1iOBq+A&MlNdS~?XHiasCqoCCn=HOk41?NRU13zt$49gX3#6mG(I2D0Qe&goN z8=S=Dtg0>+9mJs_B`y6iyQr|R2O)OMqGlVh49Z0UI=Yq#+7hnUO_~yeh~Pw|0C66i z;{D{Ze){o2DkP6w3=1oZc`!H3FDRIHWA!J_g6P>~2YdVE)fN&SKcH#|H411afG&^D zGBrU05oSwq2c#UgloT`f$)NWDlJHIy?>OLjt%gO#(w_@qz zPZLDQrKKFiZK6;0W;8M+V4H(d|Kw9*jOhmxM3OdX3u+S_QNWVJ4tv$LWvy=Im2Czi z=gf=Sox;s0on#L#-jY`0`5x!T8bcPri`D`g$rl<_-pVQSE* zk8#~m^LVs(`C|t<+f|iKhiI>D);9Vqvzrf4QSxI^^G!rd5V8k}`~pNjTzM4)!v8oB zOoH@>j86T!ePRay6H${g(r8Bk0}*(SjU5^uPJujj4CT*pKo9~!Ln9-`YG$#Elv-L^ zMBb`l(XB0zs5mXn@WtpL4O{_IBf751skxH3IK4? zalQwJErQhBtCo6e^BNF1;q0qMP(vnYOMKsiEqHNpQRL8}6L?V~h!N52S*qdj@WHo$ z@NqFdOP&J^(KV6 zc>!A>=%VxI&xe~gXCv{w4EQ1!7zR1vOa|qM661+#LsV;II8{KzU`f4pn)P3nKo+9*Am}55Dhps3;d=t$ zxs9Iw3^%;srvVWALjbTsltCmS;GSz57~Dt0Q1lX1u7=5utM#oOcH2dYQjevamVP^8tMrmHe|mUpbsR^q7M0`b+wmfgVE50- zxwm4ZB$=oxPd=L&($*;I{t};QSemM;`!?{Dy~ei7><_BDs9MF^56nnp4Sky%96clX zjcU6hGG`peK|dbFFp7gZX{4K8>k_m*L?SQxt3^zlNI& z!Wa5S>&86W{U&C*#*?8*At8VT@(Nn`RnHV2af>^(>&39}r~)PDEJ<4rQ%kZfPg8S! zF`M44^L*ddPAy3Om9S=^mS(Ol?xPF{lpd2l8MJ3Q(l}Y7-nYK(49(_!&L+^{==^_- zy>~p<-Ty!SY7$LRB1%%Btf-7A6)Ag^LRR+3j?6+LQZlnDD=RBmh0H>Ulu=S0buf>G zUHNFbm5oYPLYCBE0B~d%(o>4)43jO)=FWX*z zeR=Dm?zz$lq^^;+Fss?A#C<>X9b&f~<7-&8ZT4H&{>AHlKS!OHq^1Ux*K9#O!TTj^ zV;pazHqSpUuP*MoZx7a_ zDIZLS&2+9|fb3jtMsM7RgI#V_E2HBvxUEfC{_4HTm2;?DS7b4dbA4V+t^9G>mrbUCkd_A#jRr$1{ZPH_tMKB?c%S8jHFS|rO6T2AMn`ChpQ8f!N<^4Zq?6_XwA@mlB-EmsW_t+Ybby>8US*Po~H z_7FYi9n36z!n5tQA3jK-Q42xrvO9!?+fJPNb@D`oh1m`sTn|2j{V@Wm<9EpgI&Swv z!c&(y^zlM-6@tTb6Z9)hN}}v`n8khJYB?{?8c0=phW5#)bmpYjSV(*8W7x+Z?HjYV zu=JBia(+|(M&0mh<($dij$XcRt`AhG*x8pK(`gcZLWHP9hKwan@;{%Yrk(=vk!YMz zbjSr6&BG@+bJ!o(Bp>L$RG`2{{Pm>aIbDohZiQ+s{ns1U%8lIJZo4$Nhdi6>oO%** z@~}(iyl+jGqp4};_e|D$Y5&vUqJLc!Ik-N?JA^V0~EV#wT-y1rJcTd?rMnMIy-ya91G^vRo`gdr-9UKJ7EmS zM*Z2kzu{d2{*SZ%@O^3cVb0BPKC^q`_-fWWQdjup`2U&C%OgOY%=FKEl~S!KP{5>U z>-}EAQd`8+_riAvzoE~Es+?~|YJLg2`WqtOM|#kNv~t#l#*GY@q#plL6)#PDI4b&& z_ohMrt1qJ$_OIDxu@M^FD=hy0q#P`N3#w(vF5*TfgnIY8yL0 z&ObCJ*d5w^fiu*h`kryN`HnoLy=8a%4=c`gp0W37Lp;~VvNtlU;or(n8fr)yvaPi! zx~+0OXh7%czpuVLZnJlmTMqw@+wj{LQ=dK-nOu|qI{vCJoJE9rkId)9456K3)K&V` zHfC;+5h<> zs5rdKwS}H>%h9Y#m1f0{5AQ!BGKDg;HJ6$4x?gn2zV>B!z+K8G8`2q3AH6D?wO!bP zBapSL=bMvGMJH%9S^b|IR~jWbKS>X`S@-gIGaa8z--cH;yw5!!Z28|uCi$E8pHaaq zw3pG)VvcmSyd4p`BWa-}PXF)F;x#qD%B|EouPs>wABoz4_}f7JMywFGNIPsoo? zED!!3+ViA_;h(bq&&b@r9r(|nBoaE;Nd-w$1@2E5)PLsQ`c&od<`wt&=2#Z(1a+pD zFEhVSuq!^fENVZhNB{lab0E2?y3vn1e+Bbo9%<@)akpaqw&+41VQD?ZgHMwq|Ir%! zGr5+YEGu0eb9%pvlaw%3%4$Cxr|ib%JSLug;6HZmKQhIi->F zGHUAA#Rhe2{WBrW>vI2&o&RMwRA$2M=X#GCEX;p&kLrkvJ+e#Wl?Zu z&+5Sf-$P-$PVRZ4|6l9;to|}u=~;c}D)(C88BIcRH}@TxH9x&IdSBF(e&D^I(>^5^ zIkzduZ9pXV$@Z$-pJHsg57=znMaHA3I6d-*>f`-8-`gDhjJd7{I>eZiMJ-&uC_u?mE*ACzkf@8MB|8OQbMJP zf>sutHyGE>X#rZcw#TDrCL~L~gbIA+sZ7xQ_{Gfe(G|u26!d>)Rg=*2!VoK1M?lwC z`!)TwM$-H8qFxFluD5+&_ont%n3^Al9lVNa7`{n8pyo}i_c(AUz3`O9gq_gmOpB{j zb;Ert#RH3mWM<9Qx7-N$V*iiwX8Gmgw>%{i>diyZ1U@Vb=lE+m#o<4W-24=(7Udnm zc36q*-0HInvv01JIbyWgSMeq)_{6_>fk`z86d-ZJs^S`UM|Ek>oI@q~>-DF3R0(045zhqicZ+&^?xwA3qUEz?0ibptHY5(v?4JvQ7=2gi_U5l zN73CBs>mtel&88;mg_5?q*X@YUi2@I>d;p_NY~u(>)(YPa0#s-dMCGY?P4ZNrva$$K#Wp_{beUdh()H*r&y=pEF{ zWbO-3`M9t?Ij%v?rCWPvDtzFpq@jka^2Yh2KI|g=MH{80{!_&S52~i0qjrj2-dcX} zqc>}63g)c+VMa-=hPuZ)(C}Qvy67ArphFHdVzsUp64WL8wq;(<4f?8n;r8~!hLXXL zE^jU8aLjpbrlYWbaOc8`a9+@D1k^^R*a)#}wvlUASw zG5))>XfCqK?U((1b2Pm=F2hw@UPJQA%HoZ4OmvB+GNL&VhaZ!ZQu?nCEb*>Or*e_q zb6Ne@&!+7$Bj%QSa)!c2j2M|Gj@rsDu8nOoIlVIe%!Iyc2O9hmc(mHVea)0fik zS$`M@On7q+4Jxf&Yrdhn>!Mn!DhT=iQ&V4+EN`D9&rp9ZUN*h_x1mp((S*smE$@4) zU*>jQtXw`mkN1hY-h686evx$WvR9CFV9N0eWeF&AzV6&E67d@ZSAPrCSQp{Vm_I3U ztE=(1v7OiEF}F+5Lur^03wrMb$y29V-B3bd(52RLR7@;ts5!mt;HDm!7+R0~I4K(= zwu9hh$Zru86ePi2@Zm-T-p-PxMbW)MHAbQLpNAfdO_4vitqW~IVv7fP&SS`-SOa~) z-OeWZrnjn>dO^$w3g6qh>J)KfeK^%n`fnaWMWcgHhA z>t!913eUh@Ii z@zo8~Q^EiwUl@W}U*C0v6a@_C*?bIV7@{9R*WQMKB=7`DR5dsEhrmrANpaxTUF4Kk zQrZG4u`e*KazF`a{}Ra=o*W55#iNB3i8etfU(lbc&kn9lFBOc94^YsYr6n0T(lj~5{bJckHFX%&xdqAZ1jmF6oW~`wR3ZM znVBB=htPwU_DtKJI5T!-9Ed&;ufMAV3Fh7oiJ- zTtSp+2x5bIpC0D0E=_^PQx4=9Q+hUiY$ORtxZYa-_fqq{UchG zDG}&$g1reVGm>@Ut5HKIho!X`<$v!z}LxF?OurM1Y_TV6{KuVgUw-_we=a^ z54PUc4S(i!FtICd`;MZF#UJx6^-G?uPLu1HKeJA=~l(JV%Nxa2-uzQKqJ(LgPoemdIYKULJ@O8BoS_E0$3iZCbE=pWG$!-Fsix?!GAFt zxjf2$9arA;xRia>_gof`_#M*H)#qYQp;r`+=2QVc2B5_agan#hW)KiXg+rH0Bk^^C zs4q&OzTouV0cjsvO3xcd!Js>g-%(*q>YOWzkN1iYbISgv4hE|tqjW@@co@nnADqq} zH0GmV835#Fs9H}HZs;WckHSsR5QT=m_(;5OrJf-SiTo80s4^+Y$py^d9Yfsy1p515 zwX?TZ6A8t0j4D|(_Uwk+#h$3|0DfO80PiG%J%F9ut|fn3vf>)M_CGiqj*fzWi@3J# zK|~QDJwmcKS4oQ@_7hT?7{P+`?n1fKb3^D`0)N=SOJ!5q*;m)u!pJ-kc2hQR*pRLr zTx!4Gp*DM&j>fwV9lDTvah+!3AP?@~)A4A`Ck!&s1i6k|f)Y~)6qw63;D|}1*9;ne zW)qGwB^S8olK)OW?d;fT0Gq&aAea|s89~U^xl-bmE*NU(jtPogfcv=7qwsLm+38sz zp8oKyJ6HV%+Ch`F8eo77Xiyy9vO0gj)~8fgUvPpmSS=#hu`q!2PHO#%VNI%@hif8Q~gG1_1@Q zub`Hn0K?a^qj*QVKr623X;2WI%S3}TEM>Q0JNpQpJQ{~j*+oQ{OzmH!k-XcM-@=$Cj!sS|QO3Yv&PVDOT9Tk*@*`S}=-Tnh z_5vTS_khX(Pv1>sfgB+^Y&baJTyTJgrwT_3Vmb=3Fa7rIwE+)9LrE`F%xTQa!q(1? z*v3BrpC7#U9EM{s-oFK6FTOezekVd5JpxiCHH0hs#v6)VonOu-W@c`NZVoh2rGnW+ zt{4gTdq67L@f|S&grka#L^vCq$9I2>Uxh#qHQn>{o;M&xF;o>I6B*Uj2c#{^wUoQW z`X8)Gbh1E8&aNIrBn5c%r?Dj?&Fq*psQn~moVYI{lM|{V>vLW4z;!t+7SS`P!PA5eURdV+tJR!Gah0`HZCPOFZX@Jv@n^!X~c9H2(jt z6jr(7_K5mr)Er2Kd<`~(X> zGb`)ny?Yx;Ll%O-Z?BxT;`heY-AZY~M@IMp?P1GRVG zK1t}zUB7d+;r&$C@Uu48NvgwoMynr=7ruDzbM~y~35MsB523P|bDG93FYxuQpI?g~ z1*7#c<5kLAlsKt@rNQ>=>h#UcR3pPYndh%5pFX{M(9e&iWsT76r-~LK^{k3*v{G4T zl;X1VnlH*#(9wPg-yD#2>lr;Q*S0~1GyN>OmwFdG2lLAv)K8DN#8g8KZ8ZEXy8U%& zRKYY8=r9n^@4nuxf}JskFs$Mxa15z3A>;Gj-Kyz7Tmbs2bAhiKK7Tfl4tdLbXyRu% z)l}BR9olyTwA9o(=*bD!V-MYE^n#FTx_1$-Zy!Nejae?@u?03ge&0mEml2Np zgQ%cd%@Q}Qeb|g#&3CQv=m*v6+ibkFCrK^0^zY8$Y3H?Q$vkP9HsL&q#PWy*-0HF6 zY+ybNqFyhyZUO7z@2~Y|r-o{)3JMDa(1H}OZtS=QcZ#XQn^{?TwsN=gnmiSoQ~pXP-YxL#5_btdY%_*itjz$vv>uiXvD%RS$=^}pUXMF(RC+3CEj0SIW?rd^2d*M= zIUM7_2e0?7)4xA+C&K01JS{ITudFLZ^!h#9=RK}VIoa8a@W%b%Ol9oul^Vb4k+5xI z-SwYf(97a5)}oHByr4t-_Ro7Q!Sg&YL8a{D=YOzu|F*jkw{8Wx&k>HNEf0}uemEl~ zwe~P7uKYLD)b5w3W)Jf^SwMbjBQ3v(4z@pRG?*nUza8`LGY2_5O56ZyVgZTfUB_=! zK-C=-a2Q!Te+#&VW*V=)NZD8KG}gV&D7XNXQ?_Ft-VR^HO3DD-z%xIs6FPXGD89u} z?=X(TLB2(3juZWH6;##gmaeX|6T13nif4>mPlUL$7bmJZT#p!D%CQKdfB7C5)BSTE zZYM(IX0_d|gU%^9PbdG{w?&z&bxmtsUUf-`OySz&IF{_i#y^%njI^_83V(O+ z9*bFzxkwYJbuv6at~VwD~Bd`5>gwjf_yo&@~v*RmmmU&C~_&` zxM|i_;O3T-g@<)Or3j+|eKc-DU$7&2wk9DXY5`+z*mQmV84+P;D#+))op8w9|B$6{BNg$}_J+V;sKW zgkYllH&9|xF+yDw%F!Zs6k&n6SQLyuB$2&k9B%g75iL-?WM!e@X7g3>#)R<^BBQ4H z`T=%0>gsQyu8#vMxZgM_i~(xzm}ixzEoUxB;`|y z*(0kpkL^s`A<55_h;tH)`4+hrg%~^pVTm|;*Ci~AtBK{G9m>>No!|3S`gTACn3+W+ zcFg;uPpmG5|Bc+wfglj#{Xug`%21k^IpD;Du6y?v?Xknxuxm~nF2s@GNN9D}y8u_r z!Ts-`XEgMlOh`<$36)l~>ryoNu)RE3yUf1g&K7fQOZMch(Bu3XcOOke%hFt{W|rTAv^XBc6%0_bsj;$ib(@b*OmIEp@Cpui5VPfr zQ-&x0mgLg&IV3;C@qAq2YM~(|b}OOX$GtPv{f|m~@c8BW^vI+0-M~iwzA4(cc+HrW z+Pk&b#81n%8gG5Yo!4bp(r8 z(4_Hny84+<4k<{h*o2ILH5BLzSHLO%!hjQ~MP4^=&Z;b~s#G}(3`+awB1-4U-J3JN z0!#w)BBRU^hR@g~Cfs8m;W2^<`^3r}WGHY!N-e3@thuQhuF&v8flERn_OMnr(o7ZM zhM@eJBVyaOWknzftf0}y`7nSub`bjaL*bI3ezO&CWz*+4K??@COX=qe5=RSpEqf91 zpK$nnaoug*V=B2k>X{VGK$^f-%t&I8(jI!0CEadm%eHO%Y`@3PJpym`2+FV$ zu>(4YD{j`;-&3|wt9pP*ShfJV?uAQz`Uw(u1<(G9tkx8BJ>W?qt%tgB5=2L%rcTnFyd4 zjc)#%m9350PD!2(siqG-n}6ZsteTgxC$MWblWE%dGV$lSDsSmKt21|5=Qr!B?@&+H z3bXdvm^u5CYVWo0b)miwAC4N%+8z7TDtB~$a9qrDhs>i#OItcJ*g3gl4=L0$=&CCU zhECd7mRNhkS2Ix6N~wS_^m;NBDMcep?1zL?)zZQ7d6fk${hB*Yu54m3kg+b8F< zr=G;cu|uy;4w)z!KX8e_xz{^ERrK?v{YsY+myF}K)$sS@pmHIbA@cPzY7edb>nS*% zc!P7qnNB3~IM}N=eG9@2S@Q&@$R$8GQT9A>g^z#PGg9MELWdj$pc z>7Lmv!WM_)0Nb{^qfvDeI&3wk#i0K`X>7?_x9WYnuGD}RDPOYwX7fXI8D z$fnx~k~!sSv)&pC=wO1zA1MgmIRPITT(7IY53()n1nC$ufFC$F`~@AE=(u5zL^6K` zvOMP9`f--P)q$euV(Tq3u;Rvz8)a|bQp3E)N% zuP7tE@0q7Y(={HoJ8E3OF;H~}dBuIHJ<+H7hslkphIs{I-UFa*CUiBXTs1AYjions zGf^f6Teku-gSk!_E}F36_gz{TcBkn=Oj631wi@use(a3Hc%6eQ*U%_QrG{GSs7Gh9 z;^(%$-xEG~Vx`1iMITyq}B@gmmTkdTlXw_Ejn z`lvL68w|!iT#`vC7RAjBX*X}Mue{+~51Jik4r{)r=PJTq0u6Jgh#fUDBrZ5bzvLalYs?Rcyw5 z;3e=GXaJ8jJl7JP$M9m0!wt=d*P*AUCu_Uq>({SamyTMrtp-$22uDL-x1Sh2P=;Zl z?0UG{4{ne80H}`Lm^k44KrRF~(hz=9?GY+-c@moS?gM(dx^Kio{nHc~Vk0AIA3Lg^ zKC@RdWv%s#x^mFQm zf~I{IuXr0}^Z@PT-W%xezhsHT+S%R2wyvls=iL3ES(-GiB7U^$kae3^jfFgrz~XxG=5#GP>=UcF&nvSG24Y zo#Gfv7k1c;_sV?`ekje{5e=nzR$P{=i%Uy9(Vrr3LiLLRHK%*9{cbN+Cs|!zFYbUg zcC}LaJkIcY`vTR8iK#;xF@;8()_?n^ zGb_<^y*4O2S=Z8oeQTS}5pZExP|9C)nViqlOj2-VKIJO>Z@ z%jHV}Z`wY};p+!Ko14{LH+31|bCb2M8QzxNJv}Q96ze1&NVdnPCy{cv<1ytbMbrLB zhz?Tw{ELYaMeFgX>h8I3UVoFDQq;~59g~sV!^xQ}7OBdH4ZL0)kOz(~WfWIoN@d@fD`oaLp-}wh5@!oyt&<3DY#E^#a z19#oQ;3v7JpM&NWOG0Lk&jW}vRVd&_&qz+U>@&{U>8o!g*5vT2B?Uf|k4gDdZF1R2 z*S9sBvKwc6S4~C5(g-e1t+5Zy&=xdg=}S-9Ch*vA&1riYFnwlrF2(%!HItr(>9fXh zo491!iwgu@FOu(DokI5~!N~#uBC6{x^g=1fQ1nFhrnD%xo`|!Ff$J{ZJC60qvRb%Y z@2!swL?$IX`7jVW!RJFCii*;duCX0A6ynPP$R>lP;2kuUpt8K({GeFeysnr(xt7`h{^d2*+g&^fuwMdqEdXdn3%F9*C6`0;;c}JL3=KA zHd9$oyB~?Ad>F=|4SK-?y|IaoTQcGNU4hoM5G!}aivb3aj`e zplD;95{FsNn4?&M>xS5RXg%%qeYrIpm7^ZC05k_L&6c0uAS%HwLdQ`hV|VD z=^3>>*FLOiGRq^^ru3+JU-HEuv0%%%SL%2&UuL*?G31nIviYW6Mf#BO;4G{M!9z>7 z0j*KZ9}IXG5Io|Nid(@b9A0!FqGoWjTsPCG_~qti<;f7u@(`)~Ag(gWj+FzinsqBA z*cBBg_>~F|*lb}^KjM*7FdMgj{UgEKL$_ue^LBe2;;#81^(uCkDHWAx##K3CWd{)# z)?W8d)E~Z7QU4v_p}EWG&=Gxf$FJ8jG!Fje-utFfhW$9ajHRl-?X+iqu4;E!{JcnS zZ|sC)<3RrRVpnEVfy~TIovkTwoQT>Rh#H2;U=70(^w-L3YWyC$s0FtWDWqsy-S_XT zux#)oiVp|Z-wmf?Mdf(^oKF@oOOAG;VS;aCDvg{tPij0+n$c{}~(AC&I)Yntmi zSLRYm37=dHRS9*M*yziBF|;81^zl<-X1evpo5FwF4*NQX&8%3Jcc!!M^>v0O0m&2| z*4Ry2=g-gX@7oo)WU{2$1aTOAGgi>=p9C;-;QGRabuyij0_mBB19|6y%qc(m`jP}s zQs@=N)zy+8iKwv3J*53DAISMS@d-{{Qq|AagF(<*9kf@?WRYI&Vq&v5$3i{^N6!PA_tvhD^EwnvWGr zuIm~l+XHb;gb6RG2hy^#>y6}Q5e%g)=PXand7QAuQeFkA(2UpU4lzk`DR&gQHe^^z z$hUs-Pg1ALT@B)pW9UI#^M0>bu{l`%BG=FfK7pkfX>JRSl-yK(Z;5@mp87(6CKR0Z z@K+5N-F#e|P;wU9oLJHRt6o>>wcfaYY7diDMSsTk@M}F>J0itKCnmg#i|3PD5}MDE zPHJl+W{TVu$%H+}C$%nLepJ)JOlCd|sRQn@$=u?82yyRoUK$Dx{2zZYhTNPufE(#q zhjd*zh_=hCs&2Q3`unMnyh^l2^zwv`m*8d&QB`S1$zn}tw6N`MEscV-rWt?iqjFEX z&Cm0$mcSn)Sg}~nE)UBh_F(&wkhO~#QCcQfJFB89-Nqd zc6aZcW<(3m=If;JobsG9`pq8-<9@qPsm(NgFo3mdYcP62x|vT8Andc5k(A? zVj*z8-Gr7`Y^5^!8>y9`=GTTIs!mnsF7?b9G3HS2;fMTbW@X)E_}$rUlfnl3W1>qN zs-=Zee6C$xJvTW}%5D8XME4|xCjQTly*;V-0{9oBnl`e^hep}lUJ)9VA6V($;hBGS z;i?Fmm+CeSX&+9Fhd3py#BDo-In|hc2jz$i?XwtDvL{8+-c}1hiz#wv&HMK~dwaSe z&&aH*rrx;mg40hMz*5iv>(kgU}qr@}io>Z6K_Wr+>H%i$$#paCKxf zO(Aqt3F0NVQd#e#e{eSbg0c$B#pjwz!oR11+7n-zyz_3=fZ*<+>%HoLy#L_WJcS-> z&s(=zs{OvW@HnMm>Oc*2^5-~=iQwW!P06?Gr!(r-113w)8V88ur;`(pAmE%~qZ{uIAXmaW6`4WQ? z8gg{&0=~j>I}omP+)-yvsetljxN8Yr_CbDnr@5&^I9PH_&-j6CbTN+6Y2nF)_0(W% z+}PT5iKY3i@R6jva4z0EDRbxC`pg6Aqy_o6L*N_<0CXCQZ~Ti(H}N2DztHFUH#pw? zN(u1Bl^T+7!vf{aUAlfR$bI6+BD zJFuT>)a{c}e8PkEr-f+QS4;(rR(OO88B7foj;(w+)p6p=&EKNe*0IG4m;kI0{XK@h zFVWVci#qVd8}~EWSKUv4z$(JJ`|Xwp8>grQLmNehW_L?V9)cpZ(mwNwm;SDnYdC_tKH&bWucUX1d+K?$W?1WM=m``h2$Y}aax z?ducN4^|N)frwpPssBpJPK@@o*VikUth_cm(a)5z>$xw=RbAv!g`bO2Q9I#Lf47|# zBtNvYU)T%+$jM^w7#|-;=6w=hs4gQ4vbUT-VSfH&z>i#kS%_aqIGQ_%CGnaKLQ`GP zbp-f{rqo&g1gY3no z;^)rl<|oc0A)!4a^drIj%K3V~ljPEc5Cl~x|2uQl)B#o;YHEi*T~=?jT$hyg_!&jw z_QIXImG2Xm4i7hPH*}q|sTtcPRyt?!NZXhrVgJV-{Yj5RGzdlMdL~{^kIWpRdDj}e zdcT({3(iRb>w!Yuy~d<0sHavqk7Y~o+p5v{DK8XvY2K+(+*C5b_>0&QPV9kUwhpev zDt7j`##h1T^x@h#*oCu5Cq(H2{NNiBWjgxN#|G3=PVL^TgXPnbsq0y{W*jDgE|Z@x z4&GtKi3eWHujwd*pEu*t!F#2w< z^2J4-JagDKQz?oGW!iNrKShxv0!*yk1ilC35b>ig9Mt}&G2w1i?3y!xIv1g`1d**q z^M^d*9zc~4O$n~<;*_1Ao0-WvY6!rM=(xcY36=c@E(cOp%PrCAy2}!ha>q{F7N&ahK|>fNagL~(fq+0c(~HDh-nk}&pod#h zy(P9aCY-BO9mgf%djf#O6VgK4M4phppd__2vti%hv+b3QFsX)tnh!21UFbHcb!wjL zptc$7mcjQ??%n5!y9H=<+uY|zsYGWp#bCNYWRKtjJm_BqUqeq+z()(N6{V#V$cCHR zvSwVwVYcwbl8TC|i_~tJG!PJUfoTi|7*jF<4uAM%?1aaYot-`M`J%S9;kx@oliIg0 zhF`zxF&?LYHT*@Q@&RrT_jz>LI69nUZZ*RUe+7qSFArHtnSo2Q1H9?&9*F0q=y!E8 zx1bZ}g$<$$NODv-DkAaz&bc=OPYMs%3792pr~xop3ZH&*&;;ngP!)aLIdptS$f7H5 z42#C$aN(@h!Vf8aljkkCM=tF5I>RfRfSr&(66T4fDt2jM%SAw`@49v!_#EIA-MBMcBq zCWu#r9LlR*tXf0@fZu>%3px<>`t_%A$|XEg;6guf-5+#LN=?0s4Jun@=fyMND3K!H zvKPv10eC9l)a!%&hqzmypIZv@;1)2T(5Hm7Q`Qnb5JQsqa|D$g)MEXQk54|+$m2nO zi)}Nt6^LoejuD#R>Oodqj?;Z24>-V}T9ALm{K1OyhNBfiB1ixwO> z5XmQnPL*hk1DJ#);Zh3x{Me^3_(_Bv&duBd-lKp@HP`Pjh^xs~;vgdT*m2>4+V*a< zre`-vL=3rz;X#lW!dxrI%?5N|;U{5_())TaZeCFb8}y z!p#BfdJ_)+$f&@9pSGf^Dmzx+iM@K9F>5KOF3)dI^%(MIzX=x{1R*}2)hKB zEn+ru8eolM4+WsHi|t=G zfN=+in4DIGZvv>f0{8ER6g6(RMl&I7;nhP+;u`u2a-ua_MFg?}qaYkM z|D+-;YPv(7j<;xK=EeDa3vv!7uUQUl7*_Zt&dIA$Biv79Qg`P>{Gvj#D-zzyDBDmwY43w{x?O z((b9 z9CglMA1|+FmbsG9`?TW=^&n#dO}gpD_Pb49_j!F@De3r0-HwQiB#n!6%?`nR@F*fw zxVcvVc53(Xf*Ilke}f-5Oq%As7HItZxsp6<@Cc{&Xkkz9ivmi61H+~jD^^4a)ix0W z4Y+H;IcWnn1NbJRfelhVIKaNxK|_ZsRmo|*6898-P(Co5Rvn1BFoV;SO&3R;#kuU< z+)5b!@_`}fFwuu#4ni7>Xc4jho%xR02|cNnYPee<`d!M-?!`BC5OZ;o4%Cvk#7w#y zpbP4u;DB9rzeqd*tTXW8JwXQNN>2sgM5YcnEPhzXfa(C)v2k&o+xHUNC^1(}OHWsO zsdS~}=>fzTjedmZ;D@t<_Ol!m7|vItK=Z>t)?uL;Q2T*-7hntCa4SzyD|zYq&Xp7H zR#<-xFx))|O#)7Bgfxbh2dX1IG)YAcHq#dEg$#t+A}zEZ4Jlb(%^X8Hvd^0hT)6N8 zFMkT&2cY6mmGc2}403ueWZG1UE>ugu#){1In~d*b1>V~!d<$p2-MAp4IEMH(G&fgc zFR8Wo-GU<%Y0;=7_XHi6o)}sgQE<`t&^gT=KovR)PkH&MV>fZ#1X;1B0`Xv-P-Ccy zOlDyDlTdnWb(v;`+_MFia1wwL%4Ua_2?7ytq|t-rKZG~nM?c)*i_JdGx{QIcb`$G* zlufb^!^c<}yEzv67A7XHt*~q-+3W{izf*{Wzds1-50*Lv6mUh2vM)P7HZ}&EYO4^& zEedB1Bvoj(N~VZOsoHpPlZk%hcrojR>z6msK3S9G;#zm3;#F9*)Y%=n(IS}|71098 z2aayPHb1YZs`rRnd9f{DwLHt$Lid_wDJSV_zP)E0EZM>$v+pcaiDJmDgO2xy%5p)M2-#!b(*;2QZjE^ZYVQ-s3{%{)xQbsr{=!AzrberEib{cn0OfRvKtSr`$D z1sWpG2)I`sA}a`8Qf}FoD@(x?CZ^#BFJ-R=DOeJJk`Evry9u8GZ5=9@8kB;seh-ID z=p-%-^u?nw1g`iI63Y!?!Bap!2>r0Ku_*(8ja<<~-d*s;zQfMLvk_T5u*ll}PwY-> zbLYUoUF@q}Al{jBXsjs3R*P<*VIDf`q~dyyCiI8x-O0jY{pXu75I!B;BLL5FV7vI& z9F+vuV?!93p037W2vJ6XSSzVoB}bIBX$Yc&!e4>U^f5_c7wlW!AoU1L1)!B4fUJwr zDK-E4asz%R*!k27#i%+|0{SUzGoUMyZ-F=tAeAf_J0Hwdl+p{jWiJIFs!=a;ID{?W zCq6y?PdrER+eEYx%By`3#}KkpU{Vf_cLy}_reck-gOiIxocBvn!8O4$~Mt}qg>zUhlEwXU$`1zF)Ga6&# zXCeA2QI_|s;R0Px4NnV?WI{9k= z{Cc@;g9XXYA3rFM9e<7kn!R9kmn-aV@~8V8xl;k3yWeHT%%58G^3Ht)_D;Fg`=TN# zYj?RfePwL4Oh*^-F~#a_>KdIp=;@{_GYW;fWfv` z5|ie4vRI;COi}G;3z5UE0!0fI>weMq^d1NLQ8bxJs5-Wki9hWc{p0;Nj?Bq_t_l(Be%kYo> zMK2ZfuIw??ezd{*fqGnF%F@JEFOEd6+G97??{JSjU@Pj|Yc4xsZLG;`XQ{)UGaTmL znR%d_wc?DO0?R+@=6D+zK~&*N(Q0A9J_VWZeZSL0=}br;;mYA(tT?pqkIGfY1!ytL zU$T=Fd|7ST{EW5u=8D0iX;c{%!NS9c?t0_SuKqU3U~f}pM!AOGGowQhuFiY3_2$1b zzddvycwF{B;>?|Z%zZ!)h=___w7N_5j59=Um^^k>Iv=>Ti?H2a5%mV^HRkIJ z*xB;hzu*4lWgPIEPDHfaa&(XUB_;Wd-|NEHB|d(fCdZNRIJLaGQCyBMNqSX;YhUO0 z^1kK+dkuOno~Zp-f_gd&JCgppAP!))@MGua--32YIf!~K*{A{TDZNC`&$KMGtOztHgO0=m78V}+h9uuSIKg0A{ z&2>>>^D$xT8s+Oll2es;4J`L_zP=Xux%4|fE8pQC-&My%gf^{uDfOc_^#zUTf7I-6 z;ir$f=Y#S={I6ROiS=Bw;MT1rLnU&iLaPBi`VTagoEVv>JC{F4%wTXCIF>FEF3fXJ zU%NWXU#j>@J@_iMb=PwjcJ`JGtyM86N;%y{jd)HEZ>{|%uY4os!)GgVJ&mRpd_RJd zdi?mh^NckeM~ZJ<{*N?x*Ua1;5pMoPTFTG$p9s0)7u? z*%(%xTDymTSYJ=oC&@Fq{rXdH3)=U;+e&r~haKz*-tp2h;POe)(oYRaBL~MgF?sas zV$ALY)c*(`7;#enUeDlpSy)xE7I+WHVw-(WliVeokGTtl|JBSY_n%#SPUm}?aN8-c z`=Y`%zzci#jp;w>6Ek+lBVBxdec6o_wf`_866yD8^PP7+pY|(%rOWqf?G+XNjEg@P z&(1w=4bIti>UL?q*>?F$ma6p1@n2fm7|PpD-}y1D5vPCLOuiuh&YQHjtQNZ2)pqs& zuHNO3(020Q1#8o6)w9l+%wgM%Klgm!@3X0)v)#(SRO1rx#kObE@mZD~r`^!tu^ySo zBka>v8`58d1jjs3+<4J7f>$>94NxSq$JEoOXPs{upA-cSbYW#>t#J_myx;N9$ zJIX(Oc&M_Cb=SSu)6=G^f%gi82UlHO)i}_9r_%B8r(K;Bb{2vm!8c|{{3$z3D(=JtWnu%IequH?Mw@K z8I;ZZ_00}u*Kd!L%DaN4#m~R_F{E_1ZeH=^p?_>^mR~yF=AW{`EnLPx-sE$l6thdm zgtL6>*Qh*o#w^9ShGa1omL~PrOd{}NAFE*C9{u@~ojmmRy!WO$VHs5Zt3{&5^1Wmy zzB1i3l4&LVwtwpu-hqpExtAmsA*4CEnxTnKQe0tcq}*xR$Zb*C;~5{iew2GMeU39S zTUFFkd(1%gyYWi!`;D2z4WE1xUd;dBj?@!d3EKPo<4IZ@x%<$7%JpQn!}rMhu>W}4Vw0227vXx7=SR{w0xu-uGCh7kG9pRk0F zxi5Qz_dnlQ;p4L@xADT$<1XTxBwX96bC#M<7&2(23G1Lkhz}DsHVZ+D2%6$a#9vXL? z=G}g9xQ~W4P=nQG?oidx>wB+OT2~7G_mi)uU*33_*p0ln6#}L5>7LTyYJ2Mlt4HC)uqk~~+K+cCd!Jv{>U9Vj``M*L8|ixago00`{RZc7ub(Fo zJNEE(g!g}adR~U**MIbF%kslF<=+ynV^AIP@n?Ug!5P^ty_%0aQ5ph%XUF`Z(k;Io zmHlEB)qMu0l#rKnLvw}6EVrZz%kc&(ZVLuX#dAV5zLx|g|Lf^&_E?@c@`hgAv72J< z-J92uyyI{9x7%@cHfqd^%uY@j^nu%&vtop% z0w%p8Zn7qc@vv$EFVLn5bjcD{&Q*Mo;ixX*%xUM9zsmYzs_gZES36 zdj5uxz?XlO|HR`Dv{SIj%W-|RhEnaE(>&;i*x#SxN|>3G;|Y5L+};|XV<6rFdxUeG zl9iFu0UJxAQ?s(OLyub=Rtd!L7XR5EVc|=0vqU;cPBI9FC89cfSRBdbAZVqYX&hlJ zER%U_#49dNPFWNWP{8A$S!DS3w1$1USMJrsjxx*f0Dr}$s!1C2Eo>p5=Axz!7r775+LPdHz(-A7Xj5x216XTa5> zyUN%U5biOz`C~S=YW1vo2z7)8DGqw>%fd$}u!W`uWV)6@KppTf8!xx_3?cUF4d{;{ z`*wp4NNiAXep2=8ip9`?&@L4e6yX6!;+p`G(Kb>@7uf}?LGb&5C7#zMN+%F-y#a)A zrE{~ft$6yp3ya}yznLno}Y_5SZykw(SNpJ{DEpi6rr$H&~PpQtNW0_KwxP*;@2P1O!pG_14u_*i6GQt8HX>QmQG zcQxuiVp{gJ4fzU3{6O63vwio#<$OKzsz5uw3E11Cb8jPHO@8zv(^;k=4UW2+O45X3 z5P>=R2FL{o-@Wq&(}@@dgVpz`ZIn9|9-GLFinT}f4Rjr|-(g!$q z;9x=bM`>!1~|{khnEg?AGW#gPvAfVde>34auaQ zkVp``ZfS9jmjF0G0027DAPwLoVOjje1(38lFN?Yv*e=dH9lSI)dRq~0-me@-R{|L* z!%VS4-;Fd4fI^`m8G#OzifX%&^*qQq>siIZFM2L7V%P(jCp2Mcj>lMiGM2zj6bw= zvG+*On1SIlpV#_gACqH*k~H@k^39xBzfs#=>)*o!XY~BM%LlaDi_Bl*g3*^xhnZRheo<3E%DybO`@#`ioD6ok z!_JVbYjpDVS5;S2A)H)4?|hDYf0EcD0t8!6C!r0Az&9NOgGyf$7>1H|sDsD=8Y4f@ z*Parr>(h?K8sYRUR{bNB!x)Uqu>xxSFH`FAJDwisxGOklb-Y76uL1gz4{lW!00OnKt`e8Ulzha zg;^U2_$ey<<Eu1&|S71FvXlj@uT$P4`&`XG}SoK;G9RRuBd} zA+AjO)N7Y^ebSKc z3+CwM&{OK(eq?e|x5Sj;ddl$AJ+GpLnBk>qAruz86^AV5V%u^>g;vqlRf~l_ z>AElW@v_BFphcC1gdDQ+@ilH|W%0v41rcXAF@Fi$5(exsH8*hO&w}iSi0DCM2Es|O zIYiH5H?&5(FZb)jk2kA8YM=#xdK;EDm@A+r69X3jka}ndlWRHJ_BCUTq#*|F6#i&> zt`&I*z>^s1H+FWSx4DX#0HfPOtk?m!3yD-=x5OqxJcraTUCQeF0zxzqM-?2=1~}Yu zL>$su-6&YHGgo(wk_GJzX#bu;7uIVq>k`m^bvR{+*!}XrmQNDY;p_mCRJgG1KIC7= z&#w+P{)!2=f`TYr(sCsHAZKT1D91svNEXU9l);)dUqHXmdpO;2H(>y2*(7^ zWi>VyY?BZKP(#8Bwf@hh^bL@X@1KM4{0+@ktrMe7C3sMJdGLOJp6l=|4s~*{3rv*w z#Kh1-o_BNENLzeqjxmWEPVbH-qPZI{TANc&%`W^#nkk@a`YN=cTm$xL=SVn@1H#Su8YdN?eAx(f8;$1mj~#=}93Ar;9mz#1xD<$o zGg49vNvJ#VC%3cZ3YjQT;F}&esMHY|AIZ@*%{2ko zX>enKL^=)F*x6mhIVT+cR-DHTA(e^%5}lH!v6qe?QLsugm(jnBfWV16%?Mox^Fi!f z3{}D+A~wI<|39+cJ1*zH{U1J&mB=a$Ny=zxkp`8lXln00q=9H^pkbx7ixgTaZH2ay zv}vc(P|{XuY2DA4>-yfm@9(}pe_W5x!}U?;`F_8~aXgRbcw-qv_|UYcaZW0`?(QpNNRn z>=_JaU>?M_5DA7?<;O?~whk)?v{d%5CSDQy;)W zXJ)*sj@Zd)A)!0Vwo~jUnXApa=0?dE>dDuA`5!L8=_79u4t>Go^!~H&>Vr$XF0G9& zFRyoy{mK{1T0bS}_gC|p{aWpBIbJb|G2iUfn1z_ydUgdkF0IoEg_j5a%n((BZReYB zwlCda-j=nZfjlu5s;C~nL*H)nY_yv!)BG)e>bFxq^<_3N;v&TCxgbsZXgmDt7x4=x zj5rkWaL&3q+{{aA3ks&Zc!#Lx?V7cAQhO7+(&eTkZzy-Xe0+#w38-oqN0vjoaP`lx zi|^ffX<#q3`h*~ zWI0L?2XfRo?1F;uvQGvbhE$B0kyuAy8V-;i!25k{2X-W&<4u6o1Ys;6R+NCvw<8g` z_zy^>O;G$pz;O5OK;H0m*fVnaGxS`#a!xTcQ7hXVu_Lo zcuqAK2Pn63rQLh@@D$>sklM}XyXS=V=yiJ#9BEj@cD`I$_7j5BJ_?_Uc@8$%VaHwm zY=HX_QU^|hu#8RMiCK5zQI@oQ_?{S&8@|963#}0U zm_JgKC&D(QbbY|MzKzqa6P7nUJ#0G&u^~dxW+wXyK8y)#@{~KmiZc9hGmz`*>gCP{ zVukVcL-5saYNEsA9+?6OCn;S4K_}i_7M9>Y7h?)w7DWc2y_f3a#xFh;j%VL?1zO85E26Z2_5BG&V2+ZCqKp3o6mJKhzhQgz@hkm)NjacRb<)7=j*9cX`u}brueh z6+hkpwg1zAfDfQ!dZrZNWTrNWlT?IP89Xd~Q5ElCVd33z6HYQSLhEng&G?tJ9E10S zumTeGpbL-2O)5u&ONnJ2g?%Trn%`BFJBLjS4dDrbwqFR8z4tK>Nq#OSD^XJ{d=H^S zt94{Qc1)umJS{tsy7#D&BfdS~IJoT|OxLs3hd^~^(-|o7-qhscC%m97pYO2}rDC9~ z8>w9v^yd^M9Otu#L;01~bhbQ$w(3>&=uSRMSYV(jxNs0DR%5KkH4CiRbaJeXq8naa zWY3ObKZ&+DQTx(AKY*iTilbx1=BV=z!;*hyW&i1x%Nh|gN0l+-;gl4nCiYjBwC&OE zy}|bz8Upw|3e{T%_VruI-8}gE*T927Px>wTHy^RmcHNQDRTbFGetKj4y_5RttCoVM z{5k*LSh}LE&4lKl%vLd+e-+v^WfAQx05zmE%dLojzh1}t5t#|09CihHF9u*qlH=Jj)CfkH75qM%i{pokSa8?<)S(YHUyvGh_o}XO4Bb8 zVVA&7`3TL*=Z|LQ=EfMUjm!`_9%u?+Kq}}0;z5V78io)QDbO)6$Kh5nz#o2nesT>( z%Gmg2$28;q;TzQ6(hkAa7Z8qnL%%|k0`MlZZ3U$+BF|`~CY-@z7DUf~&@s3Jwn3s+ z9ETB&)jz%+tSJU@2Of|os#6UP;mleuQDhyafHU+W7AhgGkG&Fui--i0?LHF0S5ME2 z-|xWI1EJ6K9K9qCB9x!%ZGJc!@g?^iKdw8rfyAWPojrnTqAKMehzA&pnY?ubK7zGg z`4LA-4>Zoe!qlvu^?+ZtkB^U*LMXUIg9FO2;5CSu=73&&#R{Ml6UVWO?s04d<{5ya zpkz~(CeaUe@nQ!2&w9dj&^%aC_)_II{2w8l;;+ar~vWkOeokYTll# zLbt~(Ob>VxAIgaKgkhLL1KHl2HdAH4!l_ql*WBLXe%zJWIMKAm;O+N)Z>4-|PIc^U zY)UMq-(E7!cFybBau|0mV|3itv0!K&l^&LV3`rafG+xQCorIu8d!jJ_=Y!=a|kA^QR7q1rZLogHltYY|AJp>s=7^Zs+QW;XnMGk4uY((C0ayFE!U@}$rB ze2x*bJ?%%F-RAG8k8pNhkU7(^W6(%#R_)79*IfVPaL!L9-QJ%+*D0kR43pY>YdSM- ztM=6*UpeimO_TbcR9l_S_cJ#((--+!((zM}ZmpD9ArK|M$brji8MS7pw1p~Vd;yT-;B-0M)hv4Vxd z)6@b;q;2%;@hc^PG-7wX4izT8L~)Ozs;9|T>k$~-{ch+~h?)~UIH5lgc|g2kbnP-R zS4+7G`v`p-KL!Emo++#{vpRp^ToDoX0vRjNF{^z$7%di9-rCR*)WQ^Z{$itT0;TPSr7dJtA|{2ISonxe1v$CXC@a9QYR$|n=Fj)cWj`-`!rg^iz+fQ@j9c=Nz|hY z;dS$E=|-o*pFZW@Z2}^19?Q<1UZ4`hjb2coOBrF{hMi@p?ZCjm)OSU@C#GtGCTeB; zG=!i7r8;GQgDx2Jsz&j4X36Azarp02va61*GEIE-LrrT5jSO)Vu?eD;u({xuweK6T z=f3CqCuKidthz~TGYLuAn0S1Lww%AjuvlWgyB3&?AvUss5gv)hD2C+U^^3(%BEeIFxH+tc|s6N zf|f(b5QAo6D>b)65g{Ja4wUG5)laH5h1rZx8ph(dvhBt(c(@ciLUk(A*2@uVc|&rPr-OS!wXE|w=}{yoOe zZ@4DaweFOJO=Xf!lrX3?TlcZ9MG=dvWPX z+aC45>iO-Nk+W+uOG+NIPFZsG^clSUkZz&)OD1L8xuj!Z>*7U@x6$b@*#0;*lVD}n zIDEtR8>63Rkgk!DW=`U%jfoq?{_c}oj7!eFdy#kWmqyK_%k{TkPP%^Je*ZlZEpM6| z$8ZI;_R%ZfNjIih=qkEpx5zEv-2aLsVrdE^8$)=ffgM_gY&w1zr3v~QJmF$|^Y5Tk z`)jZIX<>D+Y1#$EK(kyOg%ykViMT&IKW45@&CGmSHty)W^FpT=XG&()J5|e-yzxR4 z&lET+l1wzfW<<4$m|`gu?VCr+KYe-$CkfEgdX3-PJeY`x*I*nx$}%4nQz_BXf1J86lP0T}*|~Tt&@T zrzA^`HSQ|&o#b0xcI1ZrMaO9_DM6K2>07T1@JVG*s}uwby?CU@vioWM#-HmtteX#x zOY|nQ7&$H6Zoa#3i2ch}sTQS&?xu?yj#RQn_M8&)R1td~LtXJ)0sYCp=U;UoJ2SHc za6mwK_&s!>AK-TihUzU?9qJ5#CpL#$3c88+FR{8`Zl3Bem@F zq?4UL)D1#iXY9E7u8ZUm+emq?V(2b@CN09lMpGWOTP2aDvpK~`xF3|h?^6RZ%Z{T5 zm9vgy3zuvhuT2;DY3}=x!@{Vqmj7eIBrO;626kb*tFX?{_v6UAq`=5rHL<<>6ajMO z*O?~7$-W{+80(&ATD{hdG{6ueyKs1Pp)eY z(N(mzh70xo1|;M>n(#WR?^-86j%9zZw>`Dl7d^^>rouKi?#9AGqADgW7^v8M@IxZesW>8#eJQ?LXp{`}~ulAOp z{?B4%{KpeeGrpEkmfo1v7i{w`Zq!7q@`^X-zvF!^j<>ZS`w!_`R!b$K6(zdAw`bu zJy~@UrD)y1fWx1&h{lvFo1GTf<`Bcxy8zA}$-%gQ_**&`W`5rVIm^tZd4Od^Ky*?S z9yjo0y*l)8%{?IW_k4X-bKSWqB%Zu^W54+F6B#=em5D#q4}gWm2;OB6lG;l^nznM+ z+Bz_i6&#{@0$XmuIX{y&-Iv3&J#ZlIIKhVB-UKoGg-IKL0`t148i^>V1To3renV@S zq;O2HsYTlo8=jbECy1pfEXn3D($jy^;?@dclcAg_a-IXheEoSg~U}(ME)N0{S?~4xv|Uj^M}tYJx=q5 zBP#cC*rcvnBT?gf46Q_@>3jfpTqC*sl%N8jS6xX$;7 zjpX#Xgl0eT`IL}$DcEchd5G8jZmI3-Z5SFXg-PRbSGPAdTmBX2FigXc7>!#{9x-@EA@n>}24 z;_S;k>Sx77gyVKR(Z1d??h+R8FhIO?LQ%OX?fkh77t^j3&im-?;71dUa@sB8nipEc%DEvGgjY6W=*xl;!!xrAP7X&4_EAg#&nr~aTc5M&9 zv5_A@RKt>lFle7yl1H{Mk&fqt!8V3*dyykBK)O^#PDWqAW@**aI%V*n+KiDTW&cUz ziJdW%%~$B2iaQ?8b08srYT7~BXdHR{0Ul?zlZahd;C19;UYj!cJhl}rdD|!?=@?!r z+jgP`eXWo$rK z;)`Jj`#A-Or@}whMa8!a9ADfTUARPLI7*InxoRtvK^<+|y@CZ3Io$2o<#zDx3Y3l+ z?PxBD6u_(*JZ>LiP3#1B>Myb5|1ixErGMKhw~H3`-he-t`A zJPc;G*oDhK+0ZF$BVuNW7|CRMMQ+XZ2g_=nygvYDSo?iB2oEON)5;- z8!|LDz5^IG%O5I#KQh{=VS*PI4g1!E*)8Wk{L36E|_Z+^GK36m2Yepl3WR@S#Pt+dJ8C7$*`9}% z(hH5Y)Ltd0sP*gCm7vGmoJotoZw#8cO;2bXsNnp%GWUZP7Bt+Q3H#Er&Dfl62sjJj zhZ5JXn>avfHy@W={OwsXZGxkvZVw-RHP^VuhdXOusU&QnbfbnRnnuh6MC?g0cTdj; z{UYoqlIZ^4#(kn}*RSRj=39YX8nDweOZ4sg55@8ExWM?%SmmML+VteS5@Y%Cf;w;JVM96C2p*Mo&(z@huRcKU1B+ zvw1!H`pDlRtMugczK`~mf9mb;W;5i7KiT*u<7DYn(MiZ83>B51#aZk!?D(#8&!uL` zxbC!~*Qp@AuP;+Go}4{z<8}Nb%L6O3XW*RiXYz@ zlPR1_k;UjO&4!^|@y%yn ziSj;covjv3@k>8^xLu?4HYbPN?S{6twfegtn7xN{%Wfw5m6eUwJQ8(?#|CQ)miw2j2N)O-Z@RYEO{+oF}ymuPWH?!Ed|~+pFq94U*cggjJY65RWUR&x{WKP z8+i4IyfE|&5V$!c+ULaq2Vue;f&mvr$!hu)tzf?YFp`vEUi+TIlnx}7`Ii^ijmJFf--i1>hlsG$6g zBnB@BF-Xf~?_y=GqZm^7p>`eGqwr|*@;Aq7_JsJ6x&`C~%|6~RulV=qq(*%Ia%g09 zbg8@g`)9$PcSWKH&nd`99NR>%UUKie+J(9b3m@yvMk50&{!T(D?2zwbu}){ycA=k;%RdU|d0Psf>Zio8?@CUBcPD zTMw~pqMSW-$}>XPh6(=p2B|Hq$e76?!CHL-141y;Rc?KYCTI#H#yFc@J9qM<>$mWd zeo-<55?BP*$;6&1>@5+jrY-~9PU5$;1XIc1V^%IXsvW8?8V~ZW8(InKO$C#n;o*vtD?wNf~aD1Qq`x?B& zQ-jWXoqa1ORW1&UgB`(ut?+&cqVkH4w0)`BW@@^+)pH9ALi++_j8?F)2~i;==b*D* z_mp$uujMj$GS?B%$~p@hwHkU(r?-$Jo5V$H;D-9?v9iFLHp9`yZ6BHD^|s@3Jaz#&ATBb(Os+k9;#&-1JX8`9ISy^qev9Y;iWmVeTOb-J`q6&qk%~ybF|e1?(P1X%;q~jKxFksA1e9W2HnKPl zG2%OkJwSBaU^EcA8wO+Q>gwdCB>n+70A2-8a2Dt%zf4rYe_Rov+yFYrVB@nu7-`?e zdjPCV2Wj6wm9C*7MP6PWRHWxHKVX2NDk=*ik|m+=`R%D&z$qgU9z7*%ot&JC!NS5X z!?yv`W(^vz2^cj;M0Bd9;Q5fecl0DZ@S(;sK{mz<{2{zUi*b>`EJ+K|D|VU1Wh(KN z!1S20Jl2yF4BFhUn4M#}ox*nm!UQd}6lh5B)42;G4S6@X#Zn%(pxpIFL9_Sx@!epz zK&N{LKFNSKy8HU5@RW%&0kP-(U6-5dhVF-?x|7k%%QKtsizh2jr`*F4yOC-&O8@kh zH@L`3ATt|CUT= z#~ro#wiAL>X(=g+#KMRW=pQF#WjB#W1AG!Ba8KgafuQa~RXQgL6!ImwB$ggin$_f& z`O>sm6<~DJi&@<{6xdZAhsY(YI?!68zHqv2Y^`(&GI1F8V|vr~|#zZCnsDXj4l9uv7f1qB6(E*KOjVnX;1c|e8pKba}T zs5F004h*j&>X%p&L zLEWQPEqb558aX!mG8-z~6^+b{jpX2=HAqz!f0=7KfJ#%(p=P;Bo%1ia^0D^)<;wj>zk&Pr z7;#eA0AwDh`R39#q5uXPzzA)UW|sNpTY=Aq!3wQxEC*i}xkEG*VsJ^s)<7*^VEjTF z3lgI!di}y`Vr30#p<7wLh%#Ha$BGHxS@OK}ViPd-%%0o@LD;EumQ4jB!KcoQ3i18j3ZXY1b;Z`MGv1I*Mtd zC4~otyg3W5cn5JF9=U!AnSRb^JxPi20b8YS1={VFSn+f(4z~^~s3tNnjc&CqX zc}&e?`@Vbso{dy#hE`S&;isV5zm*{2cqf88k#*mU;Fs1>l#icZ$GmuP8&thj6baY4 zTd09U7AorM)>9l<_G{tVHA1zNsF-wDnK4y681koFtwEg+Z-e*}MogB^&3XTQ5zhIu z(=Xn8?d!0`kDT1KpQY~j)5`PeZn|VeJu%E+;o4mCL?y~V)0cDoSZ`&_>7$SSNqi`T z`{m0Zuo_$?;);sN%Wo)Pjlt@@R3E!B*J1PlO4c=m3l0opJ)%mE`u|f1;MzO6N<-NM zR(%u-=(32l;(4o&s;YGqUpi#rnw{Nh%Eyl%Kc@r*1l&e%r{Jc1g5Mh-UB>>8mS-LR z^A8jSElp_#KMBz!W%0AoZtG&|BnN`c)~^3iS`kEUElFPa&DM;aTMTk?dV&Xb(U}Yx zQRkAy+eBDNcRW3>!CN10u4_R4D8|`UDscszUGQ6o)A;4x`~N*=wvdObQ)54E@8WC+ z8~GB~hLw3R*RaQPgZ9KJ(sN5Xqi{HAF;O^%ZTeJIG@~h>LgKWo_>TTA|0$h zF!33XP=5yk54c~G+*Hz?;19v`EXU_0<_7k(VdPx#dVJwbzX=!?9bvahYW{Fe5JXo!m-*YYNAq z$0Q>4X`jJbVEYfDLzPb2WMX47mp5O!pm-&RC+!gJ!#lfrx(C*?)pyT2(={i0=1Q*k zI*ltH%in2X!EW?Hx>0f~<@UNx1N8<=>Z9onruUg-7fh z5UFG6C8hEsi0K861duw3!Ai{7i+|s1>q`f+&g-#?+H63Qj+-u9)dEsN$O6*-FhL?Q zo1~aPwQ7RmE>aAw7*3&!5#!&ov$D7w+)s-R(rDa1fWc9sLT=;eov-reetZsMw{9y9 zzT3)Dz^u})(BJ8I(tpn@A!8AV$m3<9W?k<+-;|y+{5#!>6aOOZ3m;1I-RyOg|12J| z_wcwcdn-%GJPLmLw93}X(vmYJEhS|$<#R;^r-}#Y#)O1Ite_(5?s|J0fkNnnab(1| zgCTc0V+lH!H~y^X?*BkH#}zKxFgaDtZ`zyLuOR;Pjr)m7g95Jct>=%vn7H!A-b={Q zVbc@+@c%px^2=ZCAZJ$WeZDX{TgPx;c+r1m}y3L(I=jL@#!LKC=U&+&U0;jl?c zvZ9edrNj+=5;4gIu-py40oN>yGtzz!U~6){J%1s1%R6ObNw%g(squ%^9dtG#L!4-9 zHFxJS%%7=Dn=qDMPkkikn}b%9z4tCckTB=`-j01@Z_s8Jp*dv402 zeMSW}w_VCU! zoS#1VZzuidU2Ro?P@7RU$9@q3KE?Sh|GBp!6!&iDtdp~lvPZ`pV6@{I6mUle#`hc|R+rtSw zmYptr<>g^FePjPB(cdWe8-6GCzrP)CEcP<_0GgOet*0ykh&mKZopjy>$u)VM80Cfh zby}Qj_wUl%EOK>TSueX94)wR%M-PXAxu{3xbJmtSLawfa!R;W^O#?S}P`E86LB z{klpbt^ED&$d}I#Q+CS8=_Q{(Suk|+vY62=L2Q+=NV#X-ou8hn&m- zsC^}5$C1E;yV6y}J-@?*qiNIse)acqf%`XV;v_MD3%1I^ECpaBq=KgqqzcJ~YC|#D z;3E!Kucnve2Hw`ISi1LDMiE&aUt$4XH+9CU2Yu;yEtGX> zcSJ%ji;@5L#)*>-Cxyoyyw(-vHOe?%70KA%3YvHIe{;PW8T4xkezznS zZJm~Qz$-r4x51@leDf~mUEfDFYge{Ku(2JIlQFCcgm2l&x2ap(zN!BTzyJ5XXDS8v zgH?|1?i-!+_Uqpr%T{gq{jW;<_dhbcr!AdgKTZAqEd^=_hF6>M2?3M60MtyHDfAhS z;6-%=)RgMcO)zk5O<%Qb+ct7h6B8};FeHBwEejZa6x)jGTOJ|CwMP z^;SA{FS^s+Cr9b2%9vvVt=3wt#dZNiq-UB%WZ#)}3bmsC}+|#cOiuM3U)vzpr+-`C?vlF2#_{C<7doJ#XSfx>Ub${&HYy1wy zuV=E-4;XLOnK>}Exw%toeNnA@e!IZ0W;$5HvMn`Q=I{)gGvHp@7?f6+^hE$AhJL)2 zyOBVl=+*TLpBztzefl}XwC6vy3i%9)XAYHO`es!@{q7D9%tjHDCy#Dt(arjV1=T6^ z{(SnBE8FC#hs2ST6jF(_JjZ#9N<>u=ck-8+j>MQL>W2cH-CJzSW0NdR|JM;F{BOWC z6R2OR#&HLDcAX}=nRvD@w~C%h-Z*M6+V6gB>2W~xhddT29;`!o`-AB$%-BA(_~T?M zJ930-&diCK##$vzB}zrfYa_T9wz^LM{r;cVdwJ8p@htCIuI>vB8+RyQZ)5yslscuW zpk?f^uen2?-Ga7_CfD)q#FrPacle?d8!f?WL95KX;V6~niW)P`rHY!XwUvA&bLd>? zR!+*Z|1W!o>{b7WBcXqSOI8QS$g@MS@0Au=D1Gp$LS$x_{-V$DL<^3a%O6}XX=}r1 zLnGzvNonc#6{G#0Nsvc0P> z|9;i1Z-=yR=k-ngJLVKG%KvAv%ZFf}p>O{HsReFGPYmWku(?Y*D~y*vS69Cdc4vh+ z6qm{m;?e}@jOdPl^^maJUtdtf_W*{#GvdO!I=s&m|@uq{=+{@<|z`QSr)wvcTilZ`kW zAGA_XM4YvKh9P1vRP2DwQyxoTD56*$b%Ghh{XR7kq=n*M96%d)tjF`db@~}k{@Bj} z%6OOP-cVqS-X^j(J-3L8A&G7SBl{q11|Ouf)^Ee>9NTW@BP-Bz-yN!=v>vPi>AVw3UTFe7D{4Oq|1O`op)-$R+SxYvS6; zaUc{miA%$_jTu(D>u*?^e&STDsVx7c_x5+MTYtybf0IS&dt#t#ya1)rI$e!p`wNei zIB<*sS2YgD&OmJgP zj&<91z&WMx|6B`6L(X?xfkY>$Z!(R9d-w}8TW4ov{Idr|xBKGELwf5144{o6iop#V z+_IChS3uyRQ9AB8(|f9mPY^$ zPafL2nnL_mF$7lA>^k}t2n7iwqueSeI03)8YnbF<L%p?Jn$L=d9{!5g8Rg!ig32u#*X`(;DB|Js*jHR%<&(4kcw`GL*M29y|W z)j7a#kQX{Zw6Zavq8;BY;GJrF$C@+wc)|_ zIQ9hoV{>K43hBN$zY2Fn5FfMqMi5H>ZQl(pbg|0f>YH{+U$U*`lwEWdrEZwBs0w^i zC8e`t;lPd4%z3aF^7^Ao=!(Z!?f%b-tcCWnZ4|}KDvUw}hud0P4MEZooaIgi34=>- z{Gv5lYT{D7FxTtx$6T{wwi9AvU<-j-ijd<7jgA2=<>KIeFZh=htW0(F3OQ*~lnR7NWi~@_%toh-+d&Gy7A_X|dsJNxQof&GbRsU*M91+1fZy{S5d9>bZ zYss#ED}|o@Sxhq$;((2DY3`m&cK z#h^|{mmuOewmYv2_OY^NTJ8wl#O5U~5yZ$3b0G5H0e?RQ<#uABin20s`$j`u47*_h zhQkDaxHcZye~cE^A(>0E7NQk|)jPI>nYk*j7T3q`rP(gxj0?ZTJD@8Pt_yI<{F}3W z8eQ4o#%v(;3s`!5_|L2W$y~%@l8~Aqn(XU4w8C-&KAm#FbMTJH4@pfuJTf{e0DD^^ z8r7>)K=SWu5^)E`#u+SaqyLO3PyxThnfa@$NOEy;5i#kblZ7`T2He5iCK!EG6}~;e z^5f+q_RT(YIFn|Ec@8;gG~fMn)2V z4RbPho6^Jlal~G0(oq#>jC1w^CU?ZP1zjeQ{{MFbH8ey`P++c>2frc6>IQja#AITe z%b6qE-_P$8Zf|aDo=TV&!;m%8zkeyU=hWmonZU)43(JRwl3%7~yF!rMi;`~S2jJU7 zuO*?4LzoohP=#Ot>Zt4ygXS)&$wp8XKm#{sMa(ZqOc7+-m*K(nJAe|2-yqM30(z7#}~Ed{YM4;MK8t(fz-9**90j2}?Q&>rd@ z=*4}PbXov5*`C1H&)Gxy1jNF&6R#j+8av{`3D{p-p4HKL+7Oi=jp-Uc1Rkn$On{im zn`c45A%xl+quGOehGmp{F&HrT9O0Hi1;Y`4Pvw=hf>+;(MAdcz*BYw>+1)WQB+5ep~a{o z=HJ9sJT0fRv@`-V2Tr%mDEQ0}YR8xL$jgf=Ckm)hMCB& zfMcc7Mu>~+fhDGI^%LUIrK7G4Zuk2$AkN31DqG$QBX<1&H@9q$NNimKcxqfHqL&sIoeDldAHxr7xLvz|u&_B9 zJ5#M)n_&h?hEl6JxEP4ntidAmHfcnBJZh^FtH?@V-kvrsU%v*lNRa0++lH;-?dzF_ zyCTy~5iI2DQJFovBqTH4)71L1L({O9vqp!8hlQY`zAziGSODXM-pef>7)D{}@S*uV zQg!QvF^a5hcD#N)NGkzf!ulDu>>(SWBS(U#^7bFo$X*#ZuRlAtte=$laU~6dF_SH< z64kpAmp(B)Z6<8pT8x2~!r<~BH{p4{mpAwzpxJ*Sj^FIvc4hI+)=rk+z3q4j>AX&~t=@b`Y!=FD8&V&vZZ+Q9g15Aza`d+{aA9mm5 zoF?LJbQ(qwofG$O90eGCufsu1s!p)-aG8Aq>zMz>FS!Q(>i^*aD8Y|gT1DPt(+aQ; z>zPPE!I;EQ13o8z4KBR+YiG5!nS5wj{UH8)j(YzxqlKC?n17(;JtWJz8g~x>sM|2= z06U%dmf&aZ1AP(-O2PK?a&lg9K*`&~PC*jd23)lu+!7ur83$2{!9BnX+YfSYOs_&N z4`Xuqt095*X<`CqRJhn)Xl0r6LsX8o5yyci^e5iKogv^<>?R|5AhS3KpQ9tiXH$pv zOi%}3Oq@bNH6@Ml`QGjKe~+{YIee>~ zeCIrMAe2W7+j@0)M8pfcNAjw{)Jvr42QQVMX3cFW3d~Z8K{ZHZ=oryAf#_}O-+;5@ zG3H+XGIO(fK7KEBdq_0=5CbAQkmSS|+Sok8RBRD9n~Wk~#6}C~H!nf_pQjMZg9Y&k z?Lu2QJcZB@tS7-Kc&NHX?kxDK3=*y)AURYFyH#Q|gnC<4D+k^!cSt4$(wUJsOh^N6 z3-p-9!pD0Z=F=otLw~f25PBT)aQx-ZSaubjz}nQmyplxb91c2KXW#7uTWW4BZ=7;9 z;WJ&>c_G2xS|^o&7Hu_|!Y(|V-f40C)EVm;cV~YMvp*yl$)R~GnldbiiOVX@;eC%n z>fZCVT>)7Xm;V9BTvO={7b9}n3%$1X|MGjDdH#Eze^4U*v1@_3cTaCf{7`cu|^6A^RCn@4rz90Wo zTkE&&{^@FTfh4j;VF_Dbi0FWpU}_feBlQ7|28v2TsM30CtCx5U-~H844H(j zKR0J(Wto6>c%ccSOLmWgRVj)={qipRx2vXr_$0><~IY|p2KP)MC+Vu zs;qt(ZhVfc)V_>4(C{vth<^OI7|LvrEFUqK-tE928A{;u+sA zi8zRk+|1NC96Nj=j86jtOW^4AFr^*B!S{?6I9hv6b5MW0ZJhXAS7$-k%Yz!OOJNpJ zPDIKzAeTVd9v4ys5klFh9NIZqxlZ-; z=tqBlJLfB?%L%CV%%#mlE?Ty;o+;I9QA2ZB=f~YG9odV4nL+6@wc7uoI{ekky;>=; zTiow8wPvz!<8FGNZz6*4cZ|pZc)gg`AaKotDszf1E#mBD`uKMYZH$9|*niyZ$#Aed zbIsAw;%%JJC(2$_ZZ$H`0O>vL=SUs{5w_KyPY+%p$;moNen4eN*Mj#{#i9Z-BB3i0 zH+#xeu+rkJUaltke^OfGh zmYCCl9-gD>ppDsV2H2o86FAv|{qi!w6f^8RDj5npbDD!A=tt=iXlL%egQS| z3Ml)vNXYzQH9irM@XqgVEO(&5H}x~vyY)F{6{H`i3b}TG7lToFjuH8u5?5X|2IlAq zErg^WK781pI-QR)Jl}cBr*j-tIQ>w#n;YT7mxG+*kP$A)dFz{2o9#AkRAi5jG>8%N>UV7heFixic}jwLT-^__DQNI z&I5ph8#Pf2nWkFtj_1ZcKoBt~hNW@mNK{Jn93;bFwV7$R$FRW{697 zWc<*X|1u)*=mJq92xdJ42RB~f*#qpHHHSpq9|-2n!&lPiZOZG{PNT$Q#5Ms&TFD-T z4<0@gwW7|;&3#E60_xwsc~ID#B;8q_Eclo2%G1nmA=X*1N7b9&%Ws&!^8D8tHUeJ0 zTcvUDSKv7r8{szzE{HOFz)V zM9%sfF^qJ)4{{qB9!};zbR1v+4FmtJ&GAGxj6F5b7?AIde)`d~)f#e>+6}-Ep)>n| zh5XKaDYpdYrz}*%E_CZxmMZE6ZpIfyiF z()_WmfUj!Ztr3c~%m^~@fNQ9!7L4WcfaQpJud<*pNV*>xW>nvDNCkwS81qps#2*XGZ+n%b_9t7NZb(8unEbq9FdQB9it+UkQ zE8{C%rjfBf(}kK!UsM23>NbDky%^*>cy9UZ$%{>9E8CRYMa9LlhI2@Ky4A@2`$C@p zmJoXsVM9<()wqOi*NNQ^B$O;v!)R5uKod{`ixmO4zc*$E{3VwShrj1JDq7w|(}&#Z z=z5oWD13LCM+^=P*$LXA(@RY+0I-_4?)}ap(_d z2EcskT61SmVOuYXVyD?LmRo^_eV7FW&xkuunmXvHt0&>WdLnE?{hVX(UjWXvQ@w6) zva`Rwziu((5`=;g1$>14s@3t}8LNF{0@-843y?0y z;?kviU=VP=R?*b_oO;C)mXUl#i^@iB0@vb?mhwm18>sDnzvs;#9_mS%Wrgi^yDvtN z7$U69On+eK{@_8VyA(~T{(o19*g(^C#{sWJN2Oc@OZWGkgz*CPkI z2XrK}%r!?2Zs;T~#2VrN#vQBzT2 zBy^_^2cnaD8Z)Nd=P3ix05Ph0Qj0F%$$h6%8E;dBBP+7uAP8` z$ojVu9j^}8%963Maf$8QOJDD3ERA?rSi@1&(DR$x`L6<7g#-j{qX&YY7S(f(Ux&Bc z2qE%x5rc2sLSU52h4Y^@dWI66v-*(wSy%nlQouY_Yj4rtocg(0e3VCN`)gIW39bg1 zhXmM-lx}pgKl6(W+@|To?J=%sC(+-a$dmCAbwX$AA=lb%*9rj2Qw&$=f?P-X`aW`R z`chN#sM8&bjI=a>SnP)WjI;&0brIc6bOTD`tRMAXzxK$T+>NFWC3B{K$Xeaf5{a5G z+gVs1;h^JYDT+li17S7!{vHsKjtYKC`39rtX|sv_e0YswGUCz8@!}j7f1ZV*-lC7$v$`(knSYF z6e+fGRh_uS`OzyKXFu5RM@qJ(hM{cJhJ=Ywn=-z2zNj!^wPIw?k zIFAL9w%jIaK)s%H+)BQG|9nKIqS}kf>?=4VNDSY&R5C-f6@RoT3RRb$?XM^DPl=igqgxW+2r1cVWsnqUp!=l=A1gdO+p7 z|GScLkNjs*p{)j`b6?Tj|BU?uqLeisdAvsKn4gc-8reLavh=Ca$0~D!lFE{uiyJN0 zMcs~)QT*^d`qD95nkh>aL=cQ1G9&-%UfOKOS%s6^ zbliu=tbRT4(iDDmCPnAU?;`1z;;kxzV|*%J%El>vj9P6H9H%1 ztrn)un>WKLb2FK1qLotuI~;$a-w-LQ7^6?`t-P`lqDsf8>$*T>1dt-M8Cv1`PH0sHpvUn^BHkK^~%%wA11J*#=;*@-0I<*N4h*A8(OLNao4ebwufn=`gc%V=Hv zgX~nR5U{MkQd?nk@No!la;I1C)GrGysBra^_qb9b-Qo?>&eP*?jtC1ln6a!dnbvOD z`j+oavf+ah$^|#B@|W4BZ3$18Ts`j{UHDs^VbA&#ylnI|m$Xe*@9o=h+h^S$-(ZVU zNP+arr-7D{ge`bc4P3j}w7^>4WwCDER;ryRJKlDL?R0E_TV{3W&Ad>(G;N3WE8pyd zgf3p3QV+Gs(YB4S(QeUsdnT(t^oiBj`|$^>U*5S@eAV4!?s{MP9@x1$onWQiL;LfG zXxBWy@A~jMSBb91Mpd;$=M_rF6kJQ~xrh7t>hu59%SMf9di+7nd6W7s%m$kQ_se!Te%;m5y;zr! ztHys-jEUIt{n zcZaAA2d1>G&S55aB6^{B!Cqoaa^(rJ2*5vyov-Fwkk@(ng~AWiIA_kDb(xWY{M0eg z;HJ8IXq+buHQ%mM)s4F%O`-G$M^woeJ>NIn^}49nxR)n`c4bb9n>OW5fzLLES4=w& z$jr~!2wu0lakWoKuybsCtyfX*NZ@4`E2+rU)V^EY7Y0~d`WI3pDye_dDkP|&>JPh- z^lG+2QIhVx$FOC>rdDU-15s3}8)v9xu1K3W2YbAZ@2pMCbg=CA$aups zKYQRcHgv>T%<)(H1-S!L4Qp4j0#?d?0&{9-=Z2eJ|9t+Na=?(eOd?mG<2CYP^_+yt z^ealz6VKf7Zd^t($}6K>1@Pi3R^GMhjsMxST$bXjy}WO5uy=59Nhi;K*qoD|-v0Ho zEiH?6{&!gPmJh5P1r>H;KfTWWE?v(l$z6DR7Q>PCepA!4A|7GxA5J`em<31KeRZ=w zXf+c75l0OhVYOxhu2A+kU~p(xueMI$qW>%^3+pA{nV+-6t{ooa#VoKrjkj#$p0{q{ zVR$>$QCk$>F!<-&)4LR@m)+FG(R8~xcgx)6RtgZIxx4+atkPql&)fZ1j(!)s{O$-_ ziZ<8KSRPMC^--%}v8UT&>6+APl7>UXza@6(r10@|Bssc&AA1^0kqjte_$s}Y z?tS>EgMory5eyWV5s?N35oruSK)PG$21#j51Ox=(Al)3gyA>3rySuwP&iSo<7@p^O zU;dbx&&;^l`(A5Z>soogD)DCF;VEsY6vb@4;ye5?2oygF2&khMEv(67DKNQ(<9CGI z3kwGZqMl+Vt;fYAl$1-myc`_)9fzdb3%zk*|G~5Tl2^ZrTS2|IInq7=MqibFv`)%l zS$;z&N6%1YMl@&?{hd3%teqe=pAh*qJUk61yIz|IZ?2}xVSJ$=xY@+B{8}6CO^z_I zn6%cL=DHQ(I3UTm!;xpw-L$?-<#rGQ9Up-uLoi=Ex6AwBK|u>P9Ya>eVm4ecD0jg8 zTEn>`!su+9(-;i!Cpx;76A~4(I-B3>zG)e9ZHDKr--EzjrKZBu5z9`6% z4s>be?uxv+TBtAGl3Xbcg`PCvrx9uadVx3zac~%2FsUWZi-js#U&&@e*tyJ-Ifl$h zsItVGWE2aB;oQ8WpFjT$iMk|cJ91j&Zwg|~$lrxNsm66@u2*C4q2XUa4#iBXXiLxz}HtDakvlQI)fb#U}FYf zTr_xGhc3Tm3C_WZU0zz+T1A<5Uu1nnN5(Le{gg`r2(eVKWVaYb*J(t}3)QHcR~{vA z?+-aGJMgobH&4Ia-h|Ah2~6<)jGVT7Xahri;sFT2s+4env8pV%Al1+lxJm*fPsO@Z z)O$Xese1&f=M0#U;mxap*rrKf%BmI9LZVVq-B-1oWybD7vjzHX>kW&tS4*DgN6@O( z2?K)7i|zYkS7>pSfC;MbZX05hkV;bmLY5D)yeP&M&8nO=@>YF&z4B6h(|Vm!x9V)i zvvKHV?5)5-V%p)Cg;Kg|r!_@M&k_!;W}aO>1%))`?v{h;D}_^#q!3?x(Ck+EGyNRO zz#9>$-W2JKrQSuQ=*Q$O&h9Ke1y_FQn}sI~Ukv8|F>_zUGRxSP%EA+Bdt#sc3O9Z2 zcQ68lndz&Ac?_gfo6AIdHa~}MHKC+K0b!gU?g(@ zoy?@HV%V!bz)`doUV%_cwuhYRw+I5C?DEWuIj&kmk3{IhSK1-n@DHVTq$1T9Dyg__ zo}D{)F0vmI-K+J7S?}Jy-57ay(a{|0zXR=q3W{*6L==)##;GF#oioa^CDa&QrV*em2%z>Dd67A_vTUqBp3hmf?OZhMpa}EcK`R_K!B6*c(LK~2SxxhxtlWFJSw&bW( z=w4{83&IqL)O@64?*u zojFZ(^J@PgEYA3y8?_2a0S+jGX)n7$kv5W3#}+(5kMAgicPWtE4Ujx6dp(zo@zPk2 zupe42D^9bTtOGaoBD?lvRc|z)y#?8VCC5nw4sKzpli$~qOjw2YQq=_}@BCclo-Gg{ z%#^M5^;&TAps~|lC7}r0F7CDx+GURx=i_U~thhd|p3dm(o)@F+JRO~NuwN9Km<*~S z#!vx$)dw5r-#}Ix?Qcu4x!B*mz6;rnjWKW5D|1x?1!iFxAz@Oa=`cwqHQj6kmBluWZuL0mu@UFauN{r^ z<9EEGCA7OT@l`4X8qoqIeZ+vNl=AP*#N9gY19}?DZs5BFlU|Q;cEE||pv`i7pklG0 zpUL9kLo;aJF9Ef%1qUOF5k57|svg+|T>yg)(-C{{fAUe6o$t%h^MNRdvyb*;v5CKHkvUwJP4h>C#`D-Wc3q;j)1uIDZO7jhNsBXd)MVYs@&mPRg`E|e& zlIBHZ3b&64jZ@8qOV~v*C4Zmt{uV3Wkxeiz6ms|FTLWAjv~#6EQ_mh$554$Y5sQMG zMC3h2NZbY5@#Ko|%` zOGvy2f*qQgFd0Gn4j;}Ba=D!_DhC5_GO$TYh%zcDC;>^g!YZEJgz5v+L8s;NNE2x> zF|UkD?A~Li3B(*Z2Q1%V-j#z6le6hQ%QnFRKV`d6y)HBGtQ)Fma{yAUkbJr|D_^?| z~J{2M&Euh4cQy{Y!Fwx&jT3*|Q9FG48M48vrta#DN=uxDSKhYN?d=E($Pk4)Ziz|~?#qJqy=I(wi!E`}{< z(9!67h)lRc;>K^0R1OY0LB6o3`*(Sw4~oaU7&nG-*lrTW4E6;e)xp)HS)^+vHnXw| z*z^FK#r*C$__y5va!l>_H)}3S8A6E!N(FLHpS~{uY=Pd+`b9ekTvf@rZIU9L*F=|{ zUtNviSt@#hKnQeO21Z1r_nVhWz(NOOXrTf4>&nnbg>^;Z0kE_s4w_}j9N$#jhTl*I zh>-|)yBR@m+8kH`T*#XT>*@~S1tOO<1O?rPV1ZThCmlCZO3Dnx-FP)XC43HW6ZwNc zW9hlMRaI0}`T)Y=g%W`0YB(B=u5Je0hr0qa?0WT({{lN6I$`lKhxvdqa5;4SWE+&r zHv26q9lVO6VuBbP^hcgghYf&nAT>i6nkIzoCzx&rlp;TPcre1+X;@g2wJ>~@ghYGR zl}CwM_;U+mW?3xkotOiE(5;}PkpN)=&7C{O;I+Ad>rD2vTh!4d0q59*CDXRB!j%z# z5lECGEH;Oh+_Wpqs&qU&YCt9O7b1FLNVVQ@bWHu#PqGifRafjaj0a2dp@%ClAYjZB zjxMv~kRE77GOVB&E*brJ1J=HzK_MX(%Cb;9LPNJzJT%QuLYE;ImkPx0thN`{e*j!ANfb))} z3mmj@Z%#sjGC%bEHt*Q~DedmdT^;qHLV$`JO3u)>kN^qF6i5}pl|lY+HhECsY&Kki zR0Q%=&C=y{TVyv#8!+q?=u@YMdspBzN|A4ZMjt&ry;&%d7cAW4wOi;R5y=t94E?py zz}{?lfRjpaTq>T1?S@R7u!;oInX0fkYm(@0x4N}YB-|N7(#>l*;%;6v`tkAVfxr_a zj($8|19DZF*;Yo8eL6Q1TWG(mDHlNhX8<;;Wwz6E+7XePnw$1DtoyU+BxcH;9C^aE zbaUMoH}J9`bf|8dx|a2B6R8$PDF)iteyL|`e<+3x>ROn$xp~m3-@IIb5kN|+3#(wcXdP&7(wK9( zedo>B<-&SUxzcU!faW;( z4nv_KyLPcByBbEiUS|7!czJCKMm8ED)um=~g?FlvbNIdni&FNyh_G-dlt~#k=T{RC zhS_=i$~l1=)R)0t6me&41h!})r`}70_TIhLvbmI0F^%y6ah7|*%xa1Y+~AdSg1m3% zLVPm?@^WnQd-Q!83;$%?l_V3_vv12Cf&5Oh^m5%l0VGk)7=et6J>zv~z$obA1$~KG z4`x!4U08M7YmZ{8MNgI>3kq`~KrN`G1+(L<+?@)0JP!rBcI)SmJ}@12QwiJ*wdn)i z-3SqF(2gzQ7p6eL%`|B19hsTyIrcwX)w~NmTi%WAw{HDF=-n<@hJ!Y^Eo)B`*i|12 z%EiX0kfJ;9bo1IZLw;ad39cG;%$j9Arm*_Pa~oDCSCd4IV-5j1(Cc1KloL40pv`Kv z=3!{VmCY;{I#gfp%HJ}Dbjlwn6PkH$Lq*xl6~cssj38Xei*1}Y=Q1u2s!gG=&6aV~ zK>*4G{U)&QF$8!&<0>xo_PQ_6(p^6a9>qCF0;<&WhtB2K!m*{saFf6kv>-J?{x%GG zOQ_(PBD=^EBAe%XlRjOezW^+ZoPHs;nk?z}$(tv{ikKdq@MT|aX<}a|t$e7-<-kP~ zZltj5#Yh@yK^l5Hfmu0ee0M0@#|v(}CaTOx#`YCw&_o7(#grzpcJCb~EYA~+1cFsH z^JfHf?|QJ4JLIhxcZ?(!&5G;-Ju%3WXk6 zM-R7hej&ea7!8;C8A1zLWK%*LED91r=q9it{Bxnw1lYUg%nZb@3xRXkZ%D`8H$>_M zo+970x6K;agh9b=L$|e)Tg(yxX*WJ*^&)B5P|yW5H3dQ}Mi{k%85zVI!nxZbd=!`@ z22eN}ULzTtPeFI(HKZ*V7G$=3x$^_bWog-2r zUyq$bZdJL+xA6tRdua26+*%4C2sxKkJd8siukI+kBF2zk-8^f<@XtSw;JO%~2VZRl zZM&1JFk?)0dnTZG}K;^QJ+-E7>WU<${D=oxzf048C+r1~<8uk&e zjfpzA9`W$DaL5R8d0;+}1f6}a4=b9x4@I6noq?v%F1U#AJ~VW}%>~bp0|3H==oTbm z)B)n@^Ure#RNuvW3Yb1Y@@&Xy2|a!KI)fQDAh#fY+MwxX{c;FuMB<~-m28!>?%s89 z9P)74if9|e)e3Nv50!yD|3cZJonAUuZsk*-)p1gXelz}tvsFJw(m5=Q>Xzl0vfc9b zJ(kR@GZ#6mntiOMonF%X&8|h|b zr|{f~X+vnJLWlv4C~mL2Qf#1>1XcI0-AxIeWC&&%5D%Lvp8$PU0IRfFZ{!Sq&7Wxy zcMG7zeVCzVl!0}D=VbK3jW**?;>|3dQ0| zs(xZ8GhDr1g8Ga94^+IMK{QDT&F`1tyBu`1tRk!4-P)R3bJpKJP1k-31g1HS_>#?Ph}|3IRw^zPZrb zUVHrn3Z;dQS3`;k2s2K=uMTfZQDr1Dz1Kpb_BBtK$_)E^i~QqDI=cyVY%`M%Ry zSKqWH7D-q6_EdzUz4RFk)&7$0o~DV*d;Lz8v&tV2UEaY{Ek|A8d>tWj0G5otg*${E z7RRX&gAHKmueFhwD_nB=gV-e$O8qHa(a9Ax0ujRYQNq6~ksyLQXkJ-6GtE(GzkRzO zrh!Y*u^#-|o_fnYF@c3cTVlR+!$01hynmP>X)HTfVlhwUX;fme#}@H0M6DqNzH}uY zHl>&EjCmXPL6 zD$;Z$T>NOGIG7T|w5s6bJ6EB?Xv@}(y=DS0v-1lm9BcsPMf?B+oS^{F@t;3~KHeFI zYM>w#w}WmHYZ?=vqGQJjQ7G3`X%7<=ff7nF!q}$GOu{_3<_eW^+HU_e9EByfg3iH1 zdfZG__)Ut>RLtzG&1%W9H@(kY>{@W*PIY|!g$7$+pv%W*kZ(1}D+Rxv<$?W>0|r@* z$_JM(!6xW12wvl%+DvfzbPCu!ug$9 zrMzFeu|5=K`*1yrcI#km+EyjKUY8WPpgrT{pgmu9LuYExUaLro<8*{( z^Yuxoh?Jm>?_Zkcwzo(Urur*4`-wa@CI<3DZTb_e9$ui*KZ_d51rno$arYj8d?W{S ztY~3Q1pc11N`_m5=GrlSnR1{^lPHP|)WKEwi)qqAN}7y@h>CV?uHSs`a3nL5Ty>Ow zvpJ`#Ey`LEA6`>q-E;{X^J=p$^p!pB2Lcs0%0{((t#exDOzm`tJm5#2_gLk;iIq@4 zi(;F|N-D6M=$^NO=kqMLANQL)eEUv8XF*wMiY6u`%)*lM zyR5R5=e_Xu%Q=aEU{x_)kEp>~ob8E!kgdO*1d`C3xS$ zZqSM|xwPI&aM_PgvU=|}GjLbTg2*t_{Ur5!EwwxeIb312hzfi%{EV2&fHFo6m?sO0 zgk1Xl>?P`_H=$?Siexo&_C$kjdC}NUwsg6}KW=r`$@DyFB(Je<-`oG7wo}O%_S0}} z(M*rDr@;0=DeS)eB>(Vj0{k=Gb02>vbJt?%8DtPW4U!Fvm3qfNCzI+yv$b2$oIg=T z0^d$et#CMd3H^%*8-lSb-?8y9!~rp7-H6;4D5qLK9{ z=tgm?p9pNYd*P?Ok5YoW7<#9NMkmPs>r1DIl^f>L%FU7Jo>nP=PEWo1vH;ox5uBtg z$3clc{Fdg9CsY}27Cf@ffkAFBmUWQ_Kjb;!BM78`lXU&l@Q0o>4v?23J0{yP?NA5Z zgWWiEG}|;O=)Qa5Np{@$WRfoPz@1-yibH?CYW}&59mhrz*)C5KF=u}*b|^9UMQHZ0 zif%3k7*W9O!v03@zmVS`Eu3gnw`eCGu(r1598M(didcTI7gJH&C}CA%<^4l%e0i)l zOy5MJjlAFFFNkPoURE<@FKCno9!!O4{<(yOi@tq|DnhVr6$Lbrc5F)ThO`nR{?LM7 z+MTc}z8&G{|2I2(du=cB`vi9bLPBO8){P}EWoDnGBZp7JCMRi~Mv{at{HiStpBLV< z;!=-WKQ=};NDc5sLC56Y-OI?Yq;vDV$u+ji%f~n95rUdLkDO-mZ$5Pmfcc*8 zzhmD}8t%K(6dd;0Noa6(0?fB3lb;)pKcLtuF6O=_AU1{bn?50V-jaJm6=yd$5S>C( z!O9_bYFI5X?)sMZA(;yH?L4gbad?CLGL)&9l$4UJY(Tu6+`Ano!Yn@|hc2)CJqVmw zouPF7BV*0{Vw$a`KKp!{PFD|O%l127H~yeD z7IraVc4@Xg$X{b;`bHYiD1`(uGAMc7fD7_cf$(7fL$_=YwRS)Gyj@;c_#^T!q3-3l zWhNmmzP>O?7;Pg@h1yAY>{P@Z!FIE(_AkNSzqn6w6!a@^;!-LHrjiBNtQysQGJzN(8 z^lvUI&JMnzm>?W6P!)wC2-d>>bSSPdva(9Sl3gT84e5(;Nz8@n6#WO2GdbZit9;JC8?bnXqg`-U%Thx z8mVS88*#=aL?Y5(v}@zxy*~bY_3$Q=O@QQrUTB zQKZB~aMQ1`7S7IKq8zkmhut#sKAeYBOf8ROudD1+U@+?XBDq)JFK%;xyrDTCm&O&% z5Ok7bD-msDjIJ8BOoAL+-Jd@)ph@62@3h~awBaVJTE@fI-?Q@5#*&i0$R^atNX?0B zqwu=dKgK{$mXL_Z{@>ywW3`M#sDpqIwdfJ@Ycp}rQ9JMcHuxfJPmx|)S>AuS{&!=` z&hW4K%~Z`ojSVk{gI5X6dykd^Nu`)x4ZK#EYZGe9x|=!evzCeYj|W9@Snf?uJ>96? zIQyx9!4%m)2_l(BW7P|F9ze)D(^TY;AO_SfA2zXfrmHDIlW2t`5ptF+j!-Ov8Z{vi z5nLYT4y=59KG5BpBuwDMiWEUD!Ra+D32Hh~$5iAnc{PeDJ8Jtd&BETr#EHm2H$FvR zGmr-DH1j1u)1i}*>xV#os+thuCIm70S-84wHIBpX2>!a111L{N!uSMS&y)<^1`J$W zA*(---!6Tj{k|^P#rhf$bt zw+i-Z!#nE=eSCi%7{sN#FR`)iSFI4w2seaA56D*C76s-As}ejtJs~dhZ=coWWk+7` zpc3*b5wP_GQt6PQGj54~1JKn8d2FoRNv}unT9vw2W|Z%p1ULQA{?emMbli#hzo!2OJKQK3ylN&cTSD?ezU`#eD(?$7zCpm+z z&kYpk!mLUXq`^}mFPoy0n+TnNufHQNf?7Lw_N=t7Zal(?kln4Y!B%C^)S=qLGMP z@QfJ96BW^SiB2j#?ZlvpO@=jg_XEvu`@EU)W0@I!hcWofb1&uKQV2ie5RYE?9E$ZUylks;^toJTF zp)UR0k;96nr{=7_OjW9zHZ^o#>HCTA1znycCRx62yHpVaC5@NG9XAy|XgT?uG`UB4 z<)_n3op@k)WcsYPy@NJw&R~g2pOwp&ASXxCmATT1x)arRVID34N`!C+n#WR^v$v3ZmNG-P!Q$hm4ZGVUpN=yXNCBW5WSy3H1l9m;1jKh*)I0s683~ z*r!@;Sxn0F;8E#vS^H&Ja~bsQu4MkO<5vM=X(sqpZaU{_Fi{=BCKnyY=xK8}Rt|^n zZ@)43N;b`0@snHbv*V7aki_E@Do|$uoHT0)S1-CBeA(;%MF|U~u$c$wp1jTWyxrgW z{U-8WCYay-7KJr!ZhN9e!^|t-UbKikVy^CMq|~!`=G1Ic{^=Om4rXL##C&7_mvEUO zp;y9Vij2>sr$_W8J>+ysd}Z9n8*Xq9B&z9L@$=_tHgk&jR!bR_3q4BHm70%~%GNG4DdmOtxS5?+AlEI=l&? z+`Zl*4)!U8Dd#7b&9E7Jqnpgx-HMfor)JNXX3FgD&6D-TdC;J_4SdlH_vGN!2 zXZn_aklG!HKvUFKXa9V`okA7u$_1Q|*78B$7I{WrKH1^Q6Lim?#xPsmsppv!XK~#r zitmT8YG(+ziNp^+Sa8mdPdYgVlHuRf9fuq!>pj_PRUftt_eIDkNX$I>3JVV*pQ%vn)lfY)#v)nOY#zhx6MFYA#K&b zUA;6!{`+o7vk!u>y@&1Reky9 zCHnP`fWJUIeVVioc9H#`8HH5%&O)vgmB3OEugpE(-qK_|IkjVA7eUDBDUn41SO$mu zNVDOIJA!RM+|{pjK1nrE9=I!Rs

oUOXiMaR8%|;-)_}^F4dn9OBB20LW$hgr4U+ z*yQ{S<10FzRJ`bfH_NQ;t)ES)^iJa+I--RHR^s};`PH3g<3|Wbe(Z|3S0-sZ=MguZ z`WVs|nDb|gJjh8aH7k;jIXjNZCGB%(YIy)P!$%b6iQLj>xzZAU^iDzbEIFIb2e#b7 zqr?SdN!*TT(MjqLM8brpirjC0d=gjcs|dE4Uu#yv;karzIW>v4oVxz;Wzt@f4BO+! zOsh3JtshEu-UPfa3U_pq7UA$B&aY8glI#SD$!730i-a5<#Lk(cgYf@hQ*^SZid^VH z+prpLkfh&6_p81|sF@Ry(xR#twpM^C`>R?ZgI=GFMf@kic(EjLb`Kjx)7M!v*h4Ch z&5h}K-O!_)4jcs?4u23IxYbY1S`t`}4z}-6LW$JSQ)1V}Yb#+fw_LdMjhmcwu0G@u z0_p39O1l}kV#kumuE$=tx0ijLwP`R$%eZCKW*_GAuht9hiXfpIn{^Iz;dMxEUsab3 z{{NUK5224cs`J@3#_Jzx^&@$cYqHPT(8M-`{!;96ddI^+SG8wQ6-ZTJG8Le~|K}WD zUeRTEXd;`l)B`^t^&4Urbw zcQbV^AGujshod({kmw*V_8PS)min>RownG_OS<|*ljFX);_ zE`?;P^8~d0_ZXkxkMX=ODKX-<3LQHa6H@0j%{&{2`L*haSwJ+CCK)n43Q8*4A_9l< z*HaapXhh_Bt24~gTq%szw3DxMk>589A;#$K(lO$X(S7X$KG-R$MuHt@i>K3<2#GG_ z*N~v^%*5}_$MVPU7%lTXq0r1iZnH&b$^=OtLuK%Xkqy4A*<{B(_fZ;)}nVkGtZNqy1tjpa`%}xYq?5NUZ9wP=sv0LCo{=jLnQs z<*#Jh^t0m5gIFC$%Poe&(dbn1I$ zk4`wcS&DQflWTk8@@Pb<1+-|}@4Fm5S6kY| z@3v8Eb2V=BQu>cY*=W#BnOB?U*HiCt=I7N6K&m^uT+C#fVpB%(-$&Bo0c!Lul8RGz zF6o}jn4j0O@I17icFTW_(}wRJw)x6nJE!ZJ@H{Fetx*km$&5D|^Yhq9ALuE)S-2Eg zq3$F*6MKZx4}I|3YIr0eSt&b5*&vM|=ImGEwe;+q?OnTwn{P0r$Jw%QDtW?+k%szp z{gfp4$b~`q!n|}NPa2GYNlTudn5e+)e@>DGIY|n3Rw=7e=6lrch&#m?|I@hv{+T)3 zWp;mKE40uo^HN7vOzhHi$mM3JCg?zy1Spp}(6VWZQu$yF5># z`Zis29_N*wOE=Scc4W}s_IvIjPgk-Q`bkQ8GYR24jh|kKJAeQ6gq+W<{Q1||BP>WN zLe!K;$hL^V9U^6KwYcBHC8?7d=$_Dcjy;=o44CeG>yx0&$|e$?2NscRXx43jz#eVt zw;YE{l7trvBz<;X%u<%Gj@wb`;e!Cwh!g&BbCZabt|CWJ>C``vS`sv`s?D0|s3Lk@ zw4gf)+i{lb@e}6kcZj?oK@X1%mCsDGKW+D=-sq5h3^KsQZsWD8su2I0z@wwf z2#L>jC1rGFSIu(G?sez5^=eXQIew1y0DqZNXr@%RaUDQ)rtPj6>!7Pq%+A-S^iEc_ zuP$F8c@IDC76@;&_gb(_7$J-fpOKC8-g~fM;^|Vj&vRI|Yye!mIJfexT3%L7!0_uQ zz56qQ@9hVA5)&h5+XqPM8`*7fXBhKmM_&EsYi}H>66OZ*7x zBQB(*e=S!pbr>G-*p zqxctX82CYvVa3hrd<$iXPC2dAIZi=YSVJ87Fp1OE4N5rwq^qTh;jri)&nQ>LzPAMM z?O);z_UX0@yo~mH=YZrsy!lac!9JP0BURheg8Q}l#YWA9 zoA#0Tx)40N$Py8eKrsQg@ip2D(Kp#HJc%p$EE_%Ez|jDl$L^1E@%lhPm!)Dx+ME=h zp64Vbcl<=R*0ym*QV&v_F?ewXA~|C`Ua#UboAK2M$Utr5C67>(lg@$XP~l;xFA;5Q zKX5Coc@S^fA`&K$Q!czi#7UQ@#wot=J58moPq$L;3Y|o!bZ{ckQc_9`i@0-BPPyYc zX7N~?Mt&f!bomlA$2G)V_M&{k8)94;Z4JnLyK2@8(@i7MMh)sQ$@TGYnvF~+i zcE4AdE0KGhk2ja9@L6@w)dSA3&E?_k{bRSfj&~(GQ>1iARyO*kHJ66hGKqz!-h{s4 zn8oGAOqt4x*roDrd=61WV7G&j>m(*WQ%Q(*cu0|;P!czdG1=_*B|Xl5TSnqFfY66D zv6@VftNWjVcjo02isGiNLqdD5&D#atpZ50otU6N-07+(PChyRK=O4+NSL<&1eIry& zyLX9qkjEsUfHvX3ujqkbsm>IXi%_`r*B-SB97QQKTMHJ=b5EPjsZH~|eM`#I*GC^_ ze^5lshb!y0`b+XgTlt&QcKKI!{Pd;_R{%#P|BItd7x7>>N~{EbudDnNefBVST!1;d z`ntzv-&{9RMo16)VNupf!o!af;Lf&5R)Qz8gKF#-baXcAFs+u0uyH{Apr>E@uX{$h zZzG@WW|i(#2Ilqz~pT4UkseA zh40y$&wlb%1ShJh-CJr$uQw;3IYszPmPW~a=*GaK{98|$RxLPhw!P1Ma7?EDeK08{ z{d2G@T+(Gtk!>x?WX-EPq*eVBVFmt*-#8rgnS8D=x^3Ut1QUP$^^Yj zr^XKBY%z(4CGE%5;q^s$LD$*_`kv7`O7JE1(O2Hth)&R;djf|shHiVVpq`Q#?b^@i zn+C3A_j7ZR)I{Q+Z%VJV#^-`Iw}$%|Ta1*)poY|8e}m+hNxIu3L5V#S3uWQ&1}7ul zkPq7C(V#=ZlFy0L_ON{Wm}i>Ia0+V-f!?N4g3G#3_u5};^Ib`yk(KE_H2R@!)KuwR z%T9iK$NFxKF#KWqJUr>!i7B{e2G?W?h%dP9<1ov1ZU#<1Ga{5Sg3mwG*v<5MT455v zeMU#C^>=pZbl=ae$+23qY5qLf=XCo&HKgzxh&uU(s99cA0>j+F#4LKGs6K<_g;qT_ zVZ682EMob>dAUtYR6&i^*MPu{x>U{jOLQ|-N*b|vvioPY7a_a8`nvy7`P`$oyf6C= zoV+W;09nRp#78ylKOuh{ZG#Tq8chy$GiPL6IYYCISz3% z4@E8&>t-`n zs0W5b*Ayh76KH8IF3!Z++;NzzXxu+)bL>H1{A)+d%9aTLpg&t4B&K5rcAC*W7oFT% zV)?uw?}XGNWr}lX(BHoN&5j3>pSvd0j(bji;0;#O)Cc)oie=lR{`1vsN04lXhbD$E zBK-m>Dc=>CvpwtYFyLMNiPlyx$_E`J^+OdJ^z8<(nMI->QfF9X3^AvwX*>Dq&J3wA z1_h089n)nR7tZPivDxHVCUG{$Bw?-7w?RO zbgq4joIUt_lENUv&AdJRJE0Xl8e?~0ps>De7eWtAKUPj`a>_sh(KN-wgNk}w~X#7nXuh6tv3KCcm2Whrj)M3 z7Z)YIMVB=o`L@7#71JWk%G?M1Or38oogT zjJ}}BUic~9)E_rhZLa-o#@+P7@}OJNv5cX=3>zevnX_*eChrR|uQ1p0okZLL=MpiHT@BtSVOK5Ku>|9EN_;_C z-xrBzL?r(UN$f|)XntgkNmbmPGw~Hf`>NVn%X_Nc+pGn)6)&%OQBbf>R-&gS=wWV$Q{v(6kL_1X3xl}|m z%jWWLzG;)c2-*V)p^nMC8m(Q|)T#8o>3=apPzaCh_pH&np-3i2F}#!-y^q{11PFB zo2$5}Xl$@{?ih*~7?8u1JQ9w3?P2gxrs~UlrtDzFiujH+d9~@%xel59{pH-6oUV*c zXgpX~^>wE>Roc}(7|a0)Uj2!g45Ru$$6NjXk_30JPycEq2NJwKWWN3PQz`p`rJs%) z6Ee_iAp|6eg6(O>9%R|8ztS>tet4BplqI}nl#v>yR9It|pgU86bo~Tp95tC(zC2 z`v=x~Y09JdH&5_9es^+XDB9PH=wd+)tUu7obggXs&lkUk2ja$9c==n-f}cbY{FL`< z73e#xCh2!D#U!Rj>AU%iR!^SIzhZV7Zr|WFLOpDTFm-cigg9lwN_Fh(Rr6N2uA3|W z;zV>T;%qZXH*Zyr0yjhn701m|7+>#fBXI)d?z49GOJ6^gT1)3RBQ3r#(PhTXW{n1u zSuj>%f1EkoD@j~U?l#uWp;}Jc$tj9AL3cv@WTp1h)D^Gm5UQetthmFymdRN?UQH?e zSST~~(c&K38VXieTF8u^YGlrire~xLen-G%yyJt;!Rb2?emB19GLCrapQVh-3pDC%l2S({2pA>nShRagQ z+FVdho{X?BM^j5*x%AH)B>AV1tb52eQm$oZyk!Bg#ZfJ0MFfA8z}nn%5f`>}kOxh3 zy!Yf1kqioF^H4?brNW`uFGUZ~1sN5u&Q;xti6-q|`rY3S}vBNhu>brE0P5Xcd zInM;?@13)Oqa(6#zg)+K)Z{ZPF3_A5UZRrA*$8kxYKc^$CTtwF^(B2tF8}&-znAlJ zMQgUJg2aKGiFiYV!Q7i(#{5OglH(pN?0W_%n##hI?)QgaKVMHUpH2cS!p8tJFZLd( zO`qX3OG!C(Mpc{k3aert=P*Jw(y#*+1(YQ@{*b^6O8ti&&A9or;6IN%vdAfA9`);R z$B^FTnX~fF!LRY{g3T;WU0dYPoA&HPClD2n$Ot-n?!osf5w)2{Df4jsymqM^o6YbU z#-NC>WB7F!II^HQB$a~F>jHXZZekTivo$_MwRsr1j3?^~o0q7WvIpXC(^PJifc@0f zjo#gmt^C*JARHL$OrevBH=rsN1bZ{T6+=N2WpV8a=W~U3sjAwS-NKSCbJ3bcXcceg zA2pG#eLqvTH-0g5@@$~Or~2gNcieOjczZ~2rjz{To9)EMvZ>?2D`kdleAY=QY`EuM zx`_9}vTG0S9Ae=pQPu`gcO?c&0K?oT@(dobgwGWuBJlVFXA zZbFA9So(;1$N5Z*Q{RA&XS$S1rc{Bm_PjIFnIrO3O?yeF)@JADRj?q-f~E4Zm_6Gh zC|hv5%5FEPXMA25&~7@aKxQFvbSBVrZgnW@pr%cqrT&vbr8!LEWZRj_9sa7PYxm_K ztK8&#MsAI-%L65(9>FLyl6LK549prUPM=m_UftaHn{V}VQ__r_je`}v@FVDeZ}huo z4N^i`K;neFQxAGdBEM!#2Mbs@`OHyA2E@tI0Cp_&fP;R6(^;;Fi%Ws<#ck=i<3EDW z{E6Ex`Lk%Nsb{4#o)uuaf8(w9Tdxv^XLN)NvD!jEZ3)glkbatWmgVnHG1L-{f{*_` z7w@kdP+Tc6K;(M)_XX}h@nu@m6X7eP{2?CH&%RCb533Eq#`sXKq_5;6VU+@vd%IbQ zF6Dz#8b?rH_x=r|J3~T^{}0omlkQSBe5t8e*R5fnh17B_quuY5nw@8U{Y-8ew7;P; zw6!SaHcb(=dGbAPVfmcgdqb?zuM_UC!7E`q-sPf1TFLE{A*+X^0{=X+3O}&xXlhnko2b?8zuD{M74vBEZsW}k z(KnLyziXR@<2#+oN{jI)ijsH1f4SWW5yDe~4#inRALa&=3jLXT@&yHGa1#nH&*r;Q zI1Hox^;p#s8!ReTL6M?5^F_`0hA4((RQe!irzQ)2La?7jh|2 zXuaM^>Dq%v$tm|m4|a|{MAX(79^fC>c^x@IFAdY`PC|GvO3zHt`vf=b-c9PT{8#8L zy@mA@{WGPCdUtzc&W@rpr?7-@m8Bw!RWp{1%xJ zXi%PdrmlTcgM6;weAqP=Rl5pvH%1v12Lep;5w0oV72G?u=SDbhH5sa|Q;&$XU5otS z=WqVLRexwI0`(aS5GUcT8?Q?g{qA+nLIkR9l`KSmcJ#>U?mDeo@PVXjPMkQ5@RgZJ5mH7)7toKMZ zVjT*UHs2UTBz*J7-OVXuRtv%|Ie}rX1d#*R)G;*ZBxI-3S()1|-TWUspY%L(5mks- zR=nS&pGi?zR1|(-UHzQmH}8mo%i=#zNC(P7{3$~2J;?|p1qZPs7B^#A&X)A`4KOSoNE^A~km=4EK8#2kKz;4};QvqBbQh1F>qn{&(m6J6eshbKEt zWd!#xI+Py874rQ5gJnV_*GZGz><%&h3E|IL&(CYFr0_@Enf)BHkMf^f?#Fbf3@0w4wlx0gHuJyDTTL& z3bylhXY@1YOt4yRAMymcf6E$Fq#PXXyC9F}c#Ob#Q_6Myp&gF(eT}`>jVAE`qHAvwcf70bVrMR zC(TZeJfk?(cc;l!wsQ394)$&!Dy6!e8^=-H%q5Hlo2SU_o)L8PJ@#f5zpBUtgJi9$ zgOETl_FA8QQj;%t|8Gt8?bv|aujMIz*Ez&a+`A`i_qvXhOkZz5O{A*Ze4$UeaKq44 zH@SDWlde#P4R%qz=MF%=ds#z;sIn; z9~n{dw<*TC=Srr!ulJ4}dgtFaN>YWohWbD><6*-q!b4g;ly$Y+j@j?0ql21sZ0 zDS^l=wrTrDZ_L6IM~B0DV1yjzX)u%Vd7-AnmYd(LqS^7Uy2s<(qmH|YzE<2I@sT*9 zKAVxEW9hhEIXX>kuuBH-N{+*@lAreMIfaE4>5E1~6ht@q4u}aJm>TNRw?`^Iax6if z_k*Buj&3ycfiQaBYTlR2Q|wQ}F&!LfM&7XQ;Uv5I^`eu4A9PjFzaj+`-}rvTMX}_a z=P;VQO;&QAuZ!pHJ2KqNLuy7|yXPX24!@?$PgXqGIS?*M4s)okiWx6P0A!7j7l%EA zmk`u(3vf}!W+H6;%;oV`udfn_!Z6E64G=n z1}CvXJ(;F44K6(Ag$nwPdb^@V*tcdj%|hHa%8){p0<-<(qps5x5#!H%v9Q%Vxsj{s`C;Rwl5i`jg_Jr65ef zez{FULz4`PK=}{%6oiJB2K!t(JB7RErWGV35Cm3=F{W!M+v9U{dA1Ue$(m0nY>SbZUt;TsAxF z8X69&Gp*rvJJz5EWbCs z;B&;r38QE0I^8QIvPjwr51Q#Z^cONNE*1jRNh`)e0&VY06XAKbO0AF_!&D^E z{%?3O@*H`~McC!jm7`Zr`hWiyjXRhYK$eZHuEQ>a_IDTg=|shzJ!^sc7_cq#7p}8C zg`Hk9u$hd1XXMo3;j){+q-}v&YF3u?3I3s^QQyCp!+Q>Lx3-qGHgp*8N~b?C+t3j~ zp)B}v7B0CI?6Xsc?RJOE$z*!es+L9zI_+&LL1-luLe2DC;;KtV)z0gvZNi;CB>`HK zop27}E7xyE?@~J@shGY$KH87RM{5EnrCMMba(lL*-<lU9W~ys!UlzO5b|3`K!hC7)k?ceP^po0IF6nPk^yFv2L5f${U{dnuRXtAA6f zZGN*lZ^hT8dg6qa)U&&6_L9EDQZ7F*d?)oL*9un2+fzUI>rIgv_AET5Xy#g9*1`vp zg1g!yUSjiKrTFRh4q$^0*CYqA#}n9+<$oKl#C$G}X5t#}c7QzrG@@|dRLOisDOM0e z$7=T|oB9J{#R-=v$kX?^2Mu)3Y)vcTc`&ET2je>6@mJ7=~Fy%|I2ZDaj8SW!6*dT{JzCz7r zi=C;#aNo;qQ5e7%6B9$Grh~%f9oA1zR@)&U8l*Bj<tTJirol>^_qGRV z4H`p*(Y(9&ig!!NgWUC0Lc1!jp z-WX7^z4g}T|FQMfQB|$o7ch#*6@#mSUO_-bq(Mn(6afM0k~To3yVC+iLh2yh-3K^y zC?eh6Aky7+fN$-CzQ1pb@4MsPf8KWt&vx(US!=Gj)|wNm!?-o`E;`^Z>{@|BpCNH* zNRvXqQ7M35&>WG&0jYwJjY0g9)*s+C;3a5RI*hDsylVOy3vyeelkmd!wg%0*LHSo; z_jjQF#^%v9Nc$zaf&<+QvCHsW(hmC_RoTTA3v_x z9~_;|0^J_{4PFBS+R-jY&|TSVWUZV5wV5^u55eK#=^(j;gj|Tg0VacvGqIjtzBdFf zL!_hIj5q4d(#LY9Qx7T|AA})Kf#(cn)dt#VLIgh0y^LIn0rYpGpkeIk<@N($c*w8apX1-?*N0ZXH zYl)+mY~VF?!Af4Iw$tt;?I_9NY;ttVj18>q>CDxt=||d#>0}Qu&|wxY)*-Oy)zQM5 ztmvsQH#=EvPPN?+QDzv+_nwSP{^9_&>Ml|^96CgA{S!n3zE?L=PU5;MDuz1T;%gX2*8t4dV`kux%Y(&ppMTfI z(E63bv8-GaYk#xJHgf$Dh0WUrAg0McSJA)22MR2Vdu0%pAqs(DCi7ldPEJmjk=xPw z34XpVR^yP}eRl-8u93IBaBuk}SX+lk9=crH=kHNRCGLn*1#gb(7fS^UnL-&Tj~QH? zS9!zM$#Lf)>qwXUg~q1k{b%nm*s$^FBpi|vz6j?cpQUW&p+qIc)iN-h?9|={%e|{E zBalGZ9eqHhG8F<(Kmx3lZb3^WLIs}K3nYhuY#;u>^X0Jgp zk0vWr(4_>l%NQicKzHus=ksGA+Q-m{25bpfZ%VPE1&P-nMIzCGh`o4w2h?K954Hx? zORQ6G&(5W13r^_QeCJ+x`>WMl4=z)hS7}SB)av-E7}sL50dJJu(929}y2q<$CVZM) zutf?{QA?eG`d$I*+cZdT@iDx%E9=y`IYS#>KCpLOgH+U3kp>Td${-0DS=!sTCpSUR z6LMNNB$IsI`b%JOqDd^c6LUKi1gOZ^Uo}xV`hesaxGTDteb z4tBRe;>iXCB8Pu2m>W)w$)u5!zWBXNduMi3Xkk-U5qIE}te#NIrP|E2Jpd8E3dGSG zwuA-N>Zob5!0_q=xNlp5hRfl2ufy0I!XOkX3DhPP!2hi`vv5THR~T&CA-Y0C#g6S`q`*-l$OP2+aBc-10(Z%8T8G?B)esgqrt-tA}mnc(21R||x3lZqVVn{hryM%k#6xTj;(L}pu&R(3L z4~S4AI;VgJQsJT?`43U7gN0@p5N6o#%x1TlQ_6gr`w%p8R6qcT^!oJ-*v&&!UO+6c z2c#*xK=YXj>$M`@&tf%(Fx2i0)BdJ@bR0Lj>h5Gl+zuJE3G)g(`SK+xyJNopf>Lg)_CBW<}ci(qG3aJ@D}Keq~0Fc65G{CGQuYSZ+62G+lpcbd}_FtNV$D zL39CK-a`?at?9F*PYbht`My3}ShdiX+DzxMrnpuYggweM311?j$4yWgMU+XqCp zULO$77sjN;@KU@^V<=Np3N)_uH=SpDfG zTkp4byu9nmNHPl9EBL;#R^>}%{#5L+N}N1@cg*4sks`cVg>Vz!kkrQN|G?idXa1d- z?e8BK%X5|pvghKn%2l36@{y~}RURgm4VNOtN>qy(S?2r7LQb@1h=yI0Ojmmb|KC5Y z8=``6_c&YhI71YGRppUM9~M-o&Wxwcr5pPYaJ zk-czjO1qADY#+f?6;RVifa7gMa{zcD;CuI(O#=hxJmI9>1Nz{~Go~Y&?S;^q-~oAT z?+<5Ph8g5caiuMJcq^TDIk)s>!To+&3S8A7S8&g~=9*fOc`VehNr1!|LCq<!WK~B8ehv?p#o!d}$q3YuL-{YfoPIJ7VSI14OSKBEQ=2;mBF{Bg}E&~%=rj-Y_ z;MzqGuB?dj+_a3mR`AgA(Pi<#TuXb!8Gfz*^Sri_r7gNvUHs|SU$wrR=UJt9P$7b>Ag?Op9t(k4 zfBtsi`$HYfnV@u|f!=N*7JNIv7-beeJwLx@Ku}<{ zg&rN1ZN(oxYZ2kIqP30;bj4+^w|f_i2>5aCp~6YK@>A80n(`R#TmAruH`xp=m7(*~ zvkL$(kDNPimX}$F{QZD)5X`clqG)EDJd(j#J|H0_C53JF3|-c77ge~xT6BW3D_)zJ zdjHuzp?WlW~5zT_`ab==YD_SaJ*>otjZ7(p9dYuMQH zBlz;9WvOH;R@%NW%d2E-Sqzo?+8>R|UUA44b}%Hn9DCJlj?UADEze{&YP)wnq&muP z_!w*?1A(et42s2igBaxQWoLnM2J7Z{<}uDUJ<6J>7y z^|!p_+w5r_abdl@KlaX&gfEfWq2qS3l@-D+6Nlt~Y!^|*u%R=`PM!~sOqt+`GJA~s zQg}4?%s;4tM=eVudINCjJgCjurS>KzC8dG>kR4Iws zr~-8|Bnt%x`Tx%40==@_ZNGa+3Cs(c$8hsjJfC0XR35#s>1}Lg=)2@-mPtf3{P(3c ze1?|_;vj@RlmhA)lX>7wI^mjSBQR=a+HGf?uzY{16uFg1G(t!qG?SE{;9d&I6#TAj zk&g_zukaPA+8bgh)JZZEQeSevPl(PSdz8fLhbRcI<4|l#@oN5*jX^`H*)@j566p;{Y zj^I%Q!SxW6K?IqCBH;9N7CcFx`FUIzF15Y4X#Fys+{bt_Hh9mub(+$M*;8co@}-2) zBbQ&6^bX%2WuL|fi(a`j)0O^?FIDDIj(@9qg*a6<&)6SEJ|zNq43yq%Vu*H!s*X#d z6l=K>O0OPcG9eOBVz-%s3FpFm<&ENz1%IGyQl&!>l1jDpZ@V` zgLc_Hbp8;Bv<~}l`W~;{NNQDyjStggYO>Oos3m9)y3KZ_peX09eElb{7C~fsMA1_B zzeLY+PIJ{0Q0W{He=0EGi3sq{XS*GwBcdV87&&QDA(^KTs`2j6qCG;8|E~4Z@(H!e z)(bgg*u4KhZ;FjS?i?mZZyETxSDu^c5a!lt>#I{>Tjy|YA)72AI_M<99r)mXAPnjO zG!Nr22=Jv8n6r^~baa%bkn351h5#H(ys8cJUQbfW|4elQRb&*jpGVaW)Jd@_>n@Lo z*aEDj-a6dqK7cLY?tHpr24Fe5L&3ssXetsNp zOhzR1MD&YJ;9~$7M5Zoo-B3;1oAo48a+t+B0t~o|(Olv}C+5lxsHMp3|1+?qE zZEr3#HnKEmZ>rLz|JE2cGADwgT`kFWC!)>T*L2E+W^p0MeGa~TgZF$TevYuOjVMo! z7SdE53+){5+F6>I%$Ku+v1KBp=F3XG*#CBVjW)rvOd3K4)>Jo{w5MHE?w?MMGiX|?p- z%BJn7Mm&*T(f8%i8F5W5jf!i(>-ncIDgPv${xKg_$;FMHpw%Sf;)6CYjhW>!%Te-@ z?2TfWfYTQJI_MZHfyhc?59p>EfJGLF8XnR>z(j26USGT$G57d6!YJPN;Ts%H&E9s+ ze5XeE8R4Ji15Sse$h?kJFcKMzj8_Nw&kcwG4d|o1C!ufw$yDL#1@I>glv<|-9O*=K z4xEeV_GxIOwks;NvoFlu_N-<|Btt&D`&2IYfveR-Wu>$$=>H62u3zxH z!m6Di#ZvrZT0n)C@WORO>hw0E&YT9^M>JoWy=9?kcT`1hvemIEAo9IV-*6JBoQ=<-mST@~kS~BAdcVAg$K>XRW7{`s7SP&V#Yb;t!S25mT z@f{CO*0F0o7Jkh@S7%}OVIwii_S1Q9Z>6EWzkH9Whpf{nEIt2y;cVjB4Cq;<-fVR% zRQ&vZoQo3wliTK!pe{_m2U`4`CIb+@X4A7P6hJaqRzqJ61r!+x1qE|*ry6DE$$;CX zRnAdgYmgsjMJoeDe3@i5T9tq!*-y7K}eJhUIDZ*PG`Z~6<(PfZNBeMYs#0pgn1VXL9zj&c%uS# zM_iA9F#4Q{M*W#HG^VFcGUS~N>u!%%E#yL9&U%RC)UN-l3~IhX3b!a7W?*7c0(~Ux zFp;_BA8Kzbv7&h|O;MAUI=oyDgGn@5e@fmJ^JWe^K?Mws-e)K%TY=v{o`<~@{ zH}2K{A)3bA;3WJ+1iayty#^V5#4SoD;{MG+jI2|+o*MX2S@76?G)(QwTe}Y+*F4G8 z%&xSeZ~ep8s|DtUD4qQ>79;ymn3|BvJU>6ACV!CCx#~Afsu!x`mf4gjMd~n{sMmE* z?(hY*2#1w9?Rk5f#I>_Blr;RNhp#yOeB?Ym3b1_?xR`v1e2_-a`UiDsPW_gMOZX~- z7(F$WPfrv+U8g&1yZXim58i22=o!n?20*fvY^G~`Mn}B8E`4B;_G%+yJXdh*a6hr-u+cGbjWw-0wX{Gd)2iJ7} zodtM7o`ZW9C7bK?=d$}SnIINxxV6v^8e?9*zADh{{Opl-Rm<9cAm4x*u1654{wjoM zT7F`w*m>cG@iqM3!n$=1nP8>qWDW{z6k$hcqn=8H`Dr5xY#KdjHD?+?23pBFjD~u;BZ1iOKSwr zBPw(RIM4{%Z6IgY%ihW#zp`SUn3!k)0a+cb`>Aq3j1y_^6S(44QL!hj|JQvX)kIfO zMOI$L6;(TsQsF|d-J@~dvr6^5jw?E2(w+l; zF9WC-kO1HsDLF5AP6UY)czSxyM>#FEgQRqDXlN>w!7|_)VdPNw*{}|opV9*`u=5T^ z88?JO2)&tTXhmDIodDZIi#__wjf2C44{uLGkDYFCQPAb4+!=!V6ppJ&zG*s*{A^B3 zvJ#F?-GYD1_}!QA2q2u^fM$XoXjA%psr76ev}+YXQoVb<)ydWB(oWvONo3q{r=?pu ztB1gb^)9-2aLRDP--puZU8|r*d*QI5E-h?X9kymBZRzIgCn~vf6^k8 zgNeujUi*ySmB%ysDTaV78BZJuL~%k_fNuX_8oU{3458PBOGNOK=3d7`qF9OZx5!^V z+P^4$T$6EHr>-584_iJ%xsVe(#q4Wo&DSXn|z0q zOyFEB(cw=m?u7;?n-3m`TV+XLL4ryF<#V0e5(G8e#BbQ1l{Ss)tKjZlG6+|?-?Ts|ib1dkHQ_O@xNl*Z5h`lsGqbF?23>&Gnq ziD7h3$h!qKI;jAV<5miSHwPNj5&_z-yLuCjpb4!X#^?gW>`Hf@H7-Ho`E7DFCYuEyvkJ_%~#uOHmf#f+e3k34)+~_nCWi z3BdTPg9%eW#kprkkXvP$zP5SipUImq8NGPe^wQ*+amI_K(%!nY?Iq>83^gdLQt09y z;sc=}3r18E#*zj!8Tj{XHCCJ|+U#R6+A;gulRMr=p~Q|6DGn2MjS1 zpbaM#xtj)$vZ?o3_(MArmqM$yTayWdOP78nLnYP6vi@2OiEkAoFjEg_o*g~m zP`lYBE9;O^5Fr!EY4P^=eR!{t(b>NxMku7tDT$R2{FtrKo?7?wi#9iaHn`q2X++X7 zN^XEgN7$J0eAyvR)hr)@;$JQ#&inX`douVQSR!PLl*+~R&@S=>Zr1p_HD=lHueZ;GB4 zCF_T(HF%<7^{K}6QYq=bW*4HN&Hs>Kb{6-&uV0YrB+OTByuaSs6e2`BUqi$9_o5+< z32lR+qFg9u;)<=N_ex2}f{ayHaqIxzjy6omImWExi`mlcf6;!r{kB$J|54E|Do#Hd zmM5~R4f)9~z<&6@CnTIHxEV=8YUpaV)x^!W0i#WS$C^eFhU~qEhhzYl&itB|aaUxt z0D)gfmcsorof&>5KkYctc9@EIA8{_n44<+EHqouUbbpFyZrT2HS16Q(MLXztHK957 z!1smc*wFr=j0n?Oz;&4L^lH#dFy*EFtz|5( z36*(+}!Qq1T8wjG;{|G``!4 z6QRT-MOd#y(7q2ym~0H$FTOn11vK8;Ipp~bl;Zux@~NKdRwf+9xtSdn)(%t^-LDyP z%08a_Q`Ny8bgO+)>8|Or5;4s5Ym8Vc42>544LCdN$hU@It^(qc;2B~-!EfI-x`;;; zG=xi0bQ&Uq7nB>-nSF5LUrx^1wFBzhBQamLrpfx-d-dtfre$<6|0vM`NdDwHJs07~ z>FqqbMICmyY5ONXRYl{zF&NN|xNK{IJo^lGy|ou;ymTzq*{zU(B=;;@=>vgU1|ZUGd1sybbzbdt#^%yB4;k^mP%z4Fyz4&iZ;cNTKo6| zvBR|vE%l;R)&KosHPL@%9WZGip#Tnn014v3Fb1J$6amBoOUcX=+0u!0L&04rL^VAn zOLNNqJ2VFIgn3?TBxjy3piJo3cy(3pljmj5X2$sj)y zAtlfC3u-7DiM6dl0KO1OP4;W(f1$0idm!x?{3aeA0}LFj zG;}0N9Son8V;EFkxRwRD5|2(?U~aQ*H0)6NtQ{*;Lq-3#&y{7Jj7BnT6+@FB9|V*p zK)n6u_rQW{pDW~)q6NK*c78z_wZaXR!Ri2Jz~a85(KZ!hgIiy4TJ2R6OY!)}T{F_R&o~dl z8AJHn)1f1lB9zGbpskk z?rYnHNp6aMB?8_H2aC%_i{k~9M)7-_V7lBQhT7`UXc#W(hFXhM9x8)IRP;2`-$}t^YV0Ulucb$tiL@CJG1%|Ueg&(6cfz>OlU|u?i=WWD zdIcp~D*R#1Es}g@>MR+#nx$CQlCx9h8qv=2@L)>O-y+Kksj+0|#Cv+{*2T#Bqb8g< zfC=&|_y!WJjyvXmX!f#s1Zo*BbZ2|3@L>~7W>*q=|O#vK8EK(6rDT|L)ZbwkMJ zB%V7*&ZcmIkExMvwwa?AU#A@eYh#|`i3jQOe8zX&c*=0}LCy(tyGtRnMe&^APU{vV#TdCoptyk28=DP1jMh`1$2Xa`nNV+Sd{q?|7m9293X9)92QkB6Hd1S)t531$d z+w42@_m54*{jsE@rop#ABitH;J4`R9Jl7SW4$rmqfO%s$l*4Y75$#iXOvFN4MMUiE zNXnDf>a55T)3n6+Q#3d9uJ9NI%`qoCUL<@^r)!r=vv z&=caWXUdI|sBUGK@J&v1A|YyS-bKeL0}M0MNk`&6{Szy1@6fZq)MMA<3`E8N$j~B;+7Gn$nGzyngc5A07Btpm#fCKeDPY&jf_U`%`K+ zZ_8G<}pIU&-p-g4@ zOF96XiYk2Rsc>{))_sjDqsj3!L%@^;xQ*geP7&f3(^hpO|I2C_#-ins|+((4&X(adsk z)gH*vz1h68byZ-0@dCBcwX3NPM@7Qp--cun1r?=JWzjx!AE`f#a}HvTy2Qc15XtkA zRxLaOMHcauZagUu7Zk748zZCXF>iC3q$Wyy6dyf%?=K6RBmG$m9Y*;0Ha)*3I+jh{ z^VZjbAT$=aUFxnM76_kR!&r!64$PBab_?I_3hqw3Rs*7gaU4IyT?bdZ@Y*h<=tsM# zG@_V0K@U9NWRMG7h@Z3y?q|UNMJIt6S|D~qFw$fIW_TrFx}`gaxj@020JNKWe}7_S z?mzbF&=0%d$&|d_brm0POx$AJf(%vx{V5s*6OJRTz_w<0$?P5K#=~(!JtoZ=Kd@s;e5N(qMrZx_@d#<>2sm!YA!FDu z$$=QFA?WDEi#TL#8~l*v?S?Eo)eB95L?$$ZTGtUQ3~x(7ZJ(~~vZ*X|lqRR#$6{3- zyr_aFEZPGtPaB_QLtU%F=I()NBe={%PJSnUH7!ER!FplozA~U%%V5icAW`Y7y?zjeL9QQmmNjep*KJWce4#4+u>wz|k zE>Qc8SDHnndAYr!IFC#ent2cfcriPYUhU8bIZSwR%t(DU;xJ7Cr75_eIcTv%UkQ#Y zV0W@z@N>T*flRB)Iz8f4K^7BYEDfH<$1cs8hxjix_oauDzjDmTY)=ol<+RC$o``0V z>p@QBTw2fo4)cTVYt6&K3J0qXn;TC@aSR!Q`9oj8sX_b0o z7xG$B$UtzD_;74V-skh0HOLQM`mf9J%Rsk*zOI?6tD=UYz-Z7FZqtbC{{CX9h%Yzn zY=OKx#ZUsRz3$P$UQYC|c%Dt`(B}W`NXiW(#UI#6CIr4yHN!5e4|0vpnBNBn-m7X0 z(htY{&{|Wc@nMS)j{zTyVLx4cqU1jDyZ+n<+t{B+cePpUW(OVKSmb)SjJZ-<9qgt1 z@*h16-5^pzc;tH5=`@pJGO@#942i&c^IwRZVv76~Gu01lm3lw3R?Kv7LPv3TIy&Sw zVw|;83@vC0-aHev7>*ilTt8h-jU&5TvwGk>JKDrxaHFTy%poJQzq0asi8XYwc&w~n zOo$;=gLUi!nR2d#ytwLj-q=K%(v97pB7Z`rTqN?>JbmFh9n|>FyOxN9qNVtgJb|OdWtCKcrx5_BA${q{IX(RShM#+@MiYw)>VPT9<>Xezi6d{ zNH8>a6-Jx&K1O#{zB#Mw|fl2;i_qg27(3o+o4P=MA5r{(Xuxa>eE%;D4l>!k%= zbccJBF<#GsZjuWn;Kk>heJg>?8FIGTO(TaLS4Vu44ohiSuILR&D5KyD+WT4Akf0iK zi?7lk|EJE&kuC?AG~}DQ%^<)x`Ug9u;U`^M@#>uIg29eV=pbtAxb7xeT3T`d697JA zl|KrOoL(7$rR64Ay*18SUpdfJR}YJFoTHaOTzC})o8ueu&U%?3Ll<~TY%ow{AUFRY ztX$7l9>by+i2w#uW1E+$1??cO;4S(&sPN$TI)h&Y7JbWUTLsKD^m^FkE}j>x^mi5f z8oPZt&l~Gh@ zwpt}hU7z6eTt@93Dc``+fx+EfNlGDYOzFmXwU{)>7J+;D%AOvaKF&2QO40`}!vDrs zn{%2LKM^NmX7&8XU^EOvBH=1onyJBzs$kP<0t*^lV2GOv9P(bNv=VKNFNq2c^ES}J zYUO1XVCi`Xc*6267240-O1?ah1-2fP#yxlL%=LLv?UqQ#d_Rkf5XQ)O`JCfg18~t^ zRzUdoXfePPl2FsEHR7j3`#4Ij^>tj-f5yA*x} z_u|FrbF>Ahr9!!+v#RNl$aLe%$|}q=UY(X-<|Mgq81WSbc>kF<@yzOmegi()Q4$3+$?x?vln?v zrqk92F|ots*ZN@>X|I;gC`hM@-4}=7D!wV5)V~dv+N|;b1Ecv#!=-kNU@{6riitf> zQHe0E8?5M=;wyQbz6k>8*inD!@(kU4zRyH2`|KGBce#Kzf8SfmsS4JaG3ljrqUOxc zzIi>16DC5(WJ!ha>azf)aO5*WC@R-?oxz z`P3WAXSK|8_Q_P#=)t<>u#sJFvyEBLnn>e%p4kPr_+?_Z?D=_39b=OM`S_q*!-DJd z^t=75^?{^473J7lw}#*ZIro?=3SYTMRbpKsZllVaf zHo6YINVi1EG(pFF-}6*a(UATW*3Pmryjr_mxOFZ_^n@!{`AB7fshdsarPf!I(`UJ_ z%dhu6-Ymsn2upDtiJg;4m+1U~f=|(@bOom(HfhD&++4)AlR-6$Dcq;IYbE15t>$j} zdAzZMxQB5Z=;6mc+U&eSe8D!7d0Z_W2{JXrGSrSENnLtUHm2Qk*oB2(HC{8_iqCsB zCooZO&$qxrBLv5gsv^L zDmP%~A|0M5%Jj_r>?(?^7m0WAs>C-kMiq6auD=lwJLLEA%zm~%3aWz}2YP`pAY79}s^=T3Pf>&hj%SP!Y(F$T?|<0yRA2ivPs_m0 z+9}#?cG9p*t^niqexk%YEP>cID`Kj7QRf`-8O_V{6 z7gPrrlSYDj*gDXY?*tLB_4nIEA_1W6#Z|{`J_*-nQhOm5cUZg|?WV4g>Lx^>GyFXu z(si0bZ+SpDP2l8+?vG~auEgi0QW(yg$}a_n!KOywc=@L){GPe!ryoClJlcvy}r!8LSw>5xs@DhTo2 z>C5PDsj74+S*2;2B}H*f&uMPM1crBnYUl-%LF-qC&V?I9_V4Q2@X7^pGQXcZVA}1< zwly5CP^3JTco@rsoq!v(m){){^^lEsxBY-C&$)l`NLcEp4 zO`Hw)ncbksr0D=3X=EV<{1$P;Pm6JsD{c9^Bwr-Wx;p;#3+ZyTQ@{ z3rF!>8%&jO&66nI`lbvf3M!#De2ns0!_Ip7QDqgeG*b0VoiPsN!J$TW^0@cqPO5k& zs1Bc)|Aqw4CG&7szW_EB5yL-(F>Y&Xdtr3Gx(&KrRy+fbc$8l!-bH>_uwis6yFT(x zdh1meU2Zq>(>zZ+e`Zo7ixx5~xJS>s;foCB%hy!6SZ}=(-R&hB?c_SHFog63IXKa&~9*Ce$#Zc10T zi0W~3eJ4Ry=m8==FSik%9u_+{%*FK@{Rt_%if!iCj!Fko{Jr*Sh4_2_#lth*nwwJH z>|r%iL)T`xKJmji=lAi7JCrY882c{`721Z?Az4 zdUG%6A~$!>%V9G66l7n&if`4GtASbuKD}XYRx@p>{3|$DbFF7oLEPD?sHS>vx&6q{ ztYHLeyu@nN9dAp_nKjg>7pY@k6jxhIKc0JYHmV*$M-BVT{VnW#o7pBudz}L__u}bN#n|1}sXow6zfzHq8EwBQZ=sZ&l5QZoRcDviuIn zRPfX;fq`XmJ^I?TPR}&YI!PgmjBRrHS4sM{tWuJm+;=p5`{SOxh2yly9X|{1t=LhG z6a()~k{vrf+1Fd!PFlxZvIce_cFMHBUCu&g%IL(^l}uvaG@n*S>&PCY?ea}s)qpS_ zM=Kq0@zRU>z@I*W?2PGM`=`nzUdL2UU3eWY8QarL4?_B?jfFH0xbuT2tX5M+dd1b% zb!~k;(Cy8e?&aMYXHu-1aB3r?IY=cjemKz{L<);9H}E7LJy61%Z2W=}QR0rvqH(fW zPv`yWnA2BM6~?J-0?KeGyWbyWS{xGdHBObuWH!V)X1W>e_MekFY!cTf*tak7Jnpgu z0oZx|JnYqBNILdK1)d$-BEjjbLu4XsQYyzS=Z;O-ZINk0k>VE3noOU->RCDrrlb6T zv?DE;hL7DW3XL(fk;-=Gp0yVx^(oIxrD<%kdA)-5;SCR+k3AL^(K|0r{QMr+F`sRC zgN_-Ke2xV2hIo~3^;EacL-ic12dBjuva6!}G6ykH{Rg`tkQ9CL*3VTTV3C;np))~T{_ z3x!1!>=Vof4hqRHejKRjiE=c`l7V7?CVj0hK5g%Jpy@2>%AJ0P3&ZO@jwaHN3w{&p zsOq#14|ItlD|~W-xDWsraK-V1Op&D-&^ZxAtHkX;d@nPrVW>4M=70U>h^*-m1CiXf zVQPfp$x{`XV1+1IE?YHtig0JP`Gkp5;=?aAZbscFTqB6Lj{bbKPP>|BcOMOE)S20=A4LcS9Y4G05vj-YygT(Lo-*hw*HX_(jQ~k$p zXXmPk)DxTIs7^-=4B(1kl7S=vVO9hbM?qF1gXffeh&Y-07o_uozeJ5eaz&WrxmZGG zwXLm1-h&aR#1ZN-pMkE_3<*0;gL=ya83BGF?PNAf{y%BE^Q+|8+ekqd_4(fHUSMZ} zqM}$lt5=gbp<^2o%N`^}&~ru#`L!DyHx4sx2-Z2fqB}SRZI1!h9i6nXnHY=-O4%3LarfAygpLfsvH=16&74f(iQ14pa zdBWD$4OjU%a;wffMY6H`^2G3^pvOS%@!BnAK+!`F=7mWH+HF_ISYG)9os9QIHS{^!M$3C&_=t1iCKlbs$ZsFR?qhj~`zoD?B^ThM< z(SDA{F~U9hE&!vZulGduw-y5p!b(`o_fjk3!gJ!~86)d%P}N)(^@lsJ56EI7@Ba5nRV3CVA>g966| z{lWa%$6lXtLRjvsunB%?7aE3o|9QY{LMpUC=V7W@3I565b~_Ow;*YQDhTQ6vQW;L( z*_P9G>S&wVp4q_iXc7(mvuEU+e%GS_$abw3-RJs>chnj8N>sXXobqME)kJaH}lZc2{tGz->>>P z*hyWsF_o(cFcGv=nGhWO`NE96x3B&OOzo?9MY6)ID_m@h1xr88L` zc53MJwcM{)muf)T?}I0!DqujrTbdc0^q% zf0b>!KXD)vng z)+gF^FBR6s$7gNbonOJu!u3YzWAaVJFhsUA8feo0hUn_a!pk$tybMfD?ScEUV00M4lcJU+XWzztCd}z7m-f2;# zK-OTH$1*A_Y!u~6Uf+A-It2|BC{mTzCqsq@*vF(z%NfM{?95_RRTmmtWuhCq#~jEv zSE(sTFU_bv;Y7Yy6**E}SkLz~qmwuV1+`aKS838X&xvfkHPU&b@M6jWkb?VJ&Wy#o znbMl+FQlk>v@xnxzuUj{(txbs_toZ#WE$0H$WOcbJx{zV*~HQO`S9dWV%r%+Tn!JKn&W;~ zdXA$$K#vd2X}kE8C*nc~)Xa3eEHbYC{wVBDf&@A_p$NgB{l*@zwhdDEl&hI`r!{zA5CSx6U}HkoS2`w8u*AYWq4@ zBC}4G_i|-xYexlEVVrw3;bSGXCQ|@^B;i2;QP^;DB4t>id?o9lMOeq>WwVZsTE2AC z;GAD>$a}-9V$4zaj|N!BixfPTk6`?A)tb>=`CIV!H%K~(NkEqGi#v6tP!o0Z{f6VK z7HCpUW`6(f@4^pyxihn#yBX-#cc2%^_}${!EEibPaDV6sh$u)V02P-84xqqJZA=rN zW{H1Gt2x;U0$%QC+qp_I_qZ3$>RowuKTj=wRW;TU2Cr{Q_JIH%naGY$2AxRA89YBb zB3!6>&7^)SqOL8+4^#Jl_%j~~*YDrIe;fcT-}JZ)9%J6v8CBe;`4(lonPBMIi_y(C zbyePPU1G(sz{ORb85s^p1deD!D1+?u%3$isRVSUyJMH5rtk9tBOgHKNpYI)m`?FyZ zOC;d8d670Vq5a!X)iE%lLt?Rctg8;Qaa~=jWn?i~>8p$dnO;U&@Xy+Ke+Di3=;>O* znajWLJN1|xoE)KJs?)I|%vaX^xVbqeHM-PZ$R~4L(clRifCRv$V#qNx5=K24xPT0= z9C_nozw)z2)bYd4=jaZ~m~~cSj?()|+~>NLTsEvwO%`iGGEeji?#jt6rpT4|WEU8F za8m7!Us;rr*?yvkynA;kSW%)`gK0MU+qWOFR_w%Gx_nt+_`;X$wquNF-X8UO^ZlGy zSAlM}iG#G0!WN0fRfk%Nw6=nD8mR;4)Q+RlI0W8mPkShH?DZs}n5b*B9<2Wg4hx=+ z!;Nprp^!7mVcaK&tTBTYMR$dZQ~A;1enWHfrnQFq6Fxh5RdVWY58iPmu9Nh8uUFpb zi?sF8VpnmH>3w$t>@)=o;*7_8_UZlns941f(+2kl%DUf0&d;`}08aX*IK9yo*KB!T z`fo6O`s9w>E$Cf+xviY3u1rWw+2wu$e*-pP(qKJ5O(``1i^Xd8ka@(y107lxcv&XF zX&_kXh2j5@Uvp`y^A%Iuv*jksllpwE3tfDb74hO(u0Qm&gg58X4u`BH$pm2)j7{%Axn)|Vt4!;sM!O`fgqtbY7n&{^Ut8h*cES~KCi&*xG+CseovTyZJmYpc ztZ6%9K5ib@>+WY^e455`*rxa1ebG?eDboLn_Slhgq(A@_nc;M5$#%NaN#KBh`w<%`&rCPkbu~_>d`wyLjNUCGu|CtP2TLmQUNMa998@h~l^|{?1gS&==k64^@FWW%-^wXI58Vz=6?zWj>9A zRIZJ`-Ed+~OC+<}TmVhO(}FZe)6U<@dwWzyjSSV$?}L~G4iE42Q^@=|S#^I&LVqZ~1SaRMR1^n`jPHhZ;pSO; z#_GEq<6BdPTBUI`I)z91NiGF{^Rpx}MUwqf@;M&fP`btBPck?GG<;#MN~|@NN)Yd! zc&GVXe_p>pD>L)M`x;UJoPCQu*qACeQ4+XS7f@Nc6(jgbc0~@S?dl#dM)x1f03dJc zxh{G=J-vF4<`+~>@YS`FvgDRXH0iHb=@{YR1!4BXN3(;)RSoCsGk5zvONEsD)CIb7 z#l%BwNTKnvw>HUJXzJpM+N-}LIs~?CD|`RaC8$5j87V9DS#p>cpFDgXykZr7n0C6G zVP;O9^m8hXezqqHJfryq*mWalmmy*f9UUE#N}#B{K;ZF0*nr;tyD~0abnW(IISSU5 zHnn>+0M%ffhe|BeE%z_MAmtUc?Louwdfv8qTORh->>DyRsxpUY{t~_ylIKlndXMa7jfJ$?(t^6;zz>$t-yG?POw7QUgcxCASkx zFQ!QEk2ip!A6}7YkKG=6I97u&5ZT#2D;xE5jz6- z4zriTjUCht$q#8YaRirp=_WJ*dV4EI0y6KHJd9pGD(fq$<6DyZKql~+=$-)@ zs-58u#<8=GS`_KxCECyHW%Pc}jqk2Dv{?PY1y#$)kKQ`kPI_u`1#?zZ+e@MRLt-tAYU>SnwR(0{Gt*|y^zcGf z^{w?mu!8qEH>V09Oy+*pGpS?;v)R~w+({=!N+4uF8K?cTt7dOs1UsG75_=#$L)7iO zzgQTM0G)Dej|=kJr0Ni1kXXhNLRsK6v&Cpys7vl^)<#$=NQYthh0-8#ZEQv{y_}Rx zblp=!4mFWbc&zGPzQS;doK%IbV%&(9QysfB?uTy!d^m%spag6HkD$I5CE(P+Azi-m z{j7TK3sKiY*zZNAWgrDd^4gSSl*c*WJ3B;Wyo1YYmSi@~bhy&2e>KmBZc{GTrrp9~ zZNK^_i3IOGld!b_&FtH!8%qD*ZG?PiBP7%nDgHFsQ#SqbM>mnh>wg_NNgOWQH_L zU_77`b~kKbsfY32y?a90TBR6d9kH(NacEL1Tye2M-=N!skF>7P0LG9o3ol zMtBIsGA}OhGsv!x-UE76<{-|luw27fYX|lL)_nPx`hk?U-l`@>pmfMkAHTZnXj@xA zWb>ak#5{lzTwLA0`$SOCSHXO5!@vc*T02nTQo3MUig?n)P1k_MycKIB!JGd-%ztfE zi`MV^Hm)w}i`yg0eMc(K@bTOmX&a*wM;aaH%?HM@P0*?!Hr*GNX5h9^n|>_S`38D# zuD>3+zIyMGb~qhPzv7ndhDS1G9pJJ~joSmN|vF}a#}1v^cgQu=2u+Ayl%fW43UxyHcmWwMID|G zETWFMuplakHzsky^+ec+L&-^#Hz?<(uzQlt5;khnsyCn9>r7H(W2Y6DXcbVM8Fj)_ z875QhtP9VivaYMkHEJvKk!)_V`a^s&BOPid*oA@g47;TqG_bbCAsw2e_WLDB7-iE) zp=O5dM!P{XfW-U6#6Y6y}5`BEBu)c5>NjxZ+B58U(vYQbEvLdvaKzJDzHhH zsod8_qDt7e-tOMWOv9&Wk!1GG9l~&3J98>r+pg#`a9oc;{KT|$k!x|ZQYSVxmh9e( z_Z2(8&W8Ab&}SCx=(9)wc$u9vygd0N*5-S_Iiz5eyR%_5AiMZ-Km(mk?2^e!?5t+2 zRiN*|5F=wEW&8gyb(LXJZqa(qu~9??L_okJqBJ5P(kKD~(j|S6?(P^71w}%JlJ1UC zxZK`fb;<%71c0K{kMA?HZGkCtvi^+8`=o^bwFwwR_TD^s{H4;uC3wXz#E!yDu3>{%(3V&I0SWY@vq z8Tq@wBBN3WNEt&*?>~4Tjv}c=Tv=84DXELyJONp<=BTcLKo$E}g>=ycCV@LCrPUFA zH%U?k=Zqwg?ko%Ti-V2ww)!!4s@ZQkN-Hc_#wIfJ=)jVku1UPxO=Ey2a9tk$Qcb*y zfc?DON3&w9304Wv_IVs4{x?~ABXOP=Ox)0$0WOm>caxKx7s+N8$03`n5~Pq6mCX0vh+og@To^x;dq7j{C=gk;yzSLDfQ8ld^nYMr@V{3OIbx~+Jrj9BE z1r#?1TEVUKHC5)Eqv54=?3A%@$qg=3#W8jET5`) z;B9PALr0tdAMy#%NgbyWU)wA|mPJeZ1l05c!Zkb2{3wM&C4anSAH8zY-O+$_RMXqT zGC0m&Ay4f0lJ72kGSm@c1H~ir7DWa$cdruS>Ed_#WRAYhrJUl#i60;^e)MlG0_L-a zhK69BQZ?uhZ6RQ+dkEIhdxBIqWJN*WF+h@VZm?z9%uVRo3}@A3ODRUlt{{XXZ_blt zkZT#z*E$|gm@B6J${|nQ8nqt)4O?K^U@!oWi=lx*@wMto31=yki3VDB3k&sS6kY!u zN%K1U@dZPxIH>@Mib3ux%%M0f>Iz>kF^-REkd`|Z=`T!p zW*4SPK?_Yb2E1(Dad#ss)nH`1vj(Z6?ihz>`-LUotM}z;X=oIi%TH8$7!(ZIua5FW zrZMzzI$)+Hz+B9^_=lyS-*3A|Rwz@w$dr1OqB`d^u{TL|o|7`cv1^5Si0ndjCnzn_ z>IWNEzA%K$sD4YL6B2as!vs>REnP&15T7~1c`}>(=HUf`XId?cNwqA@AH*f zU%yHtn8Q|cs6feWsbp)L4{B-PP*pI;8C2m)1yFG9Z_>QIA@$;g9PlFdBtWCBG8ir; zFe*P}Z8^qESq@W(5wn|cwimu~WC#ZA8Cn^Dgc(++gtL$AR`w4lmx^c^+E7CMD+B?hWgl^Z^M>vBk$I^2^2ATk zOvUs-YPV%}*%B|?JRqRBJjf)=YsR0(_6J?AmfLn%HFe7#FPd5N)aS zi>icadvi1U9S66$T=O9|%YDMba*cA`G12QditlRxEOBa&X=&@w(J9lBetp;epzj;? zG7bk-64U&3VY29D#TaG+lCHHCqbh5U$24~hrHTbjsssnOpMG=bYX}TO`a7_gVsauv z3#@dYOG=6gd$uih(vK-24|o6Bpi#ua-iPPt-|T`McCA^Y3rzmr9=?F6jRueQZKWYg z-Iz=2xcgT`A-4e}Tr5_JH_7<8puZjpoz6jW7iPzKWcl3y-BM zHW|r%l*giAsPUNCA1r2bvxS`JRGsXRorI$!$K}Zb(RZgfWg-Sph>66QX;hsvpOO?LZ_Y}k<+C`Q(H|d@$`)MvKq@jgO)po zMm=7hxv)d#we{1fo*STau6yBuS)49{F z%$Ob`Bn3YzM*=^ql1Q9)j8jIk=kK4JNX{{YGdBb$2WN|`uR5`0yp9(nsAw;L_GGGJ z&0%LMXsr9e#Tz24JM!-n%+q zBD2A-x^Rs4j~_pj%4`il@3RbB<@#Gd;g4ZrA|oT=%$qB9pLK6#0lcgcccGsCtHLWB zmo>+k(<@gue6>J7y+D=;By(T(*sG~}OkX^A)7isuKew#eKgdZu`b|f9jiT1JrD6|? zN+1Ll5ioT8K7}K4#h@&I0QkR1z!W7$XuC>OIIXD|8m57sp+eB{9&n5O8SpvXpl(|7 z$Ytu!AN%#m?+Pbn^ksn9IfB{hH*AX|Dd_Uv`PWUJFZQ0&tR8n71h8-_ePgROJ`x7a zw0g8+&B1Z2c`AR!19CEL)FoBDFVDDXfaXaqgU<3PnTk7y>PV$a!D`tYHJF}}`RqMG z6t6=TZ0mva9+&`xy0cjCH1Xr)BI6FIM9%*Xc;^=Vw(RUFv zHG)$CrUIh)naSqx!}5MblozBLQB#dwjmorPOV%C{ z`n@7VjD;2&QsMhpg;bC2Fr4TVefYPEDr$G4xH)(#|?!(%~A0=slj zDHmwm<*0b=>9@Xs%uA+8PZ}zP$8MG$T}v?hiQTCEc6~!bHW=drro&Cso$zZ`f}%6r z%gHW*lqD|b=a|0S4=P~*fbsP7i{%k1{#!!=b^oQ3wgL*CrLV7uaS>Zn^i|(6JyV>_ zvb&2T@-*z=4+0HC{h;3}{NxCXLD8AkGk+Wn+dL$A-F9+7hSo-+!)0$>x4yn!0yXZw zDShtzdF}3wx>N@HA6##63R<3M;gj{W{(QLkaUfPbqO{#WS~!5h?|8zjbb_a++unSd zKJuzB?i@NAeVw#g=&$-l?K9B-HaC8&z}M4Fz$t)c02QH9zJ7gJWYY6%W(rWQQ71C(ULg)?Qlz!NWEm*P|t+V;Pfu$wi^zuX~@ za0sq)WlS=Xl?um#nQ)uqZk7wg>jrgJra4Q|N(K6`G<@DyW7|7`uwjp{rO!_t+Z#-RZmN%K+6qVbvQj?-%*M&|EabED zTE-me=`pKeVpKzmJSyPd9v0u$xh%-im*IL&>UbrP*mYq<;Q#-w0tmBZVtMH#vcTm2 zZ$Y^r0S7D!4lGt>%t=W|bOCu`k%(f~^T|@lb(I%t2m`s=*EOh8otzhj+FoG>xNO1b z{6I=P@4&oylgfj<{d)p>l+>?#BbZ+8H3BVp4EVXe@gzDZxYQJm2XR~1DX+|RKGh-No1z zzPbFo#VNQns*DaSn?7T3xb6*~N3DS=%lL4mUr3h6F&#Q%Vv`|!=DoHAc0K_!)qWz# z&Als26CV;nD*;BlmnFcNGF_l~01)Olg21@z&kqWgM9=|v^8l)M*^En-)^FlGD5D9L z3%5q(VH%H)59RqB_GaY&x_Cut`7&GMUY(hw1n*;4|K)Vqid5O!2ts2Waey5ynq*RD zK#`)e!5q&ihpq{BIsFd%QCGKF!`2ivX*cJ|_gfCNU5DfZv!cQH6Ae&H-2L+-Fww%0Xa1pesG=y(o*{5WccRbYJ4rYPg}(Z8elXIcu)PaZ{ZJ>6c)+|Pgk;C>;< zC5D+q;Z$YHjZ4*czn^qMBPgyfL)PJ;2pT1JhfYF!$p<$I3aG()~?U zkW!wJf5RfofsW1uJSP+F+M(~p{gWqZELp{JAo(QPuUt7tZE*NilAK9p+K8M#8dRXc8Q} zn^A6CkD{2+t@J@t#Nny|4}E{DTQB{6ZvWiPEcPoR?27l7YN?Q6Q@xpo{8;Q9>2O}| zTT1R~=BhK*NeZtLj@PP-%a^ttnj3j)0%ox%%L;X-r1l~Oi~k+fH|)o{IxuzNAT!nt zCJ99Zfhu1bOwC$vX2{6F4}(x6Ee2+o9g)i$(_Awg>sx1)D&B0BX&dJpF$h2^g#x0h zKpzA=H{!|f8>xijq&s=>cT`V$>Xj*X!>DdBzs{`VqkRV#E)Qu5O7peh^hvYh-8`9i z-dGv9?FOX;(gqG|<414f>NwkkjGOP}@8NTEdziULp5M zM3Oj%5E{grl*Hk|=S9MceU}HS5`?22-medG>4n8aE(2}n5Fs7Si3G*cUE-1lvy{-> zl#7@5<0(&Cqet8GKb&WHVor_mD4gXq9K&M=TjDCl@&ExT57U7`V5T%Flgd1Ji~bMS zY41sZu)h1NZXN$9i=@VHFI)}do8D4eJy`rF6^;r?>dF>x`E5_4rgs@=c@CMuJ0Eon zX!;0sTK)p*tT2G-JqMJ_%T+|Bj;FjhqDsvxPRX8Kh!6S!ec%DOdA|=ItO)N_ba52`|iyv+Zqsm_2bG#G72<1pw;N17M-Z5gZi_TK~Y% zpp?!|Ipv&J@?eKVUuz^sk;|qbmvLupk3~`_;Qy5-17z#sAsm?G_R%#~NSDZ>Rv$yv z6wh$M-j{qow7wmhfiP$((g)D>ktE2w+C+zQ zfAO?!I*I-AsW;tN`9ymCmb`udt$-G+(gvup!}oDCD3VR@d~7i~_GmIhZ~~Q)ep5|o zS&p4kIG$)Bg;!n|FA1x)2SfBhPks~B@{zuJ!2Auyu_XOzu9O1Zf38b8E|0QHzv6zH zC+N0FMx8fErt|smu-zAg$-YDpv|N3$D}M_v1%`8RrW}xlFPj_2;l$mD%+rDh#$vFV z3xU=jUU@Hq1W_3x;p=yn^V!l4OHI92M9A!*UC-tG`-RF(${v4gqzr|^Q&KkaE)*gB zxR==iwj&JtxhC8~IJUNuSZtqQ?O^Lmts4w-!djDLO3%aoQN(*#HjvzM1(XS>VS|(U`iXh7pU!jIk2;E40xlYgHmv(Bz*Qj#8 zm?*MDMuw&S)a_`F44DcS{7XkQ+cpE5`s2rSsG!4XRSQAXDb%0*ga7+S@2U`70Ofb! zt*l;+=$a~&eWW{ZKbhNipO|Ac`vJOgNe_0uK~kQtg7Il_hbLZYovve9a0) zqK=?}jb8K97>^79LikFTi8N>CoUoLD^3$dR!6TcahkW93Vz~RFPv$VPe>-Y&kye~B zL-&kS_#><8Zq+<{@o3?)&b8s;aX(JGw zspw5FwPW@=EAGfS`(@8*NNBAKqi30OYt{y_AR;eCl=N1JR#9#j>q7=)G>fP7}`>=>Cd7x5O6r2VUvj3JktK}CO`!TxQUy$N-2(y zsTV<4jdS+(6|oGP)L-GtJs$C<3&*LfW{f)v$pqY*R!!}>8t9&r-9QbnPS4vf(QCiv zV3?7=aX-p)$H{HGo~_{&&4k_t^oDk3Mi^j(5oC70=WgPR# zVjl~(iN`Lq%p?R#m6o^ZZ`UHE*$~0#jYrGLxx=noIYv=1EBNLHBnO!yT_E^GMC4iO zYXz)597n;T{bgTW)-}6rw35U(Hh#9DS$yFFZBUs1RcFTPBd|zRjgk<`?l*o3=_OT2*@8@Pz%kt@D$1m1{QMSMD+CG8TbeHzc;HMSIEX&N&LrKaKi7R)Yji+;}|aM z1=s$y%o?u4Xu6T;{F1=~O(y%*l#K7*ENa@%-lBfh+r%?jU!qXTP5=iDk8k@rT_`G(Q{BL^us{9??JpA*xDS@xioHF+k9py-T&*dNZGf(v1X%rWpy2 z0w%9-48Ao#9NUcs89QK*Evxe8Q!6OO4VrdRmKk;g2iB>_l(94?UqO59)rGyEA4lg1 zG~ePN@_^WO_B%Dq%t+8us@3l4&FvAwV>K0$fvu6CmK+gUEkEF|;O$4qJoLa#sH^)( zN~>SV%`oFFy1W@z>)T5wg}b@uKIi98R?V_M{JUg!0t(oBjG~E8N#KXT6{P~t353x1 z+R3%HMc(XAkGzlp+5WiDB-uW4FR!c(j8y)6Uk%JuNIJdIns&)HLFk2T(8>quBZJ}d z(h6clqXYkY-WfQi3Cvih%KLK_Jn!WFZKi%z&>1WhzS7vw!4xkqW+@iT<;7*jXxK~w zXh9<;-FZ0}=JqqmQQMo!i2|*$Y_eUEA&etK|13DfLTh1-{XZ+bj>ECm@p0R3i*dhp zU**@C_>z{ew$F^^T<0FkkUZK~%OCYPaByE#D%rjiN(PDj1K!?m`kqrLQi=v5S8buCh)K0^O(vtreEkz92+iGt+!d?EqVe! z&Q!wo0fh52&`QNw)nYGY$i3p~I8(qI>rmbpmEgKfB4Cq5XJ=^0xM8wtcaBwA|HYf} zDs(L1Y-@-rzuDIOUo~YjB~GDD*q&I=z>uJnKQG%Q_I$}L^CnaQiVFkA1oW(DHM<>6 z?GC#c$$Yu-?4mBgqe~YVL)0&jkQPxN=UT*mR=fXHSoG=b@0ZS=Rg%*e(*QU&OIc2- zH|ymiTKYTx=*rxB^uX=a30YkCroL}n*79=B^4eUvtAte?si%i78_Mk@ZoA$CbKx*d zQ1M`YB5Q}n@?m~{ET4lhM(>|!;twQk3O6`QI$SWN%(U_A{Wo(vcyFdQb0p<&&3+04E?mkdWjdZhwTCd*LT#T9(pf&VCA0iUIE9vo8Dm zbDNU!Hud$HT{&vs*X&|BB~tku-%RU}zT+V;ALL(q{ z{Nu;xHiGAV9|wy2QggwejzE!f6q}3Zv1c5=q(ME9>ya=0+d3tIP&E%qm~JOSQON3| zdqnU|#N5(q$Er$MO=Y){sg_&r;<#&ea2VJlGWqj@{s_slTJ&$5imhdqxy78`=d>q# z6|_<1G19j1w24mva1LH3ShJpiX6hQN$M8;9m^7L*-@;{t(_y8zM=->7qRLNT?lvvc zt7^@L2$sw6LNiGzQ_mgoX3JD!1%7K6+v3rojxd-e zJ}DQSSZ=!9qXZ9X$_THO#6I;QnDT+R<9{QWnCmEi!Ox+A0-J(uunknO#-Vamt>E3?6trENV+`O_;UhF5xTq=64qIS6le zXIU}{IDhvRQ~ATxHQ4bht5L(HEX01uK$4obx6%Cq(PC|>o56*@|A{93K*H^4k~G3* zX{rCfuZp73Iah{6^*GJ|9(AvbUh}Ufithk?PzYI7+Nn-V< zIr=*~hegboJbo$GRD(~mmK_h27#HUr?fQr8t_RBnUVi@W-L0UTTWdMJDYs~m7FxyH z`>$nBFIQ0xUT(#Yupe-M{hP&mv5^4?*dsvG8z@E3BS|pSP$Mn-pM2^PT;^vbo*d+euBf8^V733Ed z#5~QvKO@c2B+uvZdeNQjAu=|ST@)PVd_+;GpL;cynGi3qG5$YD-_V}3I(;rr&2TAM z&BJ#T54gZ!8>F+MmHUZUYDei(uY9sv!P{d`yMGVQh8Y?80_%dhQd9W;*xA!3n5vzv zXEN1HS#y=I4cFr$$AdF~8v=&7J2lNxv1e&pW>>U{dqRtRy-hr9YcG}B@zcQ=-3t~8 zGRw&&#B?Q-(i@y1+A9KcBByGW5-i)6$g%xLSYTYl&i(E98|0cBuB-$QqUjpgnI(8L6XS)9oDQpesKNxSm5XRdLO6ja+b z#T)+a^*eP@`R?MUHw(WT)38S5$4ms?4UsXPXW|*KA2&A8c$JK%zqF&FiR@zakKTM{ zt{8rauE%2OQJ^NW1@7bCYjlw8NdJ}3Aa8MTnoSWWv&HWB zaGQu)251vxQr~C$g7rX%$tz7ZO|G-~MkF(jgdXm!s}e08Oiifnzs%KE%^pwi5}W07 ze|4}{UeR4c`PUxSa=H7t2wSH0d#8`8-r+etAOB!ZKYIy&Hc(4@2(M&_xp}!7@B5C? z#`<*CHv+Ks{+O0c;AU(hUxqJxX>F zFS=QLK*-Y0NS6Iq3(!85oq`*}jd;>Zm}}Wd$YS*{fBCJs^xo^Ye>+-#({tfB<|hsI-?=$dOqJFI-%rU1CviLH$@1XV zcf^yQ=s**9(*ipUUNZ2YJoVk;ZiQQV|+U0TMjqP*R2Ge2`)7P+?HER|5)j zCtsReGKq3r;4EV)7>r!6KWi{!q`dz!R_I8+$ZD@DIBab!+ne~l^$dfxPob5P28Y$+ zy_S7d!;$VU1CNdE+CQ!nLPj@XI580cBp6MS=bTCnEL)Gg#8Og4dreCtLFKvf%@982 zSJah7rfT8cVA$lhmEp(^_pNUj?I@aVa9~@93eK^C?111f*E7KI#O}n?|91(L@DlXg z+5k`>20z60-NI-h%`8ewPv1>uwd)<*efobdylt;|{W_H0Eqv5r@fNl#l0$i`PZew$ z`*2T$t*JL4mXdO)>+K^#%|exPspCETP=xQGpgoTP_yc?&A$JBic5zns=ebM8k1-v# zf<2p*@#w9{46%SvL60bZKG`#W0ef!gvb#n!Q!4!PE0`&EqT{ADcz3ALiJG#{Z9$gh98rEj{V4EswjRQeY?Md(;PKJZ*H$8O}{gy zJ=t9Ja9#K$aEfmngrS9an}=J8jOcx+)m=VcowrM__7VNu$|%6%JMisX4sE@X@gGfw z6c&Pn4E2i#vSZ|(n%(wZG#Z|}+g3zHVS31A;h)n=*0~}zz;N)&=Jc?vcRreIRoh;1t z%ihmWlmhe=e%2}x8sH2ZcnVz6gErxLU}z~x+bX*T`zk;e?%Pwt*7_o;9Vz1;5jR*X zYa1oGE&3D1A2b!OKK|_+OzF!uD(M9%GnrIw5Zv5m&R# zUu62(c=}{8Ed9c9c^J_X_}S}kf#h!PF>2$ctRXV6c~&-6`z1?O`Yx$1py9+gY+us5 z-EFa7`PXJgi(e2orcPEMHOWfF*H-CHQmq!&MTS5j=u znPq-J?VJ2;kXo8&sp6W2;I&gzMvAix7c5G}d`)fui<33uSjM%D$}~lR<>3<(o7Lgw z8VU;>MS0bdO?AHHktaVd&G_HW6t=N}(jeXN%Yk$Amz{x{q36I-bfMsqv$GE_GNHqH zp_N>@(tL=Y?!H+0GVHp8F$%jD_V09u`+ zJn?odlI{6vHbS9R#+EP=k=IkrFE|NrwpR&CSo)u^A1X>pU+x1Cd@-n%H_~M13dFay`lRUw(HkK_t(V4{> z=Kz_TZnB%W+~|X3h#sD*5)|ML3~FiEdNS<9$<@3{gp&oyQ$q|Mc(+#&6Nhaa+S$_4 zkz>1z;^cp4DdWTejQk8<8d>O8*$!~;=(*3F?K%WazQtW|eCr&XR0Ao|V2!0(#g!oA%=DdfzB_f0}9F`^-T<(6i&C$p2>QTc|FsU7#UT{9Ph zsLb@e-u^u^%C2zy_AS(lsFXo^mgNc2ObK?eviOTu^Qq;qwc3qlmnJov?3|My*Y|u-}FTF=^R*JRR^ZB zyo`P#F}`Z{^p)mv`el}97iwo!W3n;DKepP7kO!7yIuqswug?+ zI;3B?|59$DJr;@r=NVShG*2>4uKYuV__^Yjr#JLG`HdRoMgotVXAVX`f z+>`CkTyaN3Pl%EUJwa(Nn5d!3bxUZxaT#^P0Ia&yv9yFY@XH_?ZUSS{w7A%PC(+ALhh5{_>MPp#V#_GO z4lMtCC5{phV3Kte0miqTuQ^&s2)2wCYo5br^0_rAfd`ugQ+mN)3(`qfq;Xeg*AAC8{!E z^INV)c2{9K9RPR6Bf~&ER9)EGzqH#J0h2CPYEQ!>_$5iCfl)STEQz?iSJOpfS^*@vgDP~c0BpG z=LDU_Rh*Tq(LQlJ+V)jJKFpsoXyAH1b1{~gj~ogFwPe+tC2hf{PhkRou3`hJ3E>s+ zFmXw2?mMa!jvRh#n?jz*(3tbw!mCBhRfrwFdCdA`&wwBq5>g=a|7l6=AipmvRt-2P zF(j}0%rPdxXp!)C#_-A733=q&5#I#(yt6dSmm@R5$NcSKH4P?_lIhee?-*Izw*uV$hV=j0|Mr2J)iH&yx3y`@UD zNUIJvK=8_Rx~SS0K8lOY&DgT1?LdOnGgVL6pq()OX~1G{H*jCfv2rl`uTTB-=+F4a z<4pjMM~@>4FkhjWST{?dOIavmio*$;&fwPD@u6+-VOiV!N3-o=OoHQ$LG3Y_A!{|b zNJ~#G{QjHKBI7Ose*YNgQOO*AqXjRl;VxR&NjAFX;pK(PR)g zabFPAN2~5(?a+xG?9YW!k-nOkr}d8`#^aq4C>az%BdK$iJhCg@X5?5Oj|ZIleT0|` zrR~rwU3>tI=AX-~xcTRZT5=nSUvyEO4sV08<~YDwA-jB)H@-L7S7EJcpm7;vJL6v} zGNY!3vu&5ysiDUYSo{(_-%>)=pS)%{Mb!*GD|`zwo}rRxeDpHuUzXV>z3AqsL(8;& zb3QQ4iN53>_q==6@sZNpY_xwX^3S_`-z#BjdmA`WK40{@$XJ++u0D!}S8LxPT>Dkg z1H35XF1a`Y!U)8(mDP5qY@?7kvEmx*8nN8lsW)^!25W?;7JeiSvu>&v(~b*Ejge0# zp=Tq)Rtn!U>ANGbX}OeRT!N*IWuRH@)R>rkXTk`MdnX@%TyD)WhY&M7& zKX-p`ESa_F1JP_F%{pb-%tl6+zY$3f*#8=lqk;A&rkTuAoV-$zAex_clBp->xkeei zT~8!l_Yf>%KwKMw&qjdLJ7wB-;`E`24wdAeVq(>{6OwJOglOk0`PiQZ=zL+4HO&g1 zt2xNCjrU-MQZ~OIxZpZjUMx0yd61&mysCIrh#aSNZyKKi+b(Zz1*PA<=lb7~tKVsU&%fijgxIOx+g`H6#TkGm}( zCu0=-*QWu-{|&e9Lh?);sx;HUC19N2sTjt-Y^yX+<7eGm(~`||QHVD{-n17iw$L!( z=Mb@toUo*JU;l?RSDUyQ(CYRlMHbs5b`)T_)7wYe8l_?le$Kv+z(vTB!i$i4&-A`b z2AucKR{7)QEOA!npN)Sz3Qws7^rR(6ziod2a3ux*!>%h;+B5zXsrh3SB2O5uxoEOV z^_=v|k8uD#F>drOp?1Csu<$+nm%xW7UqX70z8DSIU!d3sgmTiZS`}C2lr=5WHh>e& z!TTH2K~LwF#%w<=ZZ8?>dmXwP5K-ON(bmR+K>vnZGOMMW$0oDe6BDa^iSo#yJMlu- zaMaP8MhLCs&Ed8m#1BK~hTn^0NhQIt^n8iGX^2!UB`iC!y>1N*3(aw2vu#cC?H~9^ z=vi=9Gn0xewf;;wW0hl0F!FFrSKzS3BHx1VWeE5Zf{O-TGKjgFA%TR*3IsT8?}x90 zeDWuMcjRx+fIa7TewCuLE+rZZis_8SA{S`ch6*JvtWvV^)%H_a-%sre?EB-TrOecS zD7X;TcfdKvk+%u`-80Bl}5a=_Olf=`IXAi%UTv2jC5M&3pldRsr1|wyP>KX zL($=)4LN4*ukK}m?O(2AT)bZNWIaXA?k*)62VrUB(<9b;k?0nDm?+!^*-Ckwxx91u zC;89zA!$dpje;KS$^PVk>wLEF+dS6oUNrg_TlF2)*Ga*1Z8`~4=qhU5X8}loY&dgE zYeP+uX#5KQjkrW`*f$iAxlSIeCV0<;(-7U@+YvcruGI;y-?se^LH@>MwuHmGy#b6F z^dBlnAwVMez|Lv{)m#F|LzEYbhzM+r6a=)=eC7HRW>rKtcgnJx-iMx9!KO}G9w!pN zyHiw%1(5Zdew#nZu|IMzZ#O%k&&JA@u0P{)-@4ARbr8m3m?K$iPWU@TSf9dL-j1;9 zUQeT6%Mry_=YX9VQeT{nNSKA9#$5eh4(rWV)A(-f9=2H4l9|UC2D3Ed`f*YqX<#}t z%J(YQql%R0L!f@2KYCZ{Z=+$4s^JW$2b)yQfQVQuF%G~GwBSLYTmg)YNGe|cUc@=k zizS~pSo`b9_|5f;al~gol{%Sy_3@XktZkhE%19wL*&%PMh*L{R0jSR&FjdxtL1yZW>2D_%WI7IDx+G{idAg=MQcW04hZ`^) zeQO{=K6To3R|&WEAyiFBxi<|k{Jr6+?#mwSsv|x{{_cu9mjd=T&tnfyL(twU(3R3T z*M-4B2?!-Jk#_PVq2m4IVA4=B#-9OkwFK=0$&{Vee*YBA5! zy{|qihBbuNhBQ$Sily%Zg>U>KIPT6DsuSkAJf5dxYi{IG(Ga zw>m!j+;RDNRDnVW;DwFqom*)xwW(vhwU6&J;hj@EPE7!17l@;p6Usks{#<})6J{md z$9_9R0Tvf~X4c_5$Q(0Hl^J;pzI3wRzY^=dI|hk_nD(|skU|4nDOwM|I~6akXTt!tB647df0XoqiYZMVKs0 zU2$k7qkXv7EO|)4kZG9IVtTWrbpT}{1_Peh-AAFhmQZo&9@w!L;8Zm7FV18s{llCmf{6{OS;vrV4`JBr)`O>z6 z8L*)b96EB0rVj#1vrm$^80oj;W??vofaIF*e_SAyC!4Z}dc`@>|q^Ef0V}R`vh#VbYLbZBDxY~zV3$1__!h4avQz<6J zC%_RgDaeCkib2nW;IIqLT#F&{!*L~JjbjoLrjOP{MGf?Cxoiwf%unuZotS3~()jzzM^5?Pg_ng4&im6Pqbgu{b znyoceiLGYp=t$peji`uo=`%T=5bFH>ATHEG5%HIif~i7IIN@Se=ubdvAB2mz?pbMI zovy+!!?dc=h6)fk9AohWdcUs+-it#i>A^Hi0%`xy5OyUdiHK2;6=qU&L_biiEOT%= zoQVkIUQ5NPUs}^8b^e6x8+xrorPYURfekRk67U{c;orRTv$>hw?oz*I)>WaC|hvQ_8~uRPHNvZsD+ott!H4K86gvXEi9o-U@l87 z*HEmQ$Eo0#JFnQxu`MmyVWr5_`s8Sq?9$mnWa}%^Ts7i9CSHwp7QQ}Ev|+Y+zb|0l z)k*vGI>_L4+^EDG&`Em-y`{y}Y;6VgXHmgOk99McRXhBvCU~GCHwf>sDm8iP#52J9 zEZjwplympRsuprWB1icw~KtRo!IEtpybGeGhH-MV}#Vd=jd zt@H3v%ar_np^0BjLPhYwRqEdAr`|Jsj?B;%~l(-msc#vb8tX49BK-A&2Py7v-+w zLaL{fd3b-Qr#11%RiJalE5W^cAY|(}Za1J+u_P~w`3o9`!fe_FVh0W5q+7c$5H&KU zxvy9#2c;Q4c8MMTE*2?2L2LLsu|igdiwY;ztoVQ`;eN%pwt(`b!wP?}%=V*>Oz(pO zL3Rv`Pt;0T# z(fjX~H_jK*BqYvV;esYrAbYh-^+NR{%)DV+?{un*E!}+zY*WcZ|L+I!Ne~n41elt6 z*0lTWd=YvbwTq>0(<_D5E=+3fLwT_wIXT$sUv z4wWgg*=E0w?3k&R@P+y1p?~55I|~bc4q4`!RzRA6(U)(#c`@fyo~huw>5ddD>DDRZ z!07ew`OIS@GumUFnDex38(}aOHr5z_OdGT|sup0_c4hzI_4p`RC^ym*N<~|k5qD=V zWjk<1&TiS`Bkp&u5W^Tk*%zGGPHE6G3=hgt03BVZ5Xvx%($SlKKmLsH$T-81dckN1{aiOwz)Bn)8NuE$$@CDcnC8mwm5lS89X+FH`^=2oC zGW~~g_A`(gkp~nO?Ldj*n#_V&oX(=+#Cx;QFjW$KK-dXYZwbKw-UMZawOv6Io zV{sX5Z}K$3SH`sS<8H^z1tKxT%S-Z?pxqm$E8)PL5xu}^Tz`j(LC}`5fLSVYcQl!o zdqO{^sHHiS!@X4Uu|*at((A?b8OH^zM!RB!p+kWC#JAUD#s3D#kVN3|^kggK7HsnNm9M|rO4-U9fJk8F`vv8ST_m~Dnfm%+r-N&NY?i0v zI1_xAbvPG$RGwi0(a5H?*Lha$>_3=EqJEf~lfmD5g~BUXFN6?Q0GYWcr#}8z;QSS(AyAw3h~pZAX@jWALT#RD_-(6c z7>y^k^k=uu5#c~uZ6xG!*#Va;E9l0b&&oT?{bD*6%e0qh~)jME|wQW z-oYcQ8fqZ;4H>UWZDGR%f>H9LRM%nlEE6qzU8B?Okwkz{+K>kxLKJ z$JzNdn=C{3nF#GfT#1&5iW0rn)Spmxr#zSdY2Uv}qoQ(hj9Hz~flrZS;C+vUx>I?h z%aRtU=Mi|nszj?;dI4~bh&dRf-YSlO;U419sGGQ_XmatKWPmIe23A_y*UKf_J3P3b zu_3idGLqF_QcPNi4-c*CEVSup^m^_Oh#@HoA84%<0|?JoR~8 zsJ_Po>dX3nWRo#+wKHo$f(#(5%!JBGrCAV;g<#nj(kwKla=7_4z@>cNZ4AbJ8$uwS zl9&fUEsropLPM!!|D~a8*>K_m-u4GBPDLEjMD?3yol|9#J1X55tEO}MM6Oz|Axq!zj=~myg_q&zphWmk*&^^3^I?5zzscc^?45(> z_U))tc!lQ!_G-AY$6j7WYrisCP`Byn!+AILjQQ4=Q|8#bv0?6i6WkkuRGV6bQ})n+ zhWq10^%iqsd_1d!d;O{b_~Mf@P(Z-kAb7OJ&R+;xF3TFkg_3Zo z6~2&{j=p8o;<&1`lkgogH+Q)<(hd;Eu9Li*yjEZd?D3kk`nLcldf+Y-_F*FP*8im( z-<)V}4!3cgEMkH5v18iAxy|MqL;||GvYuu|v1;dl@)!St)R7W1np7t1Uxd|96qC^p z(VIIL%mWTL&wF4+Eb$odm;+k#I#wfC<^UYF;mS*vQdcjOUPyrZX4j{l)OTW(Ffjqm$;H}8UA1uP(!5;D#qPmI zsY=gnokEq7t1(yb%3TA?PN8z+t9}rN&mtW>hwXF8G&&l&X~uH?VS-9s;}<)hCXEFD zZ&iI#WWPBZ|5%Gb%ZuYzKA1UNadm%%XV%c=X{M|PSx#3T?fZ0>8{m(=NP{=5IA#}N zB=d=vrEq=m{nON?{>ogOS(11Cqja;UvMh>OO~GY>_e(eFlaNtXsSY4b5hd*0Et`JL zeMZNo1~g1pairKK}zT_8-eKi8h{ z^Hxy!mHDK{CK6)Db-~YYo&?r3Kvqc(XT5w@vVnnw6~c)2nEORh#$A1%LgPMLPQ%)- z+L<#BrSt2V%=bAq)4RFeORL`LUHmC-wvdo#8i)FKdW2{;dkqH4srDdkBq88vW#IZ3i=blUwFf6`Rq9C%3(ceCHl0X#3iJtT=)D)xetj+8 z6`As}PjD%^Gh}FGo3b@dS?HA#aZ-ijAcU1)$i5s%(*D$YFY{xt5t)AWiSJQSjrB?9 za=P?D{nJ#lh5w=IEu*UJzOP{r1VlQeJ48VxrR&h$T_VyY-AH!`h|(n?E!`>I-7VeS zJo|9J|MB{B=rBGU&%XAGIp_3N!xMw$MsplzkZVaKO!$LCnZ;+XE zsw|aMA7GD3_;sIR!$qf?e)iENnA3U0&XGDUghxG93>QyZSqZ?$Q4&BS3=(KQeiV{m z(qjlye~C2U<%#U!J*r>VUQ>Gf#1G5Lcy<#SX0`UgJZC#ZAlB>-H8$R@_4XG+0fCLG zS%xm@oQ%UtKl-xg@719F+g>2w#X@Ojt|l;!q&BVPWS&Rd?~Ne^(i}7gK`U%2Y6&^8 zy@&vqRw`*of67i^(qfR)Ot7uo4J3;luS%^Ki}>#$#i&a4A~-537F>BO2hO5xfjV?I zSxMV}hrom$qNG80waM<$P<{!0*8YvB+aSLeGfVUcCF|Ww*omiY7(%Tkq-FGUvjM{Z zF*t>H#z{-h1)by()(lZ(RzkVJrAbxvgXJh}xR+holmE^@2{mx4O-X~to?n<>qR@Vz ztzYr>y4xT&-_c%i!i~6i(f6 z_hbk)8xxm_XH>Bt&F;{!cC7quwUd9GlmyLr@c8B86H*@BuVyUoznf2UXt=H%EGj%Q znYBj{02+86n-v7cLqJcAof=P0FEbO%@v%S+w0QdJyy$(L2pFH!J@PYTppSLH6`fdK zHFZ`QjABxG$Bco=WU8WPXT>WAeUagf$5G}d_fwC@$S;ufh*c|#SN`h|g88Kz#x2U? z+BY=bU&Q`duQ|syHF9*(@nKIHZ=763CZ0(_y0@&aED-JGPGuv*T?!@AdnMPDd4V!3 zE;U>(HQt+UCfHl#;Q)=t|_$jJEmZ4cSmWw=Qp- zi=yJ#$8M}Y<)(&7XB+Egi}TMk0^NQo0ZJz;h8bwU-X7D znc{*0)Azd+p}nF;u?r z)ObGcV3eQc4#4Q?;?D2?ij-!QP#edz|7pYei0yK9txVlQQ!e|lKWkY%ip$~JoQgo& z0v}E*N>KLc*F?o+f?o5%*hvi7x)D$c#O{Q$=Oc#`f5E3!^N@n*fG5rP$#f1alnAh;f)U zAH()AcE?d`SXB|qrWDjx&EuW2q46L2MGX9)nyt+L_$iGRFe+rU-++yg|9IGw}M8!E+cAk8h$)wF-|-6wvnc|K1461l@;#o`F=z zHp%oe8Sg@~VC+(vYLKZ*brJpAth1Elt0#kx;sIg)wP6RmNBF8#{MM4+jqd{=%91ho z{@BIl`SsHJmQY+#gkmiRtD)(xUbw%tZsYT=9veG3zh@>N zRG|v-E833tTJ^3K=VxT(;61PZyU*^1*_~{r^>ddR{xrZ+cTNr>10L&E@57B|D@pb# zy!BSW*Hua7^N3AN)!RC#G8>nKu5uaR+9^oAtKhI+ChPijV|c@f9!^p9!4N2BW@e$|4Q1mm;Pt)TsqQY#0`;OzS{d0B5Eo$NdE>5n zS?^DduALV<1)DfBMv+jtjlUN>)F@Uv&vO}}6BP;gA4p_?{V^X~+(pWK<2|wc$C0(^ORd`^kJuHZY_Dp%R9GzG|DA~1_O^DbnhQc`(a&K?l!JO&sqfW9( zH9e1?VBE?!*a{p^?Eyxr*Y4NuD9g(qmNkLhkN0`M&oV-pMq=dn@*?@J#TPwq3ctP4 zD*_W?d;WfY$1zs5ByIk^Rxo$##_~7OtNbneOY%*Rrg=On-$#|KZ1*R<~mf1$6 zsNeYkZJPRJqn$CEPAz2L^(@TpuiLrFL1EP|(8c=bxppTVbuk&?zZjzhb%YkQS0g`sOKHcKK z*{RcJy4#ZT8Ug!dz?*V~k$fE< z7_i5P-cyj9`&JqXH*a07;uG@6UJ3#!uto|4MLF zX?P%_G3-q6F--ri00_j4xe8ju-OXJ{9tp1>BN}S{V}3TyBnf5kNkEGrhR#8|UI~3e zEdd5>jXcHI#;{-CuRBl$i_mE#z!amelRI}BULF4Uy1A3OZZcqc+qvoBDcIZI^l*1T zBLPMLP6VJ5c}}E$d5C)B+@|uF{hC{V($!@rwta7P(fiftb|Itym+>qb`I68BRPeh<5{G%t^*|!Y27b%hd*2Z#y~qIy zO(%4UzTBN_vo~y9ifxVd_H0!#^%j^wbq!oqm{}^4lvno76)e`ExMHD9){)To4?mmv zB+kwi-3OHg^9elgPexci~lAn-7L#H|5&& zcXwY^HF<%lkb10IGM_(8GI`9JmKS*IeItqb>Fw!mQmW0Q#kAvfzlz-T`;>qB;da!h z_cwLbEApz&Z=EFa3rB5SHfvE&V4Jj=y7wHruT#BhwZCaw#n0_~-H+1r{{8m%bNA=E zzvs?t=a6n}Tt+A5Sf1hG&tjcD*x0Y+>gZ@Oi>Y% zI1%MJTwHk!FppPzNWs>ygFuYr(*l7PQ;^T@&(}|D52QwDEX(LNZtulSpXGhc{z=h) zS5u_j%{jg zXIF;dP^oQZFWbz*988^RU^}jyt?G*miuZgCB@oIM{2QX1bFIB6Rvy+I3)Q6<8{y$6 z2L2H6!#-PoNzyZ~t`5vsH8I(I8OcvteSUh1Vorh=3YNaS_HJ8-fho|e=*i#7%FaeJ z*2w%?(d#`Bq#f^JY3bV(p>G{far1zxq_Pk#1yBE%GL~6@0x|aUvsmPpvB(6mlAN&& zPh%Nq#stlBeSOo>o&|8RqHGVnh4(j2Lmn0Qg$#?+;!W9gra^$xHSP^r866adeyas3 zRH;85p&TEECqqm0>YA*ZWJ(OCzZsdZm48_9bBPlF8RT_I(Pe0u?y>wd-hA`n2z`os zCi-|KGztMR^%f7ZMo}Xjf?u41k_UfTFvVuyu-CxreAgk=N6|aV#Pcd7}^+9^&RzU8w$sR1yq>t{9b%ab-TX}8u-rarsi^R&I9!GNx$u& z`0K>BTET@?#%K{YpoiW_&?!h*B7L1s{|5c(nz{%6jFC}7rA{L(UP4Sk!2ll!FNleQ zOfL+pa!|$$FYnKp8b?=oW|iLx>+n8p0_O9$>1B=p^w|7#iX_1tZ``rbJuk=`i*9Cq zTJG&vm@u;%OrIJ>NoEDr?5TETW)7cTt_mDP3c@y*{N)~-v&UA{b3;Au>WaQcf@B70 z)Bbs`hA+Sny)^29>Ty$u23uu4=R0YY6mrt}=cYu668ty@b5z#i$IKiS&GW}ydsZ3o zjJGd&jytPcY-~Jqn(Eou)af|@pYl#%Ch55RSGV8)2x7Fuu4Xz+_;?0u!j~;@? zlozd}UWFq`L)o_cO!OlzVX;yH*qE4@sH#ysNN=GMb| z=uD0SXGLw5pcV^!_)`Yxal!n`Yh5>HnUWTLn5F5{wiM^Pv&o~wHgq_cj&Pztue%*R zq-W2Lu3d991HXCa-j>q8zC-qb2|~lnfFP@=`q*i~_+gV~IImQ~XV8CXqxdpUA>yIK z9Y*)@6xoO&D`ah96$X>i=9U8au9)J9P2&$FMDm$03&xm^0`@QsgLGBgVujb=RHE7# zx`s_}uEl)jKR)%;hQh?R0D+{Zj!Hk(4_P()--@bwqGI!KMdV3Tgq}}|VV~|Tg z6mUFr;G;Rfq{g9%iGc1Hx=a9Z=^}SFHwBcHabT8yR z;p$g$>9cX`vrQYY%_a}hianUeS1wG_67`94@2aLB@In+|^x+_{Aj8|oD8uXS;V~B) zFvBPctim{b-X;idc}5w~X7NEJ>AFWzxqN0| zsqFQFh#1wcJ?W4b&aa)TieLm-#W4Mu$RB)X_u18w=S7RpN83h^I`!@`O_xn!V5D3) z&G6@t-R#PGZNDz*XVqUm5i%YmElmg(r3xfp`O*}(6vKPQbc4(MOt-<4fhj3!DYFNJ zl>wC9!EAwcUDIQ&)y~LHFl|^tR~M~r9<9cGiGKj>X4rB|j*sWay0`^ne&1Oy@F4kk zx3@#wfuGfDT>eOjV2|G`$=z(6#L9z5jzd9?v*{ST6K)C>pjQ0>xN65h234)#k@(|h}AJ_~Ncz8J3 z&P~lgdjADH_&d3K9OP2*C&L;$SDf2H*g{h;CMW3*o<|_8 z<>(?NfOBn1+f0A&#ULUMt!t_wLP$)v_L5OUwYr={b-ucdSHMt%QE_Bv=DSC7s9Hdf z($HA6%@_`<f3|nkqFev@PFb5ZI{g*Y z!pRLlGTKS98)yNK&fx8f67bM*aB)$c=e{v&UF!}eu=Va-+-&C;+khw5W>rPS#YG6l z4$^dl5v-n_nXGZ{vcEV>X1WoQL(&e6(WVeK$wEnwW{4atP1t+upk*+oWiX^=u=~4p z%E*?FY;i(Mf45Q)N0yiKi`^5rSCAHh#*mC*ny|fDJQQyeZ@Zbb^s-Rqat)!OPXx{h zd-63;qa|Zzg)>=&X$b-d`;rI5v}%ky^H~MN%kBIu#9w0&9=Q9R@db9krav2C4=4j6 zea+CACdJ?BjVr#Brvsg4c%jD)H)C@3U;Zlnan5`jg>?~mYHISkHgx$W8_Y0n$5XxL zWr35`UITM8q1}}~Q7E%x)Q~SC3~~BrxK~k<)R)d`I*Xol1uz z*)K^$(sIAFO|2HUVI>W2BP9_$1%ZFqWc-6kOKUOyiues?RV9s5{L;7JDgnx9fpX2^ z%dX7L>y_tKtE^2oBv@)ZPlLbmGf*9WBM)+3BSSnK9^i_n!z@mxz-8mraL^l z^*&gbzEplP=yJW;8ytLYwdg~kiI^2NAwk%7+2fo5g)@&qpQ|=KQMlzV*&reut{fg z-0cj|Jl=xwsNbb&wVjv!ox!kB=(uq3S9h><%sI14n1X`?AFNzFNGsM;Q23(va6P%X zYcDHY2NtZFuo4pUI3Lmt28V_UYilPtkp;=-r^Gy&9Ln8P;Py{|FbD>T`eTG69}&O> zy5hx<=x`|=wC!Hk_QSXA6Y8fh#O^)+NRo^JKS{1?Z+{*UbrvyhCZ<$ID$`H5A19PP zAYgyY86+z!V(*TUktu{xs6eT0>fb(w_M+Td5b=eBcY1$~?^M(Q-W<@LxmP{sU_EDDxjIP^ z`_7DFd-2+z@+D@!Tp^6#Jy|zFrKMOus$KMT$Buk^XXL3EIU#PA;xpBIE6Oaar8}~l z2Q>1aW)H7<^n9J>GZ9n1f+*mtL9!eGwM=G9|L_4U$zXr3W^;SHwGEwr*HPNaiV4_Z zSV>7sH5%PHQ=C_V1)K1bJ*~c&uWe~GjqBz5tOd!S64}HS z&Yx_}PhH{2iJ43j{5dm(#!0UR!JE-vy@ zD`8gGGXjfG*a@$ChO`ydf9s_yQqi)Jly3b9k{A@T@TC1P%AAgj=15BZJ}L8q&Jzui zp;6@1@}$GfdAwcSaVN=V$A7^10t61*rcAX z>)d-icNEGdI$)$LPT1*D5=R?kzjS6u?@wV~J0e;o#BC+6K%eGjx#7Xny&_+vQ5ax-RWD!bm7FlMH zXXK2I5H*8`wvF3gWDe8C!J%RnuItYtUz?0{3LV=~i9c;27&tG7;q9mM`&}sDbn>jlf<=D< zy>>zgYuCGR06a0QJUdnk$S)S}@hq5xu?4XZyQZFp>_J)p;s?d4vHDWql69wdJDcnX zzvb@pu>70+x7zB&LEY=#b**!qYWJ2?-0MJRZ2lhc{mkr5L|IjN+WIw7)a}Z1q-5S% zd@|pz0|{y@p8Lcj`EDa;ODTV;*%&?IfN`d!^JznOWfEZRn8~tO3b%Phwv=2-Hql4R zr7m}x#8(DVi;m0IOr0(q3oC6QqGW+G-nzy@#-W%b{RsA~1j7p-1;kR#Z1`4&T*Bb{2h;r?pzzVm*!= zusY!aYS=^bQ+4aRNJ{3~!c(h)yGMIN@UlA65C|(W@epY<$Qv2+bNj#bUaEZf%lXRI zb{)iQb4$i;s|M7G1j5PZP@=eI0GYRG6OWT--44nKioe$*nD9_ppKu&}TBCm3n*=)m zo+TmgRI1FEpf9hY$(B&foGn~e1jpjfi<{gQ>YvirpL)5gD=c9=#niD}Z#=AM7=9Zj zXu%B;M9eG4MoUEVnRnlQp5!CVBqptiu)J^k|yp=A_`7t z9L8jhzkhSiXJI|FKj8>;4E1;BPscv9!RO=3MDm0by=k=JX0jh3rsl808;OdFrvG80 zJX2DE^@=0DcCYDdPMXJEf>c-yTwL707qZ_M4w*BHgTz;|pE0UG-4My~7FA z0cXOrJewaHzYtJc^sNja{a$OSt0RZAhe5K3L99t1yB8%nqyMh5+3Foio@}0U#P>0K z>W>@b@7!Kv*>zAdy!^W{=|>`)f%IY))`Kyv_`u6PrNq*NVm{Fw9=MR3=zy)HC_MxakcpfrSL)F?NnV~N2Fn9aOj1C2&;MqJ;1XfTGlXbJbrs7>_Ry8B&f4+E$RMT>gYcTNK_56!>-ex#`dJ? zQ-^g6ei%@v{un9p!I4Rbk}}Y~pQ7|f64l2u2La>xXDz$W(hJ&Lw|7uf#I+wxrgC3T zirL_ZbppDSEK2sxGl;EjLg2k=N95&_bNz9z+FK?wVi*Ac8BE+y-!-paQ*pQmJyF!V z!5#xUD%JZTnf=*170!nTh&G?!%vFpkm2MwPM7^`8q0CV5``098wY0)7zA@TQJ6#~A z3eob*jJAvfmCh2In7>0`f6K}{-Ohx$hF>_vFy_-O`|5LO6d#wk&frB0-q2+J(23MP7au#;8LIaQ4cn_oksLgyd%4t$ zO`b^@F}E9iqp^4J(9;2ytS?P}wr!G-eN#95J@-EDt zID3NiY~>rH@R&@~Mr{m4cSd`}kB02P_bzMfY~&VFZ!ZIjgQ620WP+pb#r~8Mp$OvX z&5@hVeR$?|v^Y6=VcGfVqrpPL8NVGuI6GRM_qixsxx@U>gned3_VZfT1F_a8s(vC4 z!X#q|bw4q6X3So2%{COh;bAB3Hau|^p5kfG`YFs1=ieCk49ODa-$a zkV>y(%*wk8F9=vBxK(}oL-4Bw{Ivih$eClVJr^ zr1Q~}#$Pt%u3;?h;oVzaB)kan0ZCvK7*OZ1RReM5th}&N)!g?d*!f&}Ci*o3{i5KJ zR9_{Kdf@1!dM!I@50wjL73iV6D36K9g4E{jR*nmVKDf|%HhW$UI)vgy^&tJ9b7FBI zgQJq%l)c0Nisvz0y&&p*ymh_L*W&wPYwm}5{AHh6y!$=^oLp&M)pAv*%%75r_^p~K z3;sULoEVta0bIGLXl~fG72_u6xpuOn+~QclIHd$X#JdxNV(XuYlXAeeEE5)Qa~Tqo zk+|`ATNZ!z*ee0&K$UZh@Z+_GSV*&{Nj$wZP<{t2@-2vf)@1HPJLg5dNRtP&!`mkH ztQ#~q?YIIsBIOPD2<2}Y)*MkqJ~J!cpNi$$*@^wK$WZtj_?ZN+zK)1CaR-ZJbcE_? z;vh#DJL~X~?+HKhJrwQWTNrD3;ft*VI)xf1wZ-#kwR!I!4{pimuPD7JLMTr)W9)D& zRPYmCok3FoPW>~PFS*n-t(0d`X#~6^DkgdF5%Rk?RV;^JntH%mNG`OqIFYDZBRv_K zcNT(iQ=~UP*-xDw{F>*QLp#7n5!4U+;q#ZYpk*(9kY8Dvztu82>+it5ok9Vkkm6HP z?wxnqDR!JNX*<1dhMgbaFl36#zK9D8McslJlFaMYZbnNe2#6A3+Si}FGyF2OAl9c|gbSQ4Q^?4e2G64bsbapu%F^y+ePzM) zLWn7@&HeK2_I~~I^Yu@YzFCUx^GzrUCmWFi2c(c34G9mr^@axQWYNUl`*)=j&R&9; zSZsoPzruooy14%*GJ4gYA$+er^_W%TIi+TPj_je18BPmecIKZc{6==Q?cw3wd8EF> zDf0Gg$#i8?1D0_?HMNL|U&u%iEr_-0!|%iiIABe&>8LC}@UhxH(;Z(^gNGfOgtasp zI8;%ac+k}2xIM_3fGJ%~%G8exEGac~%c1**_HYA5wY09G(6HZjp1o&f$%1YCVQ;r$ zD3`UQqB6sCnDYBYd^JhV)#9sY=-3SUI4T9^;oQ&VHAHrcGq9FbJCTS=l)i^8K9pHv zftR_=zzA9l{|%3&vli+c7oX!jP9_?Crpi$kaUifQbB4lpailW1uB>rR89!^E`*<8y z%|#*MV*M7vAfAA&t3b1K=p1q`z;Wzd60;5H?0_C@gm#U}LFj%n!Dk9v)$kx_idn$P z9b$1{9(q5vfgV6P#mQ*9`+hRIR6)Z#%-j}u*^h3fsiW2u{F0+X(W=Vw64nBKFWwGF z*cW~F%&lm#;R=_sSzAG+97>}*t}eq5q8T4E6nKy;NN0U7+7>pPZ&WQ@W=}pIFbO1QOaL$~9r85B(Da9qUWY z34Y$vB9zVCC-4W^p`=mDXpnH*fMQ#hhR7q755Ym3VaFO*#?GH_VoIJBjV2=W`j#Fi z#Z{BvJpu5(a#Anw{HKG}MlUv2)bVjIQUa@O7Qk0*38%Du^CfV&nv(|?m@UZlPA4d9 zIgrPzqZ};#7h|y!bB0C6FD_Ows?nD^uP#QowfvTeU^5GG+%-sUj+&TkXr=5r&PM8eKc z4NTTDH(QZ5UPLn_jbdH+mo!8Mp~%VrR>N1SS4ZaLWyfJpo2)hKBk{bXM4#fLB0=r_ zi5@H8B{?K!Cx|Cl(!V|KRa(vN??)subf*6HpfLIU+}PjW{K_<)S|>T_^skyf{i2|x zZ2F(zXlH+;dVBJd&0`kxD7f0_E|ISws4JN~2%sxY3hzC?yjjO>Y&onS7-4I3NG2<< zOSHC!v>lNu0wobhnkA_f+atRjW%+q*nZI_f3#Kcg*QO{)*}x=Mc<;YRAMmGwy zbNwJv0vPM=vrL24WA8(1Z+=1wKtKxq4EfM3zHUjc6t=P~`})bC|ILIz=18-)W=L(_ zzE4AG_g-~wqP6H;!-W84OgN+)6tWgRw)G$Ewx%SY#osa1;6M}FNk_-kqq8afkF!me zU6Ea$H=ugVzI4I1?Z?gsfn+!@I3s8P&u|f^9kr9oV3T;Qwh)V)mIkN01`{cixz#LJ zv~Acom)~7~NC~5N?K5>GQRL}up7>y@PxYy^`1|9LWcnxb{dddx`zBuG<|ZVK7Ye4{ z)wTehSdd~6a=RZ0LLl(XD<%4W&T%gbR#>Vk*M>eqfxcm8N9)2uM8F>@{kewOz3M29 zGHBS3GJRry^@Q>bL80T^`2wlC-agleHbsp7e1(!tHxO}E05Li}`5#YfoWI07zdadk z+_NqFv^w&q*BzW|DL4bkBmdc%826A(IUM*+f1h9uNj+HE=Q8Xq^@b=pb;k3K0m-p= zQUQ-G!6`=H=XSoWY~rcP-lw-O#RuOAs4MrS4WNs6I8I`g76yKqP)y5lxm=)nj}Vj! z(=NB`kpj~m0f$4J4D4|MTLd#P!m2eP*<2rACtg6lcMOWQY$YaWFf7^2dCCY$$UHYe z!geK8)bqdvBhaH65>1rf&JwGi4)eLYRR1w*QhriU!%U_oBX#zeuld9O1^(-)J1KvL zEf>!$J>K)xpAQow);2zQvqmov@7BwcIEiGLKZ zck}`@DQrB;M)Lz;kFgDmZnlF}7is-->l5tOw>+;IwUKL-BO;paaiQ{^MG-$*3<4sR z>0TMq&EGd-zd)#LS;w$>prg68*YnkyJ?`yT&DQwAmkahHZRY$>9C5F|g=Hn)(sSCb zcd5!a99g_7q0CsOdk=52<9aRH#10luTVW$OgqP5>DV|V9$6_Po6+C6c4U~7Y;Ri|^ zIdY0#=qNfmoC3V7Cak5YHIiF3M-b!Fg88-+QnT|VXQXp z<)t6~R$YNhgR?w6JxD1@@^p2fAy=;+9xZubulhUp0i>_B?()vt1Or~MeKnP={f2)6 z$P0J*#ILW6(5P?U0mUbo;(v^g{k1NPhtxxCa#PKXU|6zC1h^rtJHTH+z|Z9$iMY-r zG0EHZS$fgVsQ0A(XCM{Z8?0vIcTS#V-AqwDnUW>M+72zhP1o!y}eZ-R%{;>bPYwcJtSq=_^ zSU`+z=yMlAb0x8TqgC;fwv79;LZ^BoI(WNebT z)S2Q8r^}71t>zps81Yzn@88$*J+$t*ibekYml{4A-txG0Pw6?o*!=l}<7cdt zIy2_j&-d~XK|$`J|M*nB@%DZz8NT+x?}G`o1yN^&M(Cs-AP!LL*W?53?np{``GdTz#Ze?^frmwypDr(VzQ#NZQm zt3U-bcC>#ei;J9=lK;iN-rDbjeRMv5>7qjg5r{FV0?zP)iGYqN_qOEhu(O5DPLt$M zEb6dfa*q$JSigPrHwy%0Budpr(E&p}i|}NGC|Z$bF;DCBqb2tonFed2okjXta>AF# z!LBS#+>u1l_Op4*bRn|7gqg&lq9m-arzIi1+=tz@SlF6i7iB?s)8_)Kk?{J3<1VC) z_o8`tV%<@D?A9fy?to7m=Qc%lE$_`a$4W{QPMhP%2!+xJ<>AJnoEW5J1WLjb(*8+V z^ameyen7n~QJo?>3X*7bi`N9F~`QE+oU97PW_i4{-82rqv&j1*a! zmbXi8CV8H|F7U2^HnVe*lkb|%u7~v_4h|(Nvsnq5KSJB5&k+H8M9d0K=53$m!0@!TTx_zYeXVkPvd4Vfoaw2;T2~}jM@NsQC29HsXc&rpF!*2dwv&_M+ zP|mOe@87i`a*KK+gx?>8J)nH+DZ{z368VQE$u&f0-{XbKCeNU~RQIW>WHtvbCNKRO z5cH0%Gw3riyMt!hEOR-h0y>UmksJq`pe4U(b_!%u>6wv{qgCF+0VhgFGol*_`fblG*1}E(lW@+Rs)Y_0(Ng$ySM~|Sn7eH<>!pC=7nUfM zKy;~TAWJcu)tQx^&i;HqVcS<|X*Q?Ka}#QeNrn|UAjc8L2C#v_jJ$S4d5npced(2)n z)gs}~)dEI|8P23yMgq{1f?EkHTj0#l{3pIMi3Vycay}(r@bYq2c%*kt&OrWfF;*T< zH;|;vx9va5B!9_W!*LwA_Gv@djNlY&J`WFKgzdXd*W>0Q5`PT>M)1>tGlTJ6R%%MF zm>;{e1-G^|e8rb@g^Y^0!Nvm~2xVW5TkMy}xAETuG<18+UmQv?iME0RKL5Lm2uU&7 zWL@s4a!gY+x0xpU^>sDe#s9sVbXX(<6=jCNzwmb9=4P8tzQWoG%KaC;f2iqhw1otT zz5!7J6uI`&9mj~xgjY*t-|zZ#ZUl<{ihB1FVJjvB6Z2KtzRkN|A&C6fD%h#s@05(m z&9_mq!ct*=XfOA7rZ1C+2pAXM#J2042{$>8eZPlc&uivn_(t&>bC^H`N949}*V@^& zbF}0cd)2uX-Fyi5^L==(x*?mvhPU^ZnX~VD`lNwSgQy}q0EC*giy7C~)mzCr^m$Hq z!- zyH+WRqNI6)Oc0Y1{T2Vq{sevd5n>h9O!92%KQk>m+`|-^$SQ& zM>XMG;v!UaLc!og-$cZTa~Khy>K4FwNXEvznxUAdm^<0yHDP6~t&j;#4Pa|h_fJ$gFb}!ekhi=cSs1%r@7k5ghY;QW|FVpM$AE92YJ62I*^g6T~hF<<3 z={ok}zIw^^du`oyWwHCF1c+MK01Y*dgRALJXCEUq6K(@nW04YQ{w^=wBU8^;m@$`O zA-t!dPTozQf0rG75;aad>OJiw=b6niN_)2P=j~;d;C}dWVuXZ{X%#px^dVzu zkWaOdeyAbDDG9+rU50?;N8TO2Xw19mC}fl*@rT-*FT4Jfp7Uuz_v*}v!o78kAI_&EJmNneHQvjA5Nxc|X<`3ViOHAV#=3>; z7u!SfWGWB{9Ht27UO2)Sv4!v%EJ?zv)5Fu*U}XkDMJxCbI#DW+dM(!Rl=r8DLmIe; zwLSY*31bBXyXe0M?3#*nz5d?uAeYp=i=i%21jnGHZ#;2EJ-z; z0SUq&ux}=LPJJj(z0T6bJZce7#R(6Sc!v~;D_`gJhT&pg$f28Ws*g6Zh$5~#>MIAV zVRo|rJAaX(_CIbn3MN@og##E9M%23$&CAR7qj1p--j~k?4%upk2--U$o@VVvQBxA4 zL2c6WfxfZuVdBvya|T@qK@1dxmQghC;UQ%=7B3Vy=KZRFItgbeM+p3Z-0$rjLgrV!ix75s>ru9=C<=g~g&HM2ON#<02bmt&A2d;wM0n$z?JX zK5JnIwOJlMHQd)?Gkzu?Wg`RupVG{ae4j(QP=1qC4f>sH1WV3FsyaO*S6oUqg&^KZrH1|kqvW!J)IY>3G_jqJob-uf52=3&aT z;2iMG$A;0-WT=S}p~v&w@})OotZ^h~mv_RMhHtY=kf!}vs`vBxKTmm2i~uJh8!BJ zr?4HD^wV=*29RQHvhthhQDC^6ehHWMIi2a4^Ts+sS2+3!G~u!){2Tt98a=Gx&Mof6 z7SF3S%51k*jDYwO6c96RA$Avyrc5>ja2-*YPD-S9m;|cGCr=b+x=u>^zJQ(vZiP}e zL^Cv;HiMJb?yI%MnSTAdC^|&vFyU^=6b1V)yeZs8(Bxph=A6ojBq_t>vbCu)gN~dr z)BW~Lex4AO#g^bfmz0kt!2O^j_c{LBv*Xv(|uPwkxI#U_`)S;A_1Igu+3tgVsi(1#|` z_7dLJ2=v$+BPBAE1QDTkt+gq=0d*{(-}CVtQ=}~3@mRq1@g0J`f8W4QJn*I(Y}@fn z{)HWP5B=LmtI?pXK|VVt5~xR>`9942v=RwRP|I}P0boTjt) zK&&mg&12SXeK9029<0s9#T*vj+A2xA_C`~=ameML**_X{7n!yBGlh&)JaFd5#Uc^Rxc$BHndT>-stmIzD8#joa9ZEge1!i+ zbF8xZgaY0LJ(kzc#9!J66u%rzR>FKG98mm; zl9K;JIl<9sM9z`MA7d*^Y-V4JY=dO zkVe|~@T@AE3MwnacQZfPT->dc6_J?fxF4eFW9J=k*7xoga*ROqM1D@KYo^d z$^8onZR5JwLkaWXX#zS7oOI|GW^f3vQ*}ac(IAt7oI4pZ%TKgvTYZx2obA7pS zfetQ+_@9V|no=qi$v^Mv+uNw6=Rvgm&X(^N=o)7Ik68S|V*GW!Go<);+O^QzZho>4 z>N>$97QYA`0uh?_^SP*TG?a3v+&ZK`N~a+EOYZH7rL+8)f5J)zPv{7DR`=x^bUDOb zOe*=qbt~P7wPZ`f34I7C)&&JBc^x&9Zyr@Bfiq||6(V_=B_Ae`ld#!PfyU*1PxfHL zEt(-uZZ*veS$H2mdW(GruYhxOJo(pox!2Ija$~)HNzU#$=C75=rvOy3{9|!!&V}P~ zQJTD6o$vjP!B4|V@X3pCFubtOBUQn3aIvUvc{-GEZJ;m(uKHs zT_!M-h<}IgQfX5Ifq|fTy~NobYv%M+PVO#BYAXIMZm{s;SvEvf9!`pc>D zPgju?1UHjd*B<~f(Um&&8f`4DGV>QCa2t!Hk2gi;*(){iXf8fvji;sJsV~BrEE#(o zt%Ros_DrmEPc{h^f1u=HA4l_g(EiRz^&b*Xzfhmu>2q>;g613>05U_W4;{DN6 zb8p^bPhNo}o($_pgJRteN&&wM4TwmumN7S_xvDB(%ivAien4eDV_;oyBkK!9uOp!f z4Hq2Q9!E-MGpFvMIhI-^HvN-Kw(H-$eyu}I!;Iv0e`mke9{8x2ALIYeozdvsku|!n z#x?F$_r0sOJ>vq@0&*{j0V!ABR;bVm?77q2)Z*2;P@#+?e<|i$;G+&gxc_*T$ko6e z{bB`4Kdw38mDSeW5yXm*I-thFMnw;OHEW2FA7GaEfyU)K(;jGxH78}uSQBvh!H3#+D) zB3EFLa{{iFS6zh7(Ny+)4a&jm!Q^cK(r{X^<0K_zg_rP3#ZSXhhCQA6{FW*ONXpl>DHzjql|R zytz}MfA{{qVQxk{WkEs(GKE|3(B+V@&E}j~`!70K$g!PydSBOzFCoFVauoyKE+8#g z*6m!N)+qaotF5nkJ)hwgfM2Jkps{4rr=gw6#`5%>GD5%s!a?W}^KMkQOpD#Cr&y z2P<>MMhe|`86cMbY#7Bjumv08UA-D|Uy~alM9mPjc7!8l{a((G587nS9Z_3uDMVkf zNxV62G3a^Aqj-wMgz`pcv<uLqV0j})dgXNv8;Ofd|@{H&LP># zGSusb@6V>5WiM{f#uo4o@` zftCw-xK=TjiXR|5c7wviOPw<5cI)J}f04l(R$>_+DVsk0=CZlS_P88O2dSwyhuZKg zE+*~k$qgMQ=Gp(;m7DdnkcoNzO{LdeAkFvp8&4K-ArROLLui^CzZTqy zZip0AKOMM}1ue8{E+#{}9Y~%MvEVIjno)8I@0u<62Uha=;bpqf;}k}8#2E7jXY8x9 z&bg5lsHj+Vc4cHnSDVqa4scsTk;EG`!FHAhhCcxZ^AEE>q6z#ttX@MPZzmz3pOo$& zP()a-hOdoo=LGsF;?Vj1!@89~MGa_Vfx-ex8gvYy4!JYMB65L$GEY8!ehDpJu+iXc zVP_64$VVLO%qmo#h}oOYbALt5KzsJ*JK%jUxrnx6vIUAh(!GFvTYe@hJV(?DiZh;! z>Tw9-FLz0va?fAM2clayfTW*RwOMM`$#7e@y$E7@iU;R@VhK_}yspSbXCeX*?aGWf zhS$R#^gX`;#o*f!x7qxo_KiTBV@a^IgyQn#Acu;VtDVjPS_k+4*Z4ZEDn87It5xqN zKFs9@NyvcKIS52HgI)+|hdmye%KM;+r+aDG1vK)55@8oZkjVPlix+3%^s`5BAf}6H zc@&CGf^HrPpr@?iB!~BP`^)E!3t!Q*CcNJj*rAwIC?OeN(2=5ixvw=?Iip zs5SD9`1q6U`wnjd)eASHJ@F(v{q6sw>a7B@Y@4oOx*O^4l9Z6{ZbVvILFw*pr4dl+ zk}hfKkPt~J=?3ZU_>YVG`M(#p+;QU?rq)_BbDY8tLh{Jxz{h6gSLG4eu^KLZ+b^pV z4Hcjrlf?rIKP6Q^!a+WP-~+Q6fm7sWoNpb#Nf@9_Rp>iIOp4x}MX+(nlilb~WH!ft zLcLL<+Y6d2bM#_X!fEwcSh-81e|NM0+|FE0M?zY=)wJ%Q>@1vEcVSt?=E?8R&hDl& zxKkRfuiMTVJe(>yZe^-PD{+JuyjOhv4Wg$S_oPlr>m)jut|AObt%o1hTxw=q&hQfr zz*#0!8(jjV zhk?W}^!p6gR6{ZjzzNP5%U~H$z1E2Y(s-RNIb#Ns`_>?1+|+Dc!e^yO;#nq=mxSDU ze@eyrp)D@->N@%9Dq98{t(YVOKE4&((jAPeVEPS>cR!0g@)t0TJM}xvlKBQ=na1mr zzn{M-2?<-_#kaCa_iv43Dbv@P1=6d^EN@w4{`^H65g&0QW0sE$=})p>8t10Tzq^y# zUQ6TOldDKQ$^6H(=$i`ol*#hlnP?@Yjsf=OB~bdp5qeCkv#!KQn9tv;dljI&rhxbZ zvP16|9AKAe76xw)=uV`-Tg1^xFy!8A2|vx{(qdy~vvqsZeenZ%uqAIn__^?$MF;Mp zn~;2fZs)z&mdh2pk=odtT%j@Hxha#B>MYL6hlVH)u5~-ERtfUvIvj7IZAWj2OEE=I zoa7^BIKfoL;e0lzw}DTLIx!kTpR#7GJ6}=kb-MLb+uBSLaMpr`OlTR`w_u=EeK)0|;Ti)Wzo`cLl;Le<(L z8gCu!=`AtczQD@Ihd#uGDztXrrpxxH35OA3yoG|Sl>IPqup0XCv$LowU7}0YNaB3j zstxMAe=(byR=64E{pu>1X!NpAk;0JQ?_h~c>uyi*3i9EX8;HW2ybFOSMbt(nkCu4a zFVC9!DlLvJd}cg0Qo|JS=OBu5cxYHBede7dBJ+7owqq;CI{AA~!kiHXKo%kBdY~QmgXdmT+QjIB$Vx=54!#Ns@ z)g^jD$*pbJ_g5pr6RbN-wib77`B6An^+#d;0DB-bJj_#wVD=Qp{z!@#Tl790B_^F| zsj>6AvMcXmx2^93j29bSw4Exm&>-eLC>!T*O*Wk=3ZG&954MBye)v~mjC3ls%$9>g<-$bmd5Vgj{ed4TV1(dU419lU^d8H@@8D!n1Y48|QT6CO@x*C?Cl63h|cK-Oww0fmgdS4(i*Nv#L-VyrlM^MrW;Yg`eu$*SMF;{ zi|#+YtO6t>g?~x^N(PxdgGTpC%Q3R@a}KS1)9g0+UJiD25Mk6pAj8ZN>du2U`^EDy zSELwgA2_@#lseu9spu4XaHIZ}X;|bt>DgCMF>;B~amfA9z9_PjML0&>KsH)tujXy5 zNeujc<;Th&oUA3-k5_9xY=F@G)3Y`PLh3RVXGZIM{s?1I)mbJ%l8cxh(8cBv|M!rl zWhJd2)Sh#k>xiszoDHE@V@({~+N4r{fKjtmxcE61li^_3$~Mkv)yhByzf+r7NF2&p z_PTR&dRI)g#%&TMdM_=|{p+(}5nXgD>Cn!rLeJB?#PXvE3#dAQa-c z6z^)fgg^1SzxJO*YCMpj6}lr_MS|Lzj;ddqd-Z%`bY;Tzdgtlv{kB7_{N<}S z0PalxFQFKRK#aOLv4?fei?U9+u%t_B>^+Cn_f^3(RvCcM0j!l)e3>k}AxA3)^yF_S z%dsMyFuSL+_SQX^j0?@M>$XE>S>TFBW#j=SNw~6gY9AT8DefapkqvHqz4c%GGu_Gd zglZ6k-H>Z2q^{FV#?xf`BDx?RPGQDmHC4@)Dl^BK+RhuymShCiGHA7D{RWf558+}r zG%=Jne91>GvT&K8`Yp$tEv4k>)zuoU{aDGfLd62?Hwz$enny6{DD@GYzkSwWX6mB} z!5uk`bCE^2gFHqg{Q^sW>)I}U*+b&(jqj81)p(3EmFTYp>)H$UPd0BVHgzDZH?`^w z*fYSyDRf542!u%oBx;ABDtHcAr|DCu4l^PSs%Jjso)4VUEBsw zAB7k;b`wh`IAI8^Q~lmya*&yK zY%1RS;KX*qzmr)-#th6TuBMs4Ye##?_x;6kbv#?TpXx(5NH_?A^CfRI2mOZ23zIoE zu}0KQyDxtXvfd`qgtNY^fZpMhT`h}tCTqVH-0R6IlXUIjW1oxMp7}j;JZxmVRp|pI zzqb)~(6}FZ55G}0oru&%0xvLoF3uD=dXd2|={@Ftw`E-tf$6Ma4%A$N`V8g0aAa+2 zf;RY{4m^lw%eSKU^Pan8Sm7BW_o@tFd#m~&3WFp|cq-sahpT13fXUd*S$S2D=r7zp z-7L9ZXaVNd;;-9Pz-%O7zILL}4q@@{cG$<%8lCv$m?^x^-p@ndgkg!rOr*Mx?{eS& z(dDmdxH0W!HeQ*8rBZ~}vczpb&oS>(e9gYpkiC(ae8RgDHy?NLYI-;Px`$x%{+Dk} z@p%LTi_o(mQ>EB@>dy<1l> zOC0JV%vyc?y>}@4{lf2w?StGW2B$p?sf3_ySaEJD7qD+jxTuqCDF|dyxRz*tRn0_@ zFP4a_XNFUFO;B{UC-EblY}59g{ibhJ9iPUxNopQD;pJ!FGBt~1Hq5Z1_@+3uZi{A+ zxs#zfxNkj#U<0OVl4y54DYlGaguZ?L9OGESbXz`oPwdMj_ifW?9zpLi64p$nzYd3BZQmb44_(yQw z6TNZ4XU;P%H7yKv@B2aoSov*WOHh~SjI$kkQkzp-Pu6)GT4)Ee__tVM`6m`_8A?d< z5LD6GXRj95%`MXJM!SC_a>pAW&Ix{5oHgoB+LKTD&CD6fa>`1+V-Jq_47z*i?@-PO zoawioXv*k5Xl{`z3U+M%Gpy(JH)$C6Tt=cK%u~cVmdzRs2Gyjf0>4 zlB@fEh%OQxq_T?D6{k^z_Qw+vH|5!LZ+3p2*bB9I;5Cg2T?U8uCOU|w%wpq~NwN8( zeq0e*IHFZ(z3lKz^SF{pJzF7315O%Qrj&HRBj_>#heaxD0Jr7jO)Fw zj(eYottwE8_&r75#^b&|#EJVoc0RdsF6b8{%<8ohT^?odq46}o{ba+!*t$-czHjVZ z@T@ycpr13DIrF$dm}0dtkunbX%_LP*b=BY-YmZ!tyTeXvDR}omKVy(dEdO|F|L(A^ zfMwdQd2qp-XTq@h55!ZfA`(c|Z?Dx~Ohs>X@SfE2YY`P__F*TJN9LBO!ATRUYisxe z06A>n(UoSk7|bn8i@f>Vw9I#WmzTlVYukcXd+MkX&icl) zE!l@O!XOO`W?!r2gY3ERd+1L8fqK2NZsk7FE+Puv?EB7`oxv>KZ*Xyb?ojEUKRc1J zjRc-;*|Pdxs}m`WUhMW^QShWUJEA+{UM}Kb$FVH?@;1=W8$=jD5is^NdruCmM3)F1 zThy8SI9G8p1m`6mQn6rODF+oWy^~c|7ZljC;Y;6*>wg_Ck1D01#VBF++VXidCLR!? z4(0e4+hXz^LvRkkeZ@e(tfMP=)6GrHc@DT9$oJhrGM%%{ydD0vj;w%ybJP=G0qsyl z!hk)HlGKiWQ@E&lp>Uo!cSw6rdt>M-(sX~u9H~4|>nBbxxtOR;eNsD}L$!#NV;5~mLgA$ZZ!LnAie~@fb8nt^=`}Itz6W8c zWGN`vJ?UW-JDO(p+xqJ$SNm!*GStg_=bh)P2^#{M+3>|*eV{~rJt*D@AlP)X4MqLf zA5;vJDYTb>6l~nbmJkN{K2bIFIjo;7^ElCk>#1-A*+qolB~zMM9)%~26>lc$8rpqH z&BXFVy^z@1Qc+E}0{456bga_XLc63stc0ZKv8GeGfmfI`&SIDU|1QA7V(iMz6Df>Q zOz5cVUw9SMX<|Om#?CwSyAIwRJ@Ms-+X2I`{S8Dw4vmx$bE7SP&bBz1&YGnB|qstFOV=lt4Ekt7GTN+RM`eol4irjQ|#B9)DDaWYWm3fZu z5ekO7kF^)=n|UC>IBfTLw#k&hJ^A@3COQ0#w5|n_Z|m{-^Rd_WQ-?PdLbERu5jsT; zLkP)gh?1`_sdfehu`1r;ntM+L&c@2)Tx=7_9?bd`+jMu_a!)yq6Py1yUKO~Z>WIwR z&=&|El)z<`{XQIHuL6Dsk`=JdV&=@dc21{*K9;*)1~DfvJNv<$409Mg3_WIJR{5!= zGElkC;FpiOi;Q!=LFK(UTcOJ)dZr-L&E!^IF5`?szo3Dkf{%Gy7w+-4ZN z7x)<>m*J-Z99+bc!iP<`;J;)S5~9;{lWxhls0+UPFvBUZse`GcD#{bjC5e^1hj^=o zUUa@Iae4 z+K{q-zcDKL{4qyM;eGd>d$tjb@65IEg~&OtQgbu!3RfaZ`=n9K;@}0! zR_E#cY*?u6OvZkhZic_tkS;=EVB(^b6o|^IJ~GZFg;kh(G5$CEJ^M?3hGr%JV3%}kA8H!*SeEmrZRbw)foX<|!KW#jX*UTy* z@Zc9-G4@U;A`XZk;ysH5LDP=JUU>Te$6tKx_oAd1Cz9Kz-*8pByHob9!>7r3UZTD?PS{n0sWr#yb6Sr@@6_-17Qv4`_f4O0Xm{s~oU+LTn(-$0ElpvDh&Ljq|NI z0HcB3Iq1{z!VuycBs_;Pw@-reer)y>0(T1N-p2NMn3M7U5lhK~nb3fr(L*psXdMh; zrBH0?-hgevt|Di~{-Zsi9*XEH>VGuo*tk+i?oi*#4Nt!B1OLg|E45!p7~64W27wBd_#;_|E!kW}Do-&>XbseosbMM`#b~KB< ze)81Uu~59V+X)%7xpzir-S?v~X>CF+s;FqdNVaUZ+@@%cDkpCfDcx=TP$MtyzI8N? zbo$x|+}B$?!Z!tl$nqVy+OOdH%&S{me+0!)bGa1a=736e$vs;xYa@^N?)|j7Ujb_T zNF{|+P|qCR3@- zr86cf9TNml5LmrJ!I$K%JY}rX`>mC8q=adku~X8EyrVrj$h9Y#Iom;1T}bG@2K_Ak zY5esC<;g}J<-vOv$`whmRr(Vd+TM39{01KFR zXNwKn<;3UHr$a4p3yW9`KroUT%%$L!(4UUj`j5hqy6k*<*@}fp`zDm~xk5}?x2Y!H zj+AWkM3&chdb*rm+6OwTb6HM|VZMAe-KDC75 zWsE@9$zGvFM&8$MiDAGH3P>LzZ}i*H`2Eh{%7HIeGSRh%=k43@i(R|L#6+ItEF4zG z^4*bTH;Tk|W8&84!jJv-L*1|Ic}1bQ%G9N#jvu$ouM>LuQQCe!#ix4o2`jae^A<6} z)YO6Mb`rN)Y!7rN7dA(-8$x&1%4Ia(FiM!BZCbHQziel#9)Bt5T&|HLuAzmYaXdlu z)&*mXJRRbd^-xH|{8DTbqtzZgrFx?DlW#QMdw}me0A0ldCdo3?Q`YNHbLoZqcCWr` z4AF#Gh?m~C%9}Lm@H^i|rwb!z><-&j^q<4SkQftZ9CKOep2Wy(n5-?|KG&969wI0Y zt_vMs0H#Dd>i%D~np>eLZq}f;Yut>ne7~>ZL8!G9YFr;}R$(?GPslDPW<6>dX1A>4 z#p%_p(4!uGb*yW2%dsVFIeEQR~Vjl9ZEdJdj)!k6aor zhak;=h3t)di!H}_Ao{DHa8F0TJ!aC%k}zgYee=|wdmt0m`QC?+`$@!|=>F#Bhe`&X zLX0gQcOqgQ2^{9}*l|{bky81QgTlJu9lAz`cPbVUcAVEg>4N8XX?L?hF{D%8 znV{h?PC?-YX))Eq$nu$awV05)3`m<_zRk;EVaGf)PQ@Us&ricla$%;IVYtbe>Sna} z8_ga0O_NH+6b1!yS4%>Skcwk0p%Z8`Z!7X(TvPa=1fQ)JpddZ=!+TSJHX7FH^Ysy9E=2K2O+ zwEbSt{1fR>jOV@@uVnoz+!@kl_QD=}rgo)NyYhODuVk|CkCj9Ft8;!%@N|kJodlBe z7$yW~j)&}S+LAl($Pi)0ZH5(S@)#G1cV@wiB_~QC^<~Y_#J?Emo1Aq)ijEeB)G^Fp zC;?`GVW!F2#I{d14$)wiCsSCAae1Lk{wlTMC}*oKgjsw@|3MPA;4eRQMyve^&bCzk z$Fyoq+i(SdW5>tXzUSpx^75a7*pa#?x>j0$TS|QrGgTXRaTqFfv$exr@Wj9^JbWGJ z5rR?PJ!bWj{L5#gSN?41Ib#0priBVPat$}ob?c{u6fphSfQf05uzl3X0|})s46EM+ zsw=Tx9gp10jvJIX56<1|3AslD(zUs}j4 zb1(553GMgwZ$f2vd5A6@w6n0Qg`qv(CBQ!zvHckjB`jzjfc#0-YwSSq5J zjKzRbtZI++AlSH(fe`aIXkW!`fTHp*`%<}THHVQobjO*WFnR%dxc#edGT57v8dr_c zLe!9&pr+YDko?az(mPKpAO`H6bPlzihEB$LHm-nABnQu4Il&ovGMOvf&1CN4)| zq4>C*c3W$uUQmuOWKc=m5FxG-9vNG}d&6(t_*9B;vzb}$7Q5w+P zGSWgRY~)t^sHLQ;A|I#A*)Nx56Lb4E&*s1Lh}f-;iT{~2XdXMPGZOV>8Dk8;&a6Ot zf7k~k|48B80p@Op{l1Ci2UL#XxwYYD<|srQtx_2fXtZU60bzARkOp%(nM0!3GGT-G zefz6u*iVx1{wh-OCg|{=Fdw&cF%x_7PNL6D6%X+tM20ZLrd1)zaIyBtz2vfpe3;~a z7!w>aW<|edchGBkzwKqeVud(7Wj3>Mak94LZvI;V&fS5Piuw!dBl^t4|Cm1W!}ZvA zg7usAuvj>~8lM7IE(W7U{ZP_Ib%bDs3(Dv3kh{3;26#|X7nG#89S1e8bR`T zmY#AHYOJau-^2!`Jc-4Z+ElCgd}2%cqM%9gE1zeQ*R|*d-@mM$iIhfkUoK?%Rw2MP zj2s7%ksce-u$M&NWWRm7V%T{3qb~FC$Wa4Sa{3kGTFj$I~@L@czYqtzof{(BX0Nf-5BH zrOXh#WYw+sUF(FK4PN1JlQD&1jgIlbAVnxXw378QNlT&Y z_&C&UJ01qZ7;yRy)i9)rFmQmDb|T+7Wr&kvlJes7Z3>@F}T zzzwn-cdXk*sh!P`m!m)xeIg8=_a(XEwOu8a_%P}hQ@uoBhnHhDTuax*^w#8Jo9;8x z(Yhp}l+?2yk7`$fVD)Y5#YthSUO8-zsr}rFK16TEj`O)Pp{ahIR>1mB7Gymd4Sm?& zi=x$VC3VS_KmK=Tj@+cop?$`!1PgALA)z$c%9bTnl1Az{Z=O;wFESjd%+7o_uXMDLY}DUNH0uOgStsXk~1{1H`T4= z9_HqhH_2}8EDRQ9opfK^Set1sE=fWd3wFN|eK&|oB z4qGP;s||Zfbs`k^{H7p;1zUOGzf>nyJl$3+w+RV(#YM@j1)Jfo6?sA?v$7?TQZO$TUI>LXA>e;6w$PKFgxvVk_f5ZjXB@$#>DP_jkRX%{Rdb4rc>y&mnC- z`!9C9SV}Dg&yQV*d))eCJI}8Wn6L@w(N`?#^q5gO$0oNyDGR}={7WR)XZRtPr{Mq< zIqmI*6l#w>U*_CvdGKns9tI2s`41bq7f|ZOPw^@kpd75AFM;WA^SPO#P_MKII~6#;QGUMk-YS&Eky_QHV^ zJ28xuY6Lbm;$5}XhmnPm^#iX^I(ixpWpWM2iEsNXtmHJ!{gHQiv@uk28hbh(bE!T6juoDDtnB(=<81Jz$uNxl&8a3fd(VAAyVpN<6T4ts;G`xoC#R)*>HGv888o}9N$NM3C}%!- zKg@D53l_}wAVrIt{F5&XJ{P4aJXUKv7@>KKEdC((Th$B3x-hurPZv6%-MZek>BgfE zs8V=RZyiSZFWQkK4En4;=mrs{ik&Kwf@yRajVuZA8K472jbiV8kTbEH2*lPITAD-k1>{1ZI;=8 zHq{t$0Z+2DvJkw~LStFV5BR1tL{H)a?ND67l^mz)Th`%GIwZ1&HvRd5L4Y{&l0(sU zo@^j1{wGq12xX@j1ht#D>EqcgT`q5o2)~=@9Cy+*{!I?gi~lK;W6JNjI37jVu)Z%w zYm|i~ee#e?uhg-rwhM!;PY#yNZLO%)T0Y#$;XpCg`u2n9Y5@BSC%1O0knFQovxh_*h~!7&5`yO#6oIkGsnwp>6xC8-<)r2N9AH18W*2 z0zR#QJQm+ySB1mW8=-pD#iCu567~^E-gdV5H-9yP=RH}xRqn`R^75s#%#5GZ zwdw}jB$Dx(wa>p3a_mz`jqlzES72w41j?~41h4OvYoWzG&91jj?CpGvUGk6u#@}PS z&$#u}$7mbVtFDx@fdBt^b_N@iD&5c`#sDWGtHXhyH=#nimJkOaj)jK784&R`l99 zu6CeTnO?~7j9r+U;hsUL0yp^J$YO(C@!?c59*56)qzTkyiE3Nmu;Z?|HkB=(kA>)gl3Kq|Nuv>hb%0J99Oeca z`W-W;*n!vw92-KWVe(dvSSc+y%r$SmF5$8obgO4Nv;vkTsAdF382VLI)i`?%$yAYI z%D?_kOycHq?i0SnhUJX|`2mL5;9XcDmMz*F;}jTVblh_G`Rz%Xle3w{&vBfP^82CL z+Ti^0R;r=KPR14?bBT~MR7lNw z#bkd9o_U+0S82LR_Z5(^yXEI*Vgl4;WOkc;Y(;xNY4O96Rtjg7M)Y+{E2;L{kHao- z<^YK%-2VoZV_olCZ=T#=_Eb%5hp%cHTpp}E5G+fR?QOzZtrp&wCt}~6gwlm!JTDNc zC8&x4P~<=9jApRy#7O{-Am}T0!b<-L^VvY;_dx_cZvwU@8?Scj#4eO`qyIZ7eXWz$D+G=casM9BqJc?=&Rc%gQG?uwc;_ zf=pFWBNis=tr3kaxJ#mHz(sup;&c?@jzDlbdCzr==OyAN79;HqO5fM`KuURr zo?*F^pd_Xk0m7$$Qf?nG-3Y>BU*+e}Nb;{?l2Wq^N|96vgrkH9ysW#Q&* zvAul3+z%!Z1v-UIruTGG5|4<#X9Q;;?dOE=?2#uGM%~FwWXR>agNyZ-y{gC#kgkFH zPl4L0+DI4s<9Bb?Z0NGi|NQiO#sgiao^(FhG}&!e86(>flWjKZ$wr}~OmJzRDM$V~ zLVQv?(Kcel`9y)vG|7U}LzyjJYokr{>9V^B%t!~rXa5N3ZNsLwtdnMBNr+t$ExcB1CX68z( z#QUFhCao5)5iVy%gBZfIR%z?jziJE3dj_rOq=FP+6Ey?DxzER3df&`CNiEL8AaCr! zb6+9XrfV|88F9}?s7^kCx-MA>1~Ee+Ov7~wCdBT@wEypp1AqMMrmNoPm$M@sgW zDMhzgzzEV#uS!VlWUu7&1sO3;bvYilG{xg;k^R3V@3mVFIBMa)VRQy^zED3$s4YcU;x_{(;{geTt9`hX|`8_eD z%qq{4D-cA(3t%CH`%Y|D&k2Ut1Qsg}neihST{ZB?%UyiEBM~yXX_|XKnJE%*?56m< zSx$_9q|OiI1Hr4{STMGF>3lMapv2t|dc==3my2VIuq?i5^899ACM3URT3pTmLzVF8 za7*9Ea=cII2)58=kTPI^>y;Z=q-5KuDzT~+Zc`Ciam>!_9C-c2Ly=_x?o6-}oBf{7W0-)GWZxCgrxG^7<(Vs0>-*B2v+k_{qYVsLu zJ^tD}R8grO;;r8~Els%7-`K{I-Av4f+qbj}F*(M?3r9`r%kks;DzBm17#OF%ct;KC z{6MViy*=TD0^{{LUMHN|V$UllMq&R?q?Ai=PB^>*Bx)YNd`njGgR!Q@t5JvWM8mSr zs8bHxeEZcqLEJ|~6KgRq`Rz3z-Sx#}$t!%B@sD^jviC)0_E3wH*b7W}!7%#b_y@~j zbDqhAG-dlw(1g|c1gU2Dj(L(j!3(mYSop6cTLO)M9a__tYfzbrjrwj+c!1VflSE!YhW9%| zG7H-DTfoKXdif^`1 zZ%+>l;za78aR=gCuwi}A)rsZ>hb`T1zNPBSdJS@2cpg}~QR_Wc4}P`;VlK4Tma^-X zuV|Ss$6r%8LN-*SRx6gEn;XupbQ!k#8%eirnS=dfJ)vHZJ^ASwi7_V5rRa9>+LFd=qC&LFq$7T%|j8j9T=5*mR{hv-lx z|Lba1F^{PB&h=H^hWf>ROzC95st~-z!U^hs0riZEHh}k)sPpTzVemj-P3@3v>$8+= z;Dju4$IloYny9z-b)EZ#YRo_`hz64<$cSM0Ldte3!HalD=qq5`!+I!TKZjabp8AwL z3~fm`x3flY@wi<+~j z#WFXAgVy)1jVq!$1TjV#X$b5!j_EWs)L&jbQlefj$i@gqg8ik^$p!+0KwP%_EQmBn zo&AS{%UJA|%IUPYti*|UynHSetwxQ}c^LCAWv;wJPD`gCCDtKPaV(fO9kNs{u2I-S zR1X()7wGS=l2E30#rnm?@tNFM0_-s^ghU^7BlK0F#ne%5Bi5JG~H>dGR ztF#BJP}Gk9+A(pxC+4(^0@ zkzSBdZW)vArcGQ!sCB!>EqG#y=!wGFOBz0&QPfRW{UT#GXk|pGzQdfR(wppndKWYK zso^}Ow)m*^fHzGVcPR6v-UOZ9OAcn~G%yxVP*Pt|QW;w=^J`XCBHfL|Ecz|M$69ea zk#dgw8h1lqDsy{Z+4!~pjS@YqZxa=^TWaMKa^?^S)Dw6t48C4iuxrA##koIT-wbA|bJ|Cj2`_Z1pPAGqVtD zc$b^O#HBG{q@{5GnyrQ%D;Y37l0yvN%x|8aj#PhP3U_;esHX2*POoSAP7uMNM_>AR=5{3n3R+DvEDt_QuBc;8J}e=pcg)!p zKfiSTv2QJ0N%(nLBLILi=*`t`AnV_h?$xisErS7ivEo18tHzMfsA z1zBGlCk{)D;g)l!M&P`bMz~3wc0^94gw_4#Q>2T(JZeG~y8lWmmxuc`C7T)E@-rTo6nF@y3vQ4cmB&3DqM-oIvc|8X3-S9(E|22 z72I%r1wnmkh@YHem0*8+8iA? z-|hmS4^-{9vP_0de(*+PKF|q-_qRfox)40Rje*Qa|L0WW#}BGaL{YIeX#$D8H3%f< z9|XHHF}pSH)`9OP<@_*(yiE{PR8XBHoTa2*Jeo5x>Ps7Wzc0Vr*WGrJ`iWz3Ic)a8 zDZ&jR`WUlnY9~|uS@$JU%KUyxXi>ohk3sik^7wrwhT?P?J=4iLj0o3{i@1n<8E%+_ z0a~DgH%hj2cB@MXVu@?=rQGsc%_Mzzx|mb{C?iC;u9Ost(d-GK;mFdky5+R@ZXCvh z_i9S^w8S_T6%?V zgH6^y+4G@?kUmC((|6sXl^eHwrITQ}(*7>({DE3fJH7^Mf8fqtkLO4x z{^r&k3`S9UNG~?3=UQeVZbb&p(1%AzJ9o&T zODZ~!Ho$LhVtFWotUGr8OuQWI%=$Xn3Jks?>f=R_sy2S4J$)=4O80?zy>Qp#*}U~@ ze`Rdjck91^;{3Z$EZ(`q#!b2>)461ecB-sF{F*A)_Y<0)oD3Ol5C)dyZ%@_4Fb31) zaFT;m=rR=SR_*5dG#6-ab1=;q42o#CzDCb&@-n1uzJim#qWHk_|5 z8DZF3(E^eUQC`dSlFBM^^HagiPiw~@-n$^(($ps2r!&L*I$Yrxq+~$lN$dq9MTGx) zjNK>xAA8+N@eNpGNdFJ{_P&y%N4ELyt9WWfpjUKl%%zYpPZNyXF$U9@83aNrVg~KjS zA-ZB)l3I49ge`P6ChmjrKR8eRF1dIOkE6nfr>1)JGzz$=3fq1B<-xDsPb=XO^pm54 zf!Cr>e#3QsdT8HgypGtuT4gqgVAAr@7~r2C-Z}nO&FH5e*jr$-@lE>7baU;29WU4 zRCnR4@mG9R4bf)-QTm8oeiW9{-R!!%qL1}tZT`LyGR^4V;Ib5ZAsy3zxD#jVT&FbG|57d}Hu5wH zMTeDA`QCjjb5p15Yq=C~13)Wn{V^|DrL=qo#BOc-vAY_7;Z>S?zFdcj{tIhW82Fk( zCz&=_%tW86mVEyGz2!Y@0wIgd%TI8QEO3L<3?{mrtH+EIl34YZTo8BywtM)azx1{M z0z!VdgJd|D6Vm)@9D9XoF3A0xhhAwFE*<%{&%O-fz}oem-)0}Q14 z0PQR~8|?Q);mui=Y>KOHuOjO>-A|G$!f2uP>0pM(y=}FVvlfD=QfxkS*~t*q#-R!!pyY$(8pGXB;o=XLuopw`H9m~ATq}*hEWiWm~)wyCX zd&*6{31i}c@Gdu;VJ(rNhezxo;XB8`+iAAh+j()#YK$o|Q8SO{zFvj`*|7ECxWwr( z1?m2sn6$VnI&0Rf<>k4Qlsx~i6fY$eO|nB)LvNHqs@(Qv`WLc}87!?NEYxlF0v|V9 z`|LfB50KPK{qtk?(L|v~sLjVOS4QyKX#yaK#Y^ZB*KLb>l0VccW>I^<(7Qcul5}bw0&VMjO_`aK7!wZhlJEB2M+} z22Se3*oa7lEN%rWqQwNF)*8H2Ikf5VQzL$V1{NxU?DRNtZ_Qim`7(DB7ay>zwyZx8 zF?r(%9p!N3!ChF0G}T0Ow~>Ju!cJkEbx9T6)`&=a*rSS>r`r;qAfQHQ#-oz=xYWbmHlXgMqP_!B#0?Na&qjGhL%>A4OAv2gY_gn$1PkUZ zdfQ)cdm>Do`DJ;gq~~Z2soos-W{@8dMYOd6{R@(qx)0{w^&+JL$@NZjzU<}F`9>a% zAeAL(0ogyoA87~OGoJfEtrRc?A9VhnVKV-C<6*E(O}#+`#HfZMMCF&lmStPgvOcYI zC`mcN91?{3lteM~#wi>La3$USBhcPQCmG ze|{dI)Bc=LHenWi47G6ZCo1`?G!71m9?FAPYWA!Mf9Q-C@5i>CW1yt^ib+TftseJs z6JMk|Ai%J-AQ?6^#P+Q@k@7b@1EgY_oR$3Y2I)$6C&SmlM2p+k$CbF_R%o4HfLc%K zQImexv|-nT0GnoJ!IS2<;Um|lyZwNk!tpU~E3jF4nt|YCWkaFV9dWIs7emR*E5^n9 z1-HV~d3wKCvk*J(dzT(ONewOfJ4~%r?5Jl`$lOmpC6s>`7W)m|_MEG#0cJU%zQ*ij zg-Jh@R+^x6k3FJD!4=fLW>ZjifXwzb7ga^J%HoMcKq?CKk0*lmU>^i5ep11()k75( z6oA$+s|&{5cNc$6nvA9_-DVX_+gS5+m%pPU4>Ml+goJzI9*g zjG>|CACFLn3?f+#=!1fJZ79AJc@As-`H-|lz9~yzW{D>e$l*es;BN!hwOGBNO`lA| zErLCwm`w9do;W*+Ey&KE@wyF%tiJ#e>>Zpof|Ds&)@Lb-D~Mg@F=W!D39y#l7JP~`7KyZJPnUzs%8AN0UkD#pVU5RncP}-`$wkyOSuWf5^}~|WRv}~$l;n9s zDdf0vAS|!InxsN79cw3KA+?b-KesYF{a@PGjpppObeR#oKk3<-q`ipSOHKnCo>X)! zEf^C+1G1Xf45F@GEyK5?l^r6yQ4*0*MMeQwh^4U~IX}a|NT{xu^tm zD*A6YFq<_4=VW9TKmC%^fM9&S^kf$+scUL3?kS7=ksh<*MnS1K;s$#_gw7 zEo=#JTI0a8e-dGSGTr@VzPVj;1B&RBGpEq#==$?_j#k&`1auFIAg>>>cfN_#c=)oC)yoD zu?5kp2BDx|5tBgM(9(Ddt?Y;0b9i9!T9X-QF)4{2qfNTSZ} zdnE9VeJW~hJX|uOs5ZQY3tJ4eF#1mvmHIGj-%1WL2s|Jl?c~BfU46?zME@6-B?_bv z83;Ofj-dBKPyulx6DU&!bp2qQfC@&IovISmNb>ZVyTcdEQ+wZuROn$llJujgYc<;u z@?mcH_!!XWVOwl2@F50)nG+>hsR2Gr-#}X*9S-kDRCncZ8T~B(p-}V#(R-nbR3KcC zlRJ?MHpDDnk>BjCa6Q@b+=)i?EVRG=t7tE6OZpMTx15st`f*868rit-&}J91XpUCW zea}MfIrcEN!DjWP&k2scf(BqW?>k~pa{`&9wLeEKFh`P$M}mtN*3?{Pzx41-M)Tbl zI^;$);Hk{BgVevWRC=jb`aKNb6v@J>Ab@q%?tOONd zs!eKJ_`I^g>J>B&;kxYpyiFmkqcI&PT5y8ka-)mK< z+~CW_d8ec$FU6^hv~cTxe6tI_RVBRN`np+V;8o1=UVYDhrZiAc!GTlP5skSI>t773 zHP5qiOrRvb-EZeLU|RFhn&Adl_TA)<@#LH*^{pa-7X!?HYDnd>iGzROqQR* z(SX4?pjO;FJbviagWfH%d1lB{An>@+7h4;2se;8Eyf{V5?g%r+eL_WiGF+aP=r69~ zATy9XL2_7p^7geR?ah!dCIk~GYmAc0DDP$ab@}JFly!U%d}qS($7n@W4!zxH<7v+* z(t<~HbT~fO}Mp`8LKo98#$0fYyHeW;{MKjEImD3PDNVk2nX0D4eYvrLl`(nz+#nKm{|wNbj{Q8!2>+nU7l6>n}?$*CsPv@=ow?)fndQhr2S|?mL9pJ1owHRCS|@v zTW7$%H93n`+kMmV)PCX{X`3F(yOVv%i>6<9K_reu_PVyDwxTx35BM~6b?~f>R(@@uDv%q&lO$n) zSs_7r%3reG9|Y6v2lA&3&xc*xuDC&ZyUTlC!)6foGqz6Lp z5KxfNgh)&1y$6VtP(r(#bI$vo^PVy8&pYn-;~Qf`_8!^Ud(AcHn)8{@YCzYI%zlY` zocQht^Ijj9TWh%5eji2nFTL$4`A^dSgls`DLr!^?nQPlm^}Ka9a!hN0Q|&x~R($po=6Y5Y=43BS7fR(K%}<+BVU=l6TJrFd^gr`0 zAsvVZ_f9J?nF>gJ>D_n@|Ky48C3Ca=Mru(0X^4x#j+$3uiJ6eG>sjA|^gx#{yep%H z7m<$l(PCwTS1yOqa_#>dZzIcr$bN-7z=I#Iku+KQ*p-1nr^>P}wfQ|gJ-shat*s3CL8NS*%O!<5kF;d`j z=^~qq5W|dG!CCY1(W7jDi!GhuC1Z~b))!K} zxaKk%g;|wr#nqPxJ*Oul>tKH!X}{|qe|T`#88=#qJ5m$ADUupG%5cAMmN z`ueothl$&{+QT)Q4lV0o*gfxW{}=^p3(=zT+1$;D_FrETY;H9-JHrJLnSM{JLDk#zFx!7ESt{^-~!DAXS+Y1RU{1~(|+%y|>T>~TW!VI&SBoxCro z`4KQ|6YDxLo_#4OutL1*yMPsjjPDY<;#v4*v)963FLb%BF3=HUPS-^SN-oU2KiR%N z1x2%*BO`k*nJP=WIo(+ss@)ckNe@A{QV+e};(LymoZ2_mvr zQJ4wATebH`4;u#?psRt}U7B%|=A~!~bGxw8PoZsi0_}r%ot5x(&Yf6^pf8;M=?1-C zO{urkm?nG%mlG6~$Fv#imV_9W?O*7w;U*CXX@bE%nyDb5t?_8J$|7*4%m`MxceK~o z&3?QXEgraq6Gv3YxhMHwns&F4VTynMm6q#*%;!5{>HF7ZcZW`ej_aFHimdxvP=2cz zGBUDKJ^Ha10S_16m2l26%7_(YZoH%P2q}$_p)Gp=5)-FY9F7{WaN<&@y?)m!yAKzk ze?xxB>4G4zNz=H8SenR;;~`NSR3zW%6WDbAfjh!@WTul$wznp6n zHR;)bO-MheS|}(s+(Ej6hQ>Aw_7_tYY(isP?-NW|*QHrdq zokit7y0_qB-Kf$d9c!hF6;8%|#iE3ScWWcX?l!zMt%jNAKIF1zxpLS1G0Y<(8Rw!q zkPLG$tmT!|%R%$&;k653VS`@IZnDE)XNh%@ zQFdq|A@2kt?>ooo=QV9&f<6#dKhEw;xh6e>C%pzFjaH_eK!MAoPh-E-O!9cYFTJ{y z$U97$tjODm&tM)<-8zDn{_-Cu1$)DmnG*+El)9T-R#CdMM=GDFE^H}oa7z`b_ z5_9^3M9f+=zW1N803h$9jUE1Kz`mA!QKwC8JoY1O@&?xAI+o^w zF(5CRG0vbiTkc(ZV+6IV{aL|cXA@Udl^ut(fRIjF>FR^QuO|OcBnsf`IV)U=5FoF(thbT zc8@M}Y%g5-ZD_czn*OU4xOG+AH>sLB{s+DNmptQh4O1r6)r41Z@~1R8+p`p^{9|?@ zrP4WQE%;c4&#}P$O*1o{9huCGw4(Ae#g7@L&n@g9Vvgvj8ofM8M6kSPhS&H3)1QWFR#5Z?S@t?RRGVFNeBI#NHShNL z?o_5CDYi9Lp@1|>i!jduwxaHJ!@#js5grRBT#*gL*g(ASE)_~jgvyQJ(Lc@BnJV9y zLfqG1TgdvG8oPTNYCcl8FNNXx z?{)Zo2CGiv`@N>6*w1S%xzX(IRR-g2O; zY1Z|rdPCwfto6k%^1rplK2n`?df@NfJuEV1+O%}#R3C8Yd{!lZL6UKcC8ZR;W^SC5Qd29$ToAb;3ea+-Eetg3tP%xyk*sndLNAEn0mem&YT zUpn(slO#5zDz+T?B&p?NMa@N?vY0NEH*EpHUwv}y4tH{31%|AkO)zTmfO7sZ+fEzA8x>loi0Os( zyRUq${*G$duW`|<1o>@fF0?_GC2%YFl(nw-m+LKH;M8=Ng-JE~k{;tmv}3kJ!KcZ0 z%6`-ynrf-BWuYJRZ_@EvA}t`R^`-C%ai?`ewTQwnI$9eDc+{Ssw;qIMpU~NYTI2-T zwV|ddlOG2IhJZ#kaJ$og0klAs7eY*Wa)l^=gwk^W`vw3KWm*o z8rw#iBbV1}c>UVfDW`n0#-E(8Qr{VUZgz^7Hy^lezBX`2gsPhW!+RGy;#6G0d16$b zy2qZ7(Y5{U{?*#kCkk1Q_T)2ym z#y*nWDyA3lj#)rkE!8|OXkU?SsAR*ys-1DD#v9QTShYGPO;b?7{$9x@X-&}Z8_Vwn zp;}R9rYSBNgOzpkUpHRv_86(1@N#mgFX)esH55gDR0~nC%E4t247wnRJVB zMeRklV8N&KcO0PuOT|=AApbYJO`#8C78SXrIb z&Ph&~=i2hlU@MMAV$^rs(5PhI{OkLinmUtD5~zaO2|m}rSo|Fwu-dwI7YkUD`0i~+&_#Bkb&%W7k$=uV=PdgaBX`3^t)L4`$^wT z*w$XSna;M1(;44|KFqGidJR=o7Zg`Ps*vjlM@J)PjlxDvst>#-whD|s2S09j?Z3Ca z{V-$WKofaotPw?zcJXc*emOC_b12N9dQH=)Sq|_4dLeQ5?6V4W=pp{&FWT|SLj@sK z48!#{^&~OR^(yyE=QRd=`3l1g%!BuKnEd7wU0*y%O4)84E!Ei@A1s(^uG7~lo3>eF z)2;glGCC6>VqxvBS7k6qq^$INvEA%Q*?U$|9o4C*GG+D!x)Mc!L?kW?h`9kLqf$1;;yP89oz_Fy(E-p&(nbv2q9k!do|yIJtYP8|-`LkAsHxc+X78 zNZI^T89x%A=mjWXvmQwm*smZL&P#8wPr|3I;!?c0OMNTMY<}6jKG0a=JD&5HAdv*k zjR-z|z1v4DHIr3YAl7;qvbqRSA1V7A&-K4UDLEj(^MW3R@4M_kngqJ(*lsf>5?`us zvG(O+8T=^omn3Z&tZ>vo5h(c6kW|r~RZTK~l1-klI9>Bj_3^U;dq+2|VQ0Moklw&; z?%Hz2yzsSm=_|puQm@uEcDWOTM|VT}hFs|(9xeG6?iS8|h(g+?&5I$f<2fB#4f9i_ zuLZr#*tQaoD$rFNF38QSu_(wfUK7X{Y@TTRylxb?ZkAuUkQvPU4+k$%`qQJiXUUfBn3%1w+ zR8;(WRm@D%3Ct*Oaus@A>fGAbC-$?0vz5>ZHZj-u7E5V_Ihl>cWlp}!sPnIRc|G0XiN{;d-I z;oRErWA(Ybv>u*Ec>Ug@o6>+ubq;31%BxR>-QWb3kJJy?A4Y ztOZ~3)fJ(Y9bS=hYb$RfB-qQ^1;spjegl;01S+h2^QNOQ4cFZ&C_UqNDu)u?formr z6q~QCQdbwtSyTo+b{Bj(KQK0!{YxN^B-ZGdcog|9t|}{Uj)c|c+!4Iw7ti|x<>Ed} z=yQdPVRf|oJ!yDy-g#u)yKRSadtD_&@Ug+u4Xl?&6R4Ul>uB>3D&0gMEg!kP<{cJ9~ z_JY2j#V>MMkzbMN%sz%Ck&Rz;yZQmhWt8!7)T{r+&~VEbvDh@{7YxT8a15h_2BVJf zIQ?_r&`@fVV?v(JDq%N$P<5*Xs#$9fNt^WEW<1VA_H6n32$1DG{~M~DDZz|exU&(h z1Eh%i&rPY{7OVuaNfh#6qr0nBFqbXT5X1BEoo@W~{9KzM2HLVJ{z8L^M)kgWsk(`D z_Bw&m?8r~I@*;pOMQXk&YCm+my`n({WF`{Mu=B&{9_^4K!P-abiCpOs_=5qA4T>Hq$GbmV%Dn9%R( zVoGkswaRP8LLZrVFP6{2x9FNw!)44%5R56HiSaM1xwnTzd+r!XRB_$7CfEsk&92{} z-l~!_5SIF3_?u#{TcfMnsCQk?XcfShRGK)9&#vJm1i&|>(;09l1BYp3)xjCjG0nNT zZYv0V?R~!?wXFJEwvyfmNLqoVi86xSSm*EZY!1$LJfwDtoT&xAp?D)^5U@Xgnx|uJ{Mhu)7 z+>i!QNYi!*DBh5Uv-G|GcEN9B$K&izN|=O5R&nACkX`t#bg1AKjM zN0@@NR|l@YeEGP2b|pg<&zdfoL)ehcnfV@)$K+B-u`+=x!8W+Qz#$yoz@7{IuU^Pf zGn_|0#Mt&;wFv7^rdM}#f56}HkQU8hQ{*+*CPK4-=!~swtfc#c+pbo@7mYsT8C#Mg*%Um z9#$JONYj2FofF2sODGpMga^F-3>a7{GSi+^3y1=W(xNoN-RQ8iNtWf`oaXJHF6{ql z!I=u-(|_XB%28mE1wj5e4$k*fLwu94-usR=u$Rl9slQ&6E4lFY5)Ffzy+%#nq9xgw zoSx;}xn_m2YTn8@M$Y#WWr~w%w9>k~K+m^yisOj^*X1z5UEZ|jUvyWJ3(wd4+cg9& z)r&j7_yEaeInT@Vzk!e}LNG<(he7ML42CYWN(I1-3fprARB>7tQ84%`RLjm7pp$Ri z4luhLgDF9TqD9^ghbn_;FN~-<^XuQXvgzc=p1I#fR$WWg#G}P$d4pH2&zAF5+T@4V zPN0-|^CO4WH;|v7`tJK#ER&-i?SC<+_j?kaDd)T)K5BZ^0+*gJ&K_OqH#SBy+62j5 zH{z`fOnC#Y>0%D&+X(ymM$2}K^zBL@X`@NsFNZM8-ixa=!q=2I;-pJlK+f-YYz+kd zP9)FpA(?<+cX>R0&kJ`h5$5mOSMz9DJGtb9LCoNfWt^DHR(4?(8W3h)N&VaHjEn{o z!`a4)u?R*++Co=@qNZ;n5~3Zf)@e_Q$n1VA3yV0sAHn*a)FLv2_q@^2Sr7-AUkTUY z)WNr9(&Z@*;|`&H^&5@#emTYFSy6$?pPa&@8YZ?%pm{%s&g5u#GgiYpJj9-esmaec zx9weOr!W<`8+Sr{TmrAVVuT{e2%umd6@Y5ICdvw)$>rWr4*3T$dM3o-Bzzq%fRNmJFeoEJ7)6Bve-dg8|BS)zP(>EW?fAA;qFRKU=R~ioM zi3)omIKTL8(t9Xjv=7nXRk@P9>yR6+qV%sY^SRMX=3|e#*BAN@bi`q&3D^uCI6`7V zAPhl@u>cYmO*LT$PknyjoHovqD}c5Um444GXE+|Mn#K9RV`jC5e7ilHL~JAT!Zx`a zvmFvb+qpnYnB-bqA9}5JyzL=NlD|X}O|A8j^iY8q6B=!SA84hBSCvlk7MuOcf0>!| zUbCjQVqjMm0o+*HTv=sgS}H)ZbXJ@nQB#Y!BL#WYy`YzmP%b0+DdAUXk^CEZ)uU2p zrfQV$#7zf1!iO-~l9vI>b3utHmB}|wFK&66x3|aW zTq+g3-jqFFhN~>bU_*nlh{wvzZ{bU~l%1jgt>kJ^x{ircx44XR)nte%C8~cbC(f5y z91m>El2$SYx@ZMXLXxz4gizbbAipIsbHBKp)gz&ng;cZ!_a*8{}gwWvGE#qVVM_u%I_NmZE`isuqLOi0Ft z%3{V6m2{XR&6roz8*nvOt#9mk2xQB^2x{8;sQoEV2`ls3fV?~|Ye@$Z&}E@kZwrf> z^d<73))BXn#bBS0*Na5|rJ!jifG`R#vs7Bkr~b}?{s1b^VrWyL=9$k`Ft(c*PI@f< zEVX+7)}k>3Wkj7FPgOP&bmSaW&;Vo`s;#;S+ue7nVr$Bqv4j}9n1UW#Po6h!$~ zPx0>6Yb(__>058#2IMk)uAkUgRh2mD6C#XWUYJxno^CTg{?!&lrP=Fx@lBLg-^wo{ z!tf+b@!(%xflPxvqsq2r*e%aeTryqXU8ZpVg=u)7TMmb^%~-f4SIWoSXsu16uqIR{ zrF;F^LsO(Gpe-2e!w@bFED44nUxK0nzD<|!(%O9a=%Iic0s2QA`+Q>agLS8C;#X%=+K^zOYY#2`~-5im-pRRJ$&<~ z&FtJjW>FGtmo|B2w+X*=+Z>Vi22!sUq5LVCv2E3Xwz%L~Ey7kRj>^c0*9?-STcqw1 zV^`M@8PAdE`35%vD$8n{8|=7cKFkiPlKO@}s782o;*EN{R=$6yU}lj10^n@VF9cef z>&=MJjaqEF_$s}y18YJ+gs1dv2dk|jyn)jI=!2K@v*%Vq?`;ByBf7xVL9l?Q@_uDH z^0EP@a|K6@L4vn`t?m{pqij54CmRc%5ufEk+q+u+7oU=Ky|x)q*IBeBFK+`zd~9^6)ceW~rjhtdb6oFDlG= z{IIL)X>XNb;ca-WVsRcLyM}vA^#&Dgg@Cv3ZuUY3APeW+_0xQ--`F@#g+Dujp>TL#;!NqgF5C0CgvQp>dh{I?~fhsC^?X=X(V@)MN6?X3a$6H}v zv{jk-Lo3C^K41-b_4KoJLGf?;l3t0^AmKPdvrj9RU^RamkT30~QjSn-FRU%xC{9F- z#vx2|aIHg9i+igu>5|OX77GnJyjY2XU5hN>!1lGJB>1RT3-DCbE4>D|@RF&b;ZcDD zX~XWCdd2h?Hlbx`dghH7{HD>x%B7Ca=Br7@KMAfTpqw#r@ntL(RlS>Y+8eoCwf6ph z(4uDzldQH@VNy>j-MeL^b6itb`jft;O!4>S-}R{H)Z01ZwA#`fOk&|>18}uY8fZyn z*GNSj>fFKGo1FPkPiZ5c(jpLg+4bt5MtMRC3u%DsNc4vkk7yk0b3~teJc~EN3e%7l zS00;~!xZaMV_xFY=h$bOwsT2L%3RmtA@bMbE(*)A)Vo+m!IP?rA&U`bt*sJs@yzwc)tLuC9u(@QZiN+>#31s2G zZZ+cpN|O-T_HSjD);f)G#r{rVH?W!7AKoY&-l!MLz}}9IzmuUWC6}pbsZi&sP@R@a z+b{L4s7xQe)qM*)Zy6F&azFqMN$&+uK7VNS>zBWa zNf^~{Uo7X0(5KIcbKhh~1!(tkXXtPSn5XUhCYl<-Ej{ze?uTCMdnEQI=ldwWzM{@L z)yE-Nzd8W$ubLH(62UavU@!4^E;ak$k%)lxGcsNlB z=&1!Z4IC=2B92xSYfBsAbk9X7{i9F%j36T`ZH#n|)1piLSWQ1@3-6VaOxLyaH0#`S z%bAPqdcYa4!EVrz_duH8Dz!(lmxlL6x4mjFdyF7gi6EEjL%!gLUmia)D=LoSGNaYc z=gC!tX=Wnw0cc&J^o8>2J<4}2?*5nE5~76kY@;bh%3}MU#>7E}MXR=(DzhzOz?=bc zs2O5G$uvl2BT;q*{+1EbqG)>=%mP@cC(3fTNt2N@aRnXh}D(` zRSvK@AGrIU-`ZE)I#ApqZ`}%O*`j+pqAR*;FlyLNQ|!L`O?5L>*Z4$N6*ycZGS}fi zLJ@9mU!GS>fV~+GPoH%wF)$u)erunoF=1X(AJ}G#O8d`P0CQqT;bxr9NUc%w4v)-= z-gfPvu8V#JFOrZU;oXBPim{`uYc0k9XGdW|XeJ zooFreDzMDo%!@+pD+&GH%WKfVP9+L31CWu2CX0lA>|D#Ae!7izX1+6))ZKifAxm1$ zTHV+%ynEv(iHIj|cmo>*?5G51|GIaq(q|PTo7;Sl(%e^pkt@v1WD^q?Ukzn)`KcmL zdWJe)cZJ3|re43!b!JLrJa3(u7WBw%;J!9`y)^A322WOe48u3o*xNbcl7H9bkSov> zE;i%H3E(JD5MZ~mdl1Wgu~Pq#J-HReP%9f$hkD|B+`D+?U9RHD&#hc{jbE?=&F^R0ldQN z7d0R5Y3?6$-I1_)Q`W=MpyXPdc#r3ILf_E3*yzZ!taaz1QLG_ycl76nleOBTKPysc z^9xv}~6z)ugQv(Q&BR-llE?S2tMUyL0nxGqTr>8;G)06dxNr* z^{V6&x#HHv^Y66a)?hu$mxlCXoNb3W`(KNWzp|HbI;t`5_zbRZug0|(&QwBI-l=eG z#0U2ylPMCDBjU@xb=Nk!e_JxF7tdA*tQs^HboHoEC>*76#v5oIN+~6eEjZ^VO4Bz6 zq}9JFv(`SGq#qI~iE)@>x-T^-VWLM~^Z-PTsz+3rj*TKJkw{acDnol`#W?k0O7x!* zv}^SBg-f2@{54!@+PsWWtKTlM4sJ{|#B1jfQG`I9seQD>1IbMM$?+j=M4Nhj@7wEg z`GP^45%BCZY46q7msrlM7Iw7nCcE=fQR?ON*B#%C)1xK2rPiL73xCiXWPG$^;$1mc z&W>EluH3CQqP@O)qz;c5;L2qRJnv;X@t9n7)aS7t;F<3)=g)&IhC=wR&IcLPGfnji|-tZ(yZRxyNVExh@+9k zf#%ODBZya9V$YK59ef#A7hGTH)#hlGEe;`^A%)L9A{t?^M40nPr>6N~#JM#O&h{NW z!aMhne!|R0OcsBk<*Gxpy_JV%O0pGjh7dy2(eRx6leXe=teFaGNDtUj^F^72qrRcJhZNfg2tf;9p|VEV78g;(GNEg(o0yS6^n3^gw?3cJqQl;05fkU_H`Qwx4x+fp0cSx;I$XnRRKO8<6&3@cb@PkM?=JjwMFW`>ET(&;6 zMOah6)pOTXzp8dH|0cOtgYlQ&(9-6Mue%O1>qa~apW=UYBzp1!u5}`NP?SwcL}Pn% zk(YUwN1s-=nB!hUvHC_bqc47yCg0Vdw%DXE7ld#zL%y_%0CF^ULCeP}yu(uC@&y<2 z&^t#3Zp_Op`g_YcC39frt%5>HL{yIuOic1njOr#5LD{9Hv_|{(&v+Au3_RSm#}z~1 z+5QBS5%h2r*DKjx?d&{Otp!x0p_W$E#Y>kKz|&I9dlTLnC!~^7Q+TB}!FlbbUsHL?@%sSM}SV!K7C*XmL3z5(w30b4%6|FoVt zVkdsPbM8t9ep~Ajs_7MU^3@cD--RH~9jc4-*QMG|Uh1Aq7X?0?e%BT>HA{}F1ELPI zl*oInYw5&oHg5kD_D)+7ILEk4+OCi1sE=qmk^9poPs(nGy6?PNJ*{fJLHx(nSv{DZ zDL1Ucq#2x)uTy8NYY2LZi0*{w^XoVx1FDFWgA!b3Q$?zfSW6f`B~HAHI-;im?#Gor z^jhCa>ZfdRqCe-3JK-H#FJimxax)GIj24i>-!~n5SQV3BAy)(%V$03ZAxE{fF6Be3HbFyt$!{?zF3;j=rM_Vh|Wi$Ac28_qm8~5*I4Cb zmg-U@T(O_-s|;eXxf>fBN@e{Ey1GUyoVvg2E*AYZ6RC|?0I5UCeSVKYmS zH!A)~Pas}TJWWq7O|O`K{I)xTNnKIVn~D*`UW%7mP4ttC3xle{*?ueI?sX{lx`5uv z_4u(l-VMyS3=6^Y7k<}oVbd?-Txo??*w;3c;B;&|Ar?bO#1Ud~1dkSrm`@n(PP?{+ zo|tYZeY}J;hpU(I#{6fqBq44wo*d=u`BG~PGe@qO0F|GN)cWoV(WN2l(L7Hyg$;ze z<~&8sJ+%UHfuRTN!85`qeFCrexv4$5nYOrCx4I}VS*yI!d1~0^zuH zA$gBsX$nA~2(ZhpdL=vB%L`2uY3Y2I=Q}N6nKZ_Aa=0F70(J?Wvlku7LO* zdA?S~VT`+Mx4IsPbbU{o7LV1WYaOd}U#WCYiq!W_>B;}tEyd;{hj$7oT2s)jUjMC5 z`+Z8R<`Z}o*D*h9^|syTaZ1uMOXtwN#6m5B8Wn*=;PA&T!ZmuQL~0_<1UmWe*ErQh z@h4Jlb$8G{bSIB0k5WDs0Luz+6X1N{@<{730fuhXginQceJSDccD!*Fv_BYp`N1q! z-OkeM=_qf8ZpmuoEQa6LNyB9fu2DVSM!4`J@6LCA?o=U4JIa3JCwj{L-&2>E)Cy+B zZQ@lBdm0i46J3W8o_*P_!-=i~nJ!YJdIL(mUPj&bxXb6fy6**be+6|vMfLht^}1H| z`upnjSzYxCU3H4RlhkqR?uE~ph>QZdaFdykELw-F z+`6gVc!%Xlp6LNJyWs^uT7J|MzVX zs}BTBrQHLDMC);7^a6Zc7;aWRfj+3y5Z;)jg;ewk_R2k)fukY)&+1Y$>K=dvIv3#G z>+o@xy74xv81RN=?Z$3xDyo}3v7JAro4uypcil^xP&e+fq|>o}q$yBiCLk&A-2fZf zkZ>F*1`BSUhLUxi)t??mY~Qb6r4JyUQg22_O_lE5WRF#Yd3y^kOUcU04z$xX^bXQc zzy(&MmIkJ#nm_OUS)--pb!Gy-$1K1R@rRNTh|uR>idVCV)3AwC%Z<|l zPVC~kAF;Uz#De5fJQOC(ij6Z+qE~;8>8e5`dp!nwxuY2EA9Sxd(q*@>4NEfO9Ev1ti(0fKN zPomrT$p8*;(Qpq2G1}AFwGf9{vRT+}`zRsE3;k$A`{9Y}6R*wT>NG1Wlk#;1p?&@i zdJVKOoV$Y_ti9mCu00lMvtSxtAo>bezx_jbT&Hzz%<)M?N&C&qzdcY}*%{wHg08}H zi=TDD{w&xl-q+m=@b)=)UahjNuAJHvHBfhdMgiHd06pziLQ!K*fOw%9N1lRso`L|u zfVk*h?(Vx#;Y+*ayM$__{$+abrb5tVcMWCJDL`WlPN1d)CqynBjZ?fqTHbJ?i%EOv zW9v%SYN3H1)_yEQH1%?GvC`npb)w#0bf>O8AOG$vtqnK*Rc+QK_qvGmj2>HdwM^aR zdv*yyX$6jP?r9NPPsXRFN|;<5-JUrA!EBO|#k+8jB?O$)+047b)BvVfdy}K^4Gt>( zaITb^g>FlK#&r{=u!)-Wt+wgzrAsZyxuXB1blrdgT;CbAyT9y~zbu{n&F*5|mo1JpW2kY+xt9!X?OA-}Z=Vd| za2jwwKR*GVjp5;8Px|_wReE-pGJ#gG#sk7y|A`#QUUkh8{b(-lFKj;}Tr{|K-Y5=< z1lOp;&HdNKXemP^G5*9Mqr8|JrxCq=v0fLuvJzlK=wHg;dI}f`43Df)chB7l+^Abx zlp)YH@3-x2^4E-)e|9W_G#Db0$fZgTIUSkJcMl4XNG(Tfk;%`$tOt0EZu(`jXVpR5 z62Ts%{ZA?^k_5?WXT0>(LG*)y++5`b0Eqn*GZRSMsK*PaX_MmzbmRjkx6^}2D=U;Z zAbx2G&|UfR-x3*|KPx|=?q@XuGKO$_@h|Z@@nAXpS}U0RY_tw_-CI zvoU|w8!&2P>j@zKZhT3AQl_pd0NA}Jee;@rg$ORMEi~5xaybLAA4ca-x_d1bV~% zE8N!)OOTCgWQmqby=kPAz2=fXv))+JPM&;VuO#qfznwmnvox)#H{&6St1ZP4-mXcY%C%8D{Tie>`4cOkyxe!L z6#@!4OF~X&1y2+%UAyxW2u=bkj4n0c>+(mXXKD5X3ecK8yYf zWt{!?Y@W)0pYiH!*!+L59s@Ir{wo66XJDSdf4?Hr1-df+zTyHVf&6_H2TbDm`|4~w z?cZ1b_jP~N#s9gL`3HV|=@z13L8@(T4rcKGk3D=Hw~evDu#870g@d>hoJNlIWZB}HD!_Cn272|1SFD;TZl`maqi37$dig)3W zKW2AMPEJ-Rr_M6SAq1#<{Qh)Td*CIrpmFc73;%l(?>7Z_%M}x*8rP}f=MQTEyG}5- z$rXQ`L~-$L{Y)rRLIV?T7Xnl`2pBz31Rlyg=9dRyu8 z?#ws}pFG*E0T@&Zk{s#lCAre+-P?0bJT#P~QCw5*m`j6JR8$mYh=OTz5NZRpXO7zH z;=Qs#UE0rw4%}OJ4fdbJe0L2#9HVm#ID`4hz z>fm0}p6o>M>2W=fiB(#(UIcFMChQd!6sWldZKn`_Dqgp@wzf83KVCIz?w8#g&+6^f z2I6~FW#zK7tA9$<;RHb-A3Als;SF0#^G4+4D1Cdbo0gVVzm)C|Fd372avZO6Y{{O7 z9DchV^c;aeL}qWh0O4YBxXh}1LhE^33Z6yE|11w&bZRZ<+s*{S#GB?*fLI+iPJkWA z(U)opc(OL09uqRWr6XWaDk`cZKra@(;^$w#IzUS_xrJvi8kx~#08Gq;mZN*-xCXQf&u(cB#l4CyY(vucIR3;x&&Cy$w2%3`sio5QWf&r+r zhyxX}=Qn-G$#{nWK6591uTf>i)3on}!F9R)&t8N{q^A^;5J>PR0j15%Qi)k*CisM% zDax3ydK_}murVfZ&8i-lS-Th@e=GdQ!K9tsHiPWZuh#!Jp7T}HL60kcf!OM|m}IwO7rbl7 z2$(2pZ?dquTKtU87d&n~&-*=R+*_X|_{goQy1Mru$lw1kmC!clhMHq0{4BF-f1t*C z(4n$3xz}U_pbr1twZ1c2MN3#V!r& z;xJFoj&i>Bi3TI+$yVaf*jOPUU`e`bX`2)lydNexHtT!o2gY*JTi<8$99@kc(P4cO z1;#K0x**w1eM3jezlKxBXi?8)>XZ7f4!r}0N_b)C~yOf1< z_I+QUfAP}*tOIu_Rcsm!l^PVKiEG}Xw?tIy?K9EE#(mzwLK%C@0N-DLND zWqM~)G2?sgQ{K^nQ{EffL2?;jm(CwaA|3$UvB(QOP!9z-R_7qL$F1wj-|0RJfTq@* zCL4WcauUr)i;kT@IYV8@X3{Tf}fDcc>94|+GGh=b5*_Ni3V>AAYKfO8Wo$F zoL!aKrup{vZjkULf-(p5u^};Xq#1AC?#e(O zUCHe}0Ef5K!|;r|mYLx%BJ0O?A~QxR>=GLj@Wl2fGyp2oEi7P zOcVw7eNhSlC>~lk&F+|MbITj@E8_mq=Pq2NS)!K@0+#mPZn+{zb9j~h;q!_re&f>5 zrBgwNz|EsIu2ADfAGc#DSE&||&vM`ah3jupq2J%U@32;3(Hg3NNtcaq6~-1r8q$m? zpe$gf+LKoKBSp99&R9;|u-(>tCoJglMd=E?HyLO|Ap-RhU|I!>%<9U~a`J~Gm=D`B zK>rxglvNZa&GQn{;N`q|p$dT3SU-xvoA;GzGVlwDP=S@Y&jtQlotzP;Y0#j91+m?- zsV8#?orw{`=S}8-U*Fm^t_1fXYbH&L-iEOT)uJ~&7JqQZB_`6rX?h>cMKT_y0{KLn z;N6}ikvy~<0B&EVLEn*iMcNW&Zr*<00QlD+BN-NPC*!EVWHHBtD+0B5)yh3r`XgE8 zgBpscPWovmLyiDMt^*Gs7D9WTsW5}AnzN#%crLL9yoAs1HJ=7jhO~Hr?+1*C8bW{u z@9zsZYWRbf){6i2cKu{Q{Mi`9pBicHHGOVhd$Hd=Wq(ZnIH63oG#OlaN zs%-(fvp(2b7#d9!gr?I0%fqX=19gS-t0O#ldoc&#!PN%uh_VBte>#&Nxmm)a_F|6! zFZs-|xhe4Qb_}Oha%B&WJUHAIlMZQ2{Fy6e6pPpe>gXY%`ShgPoH4*&%n$-vPXs61 zm7sRjV_0P2!-5u$Lp2+IdmE6eA%o1)g<_6anwakHZV8|(yv)7n& zjVEDJRx#`va{KD$S+9)GQJXwpld6`&F`$@u;j_vucVFo8E1~r=Yz?LAp7bZACGdlD z_4BkPSM#%g4DjdyYtX($+T1GG8}RlzH7*nIVd@cIkfGu27{I;NnKT?EK4|!Q92(G_cI1o|e~2e~wQPN9LJ_$@!trMT3ZX+iW1%v~RhH$;g1=u{J=< zKN!e3iA!;9lANfAb(aJm<4xN^LppdYasV&@sf;Yd#%(@h-Sx-^!wdl@2TKf+?$(=2 z-`-ww#kL1siYk8a$8vvxXn0D6S)CUKBF+p)>0BQ)m z)`2R{^?k*hwX2UN^ZKksu=ts)mBPR43cTI0*er>b*U+ik=3AmEps-%K!?hj4lsptb zK7g&H@EMohjgOD7JKSifc+vOe2Uusk7hqLH&lmZq91dkzGb$br1G8sMneCL`|v7EyU@&A1ov;0o_X& z#}ZURS6A0KkWAOje&GEO$fy^aa$MQ4~!9z_Fu0us%iqka3A*y!t1S;^h`{WSdW)g)Zo2o(n|p0k0{Fk z>{BNYF94Lg;NrDAzXSXT=g9ul77+Wkm$PDQ*M`d&gsoLih|p6pyEK?#mC&x#T{kt0 z&*7s$0VCQ$5)maDLkkY3RSs2uhy;V2cfiQ*{Qw3l7|Ik90Rl*=YBzLnJ~uH&FIKk! zjo>9xPz?YPphqrXau8TWDIRF^a-HZyuL+<>E`?ZtXb5f8(E|KLObxIn7NATQ06Szr zyeGu0(2;5Y+u^w8*!&jV_px3BXj6#Y^|2)Y6ZZfxJo&6HTu*mhw*lX1y#p+^Cgy0^ z3z)<>0H=r0!}wP~f-m~4VZ9o>Su2;=W?biP11CWiFXoQ`yv*m?$IlL3P`M5O@a8b}iQu$@k@c_nx!Q-uoOvk_O#u%F1-lIF@xN zJ}oONt6S%03C>fB7vWj5(IVqPYOJ!QGg95J)!f`aXzUR_&3x*fw%41|q~MPY@qj}_ z*T+QOk^(=;*DU$enEyk+Lm7`3a(TO-{@n4VF;-c?6AO7Yb^ZemmhA?Wk(&TPSr6XQ zYp+OKS1?1^!Z%n;pEqiG;e5#nx<${{G|H-z+U(+lbZR=O+sB6R?#4ESp7xWRD%$N5 zm8Vv`5?fsA+?o_$ol+mis|s7`+qf<5G!l?XdGlM}Z!1+^_313i#$>-Nar;+r5}sVp z*ii$~+Y8@cuG!Ip{B`^@$k)mG+p(_l)&~v7TDp!G4m){8 zLhyN%tGtRY6VCM1^rO5!4(R2$ar@e|>|I@tix(@ziSk{@K%dI5R%nVgJ1Rc6@-yeq zJY7=2!<1msIs9dDt9t`jPwv@wRq`FPPq$CAA7>m-lgz2UJu#=gh$Qvxc1c^uQ;w9; zRgZHBbSbOtp1i4Pd)$B8sp3L-*C&$?rmFotxlTfBbKC@k_3GT;7a_SScxz<=!Tc-;7s;qeD2DZmpK*3q;4iGJ+8iTH z3=}qXpP*}^yco>(`chdJR_iG(j^!PG#6~a#3?~s0JXb4<5W#$ttZ-r=176O1xXc-I zt^7Iw#C_U#Himq9o@K*%P|KJ=+3uS{rFhI~x|rOZe)P^UrL`=1cpR^@%4ikL&%N>i zq=(XsaI;@s>j^MfT`)_*NM?=#l?gYLf6-{kv8=9V^5(mOf{BqP=89Dss6u^Hju&O& z_m22BJ2hP&yrh_t?!DAo>?>Fjv?)>-W+!RmDeQ*Ypd#olBkD)|FE`NoonlJBX6%dc z?V+30ZOb$rQ7?iwQE#FD6-G8XZwBg|eeLeGyJw%S(smbrT@_bd6p6Ig;vN&HoCyk`CcAwT%l;+CUhf`RD6k~RhI z)(4NB!Ol2N^cHK)z=$5b+!g|U;n692VXXQ^2=o+sg*dF%)P?E2EeUX4Bks&8{n*GF z0{9BG_7j5YIDr$dLUdZtQ}ri`?pUn^%iRg0P;Rq$dtbzNUc;z9EQ++k7LZ4Jpjxr%? z-W0K`?9Q`MsdBE!?b37OX19=v91|OR4`CQ@7!tTYQYIeFc7?7v4?d~pkAYjO9n>^U z(~{5Gcvc8!K&4k$m;B;lk!AGJS;`g^BF7qlyumx@qGBL~jwx_-N(i(x0SYX5u zPICoNOZ1VK88vy;J^3!t&MgV+dK^N9)1O8<-12mBY-E;;A138c57MUY)EHBWu^JM* zlBy0#pwiCopWU*CI3bK3Jp2dz{ZO!2Q%T9^_|4+p{2Qkt8#anQpas+2Hq%XrOezL~u8Oa#*e68F>U&b6 zP&IPYMp{WSVT@{NjBeC~lAL$Cft6+TXfujL!xd?bkANaR280?b0Bed}0rPJU$T8$W zoiZs0GA-CIZtcBjfd2kGUM869kD|kCX@YC7?l zNu=!bDxY@-zqR0aWO{4OPE!exxM+}+aF*1x|lDu-Ki`SRr| z!b^8%6OtT;@iea9;hqAIm4^-I2MXrQE6+W@7T6>X-GA5v{J`kyQU4kJGdC8P&bcsj zByqw}8)$g8R+|1&??9e&i{0?|2{&{1+`1kyxm1wl);qJf?7g=!c6kJ_iB8sU&U=gHnx7%P3oBH|H>?GZ9t_!0&pL zrS^NqaY^2_(#7yU^TeV>?e|)Hj(@CKO!sVnElV>Py-xwvw>|YT86?sfvCmi-PSbFC zaku$~+Ln#*tR#2Z#8|Lx?)mat2`6sV8_klCGbnOM-Cd5}(tJk_GZry6fSDh&#hGz| zW$3h}G<0>ig!HjVKQcmCjCU2(Ztyw{Leb^hw#yg!#s>w@oaJj*&TRS84^ldsCtLP+ z>-xN^@|OIp$}if zbudJdwh#u3K+K2azbJdoD!Y7Pgz(r}HSIS6r@;8g8_E;7_AGiAP9g;kmFYQN$ZS$T z-z5ep<|ro|#w6AF$t8oJ(_0d>v{&iPBq5{ouv3>Q5~T817|rcY7N8sSOjGA6T7&r2A4il1ko(Hv?B(A`)MMcF z2ARX-2Tm(yTltnOr~fS{@kA05{6Som^$ma$5Mpv@?C+wuDY@C{6f6Cw|W zd4`(=YB)0SiIuXP({6M>Xo?{tzl!!c+Z#?P#k1V~YlgE-_O+ z8xN52@(o&CyJ*O(OiXv;0M6~8lk|9K`WiVVZ5SM2lYiz9@BZ3BFVT<}Wt3DMnb4&v z273EGV~^^b`>HS16qvT1HDz6t3sgb@$)GUdDd~#l`boClwa#;9fMfW)zjaZvhs(B& zTfSQ7uNYx6PNaCKJ68P-MLV-whQMm1;llgoR3oEx>(lKeGmKhHbC?dG_oONa32GY7xc)x^}pmQo?VVyIFxv62|5_j@qZv>W#_i zm+K1@@O|NJ+ESlXVi=q1)9YO~h!#*zwEP+clXTzJSWLnTrID#2(^S&vKJVe&zW4~O`seUvGN)-cj>{oN zuvFLcN88g5C%5?z{Bb~mif}E%utve$9DJa7`C*iT8CWDXbr}0lhh-YTQ8ULt2uFUy z^5=jYJ?_ysRN*x1yl$}!0OfVGd#Q=~e|pm)IFH+0deQE54K773@Qdt0oZ(e5Q>?AV zm!ni2N{bx&qff&yd_cM1h&mN5Pn&PFEYLY%U$C~*BJ5FWsA%3|7fy3o&jG=oP=C$= zY)nbc2nM?D+w;puw6Y6M9jj&a)^5xxG4V1FvVWjLx6`_R2$`quKhcY*tavi|4St55Kgg=WJG*O` z&+av45rmr#hTE8783b%>hsx{gH{U%u)|E#Hw@qD+ZDDpmPN!=4(*rWzKfyq~#~x?Z z;RZ{CpGOuRw#ax_|63PSTD1I~{Z|_i?S-!8E93%hb`}t@0F5?b5vktM{eY>m_Pb)x z7FYbRcAZ>PS}N0~Uj{gIFn3Cni;G?7`tYc)wtY*cA6_{Vatr&2LRE*vusi$YYb2s6 zTNnoxLRvhWf+)<80E6B;N9BM;ZrOiD$&lXR9Sg1L3^ioN>J1jO%p)aAyJTRBH;~4R zJQda(NTUT?I4PQ#%til7{IT=Bcx-4(oW?(O5um3^9C@gP>$Z!6NwesFqUWL9sU2My;-Sxsb!kNk zhqNo=2`_I+!BJcT9#k${Ty!U7?xw2YD@+e9=34KJl_nvOMD$QeoGvU$ylvF_K@uH;NT$9K7rFC zu@S1}I%tAh5tG=4PjcCTF&Ing$}q~*_U+qWz>6c6)qIqAF3JG=$O>2~#EeK%TQ5 zeW`rU*LrLlPUG$svN4m)*~g4=4tGStHhLhx-K+^?6h1({0Z1q+x{b#j98$>9{k|o# z9?k4BQhBzD$wbbak9RnkzA;VEFvM1#X-py^L;n2Nx!7&c@*}UFNBnu+vN?61&$q;v s|4ZrKf9vV~NtE>e*Z6-odKSdXwA;nBa+UoLNX}6Anf;W#=frRS0j7Ufe*gdg literal 0 HcmV?d00001 diff --git a/img/multi_channel_net_2x2_x2x2x2.svg b/img/multi_channel_net_2x2_x2x2x2.svg new file mode 100644 index 0000000..d099112 --- /dev/null +++ b/img/multi_channel_net_2x2_x2x2x2.svg @@ -0,0 +1,743 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.10.2 + 2019-03-22 22:36:04 +0000 + + + Canvas 1 + + + Layer 1 + + + + + w + i,1,1 + = HC(c, 0, 1, p) + + + + + + + w + i,1,0 + = HC(c, 0, 0, p) + + + + + + + w + i,0,1 + = HC(c, 0, 1, p) + + + + + + + w + i,0,0 + = HC(c, 0, 0, p) + + + + + + + MaxPool 3x3 + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + ResizingMaxPool + + + + + + + ResizingMaxPool + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + MaxPool 3x3 + + + + + + + Add + 32x32x32 + + + + + + + Add + 64x32x32 + + + + + + + MaxPool 3x3 + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + ResizingMaxPool + + + + + + + MaxPool 3x3 + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + ResizingMaxpool + + + + + + + Add + 32x32x32 + + + + + + + Add + 32x32x32 + + + + + + + MaxPool 3x3 + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + ResizingMaxPool + + + + + + + ResizingMaxPool + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + MaxPool 3x3 + + + + + + + Add + 32x16x16 + + + + + + + Add + 64x16x16 + + + + + + + MaxPool 3x3 + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + ResizingMaxPool + + + + + + + ResizingMaxPool + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + MaxPool 3x3 + + + + + + + Add + 32x8x8 + + + + + + + Add + 64x8x8 + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + Add + 768x8x8 + + + + + + + SharpSepConv + + + + + + + GlobalAvgPool + + + + + + + Linear 768 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MaxPool 3x3 + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + ResizingMaxPool + + + + + + + ResizingMaxPool + + + + + + + SharpSepConv + + + + + + + SharpSepConv + + + + + + + MaxPool 3x3 + + + + + + + Add + 32x16x16 + + + + + + + Add + 64x16x16 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Input 3x32x32 + + + + + + + Conv 3x3 + + + + + + + BatchNorm + + + + + + + Conv 3x3 + + + + + + + BatchNorm + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 422e3e44f1b1968d49b498c70cfb8d4e4a1da444 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 30 May 2019 17:38:12 -0400 Subject: [PATCH 31/40] README.md typo fixes --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 19d2610..3c4dbcb 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# sharpDARTS: Faster, More Accurate Differentiable Architecture Search +# sharpDARTS: Faster and More Accurate Differentiable Architecture Search ![Differentiable Hyperparameter Search Example Graph](img/multi_channel_net_2x2_x2x2x2.svg) -Please cite [sharpDARTS: Faster, More Accurate Differentiable Architecture Search](http://arxiv.org/abs/1903.09900) if you use this code as part of your research! By using any part of the code you are agreeing to comply with our permissive Apache 2.0 license. +Please cite [sharpDARTS: Faster and More Accurate Differentiable Architecture Search](http://arxiv.org/abs/1903.09900) if you use this code as part of your research! By using any part of the code you are agreeing to comply with our permissive Apache 2.0 license. ``` @article{hundt2019sharpdarts, @@ -52,7 +52,7 @@ python3 visualize.py SHARP_DARTS python3 visualize.py DARTS ``` -## Training a final Model +## Training a Final Model To see the configuration options, run `python3 train.py --help` from the directory `cnn`. The code is also commented. From 8e4d5f8974fdfd71ffc6249748e953c8b9de772c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 31 May 2019 12:07:25 -0400 Subject: [PATCH 32/40] README.md add svg images of cos power annealing --- README.md | 3 +- img/cos_power_annealing_imagenet.svg | 296 +++++++++++++++++++++++ img/cos_power_annealing_imagenet_log.svg | 212 ++++++++++++++++ 3 files changed, 510 insertions(+), 1 deletion(-) create mode 100644 img/cos_power_annealing_imagenet.svg create mode 100644 img/cos_power_annealing_imagenet_log.svg diff --git a/README.md b/README.md index 3c4dbcb..b306183 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,8 @@ for i in {1..5}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --b 512 ## Cosine Power Annealing -![Cosine Power Annealing Example Curve](img/cos_power_annealing_imagenet.png) +![Cosine Power Annealing Example Curve](img/cos_power_annealing_imagenet.svg) +![Cosine Power Annealing Example Logarithmic Curve](img/cos_power_annealing_imagenet_log.svg) Cosine Power annealing is designed to be a learning rate schedule which improves on cosine annealing. See `consine_power_annealing.py` for the code, and `cnn/train.py` for an example of using the API. diff --git a/img/cos_power_annealing_imagenet.svg b/img/cos_power_annealing_imagenet.svg new file mode 100644 index 0000000..f2aea73 --- /dev/null +++ b/img/cos_power_annealing_imagenet.svg @@ -0,0 +1,296 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/cos_power_annealing_imagenet_log.svg b/img/cos_power_annealing_imagenet_log.svg new file mode 100644 index 0000000..6be278a --- /dev/null +++ b/img/cos_power_annealing_imagenet_log.svg @@ -0,0 +1,212 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 0eb5d9035cf10209b01bacbaf263af276c432e86 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 4 Jun 2019 16:32:39 -0400 Subject: [PATCH 33/40] README.md caps fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b306183..b6d229e 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ Here is the key line with the settings you'll want to use: lr_schedule is a numpy array containing the learning rate at each epoch. -## The sharpDARTS Repository is based on Differentiable Architecture Search (DARTS) +## The sharpDARTS Repository is Based on Differentiable Architecture Search (DARTS) The following is a lightly edited and updated reproduction of the README.md from the original [DARTS Code](https://github.com/quark0/darts/tree/f276dd346a09ae3160f8e3aca5c7b193fda1da37) accompanying the paper [DARTS: Differentiable Architecture Search](https://arxiv.org/abs/1806.09055): From a6dc73104b035adbc3a967151d8b359d01951bb3 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 4 Jun 2019 17:48:20 -0400 Subject: [PATCH 34/40] README.md clarify steps for model search. --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b6d229e..8433be1 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,13 @@ export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --b ### Max-W Regularization Search - +``` export CUDA_VISIBLE_DEVICES="0" && python3 train_search.py --dataset cifar10 --batch_size 48 --layers_of_cells 8 --layers_in_cells 4 --save max_w_SharpSepConvDARTS_SEARCH_`git rev-parse --short HEAD` --init_channels 16 --epochs 120 --cutout --autoaugment --seed 22 --weighting_algorithm max_w --primitives DARTS_PRIMITIVES -Run `cnn/train_search.py` with your configuration. Place the best genotype into `genotypes.py`, preferably with a record of the raw weights as well and other command execution data so similar results can be reproduced in the future. +``` + +### Add Search Results to the Code + +After running `cnn/train_search.py` with your configuration, place the best genotype printout `genotype = ...` into `genotypes.py` with a unique name. We suggest also adding a record of your command, the git commit version, and a copy of the raw weights, and other command execution data so similar results can be reproduced in the future. ### Visualization @@ -91,6 +95,7 @@ python3 -m torch.distributed.launch --nproc_per_node=2 main_fp16_optimizer.py -- Resuming Training (you may need to do this if your results don't match the paper on the first run): You'll need to make sure to specify the correct checkpoint directory, which will change every run. + ``` python3 -m torch.distributed.launch --nproc_per_node=2 main_fp16_optimizer.py --fp16 --b 224 --save `git rev-parse --short HEAD`_DARTS_PRIMITIVES_DIL_IS_SEPCONV --epochs 100 --dynamic-loss-scale --workers 20 --autoaugment --auxiliary --cutout --data /home/costar/datasets/imagenet/ --learning_rate 0.0005 --learning_rate_min 0.0000075 --arch DARTS_PRIMITIVES_DIL_IS_SEPCONV --resume eval-20190213-182625-975c657_DARTS_PRIMITIVES_DIL_IS_SEPCONV-imagenet-DARTS_PRIMITIVES_DIL_IS_SEPCONV-0/checkpoint.pth.tar ``` From 91532e62aa793140b37168541838428b3688868c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 4 Jun 2019 17:51:03 -0400 Subject: [PATCH 35/40] README.md typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8433be1..2b9fdda 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ After running `cnn/train_search.py` with your configuration, place the best geno ### Visualization -Here are examples of how to visualize the cell structures from a genotype definition found int `genotypes.py`: +Here are examples of how to visualize the cell structures from a genotype definition found in `genotypes.py`: ``` cd cnn From f6580233982932448699406ef43bb4354d8c9c1c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 4 Jun 2019 18:02:18 -0400 Subject: [PATCH 36/40] README.md explain how to generate cosine power annealing charts. --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2b9fdda..e281e7a 100644 --- a/README.md +++ b/README.md @@ -124,11 +124,14 @@ for i in {1..5}; do export CUDA_VISIBLE_DEVICES="0" && python3 train.py --b 512 ## Cosine Power Annealing +Cosine Power annealing is designed to be a learning rate schedule which improves on cosine annealing. +See `cosine_power_annealing.py` for the code, and `cnn/train.py` for an example of using the API. + ![Cosine Power Annealing Example Curve](img/cos_power_annealing_imagenet.svg) ![Cosine Power Annealing Example Logarithmic Curve](img/cos_power_annealing_imagenet_log.svg) -Cosine Power annealing is designed to be a learning rate schedule which improves on cosine annealing. -See `consine_power_annealing.py` for the code, and `cnn/train.py` for an example of using the API. +To generate the charts above simply run `python cosine_power_annealing.py` from the `/cnn` directory. +An overview of the `cosine_power_annealing` API is below: ```python def cosine_power_annealing( From d0c86e465b53063bee30e26515dff1d96eaee870 Mon Sep 17 00:00:00 2001 From: Priyanka Hubli Date: Sat, 8 Jun 2019 10:16:32 -0400 Subject: [PATCH 37/40] Added support for Cross-Modal embedding --- cnn/dataset.py | 8 +- cnn/model.py | 289 +++++++++++++++++++++++++++++--------------- cnn/train_costar.py | 41 ++++--- 3 files changed, 219 insertions(+), 119 deletions(-) diff --git a/cnn/dataset.py b/cnn/dataset.py index 2101000..3057e7e 100644 --- a/cnn/dataset.py +++ b/cnn/dataset.py @@ -13,7 +13,7 @@ import torchvision.datasets as dset import torch.backends.cudnn as cudnn try: - import costar_dataset + import costar_dataset except ImportError: print('dataset.py: The costar dataset is not available, so it is being skipped. ' 'See https://github.com/ahundt/costar_dataset for details') @@ -84,14 +84,16 @@ costar_class_dict = {'translation_only': 3, 'rotation_only': 5, 'all_features': 8, - 'time_difference_images': 6} + 'time_difference_images': 6, + 'cross_modal_embeddings': 6} costar_supercube_inp_channel_dict = {'translation_only': 52, 'rotation_only': 55, 'all_features': 57} costar_vec_size_dict = {'translation_only': 44, 'rotation_only': 49, 'all_features': 49, - 'time_difference_images': 0} + 'time_difference_images': 0, + 'cross_modal_embeddings': 20} COSTAR_SET_NAMES = ['blocks_only', 'blocks_with_plush_toy'] COSTAR_SUBSET_NAMES = ['success_only', 'error_failure_only', 'task_failure_only', 'task_and_error_failure'] diff --git a/cnn/model.py b/cnn/model.py index 4b7c796..48e69ab 100644 --- a/cnn/model.py +++ b/cnn/model.py @@ -510,124 +510,215 @@ def forward(self, input, log=False): def reset_noise(self): self.classifier.reset_noise() -class residual_block(nn.Module): - """Implements a layer of ResNeXt. - - Based in https://blog.waya.ai/deep-residual-learning-9610bb62c355. - +class Classifier(nn.Module): + """ + Classifier network for both Temporal Distance Classifier (TDC) and Cross-Modal Temporal Distance Classifier (CMC) + Reference: Playing Hard Exploration Games by Watching Youtube - https://arxiv.org/abs/1805.11592 + Code Source: https://github.com/seungjaeryanlee/playing-hard-exploration-games-by-watching-youtube Args: - input: Input for the block. - channels_in: Number of channels of data for the convolutional group. - channels_out: Number of chhannels generated as output. - cardinality: Number of convolution groups. Must be divisible by channels_in - is_training: Placeholder that indicates if the model is being trained - strides: Strides. - project_shortcut: Indicates whether the input should be projected to match the output's dimensions. + Input - embedding vector of type int (Image/ Joint). Default size 1024 + The input is a product of two embeddings. + For TDC -> input = img1_embedding * img2_embedding + For CMC -> input = img_embedding * joint_embedding Returns: - A tensor with shape (-1, channels_out, width, height). - - """ - def __init__(self, channels_in, channels_out, cardinality, strides=(1, 1), project_shortcut=False): - super(residual_block, self).__init__() - self.channels_in = channels_in - self.channels_out = channels_out - self.cardinality = cardinality - self.project_shortcut = project_shortcut - self.strides = strides - - self.conv1 = nn.Conv2d(64, self.channels_in, 1 , stride = 1) - self.norm1 = nn.BatchNorm2d(self.channels_in) - self.group_conv = nn.Conv2d(self.channels_in, self.channels_in, kernel_size=(3,3),stride=self.strides, groups=1,padding=1) - - self.norm2 = nn.BatchNorm2d(self.channels_in) - self.conv2 = nn.Conv2d(self.channels_in ,self.channels_out, 1 , stride = 1) - self.norm3 = nn.BatchNorm2d(self.channels_out) - - if self.project_shortcut or self.strides != (1,1): - self.conv3 = nn.Conv2d(64, self.channels_out, 1, stride = self.strides) - self.norm4 = nn.BatchNorm2d(self.channels_out) - - def forward(self, input): - shortcut = input - x = F.relu(self.norm1(self.conv1(input))) - x = self.group_conv(x) - x = F.relu(self.norm2(x)) - x = self.norm3(self.conv2(x)) + Number of features in the output layer is equal to the number of classes. Default number + of classes(out_channels) is 6. + """ + def __init__(self, in_channels = 1024, out_channels = 6): + super(Classifier, self).__init__() + self.fc1 = nn.Linear(in_channels, 1024) + self.fc2 = nn.Linear(1024, out_channels) - if self.project_shortcut or self.strides!=(1,1): - # When the dimensions increase projection shortcut is used to match dimensions (done by 1×1 convolutions). - shortcut = self.norm4(self.conv3(shortcut)) + def forward(self, input): + x = F.relu(self.fc1(input)) + prediction = self.fc2(x) + return prediction + +class ResidualBlock(nn.Module): + """ + Implements residual connected blocks with no down sampling. + Reference: Playing Hard Exploration Games by Watching Youtube - https://arxiv.org/abs/1805.11592 + Code Source: https://github.com/seungjaeryanlee/playing-hard-exploration-games-by-watching-youtube + Args: + input: Input for the block with shape (batch_size, in_channels, height, width) + in_channels: Number of channels of data for the convolutional group. Default is 64 channels + out_channels: Number of channels generated as output. Default is 64 channels + + Returns: + A tensor with shape (batch_size, out_channels, height, width). + """ + def __init__(self, in_channels = 64, out_channels = 64): + super(ResidualBlock, self).__init__() + self.conv1 = nn.Conv2d(in_channels, out_channels, 3 ,padding=1) + self.norm1 = nn.BatchNorm2d(out_channels) + + self.conv2 = nn.Conv2d(out_channels, out_channels, 3, padding=1) + self.norm2 = nn.BatchNorm2d(out_channels) + + def forward(self, input): + x = F.relu(self.norm1(self.conv1(input))) + x = self.norm1(self.conv2(x)) + output = F.relu(x + input) - output = F.relu(torch.add(shortcut, x)) - return output + return output class TDCFeaturizer(nn.Module): - """Temporal Distance Classification featurizer - - Reference: "Playing hard exploration games by watching YouTube" - The task consists of presenting the network with 2 frames separated by n timesteps, - and making it classify the distance between the frames. - - We use the same network architecture as the paper: - 3 convolutional layers, followed by 3 residual blocks, - followed by 2 fully connected layers for the encoder. + """ + Temporal Distance Classification featurizer + + Reference: Playing hard exploration games by watching YouTube - https://arxiv.org/abs/1805.11592 + The task consists of presenting the network with 2 frames separated by n timesteps, + and making it classify the distance between the frames. + + We use the same network architecture as the paper: + 3 convolutional layers, followed by 3 residual blocks, + followed by 2 fully connected layers for the encoder. + The final embedding vector is normalized. + Args: + input: stack of images with tensor shape - (batch_size, in_channels, height, width) + in_channels: Number of channels of images. Default is 3 channels + out_channels: Number of channels generated as output is the same as the embedding size. + Default is 1024 channels + + Returns: + embedding vector - A tensor with shape (batch_size, out_channels). - For the classifier, we do a multiplication between both feature vectors - followed by a fully connected layer. - Set 'classify_frame_distance' as True if we want the predicted class from the model. - To just get the embeddings, set 'classify_frame_distance' as False and 'frame_embeddings' - as True. """ - def __init__(self): + def __init__(self, in_channels = 3, embedding_size = 1024): super(TDCFeaturizer, self).__init__() - self.feature_vector_size = 1024 - self.conv1 = nn.Conv2d(3, 32, 3, stride = 2, padding=1) - self.conv2 = nn.Conv2d(32, 64, 3, stride = 1,padding=1) - self.conv3 = nn.Conv2d(64, 64, 3, stride = 1,padding=1) - self.pool = nn.MaxPool2d(2, 2) + self.conv1 = nn.Conv2d(in_channels, 32, 3, stride = 2, padding=1) self.norm1 = nn.BatchNorm2d(32) + self.pool = nn.MaxPool2d(2, 2) + + self.conv2 = nn.Conv2d(32, 64, 3, stride = 1,padding=1) self.norm2 = nn.BatchNorm2d(64) + + self.conv3 = nn.Conv2d(64, 64, 3, stride = 1,padding=1) self.norm3 = nn.BatchNorm2d(64) - self.residual_block = residual_block(64, 64, 1) - self.fc1 = nn.Linear(3072, self.feature_vector_size) #dense layer - self.fc2 = nn.Linear(self.feature_vector_size, self.feature_vector_size) - self.fc3 = nn.Linear(1024, self.feature_vector_size) - self.fc4 = nn.Linear(self.feature_vector_size, 6) # size of each label - self.classify_frame_distance = True - self.frame_embeddings = False - - def choose_outputs(self, classify_frame_distance = True, frame_embeddings = False): - self.classify_frame_distance = classify_frame_distance - self.frame_embeddings = frame_embeddings - - def forward(self, img_1, img_2): - logits_aux = None - x = torch.cat((img_1,img_2),0) - x = self.pool(self.norm1(F.relu(self.conv1(x)))) - x = self.pool(self.norm2(F.relu(self.conv2(x)))) - x = self.pool(self.norm3(F.relu(self.conv3(x)))) + + self.residual_block = ResidualBlock(64, 64) + + self.fc1 = nn.Linear(3072, 1024) + self.fc2 = nn.Linear(1024, embedding_size) + + def forward(self, input): + x = F.relu(self.pool(self.norm1(self.conv1(input)))) + x = F.relu(self.pool(self.norm2(self.conv2(x)))) + x = F.relu(self.pool(self.norm3(self.conv3(x)))) for i in range(3): x = self.residual_block(x) + + x = x.view(x.size(0), -1) #(3072 = 64*6*8) - x = x.view(-1, 3072) # flatten layer (3072 = 64*6*8) - x = F.relu(self.fc1(x)) - - feature_vector = self.fc2(x) - feature_vector = F.normalize(feature_vector, p=2, dim=1) - if self.frame_embeddings and not self.classify_frame_distance: - return feature_vector - feature_vector_stack = feature_vector.view(-1, 2, self.feature_vector_size) + x = self.fc1(x) + embedding = F.normalize(self.fc2(x)) + return embedding + +class CMCFeaturizer(nn.Module): + """ + Cross-Modal Temporal Distance Classification featurizer - combined_embeddings = torch.mul(feature_vector_stack[:, 0, :], feature_vector_stack[:, 1, :]) + Reference: Playing hard exploration games by watching YouTube - https://arxiv.org/abs/1805.11592 + The task consists of presenting the network with one frame and joint_vectors information separated by n timesteps, + and making it classify the distance between the frames. - x = F.relu(self.fc3(combined_embeddings)) - prediction = F.relu(self.fc4(x)) + The joint_vectors is concatentation of joint_vector over M timesteps where M defaults to 10. + To set M: It is an argument to the function 'block_stacking_reader_torch.generate_cross_modal_training_data' + Args: + input: stack of joint_vector with tensor shape - (batch_size, 1, M, 20) + in_channels: Number of channels of joint_vectors. Default is 1 channel + out_channels: Number of channels generated as output is the same as the embedding size. + Default is 1024 channels - if self.classify_frame_distance and not self.frame_embeddings: - return prediction , logits_aux - if self.classify_frame_distance and self.frame_embeddings: - return (prediction, feature_vector, logits_aux) + Returns: + embedding vector - A tensor with shape (batch_size, out_channels). + + """ + def __init__(self, in_channels = 1, embedding_size = 1024): + super(CMCFeaturizer,self).__init__() + + self.conv1 = nn.Conv2d(in_channels, 32, 3, padding=1) + self.norm1 = nn.BatchNorm2d(32) + self.pool = nn.MaxPool2d(2) + + self.conv2 = nn.Conv2d(32, 64, 3, padding=1) + self.norm2 = nn.BatchNorm2d(64) + + self.conv3 = nn.Conv2d(64, 128, 3, padding=1) + self.norm3 = nn.BatchNorm2d(128) + + self.conv4 = nn.Conv2d(128, 256, 3, padding=1) + self.norm4 = nn.BatchNorm2d(256) + self.pool2 = nn.MaxPool2d(2,padding=1) + + self.fc = nn.Linear(512, embedding_size) + + def forward(self, joints): + x = F.relu(self.pool(self.norm1(self.conv1(joints)))) + x = F.relu(self.pool(self.norm2(self.conv2(x)))) + x = F.relu(self.pool(self.norm3(self.conv3(x)))) + x = F.relu(self.pool2(self.norm4(self.conv4(x)))) + + x = x.view(x.size(0), -1) + + joint_embedding = F.normalize(self.fc(x)) + return joint_embedding + +class TDC(nn.Module): + """ + Full Temporal Distance Featurizer and Classifier. + Reference: Playing hard exploration games by watching YouTube - https://arxiv.org/abs/1805.11592 + + Args: + img1 - stack of images with tensor shape - (batch_size, in_channels, height, width) + img2 - stack of images with tensor shape - (batch_size, in_channels, height, width) + + Returns: + temporal distance prediction - A tensor with shape (batch_size, out_channels) where default out_channels is 6. + + """ + def __init__(self): + super(TDC, self).__init__() + + self.featurizer = TDCFeaturizer() + self.classifier = Classifier() + + def forward(self, img1, img2): + img1_embedding = self.featurizer(img1) + img2_embedding = self.featurizer(img2) + + output = self.classifier(img1_embedding * img2_embedding) + + return output, None # Returning logits_aux as None + +class CMC(nn.Module): + """ + Full Cross Modal Temporal Distance Featurizer and Classifier. + Reference: Playing hard exploration games by watching YouTube - https://arxiv.org/abs/1805.11592 + + Args: + img - stack of images with tensor shape - (batch_size, in_channels, height, width) + joint_vec - stack of joint_vector with tensor shape - (batch_size, 1, M, 20) + + Returns: + temporal distance prediction - A tensor with shape (batch_size, out_channels) where default out_channels is 6. + + """ + def __init__(self): + super(CMC, self).__init__() + + self.img_featurizer = TDCFeaturizer() + self.joint_featurizer = CMCFeaturizer() + self.classifier = Classifier() + + def forward(self, img, joint_vec): + img_embedding = self.img_featurizer(img) + joint_embedding = self.joint_featurizer(joint_vec) + + output = self.classifier(img_embedding * joint_embedding) + + return output, None # Returning logits_aux as None diff --git a/cnn/train_costar.py b/cnn/train_costar.py index fb8ac60..4b9d510 100644 --- a/cnn/train_costar.py +++ b/cnn/train_costar.py @@ -33,7 +33,7 @@ import torchvision.datasets as datasets import torchvision.models as models import torchvision -from model import TDCFeaturizer +from model import TDC, CMC import numpy as np import random @@ -71,7 +71,8 @@ # choices=model_names, help='model architecture: ' + ' | '.join(model_names) + - 'TDCFeaturizer for feature embeddings' + 'TDC for temporal distance classifier' + 'CMC for cross modal temporal distance classifier' ' (default: SHARP_DARTS)') parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', help='number of data loading workers (default: 4)') @@ -148,8 +149,8 @@ help='which subset to use in the CoSTAR BSD. Options are "success_only", ' '"error_failure_only", "task_failure_only", or "task_and_error_failure". Defaults to "success_only"') parser.add_argument('--feature_mode', type=str, default='all_features', - help='which feature mode to use. Options are "translation_only", "rotation_only", "stacking_reward", "time_difference_images"' - 'or the default "all_features"') + help='which feature mode to use. Options are "translation_only", "rotation_only", "stacking_reward",' + '"time_difference_images", "cross_modal_embeddings" or the default "all_features"') parser.add_argument('--num_images_per_example', type=int, default=200, help='Number of times an example is visited per epoch. Default value is 200. Since the image for each visit to an ' 'example is randomly chosen, and since the number of images in an example is different, we simply visit each ' @@ -225,8 +226,13 @@ def main(): # baseline model for comparison model = NetworkResNetCOSTAR(args.init_channels, classes, args.layers, args.auxiliary, None, vector_size=VECTOR_SIZE, op_dict=op_dict, C_mid=args.mid_channels) model.drop_path_prob = 0.0 - elif args.arch == 'TDCFeaturizer': - model = TDCFeaturizer() + elif args.arch == 'TDC': + model = TDC() + # To check the learnable parameters - + #for n, p in model.named_parameters(): + # print(n, p.shape) + elif args.arch == 'CMC': + model = CMC() else: # create model genotype = eval("genotypes.%s" % args.arch) @@ -260,8 +266,9 @@ def main(): init_lr = args.learning_rate / args.warmup_lr_divisor # define loss function (criterion) and optimizer - if args.feature_mode == 'time_difference_images': + if args.feature_mode == 'time_difference_images' or args.feature_mode == 'cross_modal_embeddings': # 'time_difference_images' is a feature mode where we try to classify time intervals between two frames. + # 'cross_modal_embeddings' is another mode where we try to classify time intervals between a frame and a joint embedding. criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.Adam(model.parameters(), lr=init_lr) # Change collate function and model input shape for this feature mode @@ -358,7 +365,7 @@ def resume(): for param_group in optimizer.param_groups: param_group['lr'] = learning_rate # scheduler.step() - if args.feature_mode != 'time_difference_images': + if args.feature_mode != 'time_difference_images' and args.feature_mode != 'cross_modal_embeddings': model.drop_path_prob = args.drop_path_prob * float(epoch) / float(args.epochs) # train for one epoch train_stats = train(train_loader, model, criterion, optimizer, int(epoch), args) @@ -497,12 +504,12 @@ def train(train_loader, model, criterion, optimizer, epoch, args): # switch to train mode model.train() end = time.time() - prefetcher = data_prefetcher(train_loader, cutout=args.cutout, cutout_length=args.cutout_length) + prefetcher = data_prefetcher(train_loader, cutout=args.cutout, cutout_length=args.cutout_length) cart_error, angle_error = [], [] input_img, input_vec, target = prefetcher.next() - if args.feature_mode == 'time_difference_images': - target = target.type(torch.cuda.LongTensor) + if args.feature_mode == 'time_difference_images' or args.feature_mode == 'cross_modal_embeddings': + target = target.type(torch.cuda.LongTensor) batch_size = input_img.size(0) i = -1 if args.local_rank == 0: @@ -523,7 +530,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args): # compute output # note here the term output is equivalent to logits output, logits_aux = model(input_img, input_vec) - if args.feature_mode != 'time_difference_images': + if args.feature_mode != 'time_difference_images' and args.feature_mode != 'cross_modal_embeddings': output = sigmoid(output) loss = criterion(output, target) if logits_aux is not None and args.auxiliary: @@ -532,7 +539,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args): loss += args.auxiliary_weight * loss_aux # measure accuracy and record loss - if args.feature_mode != 'time_difference_images': + if args.feature_mode != 'time_difference_images' and args.feature_mode != 'cross_modal_embeddings': with torch.no_grad(): output_np = output.cpu().detach().numpy() target_np = target.cpu().detach().numpy() @@ -567,7 +574,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args): end = time.time() input_img, input_vec, target = prefetcher.next() - if args.feature_mode == 'time_difference_images' and target is not None: + if (args.feature_mode == 'time_difference_images' or args.feature_mode == 'cross_modal_embeddings') and target is not None: target = target.type(torch.cuda.LongTensor) if args.local_rank == 0: @@ -638,7 +645,7 @@ def validate(val_loader, model, criterion, args, prefix='val_'): cart_error, angle_error = [], [] prefetcher = data_prefetcher(val_loader) input_img, input_vec, target = prefetcher.next() - if args.feature_mode == 'time_difference_images': + if args.feature_mode == 'time_difference_images' or args.feature_mode == 'cross_modal_embeddings': target = target.type(torch.cuda.LongTensor) batch_size = input_img.size(0) i = -1 @@ -660,7 +667,7 @@ def validate(val_loader, model, criterion, args, prefix='val_'): loss = criterion(output, target) # measure accuracy and record loss - if args.feature_mode != 'time_difference_images': + if args.feature_mode != 'time_difference_images' and args.feature_mode != 'cross_modal_embeddings': batch_abs_cart_distance, batch_abs_angle_distance = accuracy(output.data.cpu().numpy(), target.data.cpu().numpy()) abs_cart_f, abs_angle_f = np.mean(batch_abs_cart_distance), np.mean(batch_abs_angle_distance) cart_error.extend(batch_abs_cart_distance) @@ -703,7 +710,7 @@ def validate(val_loader, model, criterion, args, prefix='val_'): abs_cart=abs_cart_m, abs_angle=abs_angle_m)) input_img, input_vec, target = prefetcher.next() - if args.feature_mode == 'time_difference_images' and target is not None: + if (args.feature_mode == 'time_difference_images' or args.feature_mode == 'cross_modal_embeddings') and target is not None: target = target.type(torch.cuda.LongTensor) # logger.info(' * combined_error {combined_error.avg:.3f} top5 {top5.avg:.3f}' From a00cf24f2737308d8fef9388154171143afabf7f Mon Sep 17 00:00:00 2001 From: Priyanka Hubli Date: Mon, 24 Jun 2019 01:05:40 -0700 Subject: [PATCH 38/40] added visualization and testing reader code --- README.md | 41 +++++++++ cnn/model.py | 12 +-- cnn/test_time_difference_reader.py | 79 +++++++++++++++++ cnn/train_costar.py | 31 ++----- cnn/visualize_embeddings.py | 132 +++++++++++++++++++++++++++++ 5 files changed, 266 insertions(+), 29 deletions(-) create mode 100644 cnn/test_time_difference_reader.py create mode 100644 cnn/visualize_embeddings.py diff --git a/README.md b/README.md index 3f711a0..36af51e 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,47 @@ python visualize.py DARTS ``` where `DARTS` can be replaced by any customized architectures in `genotypes.py`. +## Training Temporal Distance Classifier +To train the TDC network from scratch, run +``` +cd cnn && python train_costar.py --data --arch TDC --epochs 200 --batch_size 128 --feature_mode time_difference_images --num_images_per_example 200 +``` +Default path to dataset is '~/.keras/datasets/costar_block_stacking_dataset_v0.4' + +## Training Cross-Modal Temporal Distance Classifier +To train the CMC network from scratch, run +``` +cd cnn && python train_costar.py --data --arch CMC --epochs 200 --batch_size 128 --feature_mode cross_modal_embeddings --num_images_per_example 200 +``` +## Visualizing the embeddings using tensorboardX +This requires installation of tensorboardX which can be installed using +``` +pip install tensorboardX +``` +Or see https://github.com/lanpa/tensorboardX for details. + +To generate and save embeddings using temporal distance classifier, run +``` +python visualize_embeddings.py --data --model_path --feature_mode time_difference_images +``` +Or to generate and save embeddings using cross modal temporal distance classifier, run +``` +python visualize_embeddings.py --data --model_path --feature_mode cross_modal_embeddings +``` +This will create metadata and save it in the following manner: +``` +-> cnn + -> runs + -> yyyy-mm-dd-hh:mm + -> default + - metadata.tsv + - tensors.tsv + -> projector_config.pbtxt +``` +To visualize using tensorboard, +``` +tensorboard --logdir +``` ## Citation If you use any part of this code in your research, please cite our [paper](https://arxiv.org/abs/1806.09055): ``` diff --git a/cnn/model.py b/cnn/model.py index 48e69ab..d21614b 100644 --- a/cnn/model.py +++ b/cnn/model.py @@ -510,7 +510,7 @@ def forward(self, input, log=False): def reset_noise(self): self.classifier.reset_noise() -class Classifier(nn.Module): +class LinearBlockClassifier(nn.Module): """ Classifier network for both Temporal Distance Classifier (TDC) and Cross-Modal Temporal Distance Classifier (CMC) Reference: Playing Hard Exploration Games by Watching Youtube - https://arxiv.org/abs/1805.11592 @@ -526,7 +526,7 @@ class Classifier(nn.Module): of classes(out_channels) is 6. """ def __init__(self, in_channels = 1024, out_channels = 6): - super(Classifier, self).__init__() + super(LinearBlockClassifier, self).__init__() self.fc1 = nn.Linear(in_channels, 1024) self.fc2 = nn.Linear(1024, out_channels) @@ -684,13 +684,13 @@ def __init__(self): super(TDC, self).__init__() self.featurizer = TDCFeaturizer() - self.classifier = Classifier() + self.linear_classifier = LinearBlockClassifier() def forward(self, img1, img2): img1_embedding = self.featurizer(img1) img2_embedding = self.featurizer(img2) - output = self.classifier(img1_embedding * img2_embedding) + output = self.linear_classifier(img1_embedding * img2_embedding) return output, None # Returning logits_aux as None @@ -712,13 +712,13 @@ def __init__(self): self.img_featurizer = TDCFeaturizer() self.joint_featurizer = CMCFeaturizer() - self.classifier = Classifier() + self.linear_classifier = LinearBlockClassifier() def forward(self, img, joint_vec): img_embedding = self.img_featurizer(img) joint_embedding = self.joint_featurizer(joint_vec) - output = self.classifier(img_embedding * joint_embedding) + output = self.linear_classifier(img_embedding * joint_embedding) return output, None # Returning logits_aux as None diff --git a/cnn/test_time_difference_reader.py b/cnn/test_time_difference_reader.py new file mode 100644 index 0000000..bdfbabb --- /dev/null +++ b/cnn/test_time_difference_reader.py @@ -0,0 +1,79 @@ +import argparse +import torch +import numpy as np +from torch.utils.data import DataLoader + +from tqdm import tqdm +try: + from costar_dataset.block_stacking_reader_torch import CostarBlockStackingDataset +except ImportError: + ImportError('The costar dataset is not available. ' + 'See https://github.com/ahundt/costar_dataset for details') + +if __name__ == '__main__': + """ To test block_stacking_reader for feature_modes time_difference_images and cross_modal_embeddings""" + + args_parser = argparse.ArgumentParser() + args_parser.add_argument('--videos_path', help='Path for training data') + args_parser.add_argument('--feature_mode', help = 'cross_modal_embeddings or time_difference_images',default='time_difference_images') + args_parser.add_argument('--batch_size', help='Batch size for training', type=int, default=32) + args_parser.add_argument('--visualize', help='To view frames, set true', default=True) + args = args_parser.parse_args() + + visualize = args.visualize + + costar_dataset = CostarBlockStackingDataset.from_standard_txt( + root=args.videos_path, + version='v0.4', set_name='blocks_only', subset_name='success_only', + split='train', feature_mode=args.feature_mode, output_shape=(3, 96, 128), + num_images_per_example=200, is_training=False) + generator = DataLoader(costar_dataset, args.batch_size, shuffle=False, num_workers=4) + print("Length of the dataset: {}. Length of the loader: {}.".format(len(costar_dataset), len(generator))) + + generator_output = iter(generator) + print("-------------------op") + x1, x2, y = next(generator_output) + print("Image 1 shape: ", x1.shape, " Image 2/Joint shape: ",x2.shape, " Labels shape: ", y.shape) + + pb = tqdm(range(len(generator)-1)) + for i in pb: + pb.set_description('batch: {}'.format(i)) + + x1, x2, y = generator_output.next() + y = y.numpy() + x1 = [t.numpy() for t in x1] + x2 = [t.numpy() for t in x2] + distances = ['0', '1', '2','3 or 4', 'btw 5 and 20', 'btw 21 and 150'] + if visualize: + import matplotlib + import matplotlib.pyplot as plt + fig = plt.figure() + if (args.feature_mode == 'time_difference_images'): + title = "Interval between frames is " + str(distances[y[0]]) + else: + title = "Interval between frame and joint_vector is " + str(distances[y[0]]) + plt.title(title) + img1 = np.moveaxis(x1[0], 0, 2) + img2 = np.moveaxis(x2[0], 0, 2) + + # image 1 + fig1 = fig.add_subplot(1,2,1) + fig1.set_title("Frame 1") + plt.imshow(img1) + plt.draw() + plt.pause(0.25) + + if (args.feature_mode == 'time_difference_images'): + # image 2 + fig2 = fig.add_subplot(1,2,2) + fig2.set_title("Frame 2") + plt.imshow(img2) + plt.draw() + plt.pause(0.25) + # uncomment the following line to wait for one window to be closed before showing the next + plt.show() + + assert np.all(x1[0] <= 1) and np.all(x1[0] >= -1), "x1[0] is not within range!" + assert np.all(x1[1] <= 1) and np.all(x1[1] >= -1), "x1[1] is not within range!" + assert np.all(x1[2] <= 1) and np.all(x1[2] >= 0), "x1[2] is not within range!" + assert np.all(y <= 5) and np.all(y >= 0), "y is not within range!" diff --git a/cnn/train_costar.py b/cnn/train_costar.py index 4b9d510..10ad27f 100644 --- a/cnn/train_costar.py +++ b/cnn/train_costar.py @@ -149,7 +149,7 @@ help='which subset to use in the CoSTAR BSD. Options are "success_only", ' '"error_failure_only", "task_failure_only", or "task_and_error_failure". Defaults to "success_only"') parser.add_argument('--feature_mode', type=str, default='all_features', - help='which feature mode to use. Options are "translation_only", "rotation_only", "stacking_reward",' + help='which feature mode to use. Options are "translation_only", "rotation_only", "stacking_reward",' '"time_difference_images", "cross_modal_embeddings" or the default "all_features"') parser.add_argument('--num_images_per_example', type=int, default=200, help='Number of times an example is visited per epoch. Default value is 200. Since the image for each visit to an ' @@ -179,13 +179,11 @@ def fast_collate(batch): # data is a list of [image_0, image_1, vector] return torch.cat((data[0], data[1]), dim=1), data[2], targets - if args.deterministic: cudnn.benchmark = False cudnn.deterministic = True torch.manual_seed(args.local_rank) - def main(): global best_combined_error, args, logger @@ -207,7 +205,7 @@ def main(): # note the gpu is used for directory creation and log files # which is needed when run as multiple processes args = utils.initialize_files_and_args(args) - logger = utils.logging_setup(args.log_file_path) + logger = utils.logging_setup(args.log_file_path) # # load the correct ops dictionary op_dict_to_load = "operations.%s" % args.ops @@ -228,7 +226,7 @@ def main(): model.drop_path_prob = 0.0 elif args.arch == 'TDC': model = TDC() - # To check the learnable parameters - + # To check the learnable parameters - #for n, p in model.named_parameters(): # print(n, p.shape) elif args.arch == 'CMC': @@ -259,8 +257,6 @@ def main(): # model = DDP(model) # delay_allreduce delays all communication to the end of the backward pass. model = DDP(model, delay_allreduce=True) - - # Scale learning rate based on global batch size args.learning_rate = args.learning_rate * float(args.batch_size * args.world_size)/256. init_lr = args.learning_rate / args.warmup_lr_divisor @@ -274,14 +270,13 @@ def main(): # Change collate function and model input shape for this feature mode fast_collate = torch.utils.data.dataloader.default_collate model_input_shape = (3,96,128) - else: + else: criterion = nn.MSELoss().cuda() # NOTE(rexxarchl): MSLE loss, indicated as better for rotation in costar_hyper/costar_block_stacking_train_regression.py # is not available in PyTorch by default optimizer = torch.optim.SGD(model.parameters(), init_lr, momentum=args.momentum, weight_decay=args.weight_decay) - # epoch_count = args.epochs - args.start_epoch # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(epoch_count)) # scheduler = warmup_scheduler.GradualWarmupScheduler( @@ -341,7 +336,7 @@ def resume(): validate(val_loader, model, criterion, args, prefix='evaluate_val_') validate(test_loader, model, criterion, args, prefix='evaluate_test_') return - + lr_schedule = cosine_power_annealing( epochs=args.epochs, max_lr=args.learning_rate, min_lr=args.learning_rate_min, warmup_epochs=args.warmup_epochs, exponent_order=args.lr_power_annealing_exponent_order, @@ -451,7 +446,6 @@ def resume(): validate(test_loader, model, criterion, args, prefix='best_final_test_') logger.info("Final evaluation complete! Save dir: ' + str(args.save)") - class data_prefetcher(): def __init__(self, loader, cutout=False, cutout_length=112, cutout_cuts=1): self.loader = iter(loader) @@ -487,7 +481,6 @@ def next(self): self.preload() return input_img, input_vec, target - def train(train_loader, model, criterion, optimizer, epoch, args): loader_len = len(train_loader) if loader_len < 2: @@ -504,12 +497,12 @@ def train(train_loader, model, criterion, optimizer, epoch, args): # switch to train mode model.train() end = time.time() - prefetcher = data_prefetcher(train_loader, cutout=args.cutout, cutout_length=args.cutout_length) + prefetcher = data_prefetcher(train_loader, cutout=args.cutout, cutout_length=args.cutout_length) cart_error, angle_error = [], [] input_img, input_vec, target = prefetcher.next() if args.feature_mode == 'time_difference_images' or args.feature_mode == 'cross_modal_embeddings': - target = target.type(torch.cuda.LongTensor) + target = target.type(torch.cuda.LongTensor) batch_size = input_img.size(0) i = -1 if args.local_rank == 0: @@ -558,7 +551,7 @@ def train(train_loader, model, criterion, optimizer, epoch, args): losses.update(reduced_loss, batch_size) abs_cart_m.update(abs_cart_f, batch_size) abs_angle_m.update(abs_angle_f, batch_size) - + else: reduced_loss = reduce_tensor(loss.data) if args.distributed else loss.data losses.update(reduced_loss) @@ -609,7 +602,6 @@ def train(train_loader, model, criterion, optimizer, epoch, args): del progbar return stats - def get_stats(progbar, prefix, args, batch_time, data_time, abs_cart, abs_angle, losses, speed): stats = {} if progbar is not None: @@ -625,7 +617,6 @@ def get_stats(progbar, prefix, args, batch_time, data_time, abs_cart, abs_angle, }) return stats - def validate(val_loader, model, criterion, args, prefix='val_'): loader_len = len(val_loader) if loader_len < 2: @@ -729,14 +720,12 @@ def validate(val_loader, model, criterion, args, prefix='val_'): # Return the weighted sum of absolute cartesian and angle errors as the metric return (args.cart_weight * abs_cart_m.avg + (1-args.cart_weight) * abs_angle_m.avg), stats - def save_checkpoint(state, is_best, path='', filename='checkpoint.pth.tar', best_filename='model_best.pth.tar'): new_filename = os.path.join(path, filename) torch.save(state, new_filename) if is_best: shutil.copyfile(new_filename, os.path.join(path, best_filename)) - class AverageMeter(object): """Computes and stores the average and current value""" def __init__(self): @@ -754,7 +743,6 @@ def update(self, val, n=1): self.count += n self.avg = self.sum / self.count - def adjust_learning_rate(optimizer, epoch, step, len_epoch): """LR schedule that should yield 76% converged accuracy with batch size 256""" factor = epoch // 30 @@ -774,7 +762,6 @@ def adjust_learning_rate(optimizer, epoch, step, len_epoch): for param_group in optimizer.param_groups: param_group['lr'] = lr - def accuracy(output, target): """Computes the absolute cartesian and angle distance between output and target""" batch_size, out_channels = target.shape @@ -799,13 +786,11 @@ def accuracy(output, target): return abs_cart_distance, abs_angle_distance - def reduce_tensor(tensor): rt = tensor.clone() dist.all_reduce(rt, op=dist.reduce_op.SUM) rt /= args.world_size return rt - if __name__ == '__main__': main() diff --git a/cnn/visualize_embeddings.py b/cnn/visualize_embeddings.py new file mode 100644 index 0000000..1466f10 --- /dev/null +++ b/cnn/visualize_embeddings.py @@ -0,0 +1,132 @@ +import argparse +import os +import datetime + +from model import CMC, TDC, TDCFeaturizer, CMCFeaturizer +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable +from torch.utils.data import DataLoader + +try: + from tensorboardX import SummaryWriter + from tensorboardX import embedding + from tensorboardX.x2num import make_np +except ImportError: + ImportError('Install tensorboardX-1.7 by using "pip install tensorboardX" or' + ' see https://github.com/lanpa/tensorboardX for details') +try: + from costar_dataset.block_stacking_reader_torch import CostarBlockStackingDataset +except ImportError: + ImportError('The costar dataset is not available. ' + 'See https://github.com/ahundt/costar_dataset for details') + +def save_features_to_visualize(embeddings, experiment_name='default'): + """ + Save the embeddings to be visualised using t-sne on TensorBoardX + Reference: https://medium.com/@vegi/visualizing-higher-dimensional-data-using-t-sne-on-tensorboard-7dbf22682cf2 + Args: embeddings - of shape - (length of video, length of feature_vector) + default length of feature_vector = 1024 for TDC and 2048 for CMC + """ + vid_embed = Variable(torch.tensor(np.squeeze(np.concatenate(embeddings, 0)))) + + # Generate metadata + metadata = 'video_index\tframe_index\n' + for video_index, video_embedding in enumerate(embeddings): + print(video_index,' video_embedding length: ', len(video_embedding)) + for frame_index, frame_embedding in enumerate(video_embedding): + metadata += '{}\t{}\n'.format(video_index, frame_index) + + currentDT = datetime.datetime.now() + path = 'runs/' + str(currentDT)[:10] + '-' + str(currentDT)[11:16] + subdir = experiment_name + save_path = os.path.join(path,subdir) + try: + os.makedirs(save_path) + except OSError: + print('warning: Embedding dir exists, did you set global_step for add_embedding()?') + + metadata_path = os.path.join(save_path, 'metadata.tsv') + with open(metadata_path, 'w') as metadata_file: + metadata_file.write(metadata) + + writer = SummaryWriter(write_to_disk=False) + mat = make_np(vid_embed) + embedding.make_mat(mat,save_path) + embedding.append_pbtxt(metadata,label_img=None,save_path=path,subdir=subdir,global_step=0,tag='default') + writer.close() + print("Saved embeddings in: ",save_path) + return + +def generate_features(videos_path, model_path, batch_size,feature_mode, version, set_name, subset_name): + """ + Generates and saves the embeddings for all the videos listed in the file -videos_path. + Args: videos_path - path to the file where the videos are listed + model_path - path to the best trained model, assumes it is named model_best.pth.tar + batch_size - batch size per process. Defaults to 32 + feature_mode - "cross_modal_embeddings" or "time_difference_images". Default is 'cross_modal_embeddings' + version - the CoSTAR BSD version to use. Defaults to "v0.4" + set_name - which set to use in the CoSTAR BSD. Options are "blocks_only" or "blocks_with_plush_toy". + Defaults to "blocks_only" + subset_name - which subset to use in the CoSTAR BSD. Options are "success_only", + "error_failure_only", "task_failure_only", or "task_and_error_failure". Defaults to "success_only" + """ + datasets = CostarBlockStackingDataset.from_standard_txt(root=videos_path, + version=version, set_name=set_name, subset_name=subset_name, + split='test', feature_mode=feature_mode, output_shape=(3, 96, 128), + num_images_per_example=200, is_training=False,visual_mode=True) + + loaders = [DataLoader(dataset, batch_size=32, shuffle=False, num_workers=1) for dataset in datasets] + print("Length of the dataset: {}. Length of the loader: {}.".format(len(datasets), len(loaders))) + + best_model_path = os.path.join(args.model_path, 'model_best.pth.tar') + pretrained_model = torch.load(best_model_path,map_location='cpu') + + model_tdc = TDCFeaturizer().double() + model_tdc_dict = model_tdc.state_dict() + pretrained_tdc_dict = {k: v for k, v in pretrained_model.items() if k in model_tdc_dict} + model_tdc_dict.update(pretrained_tdc_dict) + model_tdc.load_state_dict(model_tdc_dict) + + if(feature_mode == 'cross_modal_embeddings'): + model_cmc = CMCFeaturizer().double() + model_cmc_dict = model_cmc.state_dict() + pretrained_cmc_dict = {k: v for k, v in pretrained_model.items() if k in model_cmc_dict} + model_cmc_dict.update(pretrained_cmc_dict) + model_cmc.load_state_dict(model_cmc_dict) + + features_all = [] + for loader in loaders: + feature_vectors = [] + for _,batch in enumerate(loader): + if (feature_mode == 'cross_modal_embeddings'): + frame,joint = batch + frame_features = model_tdc(frame) + joint_features = model_cmc(joint) + features = torch.cat((frame_features, joint_features),1) + else: + frame = batch + features = model_tdc(frame) + features = F.normalize(features).cpu().detach().numpy() + feature_vectors.append(features) + feature_vecs = np.concatenate(feature_vectors) + features_all.append(feature_vecs) + return features_all + +if __name__ == '__main__': + + args_parser = argparse.ArgumentParser() + args_parser.add_argument('--data', help='Path for training data') + args_parser.add_argument('--model_path', help='Path for best model',default='') + args_parser.add_argument('--batch_size', help='Batch size for visualizing', type=int, default=32) + args_parser.add_argument('--feature_mode', help = 'cross_modal_embeddings or time_difference_images',default='cross_modal_embeddings') + args_parser.add_argument('--version', help='Path for best model',default='v0.4') + args_parser.add_argument('--set_name', help='Path for best model',default='blocks_only') + args_parser.add_argument('--subset_name', help='Path for best model',default='success_only') + args = args_parser.parse_args() + + features_all = generate_features(args.data, args.model_path, args.batch_size, args.feature_mode, + args.version, args.set_name, args.subset_name) + save_features_to_visualize(features_all) \ No newline at end of file From 7be01d37963ab2ac9be814de1e99ca384e6f2b29 Mon Sep 17 00:00:00 2001 From: Priyanka Hubli Date: Mon, 24 Jun 2019 17:40:03 -0700 Subject: [PATCH 39/40] moved test reader to costar_dataset --- cnn/test_time_difference_reader.py | 79 ------------------------------ 1 file changed, 79 deletions(-) delete mode 100644 cnn/test_time_difference_reader.py diff --git a/cnn/test_time_difference_reader.py b/cnn/test_time_difference_reader.py deleted file mode 100644 index bdfbabb..0000000 --- a/cnn/test_time_difference_reader.py +++ /dev/null @@ -1,79 +0,0 @@ -import argparse -import torch -import numpy as np -from torch.utils.data import DataLoader - -from tqdm import tqdm -try: - from costar_dataset.block_stacking_reader_torch import CostarBlockStackingDataset -except ImportError: - ImportError('The costar dataset is not available. ' - 'See https://github.com/ahundt/costar_dataset for details') - -if __name__ == '__main__': - """ To test block_stacking_reader for feature_modes time_difference_images and cross_modal_embeddings""" - - args_parser = argparse.ArgumentParser() - args_parser.add_argument('--videos_path', help='Path for training data') - args_parser.add_argument('--feature_mode', help = 'cross_modal_embeddings or time_difference_images',default='time_difference_images') - args_parser.add_argument('--batch_size', help='Batch size for training', type=int, default=32) - args_parser.add_argument('--visualize', help='To view frames, set true', default=True) - args = args_parser.parse_args() - - visualize = args.visualize - - costar_dataset = CostarBlockStackingDataset.from_standard_txt( - root=args.videos_path, - version='v0.4', set_name='blocks_only', subset_name='success_only', - split='train', feature_mode=args.feature_mode, output_shape=(3, 96, 128), - num_images_per_example=200, is_training=False) - generator = DataLoader(costar_dataset, args.batch_size, shuffle=False, num_workers=4) - print("Length of the dataset: {}. Length of the loader: {}.".format(len(costar_dataset), len(generator))) - - generator_output = iter(generator) - print("-------------------op") - x1, x2, y = next(generator_output) - print("Image 1 shape: ", x1.shape, " Image 2/Joint shape: ",x2.shape, " Labels shape: ", y.shape) - - pb = tqdm(range(len(generator)-1)) - for i in pb: - pb.set_description('batch: {}'.format(i)) - - x1, x2, y = generator_output.next() - y = y.numpy() - x1 = [t.numpy() for t in x1] - x2 = [t.numpy() for t in x2] - distances = ['0', '1', '2','3 or 4', 'btw 5 and 20', 'btw 21 and 150'] - if visualize: - import matplotlib - import matplotlib.pyplot as plt - fig = plt.figure() - if (args.feature_mode == 'time_difference_images'): - title = "Interval between frames is " + str(distances[y[0]]) - else: - title = "Interval between frame and joint_vector is " + str(distances[y[0]]) - plt.title(title) - img1 = np.moveaxis(x1[0], 0, 2) - img2 = np.moveaxis(x2[0], 0, 2) - - # image 1 - fig1 = fig.add_subplot(1,2,1) - fig1.set_title("Frame 1") - plt.imshow(img1) - plt.draw() - plt.pause(0.25) - - if (args.feature_mode == 'time_difference_images'): - # image 2 - fig2 = fig.add_subplot(1,2,2) - fig2.set_title("Frame 2") - plt.imshow(img2) - plt.draw() - plt.pause(0.25) - # uncomment the following line to wait for one window to be closed before showing the next - plt.show() - - assert np.all(x1[0] <= 1) and np.all(x1[0] >= -1), "x1[0] is not within range!" - assert np.all(x1[1] <= 1) and np.all(x1[1] >= -1), "x1[1] is not within range!" - assert np.all(x1[2] <= 1) and np.all(x1[2] >= 0), "x1[2] is not within range!" - assert np.all(y <= 5) and np.all(y >= 0), "y is not within range!" From b20fa1a3bdc4d387289ad7f74c6b2d9a57c6f6cd Mon Sep 17 00:00:00 2001 From: Priyanka Hubli Date: Wed, 10 Jul 2019 12:28:58 -0700 Subject: [PATCH 40/40] added default dataset path in visualize_embeddings.py --- cnn/visualize_embeddings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cnn/visualize_embeddings.py b/cnn/visualize_embeddings.py index 1466f10..67fe428 100644 --- a/cnn/visualize_embeddings.py +++ b/cnn/visualize_embeddings.py @@ -118,7 +118,7 @@ def generate_features(videos_path, model_path, batch_size,feature_mode, version, if __name__ == '__main__': args_parser = argparse.ArgumentParser() - args_parser.add_argument('--data', help='Path for training data') + args_parser.add_argument('--data', help='Path to dataset', default='~/.keras/datasets/costar_block_stacking_dataset_v0.4') args_parser.add_argument('--model_path', help='Path for best model',default='') args_parser.add_argument('--batch_size', help='Batch size for visualizing', type=int, default=32) args_parser.add_argument('--feature_mode', help = 'cross_modal_embeddings or time_difference_images',default='cross_modal_embeddings')