[WIP] Byol Repro (v2)

configs/config/benchmark/linear_image_classification/imagenet1k/byol_transfer_in1k_linear.yaml

@@ -0,0 +1,129 @@
+# @package _global_
+# BYOL:Bootstrap Your Own Latent - https://arxiv.org/abs/2006.07733
+# Configuration based on BYOL paper - Appendix C -- C.1
+config:
+ VERBOSE: True
+ LOG_FREQUENCY: 200
+ TEST_ONLY: False
+ TEST_EVERY_NUM_EPOCH: 1
+ TEST_MODEL: True
+ SEED_VALUE: 1
+ MULTI_PROCESSING_METHOD: forkserver
+ HOOKS:
+ PERF_STATS:
+ MONITOR_PERF_STATS: True
+ DATA:
+ NUM_DATALOADER_WORKERS: 5
+ TRAIN:
+ DATA_SOURCES: [disk_folder]
+ LABEL_SOURCES: [disk_folder]
+ DATASET_NAMES: [imagenet1k_folder]
+ BATCHSIZE_PER_REPLICA: 32
+ TRANSFORMS:
+ - name: RandomResizedCrop
+ size: 224
+ interpolation: 3
+ - name: RandomHorizontalFlip
+ - name: ToTensor
+ - name: Normalize
+ mean: [0.485, 0.456, 0.406]
+ std: [0.229, 0.224, 0.225]
+ MMAP_MODE: True
+ COPY_TO_LOCAL_DISK: False
+ COPY_DESTINATION_DIR: /tmp/imagenet1k/
+ TEST:
+ DATA_SOURCES: [disk_folder]
+ LABEL_SOURCES: [disk_folder]
+ DATASET_NAMES: [imagenet1k_folder]
+ BATCHSIZE_PER_REPLICA: 32
+ TRANSFORMS:
+ - name: Resize
+ size: 256
+ interpolation: 3
+ - name: CenterCrop
+ size: 224
+ - name: ToTensor
+ - name: Normalize
+ mean: [0.485, 0.456, 0.406]
+ std: [0.229, 0.224, 0.225]
+ MMAP_MODE: True
+ COPY_TO_LOCAL_DISK: False
+ COPY_DESTINATION_DIR: /tmp/imagenet1k/
+ METERS:
+ name: accuracy_list_meter
+ accuracy_list_meter:
+ num_meters: 5
+ topk_values: [1, 5]
+ TRAINER:
+ TRAIN_STEP_NAME: standard_train_step
+ MODEL:
+ FEATURE_EVAL_SETTINGS:
+ EVAL_MODE_ON: True
+ FREEZE_TRUNK_ONLY: True
+ SHOULD_FLATTEN_FEATS: False
+ LINEAR_EVAL_FEAT_POOL_OPS_MAP: [
+ ["conv1", ["AvgPool2d", [[10, 10], 10, 4]]],
+ ["res2", ["AvgPool2d", [[16, 16], 8, 0]]],
+ ["res3", ["AvgPool2d", [[13, 13], 5, 0]]],
+ ["res4", ["AvgPool2d", [[8, 8], 3, 0]]],
+ ["res5", ["AvgPool2d", [[6, 6], 1, 0]]],
+ ]
+ TRUNK:
+ NAME: resnet
+ RESNETS:
+ DEPTH: 50
+ HEAD:
+ PARAMS: [
+ ["eval_mlp", {"in_channels": 64, "dims": [9216, 1000]}],
+ ["eval_mlp", {"in_channels": 256, "dims": [9216, 1000]}],
+ ["eval_mlp", {"in_channels": 512, "dims": [8192, 1000]}],
+ ["eval_mlp", {"in_channels": 1024, "dims": [9216, 1000]}],
+ ["eval_mlp", {"in_channels": 2048, "dims": [8192, 1000]}],
+ ]
+ WEIGHTS_INIT:
+ PARAMS_FILE: "specify the model weights"
+ STATE_DICT_KEY_NAME: classy_state_dict
+ SYNC_BN_CONFIG:
+ CONVERT_BN_TO_SYNC_BN: False
+ SYNC_BN_TYPE: apex
+ GROUP_SIZE: 8
+ LOSS:
+ name: cross_entropy_multiple_output_single_target
+ cross_entropy_multiple_output_single_target:
+ ignore_index: -1
+ OPTIMIZER:
+ name: sgd
+ momentum: 0.9
+ num_epochs: 80
+ weight_decay: 0
+ nesterov: True
+ regularize_bn: False
+ regularize_bias: True
+ param_schedulers:
+ lr:
+ auto_lr_scaling:
+ # if set to True, learning rate will be scaled.
+ auto_scale: True
+ # base learning rate value that will be scaled.
+ base_value: 0.2
+ # batch size for which the base learning rate is specified. The current batch size
+ # is used to determine how to scale the base learning rate value.
+ # scaled_lr = ((batchsize_per_gpu * world_size) * base_value ) / base_lr_batch_size
+ base_lr_batch_size: 256
+ # scaling_type can be set to "sqrt" to reduce the impact of scaling on the base value
+ scaling_type: "linear"
+ name: constant
+ update_interval: "epoch"
+ value: 0.2
+ DISTRIBUTED:
+ BACKEND: nccl
+ NUM_NODES: 4
+ NUM_PROC_PER_NODE: 8
+ INIT_METHOD: tcp
+ RUN_ID: auto
+ MACHINE:
+ DEVICE: gpu
+ CHECKPOINT:
+ DIR: "."
+ AUTO_RESUME: True
+ CHECKPOINT_FREQUENCY: 1

configs/config/pretrain/byol/byol_1gpu_resnet18_imagenette.yaml

@@ -0,0 +1,123 @@
+# @package _global_
+# BYOL: Bootstrap Your Own Latent - https://arxiv.org/pdf/2006.07733.pdf
+# Official BYOL implementation - https://github.com/deepmind/deepmind-research/blob/master/byol
+config:
+ VERBOSE: False
+ LOG_FREQUENCY: 200
+ TEST_ONLY: False
+ TEST_MODEL: False
+ SEED_VALUE: 1337 # BYOL Paper - Page 33 - Appendix J.1
+ MULTI_PROCESSING_METHOD: forkserver
+ HOOKS:
+ PERF_STATS:
+ MONITOR_PERF_STATS: True
+ ROLLING_BTIME_FREQ: 313
+ TENSORBOARD_SETUP:
+ USE_TENSORBOARD: False
+ EXPERIMENT_LOG_DIR: "byol_reference"
+ LOG_PARAMS: False
+ FLUSH_EVERY_N_MIN: 20
+ DATA:
+ NUM_DATALOADER_WORKERS: 5
+ TRAIN:
+ DATA_SOURCES: [disk_folder]
+ DATASET_NAMES: [imagenette_160_folder]
+ BATCHSIZE_PER_REPLICA: 256
+ LABEL_TYPE: sample_index # Just an implementation detail. Label isn't used
+ TRANSFORMS: # BYOL Paper - Page 16 - Appendix B
+ - name: ImgReplicatePil
+ num_times: 2
+ - name: RandomResizedCrop
+ size: 128
+ interpolation: 3
+ - name: RandomHorizontalFlip
+ p: 0.5
+ - name: ImgPilColorDistortion
+ strength: 0.5
+ saturation: 0.4
+ - name: ImgPilMultiCropRandomApply
+ transforms:
+ - name: ImgPilGaussianBlur
+ p: 1
+ radius_min: 0.1
+ radius_max: 2.0
+ prob: [ 1.0, 0.1 ]
+ - name: ImgPilMultiCropRandomApply
+ transforms:
+ - name: ImgPilRandomSolarize
+ p: 1.0
+ prob: [ 0.0, 0.2 ]
+ - name: ToTensor
+ - name: Normalize
+ mean: [ 0.485, 0.456, 0.406 ]
+ std: [ 0.229, 0.224, 0.225 ]
+ COLLATE_FUNCTION: simclr_collator
+ MMAP_MODE: True
+ COPY_TO_LOCAL_DISK: False
+ COPY_DESTINATION_DIR: /tmp/imagenet1k/
+ DROP_LAST: True
+ USE_DEBUGGING_SAMPLER: False
+ TRAINER:
+ TRAIN_STEP_NAME: standard_train_step
+ METERS:
+ name: ""
+ MODEL: # BYOL Paper - Page 5 - 3.3 Implementation details -- Architecture
+ TRUNK:
+ NAME: resnet
+ RESNETS:
+ DEPTH: 18
+ ZERO_INIT_RESIDUAL: False
+ HEAD:
+ PARAMS: [
+ ["mlp", {"dims": [2048, 4096, 256], "use_relu": True, "use_bn": True}],
+ ["mlp", {"dims": [256, 4096, 256], "use_relu": True, "use_bn": True}]
+ ]
+ SYNC_BN_CONFIG:
+ CONVERT_BN_TO_SYNC_BN: True
+ SYNC_BN_TYPE: pytorch
+ AMP_PARAMS:
+ USE_AMP: False
+ LOSS: # BYOL Paper - Page 33 - Appendix J.1
+ name: byol_loss
+ byol_loss:
+ embedding_dim: 256
+ momentum: 0.99
+ OPTIMIZER:
+ name: lars
+ eta: 0.001
+ weight_decay: 1.0e-6
+ momentum: 0.9
+ nesterov: False
+ num_epochs: 1000
+ regularize_bn: False
+ regularize_bias: False
+ exclude_bias_and_norm: True
+ param_schedulers:
+ lr:
+ auto_lr_scaling:
+ auto_scale: true
+ base_value: 2.0
+ base_lr_batch_size: 256
+ name: composite
+ schedulers:
+ - name: linear
+ start_value: 0.00002
+ end_value: 2.0 # Automatically rescaled if needed
+ - name: cosine
+ start_value: 2.0 # Automatically rescaled if needed
+ end_value: 0.000002 # Automatically rescaled if needed
+ update_interval: step
+ interval_scaling: [rescaled, rescaled]
+ lengths: [0.001, 0.999] # 10ep
+ DISTRIBUTED:
+ BACKEND: nccl
+ NUM_NODES: 1
+ NUM_PROC_PER_NODE: 1
+ INIT_METHOD: tcp
+ RUN_ID: auto
+ MACHINE:
+ DEVICE: gpu
+ CHECKPOINT:
+ AUTO_RESUME: True
+ CHECKPOINT_FREQUENCY: 10
+ CHECKPOINT_ITER_FREQUENCY: -1 # set this variable to checkpoint every few iterations

configs/config/pretrain/byol/byol_4node_resnet.yaml

@@ -0,0 +1,123 @@
+# @package _global_
+# BYOL: Bootstrap Your Own Latent - https://arxiv.org/pdf/2006.07733.pdf
+# Official BYOL implementation - https://github.com/deepmind/deepmind-research/blob/master/byol
+config:
+ VERBOSE: False
+ LOG_FREQUENCY: 200
+ TEST_ONLY: False
+ TEST_MODEL: False
+ SEED_VALUE: 1337 # BYOL Paper - Page 33 - Appendix J.1
+ MULTI_PROCESSING_METHOD: forkserver
+ HOOKS:
+ PERF_STATS:
+ MONITOR_PERF_STATS: True
+ ROLLING_BTIME_FREQ: 313
+ TENSORBOARD_SETUP:
+ USE_TENSORBOARD: False
+ EXPERIMENT_LOG_DIR: "byol_reference"
+ LOG_PARAMS: False
+ FLUSH_EVERY_N_MIN: 20
+ DATA:
+ NUM_DATALOADER_WORKERS: 5
+ TRAIN:
+ DATA_SOURCES: [disk_filelist]
+ DATASET_NAMES: [imagenet1k_filelist]
+ BATCHSIZE_PER_REPLICA: 128
+ LABEL_TYPE: sample_index # Just an implementation detail. Label isn't used
+ TRANSFORMS: # BYOL Paper - Page 16 - Appendix B
+ - name: ImgReplicatePil
+ num_times: 2
+ - name: RandomResizedCrop
+ size: 224
+ interpolation: 3
+ - name: RandomHorizontalFlip
+ p: 0.5
+ - name: ImgPilColorDistortion
+ strength: 0.5
+ saturation: 0.4
+ - name: ImgPilMultiCropRandomApply
+ transforms:
+ - name: ImgPilGaussianBlur
+ p: 1
+ radius_min: 0.1
+ radius_max: 2.0
+ prob: [ 1.0, 0.1 ]
+ - name: ImgPilMultiCropRandomApply
+ transforms:
+ - name: ImgPilRandomSolarize
+ p: 1.0
+ prob: [ 0.0, 0.2 ]
+ - name: ToTensor
+ - name: Normalize
+ mean: [ 0.485, 0.456, 0.406 ]
+ std: [ 0.229, 0.224, 0.225 ]
+ COLLATE_FUNCTION: simclr_collator
+ MMAP_MODE: True
+ COPY_TO_LOCAL_DISK: False
+ COPY_DESTINATION_DIR: /tmp/imagenet1k/
+ DROP_LAST: True
+ USE_DEBUGGING_SAMPLER: False
+ TRAINER:
+ TRAIN_STEP_NAME: standard_train_step
+ METERS:
+ name: ""
+ MODEL: # BYOL Paper - Page 5 - 3.3 Implementation details -- Architecture
+ TRUNK:
+ NAME: resnet
+ RESNETS:
+ DEPTH: 50
+ ZERO_INIT_RESIDUAL: True
+ HEAD:
+ PARAMS: [
+ ["mlp", {"dims": [2048, 4096, 256], "use_relu": True, "use_bn": True}],
+ ["mlp", {"dims": [256, 4096, 256], "use_relu": True, "use_bn": True}]
+ ]
+ SYNC_BN_CONFIG:
+ CONVERT_BN_TO_SYNC_BN: True
+ SYNC_BN_TYPE: pytorch
+ AMP_PARAMS:
+ USE_AMP: False
+ LOSS: # BYOL Paper - Page 33 - Appendix J.1
+ name: byol_loss
+ byol_loss:
+ embedding_dim: 256
+ momentum: 0.99
+ OPTIMIZER:
+ name: lars # CHECKED
+ eta: 0.001
+ weight_decay: 1.0e-6 # CHECKED
+ momentum: 0.9 # CHECKED
+ nesterov: False # CHECKED
+ num_epochs: 300 # CHECKED
+ regularize_bn: False # CHECKED
+ regularize_bias: False # CHECKED
+ exclude_bias_and_norm: True # CHECKED
+ param_schedulers:
+ lr:
+ auto_lr_scaling:
+ auto_scale: true
+ base_value: 0.3
+ base_lr_batch_size: 256
+ name: composite
+ schedulers:
+ - name: linear
+ start_value: 0.00001
+ end_value: 0.3 # Automatically rescaled if needed
+ - name: cosine
+ start_value: 0.3 # Automatically rescaled if needed
+ end_value: 0.000003 # Automatically rescaled if needed
+ update_interval: step
+ interval_scaling: [rescaled, rescaled]
+ lengths: [0.0334, 0.9666] # 10ep
+ DISTRIBUTED:
+ BACKEND: nccl
+ NUM_NODES: 8
+ NUM_PROC_PER_NODE: 8
+ INIT_METHOD: tcp
+ RUN_ID: auto
+ MACHINE:
+ DEVICE: gpu
+ CHECKPOINT:
+ AUTO_RESUME: True
+ CHECKPOINT_FREQUENCY: 10
+ CHECKPOINT_ITER_FREQUENCY: -1 # set this variable to checkpoint every few iterations