add pl---hydra implementation

pl-hydra/configs/callbacks/default.yaml

@@ -0,0 +1,28 @@
+model_checkpoint:
+  _target_: pytorch_lightning.callbacks.ModelCheckpoint
+  monitor: "val/acc" # name of the logged metric which determines when model is improving
+  mode: "max" # "max" means higher metric value is better, can be also "min"
+  save_top_k: 3 # save k best models (determined by above metric)
+  save_last: True # additionaly always save model from last epoch
+  verbose: False
+  dirpath: "checkpoints/"
+  filename: "epoch_{epoch:03d}"
+  auto_insert_metric_name: False
+
+early_stopping:
+  _target_: pytorch_lightning.callbacks.EarlyStopping
+  monitor: "val/acc" # name of the logged metric which determines when model is improving
+  mode: "max" # "max" means higher metric value is better, can be also "min"
+  patience: 100 # how many validation epochs of not improving until training stops
+  min_delta: 0 # minimum change in the monitored metric needed to qualify as an improvement
+
+model_summary:
+  _target_: pytorch_lightning.callbacks.RichModelSummary
+  max_depth: -1
+
+rich_progress_bar:
+  _target_: pytorch_lightning.callbacks.RichProgressBar
+
+learning_rate_monitor:
+  _target_: pytorch_lightning.callbacks.LearningRateMonitor
+  logging_interval: epoch

pl-hydra/configs/datamodule/cifar10.yaml

@@ -0,0 +1,11 @@
+_target_: src.datamodules.cifar10_datamodule.CIFAR10DataModule
+
+data_dir: ${data_dir} # data_dir is specified in config.yaml
+batch_size: 128
+num_workers: 4
+pin_memory: True
+data_mean: [0.49421428, 0.48513139, 0.45040909]
+data_std: [0.24665252, 0.24289226, 0.26159238]
+image_size: [32, 32]
+scale_bounds: [0.8, 1.0]
+aspect_bounds: [0.9, 1.1]

pl-hydra/configs/debug/default.yaml

@@ -0,0 +1,28 @@
+# @package _global_
+
+# default debugging setup, runs 1 full epoch
+# other debugging configs can inherit from this one
+
+defaults:
+  - override /log_dir: debug.yaml
+
+trainer:
+  max_epochs: 1
+  gpus: 0 # debuggers don't like gpus
+  detect_anomaly: true # raise exception if NaN or +/-inf is detected in any tensor
+  track_grad_norm: 2 # track gradient norm with loggers
+
+datamodule:
+  num_workers: 0 # debuggers don't like multiprocessing
+  pin_memory: False # disable gpu memory pin
+
+# sets level of all command line loggers to 'DEBUG'
+# https://hydra.cc/docs/tutorials/basic/running_your_app/logging/
+hydra:
+  verbose: True
+
+  # use this to set level of only chosen command line loggers to 'DEBUG':
+  # verbose: [src.train, src.utils]
+
+# config is already printed by hydra when `hydra/verbose: True`
+print_config: False

pl-hydra/configs/debug/limit_batches.yaml

@@ -0,0 +1,12 @@
+# @package _global_
+
+# uses only 1% of the training data and 5% of validation/test data
+
+defaults:
+  - default.yaml
+
+trainer:
+  max_epochs: 3
+  limit_train_batches: 0.01
+  limit_val_batches: 0.05
+  limit_test_batches: 0.05

pl-hydra/configs/debug/overfit.yaml

@@ -0,0 +1,10 @@
+# @package _global_
+
+# overfits to 3 batches
+
+defaults:
+  - default.yaml
+
+trainer:
+  max_epochs: 20
+  overfit_batches: 3

pl-hydra/configs/debug/profiler.yaml

@@ -0,0 +1,12 @@
+# @package _global_
+
+# runs with execution time profiling
+
+defaults:
+  - default.yaml
+
+trainer:
+  max_epochs: 1
+  profiler: "simple"
+  # profiler: "advanced"
+  # profiler: "pytorch"

pl-hydra/configs/debug/step.yaml

@@ -0,0 +1,9 @@
+# @package _global_
+
+# runs 1 train, 1 validation and 1 test step
+
+defaults:
+  - default.yaml
+
+trainer:
+  fast_dev_run: true

pl-hydra/configs/debug/test_only.yaml

@@ -0,0 +1,9 @@
+# @package _global_
+
+# runs only test epoch
+
+defaults:
+  - default.yaml
+
+train: False
+test: True

pl-hydra/configs/experiment/example.yaml

@@ -0,0 +1,38 @@
+# @package _global_
+
+# to execute this experiment run:
+# python train.py experiment=example
+
+defaults:
+  - override /datamodule: mnist.yaml
+  - override /model: mnist.yaml
+  - override /callbacks: default.yaml
+  - override /logger: null
+  - override /trainer: default.yaml
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+
+# name of the run determines folder name in logs
+name: "simple_dense_net"
+
+seed: 12345
+
+trainer:
+  min_epochs: 10
+  max_epochs: 10
+  gradient_clip_val: 0.5
+
+model:
+  lr: 0.002
+  net:
+    lin1_size: 128
+    lin2_size: 256
+    lin3_size: 64
+
+datamodule:
+  batch_size: 64
+
+logger:
+  wandb:
+    tags: ["mnist", "${name}"]

pl-hydra/configs/hparams_search/mnist_optuna.yaml

@@ -0,0 +1,60 @@
+# @package _global_
+
+# example hyperparameter optimization of some experiment with Optuna:
+# python train.py -m hparams_search=mnist_optuna experiment=example
+
+defaults:
+  - override /hydra/sweeper: optuna
+
+# choose metric which will be optimized by Optuna
+# make sure this is the correct name of some metric logged in lightning module!
+optimized_metric: "val/acc_best"
+
+# here we define Optuna hyperparameter search
+# it optimizes for value returned from function with @hydra.main decorator
+# docs: https://hydra.cc/docs/next/plugins/optuna_sweeper
+hydra:
+  sweeper:
+    _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper
+
+    # storage URL to persist optimization results
+    # for example, you can use SQLite if you set 'sqlite:///example.db'
+    storage: null
+
+    # name of the study to persist optimization results
+    study_name: null
+
+    # number of parallel workers
+    n_jobs: 1
+
+    # 'minimize' or 'maximize' the objective
+    direction: maximize
+
+    # total number of runs that will be executed
+    n_trials: 25
+
+    # choose Optuna hyperparameter sampler
+    # docs: https://optuna.readthedocs.io/en/stable/reference/samplers.html
+    sampler:
+      _target_: optuna.samplers.TPESampler
+      seed: 12345
+      n_startup_trials: 10 # number of random sampling runs before optimization starts
+
+    # define range of hyperparameters
+    search_space:
+      datamodule.batch_size:
+        type: categorical
+        choices: [32, 64, 128]
+      model.lr:
+        type: float
+        low: 0.0001
+        high: 0.2
+      model.net.lin1_size:
+        type: categorical
+        choices: [32, 64, 128, 256, 512]
+      model.net.lin2_size:
+        type: categorical
+        choices: [32, 64, 128, 256, 512]
+      model.net.lin3_size:
+        type: categorical
+        choices: [32, 64, 128, 256, 512]

pl-hydra/configs/log_dir/debug.yaml

@@ -0,0 +1,8 @@
+# @package _global_
+
+hydra:
+  run:
+    dir: logs/debugs/runs/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
+  sweep:
+    dir: logs/debugs/multiruns/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
+    subdir: ${hydra.job.num}

pl-hydra/configs/log_dir/default.yaml

@@ -0,0 +1,15 @@
+# @package _global_
+
+hydra:
+  run:
+    dir: logs/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
+  sweep:
+    dir: logs/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+
+# hydra:
+#   run:
+#     dir: logs/experiments/runs/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
+#   sweep:
+#     dir: logs/experiments/multiruns/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
+#     subdir: ${hydra.job.num}

pl-hydra/configs/log_dir/evaluation.yaml

@@ -0,0 +1,8 @@
+# @package _global_
+
+hydra:
+  run:
+    dir: logs/evaluations/runs/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
+  sweep:
+    dir: logs/evaluations/multiruns/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
+    subdir: ${hydra.job.num}

pl-hydra/configs/logger/comet.yaml

@@ -0,0 +1,7 @@
+# https://www.comet.ml
+
+comet:
+  _target_: pytorch_lightning.loggers.comet.CometLogger
+  api_key: ${oc.env:COMET_API_TOKEN} # api key is loaded from environment variable
+  project_name: "template-tests"
+  experiment_name: ${name}

pl-hydra/configs/logger/csv.yaml

@@ -0,0 +1,7 @@
+# csv logger built in lightning
+
+csv:
+  _target_: pytorch_lightning.loggers.csv_logs.CSVLogger
+  save_dir: "."
+  name: "csv/"
+  prefix: ""

pl-hydra/configs/logger/many_loggers.yaml

@@ -0,0 +1,9 @@
+# train with many loggers at once
+
+defaults:
+  # - comet.yaml
+  - csv.yaml
+  # - mlflow.yaml
+  # - neptune.yaml
+  - tensorboard.yaml
+  - wandb.yaml

pl-hydra/configs/logger/mlflow.yaml

@@ -0,0 +1,9 @@
+# https://mlflow.org
+
+mlflow:
+  _target_: pytorch_lightning.loggers.mlflow.MLFlowLogger
+  experiment_name: ${name}
+  tracking_uri: ${original_work_dir}/logs/mlflow/mlruns # run `mlflow ui` command inside the `logs/mlflow/` dir to open the UI
+  tags: null
+  prefix: ""
+  artifact_location: null

pl-hydra/configs/logger/neptune.yaml

@@ -0,0 +1,11 @@
+# https://neptune.ai
+
+neptune:
+  _target_: pytorch_lightning.loggers.neptune.NeptuneLogger
+  api_key: ${oc.env:NEPTUNE_API_TOKEN} # api key is loaded from environment variable
+  project_name: your_name/template-tests
+  close_after_fit: True
+  offline_mode: False
+  experiment_name: ${name}
+  experiment_id: null
+  prefix: ""

pl-hydra/configs/logger/tensorboard.yaml

@@ -0,0 +1,11 @@
+# https://www.tensorflow.org/tensorboard/
+
+tensorboard:
+  _target_: pytorch_lightning.loggers.tensorboard.TensorBoardLogger
+  save_dir: "tensorboard/"
+  name: null
+  version: ${name}
+  log_graph: True
+  default_hp_metric: True
+  prefix: ""
+

pl-hydra/configs/logger/wandb.yaml

@@ -0,0 +1,15 @@
+# https://wandb.ai
+
+wandb:
+  _target_: pytorch_lightning.loggers.wandb.WandbLogger
+  project: "template-tests"
+  # name: ${name}
+  save_dir: "."
+  offline: False # set True to store all logs only locally
+  id: null # pass correct id to resume experiment!
+  # entity: ""  # set to name of your wandb team
+  log_model: False
+  prefix: ""
+  job_type: "train"
+  group: ""
+  tags: []

pl-hydra/configs/model/cifar10_densenet.yaml

@@ -0,0 +1,10 @@
+_target_: src.models.cifar10_module.CIFAR10LitModule
+
+net:
+  _target_: src.models.components.densenet.DenseNet
+  num_classes: 10
+  num_layers: [6,6,6,6]
+  bn_size: 2
+  growth_rate: 16
+  act_fn_by_name: relu
+

pl-hydra/configs/model/cifar10_googlenet.yaml

@@ -0,0 +1,6 @@
+_target_: src.models.cifar10_module.CIFAR10LitModule
+
+net:
+  _target_: src.models.components.googlenet.GoogleNet
+  num_classes: 10
+  act_fn_by_name: relu

pl-hydra/configs/model/cifar10_resnet.yaml

@@ -0,0 +1,8 @@
+_target_: src.models.cifar10_module.CIFAR10LitModule
+
+net:
+  _target_: src.models.components.resnet.ResNet
+  num_classes: 10
+  num_blocks: [3,3,3]
+  c_hidden: [16,32,64]
+  act_fn_by_name: relu

pl-hydra/configs/model/cifar10_vgg11.yaml

@@ -0,0 +1,6 @@
+_target_: src.models.cifar10_module.CIFAR10LitModule
+
+net:
+  _target_: src.models.components.vgg.VGG11
+  num_classes: 10
+  act_fn_by_name: relu

pl-hydra/configs/model/cifar10_vit.yaml

@@ -0,0 +1,13 @@
+_target_: src.models.cifar10_module.CIFAR10LitModule
+
+net:
+  _target_: src.models.components.vit.VisionTransformer
+  num_classes: 10
+  num_heads: 8
+  num_layers: 6
+  num_channels: 3
+  num_patches: 64
+  patch_size: 4
+  embed_dim: 256
+  hidden_dim: 512
+  dropout: 0.2

pl-hydra/configs/optim/optim_adam.yaml

@@ -0,0 +1,11 @@
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 1e-3
+  weight_decay: 1e-4
+
+use_lr_scheduler: True
+
+lr_scheduler:
+  _target_: torch.optim.lr_scheduler.MultiStepLR
+  milestones: [90,130]
+  gamma: 0.1