Update experiments.

configs/benchmark.yaml

@@ -44,6 +44,7 @@ baseline: "MONAKHOVA 100iter"
 
 save_idx: [0, 1, 2, 3, 4]   # provide index of files to save e.g. [1, 5, 10]
 save_intermediate: False   # save intermediate results, i.e. after pre-processor and after camera inversion
+swap_channels: False   # list of two RGB channels to swap, e.g. [0, 1] for swapping red and green
 gamma_psf: 1.5    # gamma factor for PSF
 
 

configs/benchmark_diffusercam_mirflickr.yaml

@@ -23,16 +23,30 @@ algorithms: [
   # "ADMM",
 
   ## -- reconstructions trained on DiffuserCam measured
-  # "hf:diffusercam:mirflickr:U5+Unet8M",
+  "hf:diffusercam:mirflickr:U5+Unet8M",
   "hf:diffusercam:mirflickr:Unet8M+U5",
-  # "hf:diffusercam:mirflickr:TrainInv+Unet8M",
-  # "hf:diffusercam:mirflickr:MMCN4M+Unet4M",
-  # "hf:diffusercam:mirflickr:MWDN8M",
-  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M",
-  # "hf:diffusercam:mirflickr:Unet4M+TrainInv+Unet4M",
-  # "hf:diffusercam:mirflickr:Unet2M+MMCN+Unet2M",
-  # "hf:diffusercam:mirflickr:Unet2M+MWDN6M",
-  # "hf:diffusercam:mirflickr:Unet4M+U10+Unet4M",
+  "hf:diffusercam:mirflickr:TrainInv+Unet8M",
+  "hf:diffusercam:mirflickr:MMCN4M+Unet4M",
+  "hf:diffusercam:mirflickr:MWDN8M",
+  "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M",
+  "hf:diffusercam:mirflickr:Unet4M+TrainInv+Unet4M",
+  "hf:diffusercam:mirflickr:Unet2M+MMCN+Unet2M",
+  "hf:diffusercam:mirflickr:Unet2M+MWDN6M",
+  "hf:diffusercam:mirflickr:Unet4M+U10+Unet4M",
+  "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psfNN",
+
+  # # -- benchmark PSF error
+  # "hf:diffusercam:mirflickr:U5+Unet8M_psf0dB",
+  # "hf:diffusercam:mirflickr:U5+Unet8M_psf-5dB",
+  # "hf:diffusercam:mirflickr:U5+Unet8M_psf-10dB",
+  # "hf:diffusercam:mirflickr:U5+Unet8M_psf-20dB",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psf-0dB",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psf-5dB",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psf-10dB",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psf-20dB",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psfNN_psf-0dB",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psfNN_psf-10dB",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psfNN_psf-20dB",
 
   # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_ft_tapecam",
   # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_ft_tapecam_post",
@@ -52,7 +66,9 @@ algorithms: [
   # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave",
   # "hf:digicam:celeba_26k:Unet4M+U5+Unet4M_wave",
   # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M",
+  # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_psfNN",
   # "hf:digicam:mirflickr_single_25k:Unet4M+U10+Unet4M_wave",
+  # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_psfNN",
   # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_flips",
   # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_flips_rotate10",
   # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_aux1",

configs/benchmark_digicam_celeba.yaml

@@ -6,22 +6,23 @@ defaults:
 
 dataset: HFDataset
 batchsize: 10
-device: "cuda:0"
+device: "cuda:1"
 
 algorithms: [
   # "ADMM",
 
   ## -- reconstructions trained on measured data
-  # "hf:digicam:celeba_26k:U5+Unet8M_wave",
+  "hf:digicam:celeba_26k:U5+Unet8M_wave",
   "hf:digicam:celeba_26k:Unet8M+U5_wave",
-  # "hf:digicam:celeba_26k:TrainInv+Unet8M_wave",
-  # "hf:digicam:celeba_26k:MWDN8M_wave",
-  # "hf:digicam:celeba_26k:MMCN4M+Unet4M_wave",
-  # "hf:digicam:celeba_26k:Unet2M+MWDN6M_wave",
-  # "hf:digicam:celeba_26k:Unet4M+TrainInv+Unet4M_wave",
-  # "hf:digicam:celeba_26k:Unet2M+MMCN+Unet2M_wave",
-  # "hf:digicam:celeba_26k:Unet4M+U5+Unet4M_wave",
-  # "hf:digicam:celeba_26k:Unet4M+U10+Unet4M_wave",
+  "hf:digicam:celeba_26k:TrainInv+Unet8M_wave",
+  "hf:digicam:celeba_26k:MWDN8M_wave",
+  "hf:digicam:celeba_26k:MMCN4M+Unet4M_wave",
+  "hf:digicam:celeba_26k:Unet2M+MWDN6M_wave",
+  "hf:digicam:celeba_26k:Unet4M+TrainInv+Unet4M_wave",
+  "hf:digicam:celeba_26k:Unet2M+MMCN+Unet2M_wave",
+  "hf:digicam:celeba_26k:Unet4M+U5+Unet4M_wave",
+  "hf:digicam:celeba_26k:Unet4M+U5+Unet4M_wave_psfNN",
+  "hf:digicam:celeba_26k:Unet4M+U10+Unet4M_wave",
 
   # # -- reconstructions trained on other datasets/systems
   # "hf:diffusercam:mirflickr:Unet4M+U10+Unet4M",

configs/benchmark_digicam_mirflickr_multi.yaml

@@ -25,11 +25,14 @@ algorithms: [
 
   ## -- reconstructions trained on measured data 
   "hf:digicam:mirflickr_multi_25k:Unet4M+U5+Unet4M_wave",
-  # "hf:digicam:mirflickr_multi_25k:Unet4M+U10+Unet4M_wave",
+  "hf:digicam:mirflickr_multi_25k:Unet4M+U5+Unet4M_wave_psfNN",
+  "hf:digicam:mirflickr_multi_25k:Unet4M+U10+Unet4M_wave",
   "hf:digicam:mirflickr_multi_25k:Unet4M+U5+Unet4M_wave_aux1",
-  # "hf:digicam:mirflickr_multi_25k:Unet4M+U5+Unet4M_wave_flips",
+  "hf:digicam:mirflickr_multi_25k:Unet4M+U5+Unet4M_wave_flips",
+  "hf:digicam:mirflickr_multi_25k:Unet8M_wave_v2",
 
   # ## -- reconstructions trained on other datasets/systems
+  # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_psfNN",
   # "hf:diffusercam:mirflickr:Unet4M+U10+Unet4M",
   # "hf:tapecam:mirflickr:Unet4M+U10+Unet4M",
   # "hf:digicam:mirflickr_single_25k:Unet4M+U10+Unet4M_wave",
@@ -42,6 +45,9 @@ algorithms: [
   # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M",
   # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_ft_flips",
   # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_ft_flips_rotate10",
+  # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_psfNN",
+  # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_psfNN",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psfNN",
 ]
 
 # # -- to only use output from unrolled

configs/benchmark_digicam_mirflickr_pnp.yaml

@@ -0,0 +1,44 @@
+# python scripts/eval/benchmark_recon.py -cn benchmark_digicam_mirflickr_pnp
+defaults:
+  - benchmark
+  - _self_
+
+
+dataset: HFDataset
+batchsize: 1
+device: "cuda:0"
+
+huggingface:
+  repo: "bezzam/DigiCam-Mirflickr-MultiMask-25K"
+  psf: null   # null for simulating PSF
+  image_res: [900, 1200]  # used during measurement
+  rotate: True   # if measurement is upside-down
+  flipud: False
+  flip_lensed: False   # if rotate or flipud is True, apply to lensed
+  alignment:
+    top_left: [80, 100]  # height, width
+    height: 200
+  downsample: 1
+
+algorithms: [
+  "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_psfNN",
+  "hf:digicam:mirflickr_multi_25k:Unet4M+U5+Unet4M_wave_psfNN",
+]
+
+pnp:
+  mu: 1e-3   # weight for distance from original model parameters
+  model_path: null    # leave null to be overwritten in script
+  # lr: 1e-2
+  # n_iter: 10
+  lr: 3e-3   # learning rate for SGD
+  n_iter: 10   # number of iterations
+
+
+save_idx: [1, 2, 4, 5, 9, 24, 33, 61]
+
+# simulating PSF
+simulation:
+  use_waveprop: True
+  deadspace: True
+  scene2mask: 0.3
+  mask2sensor: 0.002

configs/benchmark_digicam_mirflickr_single.yaml

@@ -25,18 +25,20 @@ algorithms: [
   # "ADMM",
 
   # # -- reconstructions trained on measured data
-  # "hf:digicam:mirflickr_single_25k:U5+Unet8M_wave",
+  "hf:digicam:mirflickr_single_25k:U5+Unet8M_wave",
   "hf:digicam:mirflickr_single_25k:Unet8M+U5_wave",
-  # "hf:digicam:mirflickr_single_25k:TrainInv+Unet8M_wave",
-  # "hf:digicam:mirflickr_single_25k:MMCN4M+Unet4M_wave",
-  # "hf:digicam:mirflickr_single_25k:MWDN8M_wave",
-  # "hf:digicam:mirflickr_single_25k:Unet4M+TrainInv+Unet4M_wave",
-  # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave",
-  # "hf:digicam:mirflickr_single_25k:Unet2M+MMCN+Unet2M_wave",
-  # "hf:digicam:mirflickr_single_25k:Unet2M+MWDN6M_wave",
-  # "hf:digicam:mirflickr_single_25k:Unet4M+U10+Unet4M_wave",
-  # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_flips",
-  # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_flips_rotate10",
+  "hf:digicam:mirflickr_single_25k:TrainInv+Unet8M_wave",
+  "hf:digicam:mirflickr_single_25k:MMCN4M+Unet4M_wave",
+  "hf:digicam:mirflickr_single_25k:MWDN8M_wave",
+  "hf:digicam:mirflickr_single_25k:Unet4M+TrainInv+Unet4M_wave",
+  "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave",
+  "hf:digicam:mirflickr_single_25k:Unet2M+MMCN+Unet2M_wave",
+  "hf:digicam:mirflickr_single_25k:Unet2M+MWDN6M_wave",
+  "hf:digicam:mirflickr_single_25k:Unet4M+U10+Unet4M_wave",
+  "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_psfNN",
+  "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_flips",
+  "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_flips_rotate10",
+  "hf:digicam:mirflickr_single_25k:Unet8M_wave_v2",
 
   # ## -- reconstructions trained on other datasets/systems
   # "hf:diffusercam:mirflickr:Unet4M+U10+Unet4M",
@@ -46,6 +48,9 @@ algorithms: [
   # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M",
   # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M",
   # "hf:digicam:mirflickr_multi_25k:Unet4M+U5+Unet4M_wave",
+  # "hf:digicam:mirflickr_multi_25k:Unet4M+U5+Unet4M_wave_psfNN",
+  # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_psfNN",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psfNN",
 ]
 
 save_idx: [1, 2, 4, 5, 9, 24, 33, 61]

configs/benchmark_tapecam_mirflickr.yaml

@@ -5,7 +5,7 @@ defaults:
 
 dataset: HFDataset
 batchsize: 4
-device: "cuda:0"
+device: "cuda:1"
 
 huggingface:
   repo: "bezzam/TapeCam-Mirflickr-25K"
@@ -26,18 +26,19 @@ algorithms: [
   # "ADMM",
 
   # -- reconstructions trained on measured data
-  # "hf:tapecam:mirflickr:U5+Unet8M",
+  "hf:tapecam:mirflickr:U5+Unet8M",
   "hf:tapecam:mirflickr:Unet8M+U5",
-  # "hf:tapecam:mirflickr:TrainInv+Unet8M",
-  # "hf:tapecam:mirflickr:MMCN4M+Unet4M",
-  # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M",
-  # "hf:tapecam:mirflickr:Unet4M+TrainInv+Unet4M",
-  # "hf:tapecam:mirflickr:Unet2M+MMCN+Unet2M",
-  # "hf:tapecam:mirflickr:Unet4M+U10+Unet4M",
-  # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_flips_rotate10",
-  # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_aux1",
-  # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_flips",
-  # "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_flips_rotate10",
+  "hf:tapecam:mirflickr:TrainInv+Unet8M",
+  "hf:tapecam:mirflickr:MMCN4M+Unet4M",
+  "hf:tapecam:mirflickr:Unet4M+U5+Unet4M",
+  "hf:tapecam:mirflickr:Unet4M+TrainInv+Unet4M",
+  "hf:tapecam:mirflickr:Unet2M+MMCN+Unet2M",
+  "hf:tapecam:mirflickr:Unet4M+U10+Unet4M",
+  "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_flips_rotate10",
+  "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_aux1",
+  "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_flips",
+  "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_flips_rotate10",
+  "hf:tapecam:mirflickr:Unet4M+U5+Unet4M_psfNN",
 
   # # below models need `single_channel_psf = True`
   # "hf:tapecam:mirflickr:MWDN8M",    
@@ -59,7 +60,9 @@ algorithms: [
   # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_ft_tapecam_pre",
   # "hf:diffusercam:mirflickr_sim:Unet4M+U5+Unet4M_ft_digicam_multi_pre",
   # "hf:diffusercam:mirflickr_sim:Unet4M+U5+Unet4M_ft_digicam_multi",
+  # "hf:diffusercam:mirflickr:Unet4M+U5+Unet4M_psfNN",
+  # "hf:digicam:mirflickr_single_25k:Unet4M+U5+Unet4M_wave_psfNN",
 ]
 
 save_idx: [1, 2, 4, 5, 9]
-n_iter_range: [100]    # for ADM
+n_iter_range: [100]    # for ADMM

configs/recon_digicam_mirflickr.yaml

@@ -26,11 +26,12 @@ fn: null  # if not null, download this file from https://huggingface.co/datasets
 # model: MMCN4M+Unet4M_wave
 # model: MWDN8M_wave
 # model: U5+Unet8M_wave
-model: Unet8M+U5_wave
+# model: Unet8M+U5_wave
 # model: Unet4M+TrainInv+Unet4M_wave
 # model: Unet2M+MMCN+Unet2M_wave
 # model: Unet4M+U5+Unet4M_wave
 # model: Unet4M+U10+Unet4M_wave
+model: Unet4M+U5+Unet4M_wave_psfNN
 
 # # --- dataset: mirflickr_multi_25k
 # model: Unet4M+U5+Unet4M_wave

configs/recon_digicam_mirflickr_err.yaml

@@ -9,14 +9,20 @@ hf_repo: null    # by default use one in model config
 
 # set model
 # -- for learning-based methods (comment if using ADMM)
-model: Unet4M+U5+Unet4M_wave
+model: Unet4M+U5+Unet4M_wave_psfNN
 
 # # -- for ADMM with fixed parameters
 # model: admm
-# n_iter: 10
+n_iter: 10
 
 device: cuda:1
 save_idx: [1, 2, 4, 5, 9]
 n_files: null
 percent_pixels_wrong: [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
-flip: True   # whether to flip mask values (True) or reset them (False)
+plot_vs_percent_wrong: False   # whether to plot again percent wrong or correct
+flip: False   # whether to flip mask values (True) or reset them (False)
+
+compare_aes: [128, 256]  # key lengths
+digicam_ratio: 0.6   # approximate ratio of pixels that need to be correct
+bit_depth: 8
+n_pixel: 1404

configs/sim_digicam_psf.yaml

@@ -5,7 +5,7 @@ hydra:
 
 use_torch: True
 dtype: float32
-torch_device: cuda
+torch_device: cuda:1
 requires_grad: False
 
 # if repo not provided, check for local file at `digicam.pattern`

configs/train_digicam_celeba.yaml

@@ -10,8 +10,9 @@ eval_disp_idx: [0, 2, 3, 4, 9]
 # Dataset
 files:
   dataset: bezzam/DigiCam-CelebA-26K
-  huggingface_psf: "psf_simulated.png"
+  huggingface_psf: "psf_simulated_waveprop.png"
   huggingface_dataset: True
+  cache_dir: /dev/shm
   split_seed: 0
   test_size: 0.15
   downsample: 2

configs/train_mirflickr_diffuser_sim.yaml

@@ -0,0 +1,12 @@
+# python scripts/recon/train_learning_based.py -cn train_mirflickr_diffuser_sim
+defaults:
+  - train_mirflickr_diffuser
+  - _self_
+
+torch_device: 'cuda:0'
+device_ids: [0, 1, 2, 3]
+eval_disp_idx: [0, 1, 3, 4, 8]
+
+# Dataset
+files:
+  hf_simulated: True

configs/train_unrolledADMM.yaml

@@ -34,6 +34,7 @@ files:
   downsample: 2    # factor by which to downsample the PSF, note that for DiffuserCam the PSF has 4x the resolution
   downsample_lensed: 2   # only used if lensed if measured
   input_snr: null    # adding shot noise at input (for measured dataset) at this SNR in dB
+  psf_snr: null    # adding noise to PSF at this SNR in dB
   background_fp: null
   background_snr_range: null
   vertical_shift: null
@@ -92,7 +93,7 @@ reconstruction:
 
   # processing PSF
   psf_network: False  # False, or set number of channels for UnetRes, e.g. [8,16,32,64], with skip connection
-  psf_residual: True  # if psf_network used, whether to use residual connection for original PSF estimate
+  psf_residual: False  # if psf_network used, whether to use residual connection for original PSF estimate
 
   # background subtraction (if dataset has corresponding background images)
   direct_background_subtraction: False    # True or False
@@ -196,7 +197,7 @@ training:
 optimizer:
   type: AdamW  # Adam, SGD... (Pytorch class)
   lr: 1e-4
-  lr_step_epoch: True   # True -> update LR at end of each epoch, False at the end of each mini-batch
+  lr_step_epoch: False   # True -> update LR at end of each epoch, False at the end of each mini-batch
   cosine_decay_warmup: True  # if set, cosine decay with warmup of 5%
   final_lr: False   # if set, exponentially decay *to* this value
   exp_decay: False  # if set, exponentially decay *with* this value