Adding new SR jobs

CosmoStat · Sep 8, 2021 · 3029359 · 3029359
1 parent f493173
commit 3029359
Show file tree

Hide file tree

Showing 4 changed files with 368 additions and 0 deletions.
diff --git a/method-comparison/config_files/mccd_configs/config_MCCD_SR_wf_exp_id09.ini b/method-comparison/config_files/mccd_configs/config_MCCD_SR_wf_exp_id09.ini
@@ -0,0 +1,149 @@
+# Configuration file for the MCCD method
+
+[INPUTS]
+INPUT_DIR = /n05data/tliaudat/wf_exps/datasets/rca_shifts/all_stars/
+INPUT_REGEX_FILE_PATTERN = train_stars-*-*.fits
+INPUT_SEPARATOR = -
+MIN_N_STARS = 2
+OUTLIER_STD_MAX = 100.
+USE_SNR_WEIGHTS = False
+PREPROCESSED_OUTPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/inputs/model_id09/
+OUTPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/models/model_id09/
+
+[INSTANCE]
+N_COMP_LOC = 4
+D_COMP_GLOB = 6
+KSIG_LOC = 1.00
+KSIG_GLOB = 1.00
+FILTER_PATH = None
+D_HYB_LOC = 2
+MIN_D_COMP_GLOB = None
+RMSE_THRESH = 10.
+CCD_STAR_THRESH = 0.9
+FP_GEOMETRY = EUCLID
+UPFACT = 3
+
+[FIT]
+LOC_MODEL = hybrid
+PSF_SIZE = 4.
+PSF_SIZE_TYPE = R2
+N_EIGENVECTS = 5
+N_ITER_RCA = 1
+N_ITER_GLOB = 2
+N_ITER_LOC = 2
+NB_SUBITER_S_LOC = 300
+NB_SUBITER_A_LOC = 400
+NB_SUBITER_S_GLOB = 100
+NB_SUBITER_A_GLOB = 200
+
+[VALIDATION]
+VAL_MODEL_INPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/models/model_id09/
+VAL_DATA_INPUT_DIR = /n05data/tliaudat/wf_exps/datasets/rca_shifts/test/
+VAL_PREPROCESSED_OUTPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/inputs/model_id09/
+VAL_REGEX_FILE_PATTERN = test_stars-*-*.fits
+VAL_SEPARATOR = -
+VAL_OUTPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/validation/model_id09/
+APPLY_DEGRADATION = True
+MCCD_DEBUG = False
+GLOBAL_POL_INTERP = False
+
+
+# Parameter description:
+#
+#
+# [INPUTS]
+# INPUT_DIR : (Required) Must be a valid directory containing the input
+#              MCCD files.
+# INPUT_REGEX_FILE_PATTERN : File pattern of the input files to use. It should
+#                            follow regex (regular expression) standards.
+# INPUT_SEPARATOR : Separator of the different fields in the filename,
+#                   ie sexcat[SEP]catalog_id[SEP]CCD_id.fits
+# MIN_N_STARS : Minimum number of stars to keep a CCD for the training.
+# OUTLIER_STD_MAX : Maximum standard deviation used for the outlier rejection.
+#                   Should not be too low as a hihg quantity of low quality
+#                   stars will be rejected. ie 9 is a conservative rejection.
+# USE_SNR_WEIGHTS : Boolean to determine if the SNR weighting strategy will
+#                   be used.
+#                   For now, it needs the SNR estimations from SExtractor.
+# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the
+#                           preprocessed input files.
+# OUTPUT_DIR : (Required) Must be a valid directory to write the output files.
+#              The constructed models will be saved.
+#
+#
+# [INSTANCE]
+# N_COMP_LOC : Number of components of the Local model. If LOC_MODEL is poly,
+#              will be the max degree D of the polynomial.
+# D_COMP_GLOB : Max degree of the global polynomial model.
+# KSIG_LOC : Denoising parameter of the local model.
+#            ie 1 is a normal denoising, 3 is a hard denoising.
+# KSIG_GLOB : Denoising parameter of the global model.
+#             ie 1 is a normal denoising, 3 is a hard denoising.
+# FILTER_PATH : Path for predefined filters.
+# D_HYB_LOC : Degree of the polynomial component for the local part in case
+#             the LOC_MODEL used is 'hybrid'.
+# MIN_D_COMP_GLOB : The minimum degree of the polynomial for the global
+#                   component. For example, if the paramter is set to 1, the
+#                   polynomials of degree 0 and 1 will be excluded from the
+#                   global polynomial variations. ``None`` means that we are
+#                   not excluding any degree.
+# RMSE_THRESH : Parameter concerning the CCD outlier rejection. Once the PSF
+#               model is calculated we perform an outlier check on the training
+#               stars. We divide each star in two parts with a given circle.
+#               The inner part corresponds to the most of the PSF/star energy
+#               while the outer part corresponds to the observation background.
+#               The outer part is used to calculate the noise level and the inner
+#               part to calculate the model residual
+#               (star observation - PSF model reconstruction). If the RMSE error
+#               of the residual divided by the noise level is over the RMSE_THRESH,
+#               the star will be considered an outlier. A perfect reconstruction
+#               would have RMSE_THRESH equal to 1.
+# CCD_STAR_THRESH : Parameter concerning the CCD outlier rejection. If the
+#                   percentage of outlier stars in a single CCD is bigger than
+#                   CCD_STAR_THRESH, the CCD is considered to be an outlier.
+#                   In this case, the CCD is rejected from the PSF model.
+#                   A value lower than 0 means that no outlier rejection
+#                   will be done.
+# FP_GEOMETRY : Defines the geometry of the focal plane. For the moment the two
+#               available options are 'CFIS' and 'EUCLID'. If the parameter is
+#               not specified it defaults to 'CFIS'.
+#
+#
+#
+# [FIT]
+# LOC_MODEL : Defines the type of local model to use, it can be: 'rca',
+#             'poly' or 'hybrid'.
+#             When the poly model is used, N_COMP_LOC should be used
+#             as the D_LOC (max degree of the poly model)
+# PSF_SIZE : First guess of the PSF size. A size estimation is done anyways.
+# PSF_SIZE_TYPE : Type of the size information. It can be: fwhm, R2, sigma
+# N_EIGENVECTS : Number of eigenvectors to keep for the graph constraint
+#                construction.
+# N_ITER_RCA : Number of global epochs in the algorithm. Alternation between
+#              global and local estimations.
+# N_ITER_GLOB : Number of epochs for each global optimization. Alternations
+#               between A_GLOB and S_GLOB.
+# N_ITER_LOC : Number of epochs for each local optimization. Alternations
+#              between the different A_LOC and S_LOC.
+# NB_SUBITER_S_LOC : Iterations for the optimization algorithm over S_LOC.
+# NB_SUBITER_A_LOC : Iterations for the optimization algorithm over A_LOC.
+# NB_SUBITER_S_GLOB : Iterations for the optimization algorithm over S_GLOB.
+# NB_SUBITER_A_GLOB : Iterations for the optimization algorithm over A_GLOB.
+#
+#
+# [VALIDATION]
+# MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the
+#                   saved trained models.
+# VAL_DATA_INPUT_DIR : (Required) Must be a valid directory which contains the
+#                      validation input data (test dataset).
+# VAL_REGEX_FILE_PATTERN : Same as INPUT_REGEX_FILE_PATTERN but for validation.
+# VAL_SEPARATOR : Same as INPUT_SEPARATOR but for validation.
+# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the
+#                  validation outputs, test PSFs and interpolated PSFs.
+# APPLY_DEGRADATION : Whether the PSF models should be degraded
+#                     (sampling/shifts/flux) to match stars; use True if you
+#                     plan on making pixel-based comparisons (residuals etc.).
+# MCCD_DEBUG : Debug mode. Returns the local and global contributions.
+# GLOBAL_POL_INTERP : Uses polynomial interpolation for the global model
+#                     instead of RBF kernel interpolation.
+#
diff --git a/method-comparison/config_files/mccd_configs/config_MCCD_SR_wf_exp_id10.ini b/method-comparison/config_files/mccd_configs/config_MCCD_SR_wf_exp_id10.ini
@@ -0,0 +1,149 @@
+# Configuration file for the MCCD method
+
+[INPUTS]
+INPUT_DIR = /n05data/tliaudat/wf_exps/datasets/rca_shifts/all_stars/
+INPUT_REGEX_FILE_PATTERN = train_stars-*-*.fits
+INPUT_SEPARATOR = -
+MIN_N_STARS = 2
+OUTLIER_STD_MAX = 100.
+USE_SNR_WEIGHTS = False
+PREPROCESSED_OUTPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/inputs/model_id10/
+OUTPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/models/model_id10/
+
+[INSTANCE]
+N_COMP_LOC = 4
+D_COMP_GLOB = 6
+KSIG_LOC = 3.00
+KSIG_GLOB = 3.00
+FILTER_PATH = None
+D_HYB_LOC = 2
+MIN_D_COMP_GLOB = None
+RMSE_THRESH = 10.
+CCD_STAR_THRESH = 0.9
+FP_GEOMETRY = EUCLID
+UPFACT = 3
+
+[FIT]
+LOC_MODEL = hybrid
+PSF_SIZE = 4.
+PSF_SIZE_TYPE = R2
+N_EIGENVECTS = 5
+N_ITER_RCA = 1
+N_ITER_GLOB = 2
+N_ITER_LOC = 2
+NB_SUBITER_S_LOC = 300
+NB_SUBITER_A_LOC = 400
+NB_SUBITER_S_GLOB = 100
+NB_SUBITER_A_GLOB = 200
+
+[VALIDATION]
+VAL_MODEL_INPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/models/model_id10/
+VAL_DATA_INPUT_DIR = /n05data/tliaudat/wf_exps/datasets/rca_shifts/test/
+VAL_PREPROCESSED_OUTPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/inputs/model_id10/
+VAL_REGEX_FILE_PATTERN = test_stars-*-*.fits
+VAL_SEPARATOR = -
+VAL_OUTPUT_DIR = /n05data/tliaudat/wf_exps/outputs/mccd_shifts/validation/model_id10/
+APPLY_DEGRADATION = True
+MCCD_DEBUG = False
+GLOBAL_POL_INTERP = False
+
+
+# Parameter description:
+#
+#
+# [INPUTS]
+# INPUT_DIR : (Required) Must be a valid directory containing the input
+#              MCCD files.
+# INPUT_REGEX_FILE_PATTERN : File pattern of the input files to use. It should
+#                            follow regex (regular expression) standards.
+# INPUT_SEPARATOR : Separator of the different fields in the filename,
+#                   ie sexcat[SEP]catalog_id[SEP]CCD_id.fits
+# MIN_N_STARS : Minimum number of stars to keep a CCD for the training.
+# OUTLIER_STD_MAX : Maximum standard deviation used for the outlier rejection.
+#                   Should not be too low as a hihg quantity of low quality
+#                   stars will be rejected. ie 9 is a conservative rejection.
+# USE_SNR_WEIGHTS : Boolean to determine if the SNR weighting strategy will
+#                   be used.
+#                   For now, it needs the SNR estimations from SExtractor.
+# PREPROCESSED_OUTPUT_DIR : (Required) Must be a valid directory to write the
+#                           preprocessed input files.
+# OUTPUT_DIR : (Required) Must be a valid directory to write the output files.
+#              The constructed models will be saved.
+#
+#
+# [INSTANCE]
+# N_COMP_LOC : Number of components of the Local model. If LOC_MODEL is poly,
+#              will be the max degree D of the polynomial.
+# D_COMP_GLOB : Max degree of the global polynomial model.
+# KSIG_LOC : Denoising parameter of the local model.
+#            ie 1 is a normal denoising, 3 is a hard denoising.
+# KSIG_GLOB : Denoising parameter of the global model.
+#             ie 1 is a normal denoising, 3 is a hard denoising.
+# FILTER_PATH : Path for predefined filters.
+# D_HYB_LOC : Degree of the polynomial component for the local part in case
+#             the LOC_MODEL used is 'hybrid'.
+# MIN_D_COMP_GLOB : The minimum degree of the polynomial for the global
+#                   component. For example, if the paramter is set to 1, the
+#                   polynomials of degree 0 and 1 will be excluded from the
+#                   global polynomial variations. ``None`` means that we are
+#                   not excluding any degree.
+# RMSE_THRESH : Parameter concerning the CCD outlier rejection. Once the PSF
+#               model is calculated we perform an outlier check on the training
+#               stars. We divide each star in two parts with a given circle.
+#               The inner part corresponds to the most of the PSF/star energy
+#               while the outer part corresponds to the observation background.
+#               The outer part is used to calculate the noise level and the inner
+#               part to calculate the model residual
+#               (star observation - PSF model reconstruction). If the RMSE error
+#               of the residual divided by the noise level is over the RMSE_THRESH,
+#               the star will be considered an outlier. A perfect reconstruction
+#               would have RMSE_THRESH equal to 1.
+# CCD_STAR_THRESH : Parameter concerning the CCD outlier rejection. If the
+#                   percentage of outlier stars in a single CCD is bigger than
+#                   CCD_STAR_THRESH, the CCD is considered to be an outlier.
+#                   In this case, the CCD is rejected from the PSF model.
+#                   A value lower than 0 means that no outlier rejection
+#                   will be done.
+# FP_GEOMETRY : Defines the geometry of the focal plane. For the moment the two
+#               available options are 'CFIS' and 'EUCLID'. If the parameter is
+#               not specified it defaults to 'CFIS'.
+#
+#
+#
+# [FIT]
+# LOC_MODEL : Defines the type of local model to use, it can be: 'rca',
+#             'poly' or 'hybrid'.
+#             When the poly model is used, N_COMP_LOC should be used
+#             as the D_LOC (max degree of the poly model)
+# PSF_SIZE : First guess of the PSF size. A size estimation is done anyways.
+# PSF_SIZE_TYPE : Type of the size information. It can be: fwhm, R2, sigma
+# N_EIGENVECTS : Number of eigenvectors to keep for the graph constraint
+#                construction.
+# N_ITER_RCA : Number of global epochs in the algorithm. Alternation between
+#              global and local estimations.
+# N_ITER_GLOB : Number of epochs for each global optimization. Alternations
+#               between A_GLOB and S_GLOB.
+# N_ITER_LOC : Number of epochs for each local optimization. Alternations
+#              between the different A_LOC and S_LOC.
+# NB_SUBITER_S_LOC : Iterations for the optimization algorithm over S_LOC.
+# NB_SUBITER_A_LOC : Iterations for the optimization algorithm over A_LOC.
+# NB_SUBITER_S_GLOB : Iterations for the optimization algorithm over S_GLOB.
+# NB_SUBITER_A_GLOB : Iterations for the optimization algorithm over A_GLOB.
+#
+#
+# [VALIDATION]
+# MODEL_INPUT_DIR : (Required) Must be a valid directory which contains the
+#                   saved trained models.
+# VAL_DATA_INPUT_DIR : (Required) Must be a valid directory which contains the
+#                      validation input data (test dataset).
+# VAL_REGEX_FILE_PATTERN : Same as INPUT_REGEX_FILE_PATTERN but for validation.
+# VAL_SEPARATOR : Same as INPUT_SEPARATOR but for validation.
+# VAL_OUTPUT_DIR : (Required) Must be a valid directory where to save the
+#                  validation outputs, test PSFs and interpolated PSFs.
+# APPLY_DEGRADATION : Whether the PSF models should be degraded
+#                     (sampling/shifts/flux) to match stars; use True if you
+#                     plan on making pixel-based comparisons (residuals etc.).
+# MCCD_DEBUG : Debug mode. Returns the local and global contributions.
+# GLOBAL_POL_INTERP : Uses polynomial interpolation for the global model
+#                     instead of RBF kernel interpolation.
+#
diff --git a/method-comparison/jobs/mccd_SR_job_id09.sh b/method-comparison/jobs/mccd_SR_job_id09.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+##########################
+# SMP Script for CANDIDE #
+##########################
+
+# Receive email when job finishes or aborts
+#PBS -M [email protected]
+#PBS -m ea
+# Set a name for the job
+#PBS -N mccd_SR_id09
+# Join output and errors in one file
+#PBS -j oe
+# Set maximum computing time (e.g. 5min)
+#PBS -l walltime=99:00:00
+# Request number of cores
+#PBS -l nodes=n03:ppn=04:hasgpu
+
+# Activate conda environment
+# module load intelpython/3-2020.1
+module load tensorflow/2.4
+module load intel/19.0/2
+source activate new_shapepipe
+
+cd /home/tliaudat/github/wf-psf/method-comparison/scripts/
+
+python ./mccd_script_SR.py \
+    --config_file /home/tliaudat/github/wf-psf/method-comparison/config_files/mccd_configs/config_MCCD_SR_wf_exp_id09.ini \
+    --repo_base_path /home/tliaudat/github/wf-psf/ \
+    --run_id mccd_SR_id09 \
+    --psf_out_dim 64 \
+
+
+# Return exit code
+exit 0
diff --git a/method-comparison/jobs/mccd_SR_job_id10.sh b/method-comparison/jobs/mccd_SR_job_id10.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+##########################
+# SMP Script for CANDIDE #
+##########################
+
+# Receive email when job finishes or aborts
+#PBS -M [email protected]
+#PBS -m ea
+# Set a name for the job
+#PBS -N mccd_SR_id10
+# Join output and errors in one file
+#PBS -j oe
+# Set maximum computing time (e.g. 5min)
+#PBS -l walltime=99:00:00
+# Request number of cores
+#PBS -l nodes=n16:ppn=04:hasgpu
+
+# Activate conda environment
+# module load intelpython/3-2020.1
+module load tensorflow/2.4
+module load intel/19.0/2
+source activate new_shapepipe
+
+cd /home/tliaudat/github/wf-psf/method-comparison/scripts/
+
+python ./mccd_script_SR.py \
+    --config_file /home/tliaudat/github/wf-psf/method-comparison/config_files/mccd_configs/config_MCCD_SR_wf_exp_id10.ini \
+    --repo_base_path /home/tliaudat/github/wf-psf/ \
+    --run_id mccd_SR_id10 \
+    --psf_out_dim 64 \
+
+
+# Return exit code
+exit 0