Merge branch 'v2.0-devel' into 'master'

lofar-astron · Oct 18, 2016 · ddf7072 · ddf7072
2 parents 6dee243 + 1ad7346
commit ddf7072
Show file tree

Hide file tree

Showing 16 changed files with 1,087 additions and 964 deletions.
diff --git a/Initial-Subtract-Deep.parset b/Initial-Subtract-Deep.parset
@@ -39,7 +39,9 @@
 # name of the direction independent parmDBs inside the measurement sets
 # needs to include the leading "/" to indicate that the parmDB is a subdirectory of the MS
 ! direction_indep_parmDBs = /instrument_directionindependent
+# where to put the inspection plots generated by the pipeline
 ! inspection_directory = /media/scratch/test/username/WhereYouWantImagesAndInspectionPlots/
+# scratch directory for wsclean (can be local to the processing nodes!)
 ! local_scratch_dir        =  /local/username
 
 ##### imaging parameters (Feel free to leave them untouched.)

diff --git a/Initial-Subtract.parset b/Initial-Subtract.parset
@@ -34,7 +34,10 @@
 # name of the direction independent parmDBs inside the measurement sets
 # needs to include the leading "/" to indicate that the parmDB is a subdirectory of the MS
 ! direction_indep_parmDBs = /instrument_directionindependent
+# where to put the inspection plots generated by the pipeline
 ! inspection_directory = /media/scratch/test/username/WhereYouWantImagesAndInspectionPlots/
+# scratch directory for wsclean (can be local to the processing nodes!)
+! local_scratch_dir        =  /local/username
 
 ##### imaging parameters (Feel free to leave them untouched.)
 #### specify  the image parameters here
@@ -197,7 +200,7 @@ wsclean_high2.control.max_per_node              =   {{ max_imagers_per_node }}
 wsclean_high2.control.error_tolerance           =   {{ error_tolerance }}
 wsclean_high2.control.mapfiles_in               =   [do_magic_maps.output.groupmap,mask_high.output.mapfile,do_magic_maps.output.high_padsize_map,do_magic_maps.output.high_size_map,mask_high.output.threshold_5sig.mapfile]
 wsclean_high2.control.inputkeys                 =   [msfile,fitsmask,paddedsize,outputsize,threshold]
-wsclean_high2.argument.flags                    =   [-update-model-required,-reorder,-fitbeam,msfile]
+wsclean_high2.argument.flags                    =   [-no-update-model-required,-reorder,-fitbeam,msfile]
 wsclean_high2.argument.fitsmask                 =   fitsmask
 wsclean_high2.argument.size                     =   paddedsize
 wsclean_high2.argument.trim                     =   outputsize

diff --git a/Pre-Facet-Cal-RawData-PreAvg.parset b/Pre-Facet-Cal-RawData-PreAvg.parset
diff --git a/Pre-Facet-Cal-RawData-Single.parset b/Pre-Facet-Cal-RawData-Single.parset
diff --git a/Pre-Facet-Cal.parset b/Pre-Facet-Cal.parset
@@ -61,10 +61,10 @@
 
 ## pathes to the scripts etc.
 # #### ToDo: get the scripts onto CEP3 and adjust the pathes here!
+! calibrator_parset        = /homea/htb00/htb001/prefactor/Pre-Facet-Calibrator.parset
+! target_parset            = /homea/htb00/htb001/prefactor/Pre-Facet-Target.parset
 ! calib_cal_parset         = /homea/htb00/htb001/prefactor/parsets/calibcal.parset
 ! find_skymodel_cal_auto   = /homea/htb00/htb001/prefactor/scripts/find_skymodel_cal.py
-! ATeam_predict_parset     = /homea/htb00/htb001/prefactor/parsets/ateamclip.parset
-! gsm_cal_parset           = /homea/htb00/htb001/prefactor/parsets/gsmcal.parset
 ! ATeam_skymodel           = /homea/htb00/htb001/prefactor/skymodels/Ateam_LBA_CC.skymodel
 ! losoto_importer          = /homea/htb00/htb001/prefactor/scripts/losotoImporter.py
 ! fitclock_script          = /homea/htb00/htb001/prefactor/scripts/fit_clocktec_initialguess_losoto.py
@@ -76,10 +76,12 @@
 ! get_tgss_skymodel_script = /homea/htb00/htb001/prefactor/scripts/download_tgss_skymodel_target.py
 ! sortmap_script           = /homea/htb00/htb001/prefactor/scripts/sort_times_into_freqGroups.py
 ! check_flagged_script     = /homea/htb00/htb001/prefactor/scripts/check_unflagged_fraction.py
-! structurefunction_script = /cep3home/horneffer/Pre-Facet-Cal/bin/getStructure_from_phases.py
+! structurefunction_script = /homea/htb00/htb001/prefactor/scripts/getStructure_from_phases.py
 ! plotphases_script        = /homea/htb00/htb001/prefactor/scripts/plot_solutions_all_stations.py
 ! losoto_executable        = /opt/cep/losoto/current/bin/losoto
-! flagging_strategy        = /opt/cep/lofar/lofar_versions/LOFAR-Release-2_15_2/lofar_build/install/gnu_opt/share/rfistrategies/HBAdefault
+! flagging_strategy        = /homea/htb00/htb003/lofar_jureca_2.17_stack2016a/share/rfistrategies/HBAdefault
+! makesourcedb             = /homea/htb00/htb003/lofar_jureca_2.17_stack2016a/bin/makesourcedb
+
 
 # number of processes to use per step per node
 ! num_proc_per_node        = 10
@@ -97,7 +99,7 @@ pipeline.steps = [calibrator_pipeline, target_pipeline]
 
 # run the calibrator pipeline
 calibrator_pipeline.control.kind                      =  pipeline
-calibrator_pipeline.control.type                      =  Pre-Facet-Calibrator.parset
+calibrator_pipeline.control.type                      =  {{ calibrator_parset }}
 calibrator_pipeline.argument.avg_timestep             =  {{ avg_timestep }}
 calibrator_pipeline.argument.avg_freqstep             =  {{ avg_freqstep }}
 calibrator_pipeline.argument.cal_input_path           =  {{ cal_input_path }}
@@ -123,7 +125,7 @@ calibrator_pipeline.argument.error_tolerance          =  {{ error_tolerance }}
 
 # run the target pipeline
 target_pipeline.control.kind                      =  pipeline
-target_pipeline.control.type                      =  Pre-Facet-Target.parset
+target_pipeline.control.type                      =  {{ target_parset }}
 target_pipeline.argument.avg_timestep             =  {{ avg_timestep }}
 target_pipeline.argument.avg_freqstep             =  {{ avg_freqstep }}
 target_pipeline.argument.target_input_path        =  {{ target_input_path }}
@@ -140,8 +142,6 @@ target_pipeline.argument.ionex_path               =  {{ ionex_path }}
 target_pipeline.argument.flag_baselines           =  {{ flag_baselines }}
 target_pipeline.argument.min_unflagged_fraction   =  {{ min_unflagged_fraction }}
 target_pipeline.argument.reference_station        =  {{ reference_station }}
-target_pipeline.argument.ATeam_predict_parset     =  {{ ATeam_predict_parset }}
-target_pipeline.argument.gsm_cal_parset           =  {{ gsm_cal_parset }}
 target_pipeline.argument.ATeam_skymodel           =  {{ ATeam_skymodel }}
 target_pipeline.argument.losoto_importer          =  {{ losoto_importer }}
 target_pipeline.argument.transfer_script          =  {{ transfer_script }}
@@ -153,6 +153,7 @@ target_pipeline.argument.structurefunction_script =  {{ structurefunction_script
 target_pipeline.argument.plotphases_script        =  {{ plotphases_script }}
 target_pipeline.argument.losoto_executable        =  {{ losoto_executable }}
 target_pipeline.argument.flagging_strategy        =  {{ flagging_strategy }}
+target_pipeline.argument.makesourcedb             =  {{ makesourcedb }}
 target_pipeline.argument.num_proc_per_node        =  {{ num_proc_per_node }}
 target_pipeline.argument.num_proc_per_node_limit  =  {{ num_proc_per_node_limit }}
 target_pipeline.argument.max_dppp_threads         =  {{ max_dppp_threads }}

diff --git a/Pre-Facet-Calibrator-RawCombine.parset b/Pre-Facet-Calibrator-RawCombine.parset
@@ -4,6 +4,7 @@
 # - Works on raw data, that has not been processed with NDPPP before.
 # - Combines several subbands into one MS in the first NDPPP step to 
 #   reduce the number of files. (May be more efficient on some machines.)
+# - requires LOFAR software version >= 2.17
 # - Expects shared filesystem, that all nodes can reach all files!
 #   (E.g. a single workstation or compute cluster with shared filesystem
 #   doesn't work on multiple nodes on CEP-2 or CEP3.)
@@ -24,7 +25,7 @@
 # where to put the files with the calibration values that are to be transferred to the target
 ! cal_values_directory = /media/scratch/test/username/WhereYouWantToStoreTheValuesFromTheCalibrator/
 
-# NDPPP-compatible pattern for baselines or stations to be flagged
+# NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: [] )
 ! flag_baselines         = [ CS013HBA* ]
 # name of the station that will be used as a reference for the phase-plots
 ! reference_station      = CS001HBA0
@@ -39,7 +40,7 @@
 ! plotsols_script      = /homea/htb00/htb001/prefactor/scripts/examine_npys.py
 ! fit_XYoffset_script  = /homea/htb00/htb001/prefactor/scripts/find_cal_global_phaseoffset_losoto.py
 ! plotphases_script    = /homea/htb00/htb001/prefactor/scripts/plot_solutions_all_stations.py
-! losoto_executable    = /opt/cep/losoto/current/bin/losoto
+! losoto_executable    = /homea/htb00/htb003/local_jureca/bin/losoto
 
 # number of processes to use per step per node
 ! num_proc_per_node        = 24
@@ -53,7 +54,7 @@
 
 ### Stuff that you probably don't need to modify
 # which steps to run 
-pipeline.steps=[createmap_cal, combine_cal_map, sortmap_cal, do_sortmap_cal_maps, ndppp_prep_cal, calib_cal, h5_imp_cal_map, h5imp_cal, mk_inspect_dir, copy_h5imp_cal, fitclock, ampl, plots, phase, plot_cal_phases, createmap_cal_pngs, copy_cal_pngs, createmap_cal_pdfs, copy_cal_pdfs, mk_cal_values_dir, createmap_cal_npys, copy_cal_npys]
+pipeline.steps=[createmap_cal, combine_cal_map, sortmap_cal, do_sortmap_cal_maps, ndppp_prep_cal, calib_cal, h5_imp_cal_map, h5imp_cal, mk_inspect_dir, copy_h5imp_cal, fitclock, ampl, plots, phase, plot_cal_phases, createmap_cal_pngs, copy_cal_pngs, createmap_cal_txts, copy_cal_txts, mk_cal_values_dir, createmap_cal_npys, copy_cal_npys]
 
 # generate a mapfile of all the calibrator data
 createmap_cal.control.kind            =   plugin
@@ -246,23 +247,23 @@ copy_cal_pngs.control.mapfile_in         =  createmap_cal_pngs.output.mapfile
 copy_cal_pngs.control.inputkey           =  source
 copy_cal_pngs.control.arguments          =  [source,{{ inspection_directory }}]
 
-# generate a mapfile of all the diagnostic pdfs
-createmap_cal_pdfs.control.kind            =   plugin
-createmap_cal_pdfs.control.type            =   createMapfile
-createmap_cal_pdfs.control.method          =   mapfile_from_folder
-createmap_cal_pdfs.control.mapfile_dir     =   input.output.mapfile_dir
-createmap_cal_pdfs.control.filename        =   diagnostic_pdfs.mapfile
-createmap_cal_pdfs.control.folder          =   input.output.working_directory/input.output.job_name
-createmap_cal_pdfs.control.pattern         =   *.pdf
+# generate a mapfile of all the diagnostic txts
+createmap_cal_txts.control.kind            =   plugin
+createmap_cal_txts.control.type            =   createMapfile
+createmap_cal_txts.control.method          =   mapfile_from_folder
+createmap_cal_txts.control.mapfile_dir     =   input.output.mapfile_dir
+createmap_cal_txts.control.filename        =   diagnostic_txts.mapfile
+createmap_cal_txts.control.folder          =   input.output.working_directory/input.output.job_name
+createmap_cal_txts.control.pattern         =   *.txt
 
-# copy the diagnostic pdfs to the inspection directory
-copy_cal_pdfs.control.kind               =  recipe
-copy_cal_pdfs.control.type               =  executable_args
-copy_cal_pdfs.control.executable         =  /bin/cp
-copy_cal_pdfs.control.max_per_node       =  {{ num_proc_per_node_limit }}
-copy_cal_pdfs.control.mapfile_in         =  createmap_cal_pdfs.output.mapfile
-copy_cal_pdfs.control.inputkey           =  source
-copy_cal_pdfs.control.arguments          =  [source,{{ inspection_directory }}]
+# copy the diagnostic txts to the inspection directory
+copy_cal_txts.control.kind               =  recipe
+copy_cal_txts.control.type               =  executable_args
+copy_cal_txts.control.executable         =  /bin/cp
+copy_cal_txts.control.max_per_node       =  {{ num_proc_per_node_limit }}
+copy_cal_txts.control.mapfile_in         =  createmap_cal_txts.output.mapfile
+copy_cal_txts.control.inputkey           =  source
+copy_cal_txts.control.arguments          =  [source,{{ inspection_directory }}]
 
 # create the cal_values_directory if needed
 mk_cal_values_dir.control.kind               =  plugin

diff --git a/Pre-Facet-Calibrator-RawSingle.parset b/Pre-Facet-Calibrator-RawSingle.parset
@@ -2,6 +2,7 @@
 #
 # Calibrator part of the Pre-Facet calibration pipeline for raw data:
 # - Works on raw data, that has not been processed with NDPPP before.
+# - requires LOFAR software version >= 2.17
 # - Expects shared filesystem, that all nodes can reach all files!
 #   (E.g. a single workstation or compute cluster with shared filesystem
 #   doesn't work on multiple nodes on CEP-2 or CEP3.)
@@ -20,7 +21,7 @@
 # where to put the files with the calibration values that are to be transferred to the target
 ! cal_values_directory = /media/scratch/test/username/WhereYouWantToStoreTheValuesFromTheCalibrator/
 
-# NDPPP-compatible pattern for baselines or stations to be flagged
+# NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: [] )
 ! flag_baselines         = [ CS013HBA* ]
 # name of the station that will be used as a reference for the phase-plots
 ! reference_station      = CS001HBA0
@@ -34,7 +35,7 @@
 ! plotsols_script      = /homea/htb00/htb001/prefactor/scripts/examine_npys.py
 ! fit_XYoffset_script  = /homea/htb00/htb001/prefactor/scripts/find_cal_global_phaseoffset_losoto.py
 ! plotphases_script    = /homea/htb00/htb001/prefactor/scripts/plot_solutions_all_stations.py
-! losoto_executable    = /opt/cep/losoto/current/bin/losoto
+! losoto_executable    = /homea/htb00/htb003/local_jureca/bin/losoto
 
 # number of processes to use per step per node
 ! num_proc_per_node        = 24
@@ -48,7 +49,7 @@
 
 ### Stuff that you probably don't need to modify
 # which steps to run 
-pipeline.steps=[createmap_cal, ndppp_prep_cal, calib_cal, h5_imp_cal_map, h5imp_cal, mk_inspect_dir, copy_h5imp_cal, fitclock, ampl, plots, phase, plot_cal_phases, createmap_cal_pngs, copy_cal_pngs, createmap_cal_pdfs, copy_cal_pdfs, mk_cal_values_dir, createmap_cal_npys, copy_cal_npys]
+pipeline.steps=[createmap_cal, ndppp_prep_cal, calib_cal, h5_imp_cal_map, h5imp_cal, mk_inspect_dir, copy_h5imp_cal, fitclock, ampl, plots, phase, plot_cal_phases, createmap_cal_pngs, copy_cal_pngs, createmap_cal_txts, copy_cal_txts, mk_cal_values_dir, createmap_cal_npys, copy_cal_npys]
 
 # generate a mapfile of all the calibrator data
 createmap_cal.control.kind            =   plugin
@@ -96,7 +97,6 @@ calib_cal.control.type             =  python-calibrate-stand-alone
 calib_cal.control.max_per_node     =  {{ num_proc_per_node }}
 calib_cal.control.error_tolerance  =  {{ error_tolerance }}
 calib_cal.argument.force           =  True
-calib_cal.argument.numthreads      =  5
 calib_cal.argument.observation     =  ndppp_prep_cal.output.mapfile  # mapfile for the NDPPP-ed calibrator data
 calib_cal.argument.parset          =  {{ calib_cal_parset }}
 calib_cal.argument.catalog         =  {{ calibrator_skymodel }}
@@ -209,23 +209,23 @@ copy_cal_pngs.control.mapfile_in         =  createmap_cal_pngs.output.mapfile
 copy_cal_pngs.control.inputkey           =  source
 copy_cal_pngs.control.arguments          =  [source,{{ inspection_directory }}]
 
-# generate a mapfile of all the diagnostic pdfs
-createmap_cal_pdfs.control.kind            =   plugin
-createmap_cal_pdfs.control.type            =   createMapfile
-createmap_cal_pdfs.control.method          =   mapfile_from_folder
-createmap_cal_pdfs.control.mapfile_dir     =   input.output.mapfile_dir
-createmap_cal_pdfs.control.filename        =   diagnostic_pdfs.mapfile
-createmap_cal_pdfs.control.folder          =   input.output.working_directory/input.output.job_name
-createmap_cal_pdfs.control.pattern         =   *.pdf
-
-# copy the diagnostic pdfs to the inspection directory
-copy_cal_pdfs.control.kind               =  recipe
-copy_cal_pdfs.control.type               =  executable_args
-copy_cal_pdfs.control.executable         =  /bin/cp
-copy_cal_pdfs.control.max_per_node       =  {{ num_proc_per_node_limit }}
-copy_cal_pdfs.control.mapfile_in         =  createmap_cal_pdfs.output.mapfile
-copy_cal_pdfs.control.inputkey           =  source
-copy_cal_pdfs.control.arguments          =  [source,{{ inspection_directory }}]
+# generate a mapfile of all the diagnostic txts
+createmap_cal_txts.control.kind            =   plugin
+createmap_cal_txts.control.type            =   createMapfile
+createmap_cal_txts.control.method          =   mapfile_from_folder
+createmap_cal_txts.control.mapfile_dir     =   input.output.mapfile_dir
+createmap_cal_txts.control.filename        =   diagnostic_txts.mapfile
+createmap_cal_txts.control.folder          =   input.output.working_directory/input.output.job_name
+createmap_cal_txts.control.pattern         =   *.txt
+
+# copy the diagnostic txts to the inspection directory
+copy_cal_txts.control.kind               =  recipe
+copy_cal_txts.control.type               =  executable_args
+copy_cal_txts.control.executable         =  /bin/cp
+copy_cal_txts.control.max_per_node       =  {{ num_proc_per_node_limit }}
+copy_cal_txts.control.mapfile_in         =  createmap_cal_txts.output.mapfile
+copy_cal_txts.control.inputkey           =  source
+copy_cal_txts.control.arguments          =  [source,{{ inspection_directory }}]
 
 # create the cal_values_directory if needed
 mk_cal_values_dir.control.kind               =  plugin