Pre-Facet-Target-RawSingle.parset

# Pre-Facet Target Calibration Pipeline
#
# Target part of the basic Pre-Facet calibration pipeline
# This version works on raw (non averaged) data, and combines MSs only
# after initial round of averaging, flagging and A-team clipping.
#
# - no demixing but A-team flagging,
# - calibration transfer and averaging of target data in one go.
# - checks frequncies in MSs to group files
# - requires LOFAR software version >= 2.17
# - expects shared filesystem, that all nodes can reach all files!
#   (E.g. a single workstation or compute cluster with shared filesystem
#   doesn't work on multiple nodes on CEP-2 or CEP3.)

### parameters you will need to adjust.
# parameters you will need to adjust.
# averaging for the target data
! avg_timestep         = 4   # averaging step needed to average the data to 4 seconds time resolution
! avg_freqstep         = 16   # averaging step needed to average the data to 4 ch/SB frequency resolution
# where to find the target data
! target_input_path    = /data/scratch/username/PathToYourTargetData/
! target_input_pattern = L*.MS
# path to the skymodel for the phase-only calibration of the target
# (Existing file or path (incl. filename) where TGSS model is stored.)
! target_skymodel      = /cep3home/username/Pre-Facet-Cal/skymodels/PleaseProvideTarget.skymodel
# download the phase-only calibration skymodel from TGSS
# "Force" : always download , "True" download if {{ target_skymodel }} does not exist , "False" : never download
! use_tgss_target      = True
# how many subbands to concatenate into on frequency band (usually 10 or 12)
! num_SBs_per_group    = 10 # make concatenated measurement-sets with that many subbands
# where to put the inspection plots generated by the pipeline
! inspection_directory = /media/scratch/test/username/WhereYouWantInspectionPlotsEtc/
# where the files with the calibration values from the calibrator pipeline are
! cal_values_directory = /media/scratch/test/username/WhereYouWantToStoreTheValuesFromTheCalibrator/
# where to put the resulting measurement sets generated by the pipeline
! results_directory    = /media/scratch/test/username/WhereYouWantYourProcessedData/

# NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: [] )
! flag_baselines         = [ CS013HBA* ]
# minimum fraction of unflagged data after RFI flagging and A-team clipping
! min_unflagged_fraction = 0.5
# name of the station that will be used as a reference for the phase-plots
! reference_station      = CS001HBA0

### Values needed for RMextract
# the URL of the server where  the IONEX files can be downloaded
# leave it at "None" to disable downloads, or set it to:
#   ftp://ftp.unibe.ch/aiub/CODE/
# to download from the "standard" server
! ionex_server  = None
# the prefix of the IONEX files
! ionex_prefix  = CODG
# path where the IONEX files can be stored or are already stored
! ionex_path    = /media/scratch/test/username/WhereYouWantToStoreTheValuesFromTheCalibrator/

# pathes to the scripts etc.
# #### ToDo: get the scripts onto CEP3 and adjust the pathes here!
! ATeam_skymodel           = /homea/htb00/htb001/prefactor/skymodels/Ateam_LBA_CC.skymodel
! losoto_importer          = /homea/htb00/htb001/prefactor/scripts/losotoImporter.py
! get_ion_script           = /homea/htb00/htb001/prefactor/bin/download_IONEX.py
! transfer_script          = /homea/htb00/htb001/prefactor/scripts/transfer_gains_RMextract.py
! ATeam_Clipper            = /homea/htb00/htb001/prefactor/scripts/Ateamclipper.py
! get_tgss_skymodel_script = /homea/htb00/htb001/prefactor/scripts/download_tgss_skymodel_target.py
! sortmap_script           = /homea/htb00/htb001/prefactor/scripts/sort_times_into_freqGroups.py
! check_flagged_script     = /homea/htb00/htb001/prefactor/scripts/check_unflagged_fraction.py
! structurefunction_script = /homea/htb00/htb001/prefactor/scripts/getStructure_from_phases.py
! plotphases_script        = /homea/htb00/htb001/prefactor/scripts/plot_solutions_all_stations.py
! losoto_executable        = /homea/htb00/htb003/local_jureca/bin/losoto
! makesourcedb             = /homea/htb00/htb003/lofar_jureca_2.17_stack2016a/bin/makesourcedb
! flagging_strategy        = /homea/htb00/htb003/lofar_jureca_2.17_stack2016a/share/rfistrategies/HBAdefault

# number of processes to use per step per node
! num_proc_per_node        = 24
# number of processes to use per step per node for tasks with high i/o (dppp or cp) or memory (eg calibration)
! num_proc_per_node_limit  = 4
# number of threads per process for NDPPP
! max_dppp_threads         = 8

# set this to True if you want the pipeline run to continue if single bands fail
! error_tolerance          = False

### Stuff that you probably don't need to modify
# which steps to run 
pipeline.steps=[mk_inspect_dir, createmap_target, combine_data_target_map, get_ion_files, trans, ndppp_prep_target, create_ateam_model_map, make_sourcedb_ateam, expand_sourcedb_ateam, predict_ateam, ateamcliptar, combine_target_map, sortmap_target, do_sortmap_maps, dpppconcat, check_unflagged, check_unflagged_map, sky_tar, create_target_model_map, make_sourcedb_target, expand_sourcedb_target, gsmcal_parmmap, gsmcal_solve, gsmcal_apply, h5_imp_gsmsol_map, h5imp_gsmsol, plot_gsm_phases, gsmcal_antmap, make_structurefunction, old_plot_gsmphases, createmap_plots, copy_plots, mk_results_dir, make_results_mapfile, move_results]

# create the inspection_directory if needed
mk_inspect_dir.control.kind               =  plugin
mk_inspect_dir.control.type               =  makeDirectory
mk_inspect_dir.control.directory          =  {{ inspection_directory }}

# generate a mapfile of all the target data
createmap_target.control.kind            =   plugin
createmap_target.control.type            =   createMapfile
createmap_target.control.method          =   mapfile_from_folder
createmap_target.control.mapfile_dir     =   input.output.mapfile_dir
createmap_target.control.filename        =   createmap_target.mapfile
createmap_target.control.folder          =   {{ target_input_path }}
createmap_target.control.pattern         =   {{ target_input_pattern }}

# combine all entries into one mapfile, for the sortmap script
combine_data_target_map.control.kind            =   plugin
combine_data_target_map.control.type            =   createMapfile
combine_data_target_map.control.method          =   mapfile_all_to_one
combine_data_target_map.control.mapfile_dir     =   input.output.mapfile_dir
combine_data_target_map.control.filename        =   combine_data_tar_map.mapfile
combine_data_target_map.control.mapfile_in      =   createmap_target.output.mapfile

# get ionex files once for every day that is covered by one of the input MSs
get_ion_files.control.type            = pythonplugin
get_ion_files.control.executable      = {{ get_ion_script }}
get_ion_files.control.max_per_node    =  1
get_ion_files.control.error_tolerance = {{ error_tolerance }}
get_ion_files.argument.flags          = [combine_data_target_map.output.mapfile]
get_ion_files.argument.ionex_server   = {{ ionex_server }}
get_ion_files.argument.ionex_prefix   = {{ ionex_prefix }}
get_ion_files.argument.ionexPath      = {{ ionex_path }}

# generate parmDB with the interpolated calibrator data to apply to the traget
trans.control.type              =  pythonplugin
trans.control.executable        =  {{ transfer_script }}
trans.control.max_per_node      =  {{ num_proc_per_node }}
trans.control.error_tolerance   =  {{ error_tolerance }}
trans.control.mapfile_in        =  createmap_target.output.mapfile
trans.control.inputkey          =  indata
trans.argument.flags            =  [indata]
trans.argument.store_basename   =  caldata_transfer
trans.argument.store_directory  =  {{ cal_values_directory }}
trans.argument.newparmdbext     =  /instrument_amp_clock_offset
trans.argument.ionex_server     =  {{ ionex_server }}
trans.argument.ionex_prefix     =  {{ ionex_prefix }}
trans.argument.ionexPath        =  {{ ionex_path }}

# run NDPPP on the target data to flag, transfer calibrator values, and average
ndppp_prep_target.control.type                      = dppp
ndppp_prep_target.control.max_per_node              = {{ num_proc_per_node_limit }}
ndppp_prep_target.control.error_tolerance           = {{ error_tolerance }}
ndppp_prep_target.control.mapfiles_in               = [createmap_target.output.mapfile, trans.output.transfer_parmDB.mapfile]
ndppp_prep_target.control.inputkeys                 = [input_file, parmdb_file]
ndppp_prep_target.argument.numthreads               = {{ max_dppp_threads }}
ndppp_prep_target.argument.msin                     = input_file
ndppp_prep_target.argument.msin.datacolumn          = DATA
ndppp_prep_target.argument.msin.baseline            = CS*&; RS*&; CS*&RS*
ndppp_prep_target.argument.msin.autoweight          = True   # recomended for processing raw input data 
ndppp_prep_target.argument.msout.datacolumn         = DATA
ndppp_prep_target.argument.msout.writefullresflag   = False
ndppp_prep_target.argument.msout.overwrite          = True
ndppp_prep_target.argument.steps                    = [autoflag,flagedge,flag1,filter,aoflag,flagamp,applyclock,applygain,applybeam,applyrotate,count,flag2,count,avg]
ndppp_prep_target.argument.autoflag.type            = preflagger
ndppp_prep_target.argument.autoflag.corrtype        = auto
ndppp_prep_target.argument.flagedge.type            = preflagger
ndppp_prep_target.argument.flagedge.chan            = [0..nchan/32-1,31*nchan/32..nchan-1]  # we are running on a single subband
ndppp_prep_target.argument.flag1.type               = preflagger
ndppp_prep_target.argument.flag1.baseline           = {{ flag_baselines }}
ndppp_prep_target.argument.filter.type              = filter
ndppp_prep_target.argument.filter.baseline          = CS*, RS*&&
ndppp_prep_target.argument.filter.remove            = true                     # fully kick out the international stations.
ndppp_prep_target.argument.aoflag.type              = aoflagger
ndppp_prep_target.argument.aoflag.memoryperc        = 10
ndppp_prep_target.argument.aoflag.keepstatistics    = false
ndppp_prep_target.argument.flagamp.type             = preflagger
ndppp_prep_target.argument.flagamp.amplmin          = 1e-30
ndppp_prep_target.argument.applyclock.type          = applycal
ndppp_prep_target.argument.applyclock.parmdb        = parmdb_file
ndppp_prep_target.argument.applyclock.correction    = clock
ndppp_prep_target.argument.applygain.type           = applycal
ndppp_prep_target.argument.applygain.parmdb         = parmdb_file
ndppp_prep_target.argument.applygain.correction     = gain
ndppp_prep_target.argument.applybeam.type           = applybeam
ndppp_prep_target.argument.applybeam.usechannelfreq = True
ndppp_prep_target.argument.applyrotate.type         = applycal
ndppp_prep_target.argument.applyrotate.parmdb       = parmdb_file
ndppp_prep_target.argument.applyrotate.correction   = commonrotationangle
ndppp_prep_target.argument.flag2.type               = aoflagger
ndppp_prep_target.argument.flag2.keepstatistics     = false
ndppp_prep_target.argument.flag2.memoryperc         = 10
ndppp_prep_target.argument.flag2.strategy           = {{ flagging_strategy }}
ndppp_prep_target.argument.avg.type                 = average
ndppp_prep_target.argument.avg.timestep             = {{ avg_timestep }}
ndppp_prep_target.argument.avg.freqstep             = {{ avg_freqstep }}

# create a mapfile with the A-Team skymodel, length = 1
create_ateam_model_map.control.kind              =  plugin
create_ateam_model_map.control.type              =  addListMapfile
create_ateam_model_map.control.hosts             =  ['localhost']
create_ateam_model_map.control.files             =  [ {{ ATeam_skymodel }} ]
create_ateam_model_map.control.mapfile_dir       =  input.output.mapfile_dir
create_ateam_model_map.control.filename          =  ateam_model_name.mapfile

# make sourcedbs from the A-Team skymodel, length = 1
# outtype = blob, because NDPPP likes that 
make_sourcedb_ateam.control.kind                 =  recipe
make_sourcedb_ateam.control.type                 =  executable_args
make_sourcedb_ateam.control.executable           =  {{ makesourcedb }}
make_sourcedb_ateam.control.error_tolerance      =  {{ error_tolerance }}
make_sourcedb_ateam.control.args_format          =  lofar
make_sourcedb_ateam.control.outputkey            =  out
make_sourcedb_ateam.control.mapfile_in           =  create_ateam_model_map.output.mapfile
make_sourcedb_ateam.control.inputkey             =  in
make_sourcedb_ateam.argument.format              =  <
make_sourcedb_ateam.argument.outtype             =  blob

# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb_ateam.control.kind               =  plugin
expand_sourcedb_ateam.control.type               =  expandMapfile
expand_sourcedb_ateam.control.mapfile_in         =  make_sourcedb_ateam.output.mapfile
expand_sourcedb_ateam.control.mapfile_to_match   =  ndppp_prep_target.output.mapfile
expand_sourcedb_ateam.control.mapfile_dir        =  input.output.mapfile_dir
expand_sourcedb_ateam.control.filename           =  expand_sourcedb_ateam.datamap

# Predict, corrupt, and predict the ateam-resolution model, length = nfiles
predict_ateam.control.type                       =  dppp
predict_ateam.control.mapfiles_in                =  [ndppp_prep_target.output.mapfile, expand_sourcedb_ateam.output.mapfile]
predict_ateam.control.inputkeys                  =  [msin,sourcedb]
predict_ateam.control.inplace                    =  True
predict_ateam.control.max_per_node               =  {{ num_proc_per_node_limit }}
predict_ateam.argument.numthreads                =  {{ max_dppp_threads }}
predict_ateam.control.error_tolerance            =  {{ error_tolerance }}
predict_ateam.argument.msin.datacolumn           =  DATA
predict_ateam.argument.msout                     =  .
predict_ateam.argument.msout.datacolumn          =  MODEL_DATA
predict_ateam.argument.steps                     =  [predict]
predict_ateam.argument.predict.type              =  predict
predict_ateam.argument.predict.operation         =  replace
predict_ateam.argument.predict.sourcedb          =  sourcedb
predict_ateam.argument.predict.sources           =  [VirA_4_patch,CygAGG,CasA_4_patch,TauAGG]
predict_ateam.argument.predict.usebeammodel      =  True
# This is run on single subbands, which means that it is ineed "more correct" 
# to set usechannelfreq to false
predict_ateam.argument.predict.usechannelfreq  =  false

# run the a-team clipper to flag data affected by the a-team
ateamcliptar.control.kind               =  recipe
ateamcliptar.control.type               =  executable_args
ateamcliptar.control.max_per_node       =  {{ num_proc_per_node }}   
ateamcliptar.control.mapfile_in         =  ndppp_prep_target.output.mapfile
ateamcliptar.control.executable         =  {{ ATeam_Clipper }}
ateamcliptar.control.error_tolerance    =  {{ error_tolerance }}
ateamcliptar.control.arguments          =  [allms]
ateamcliptar.control.inputkey           =  allms

# combine all entries into one mapfile, for the sortmap script
combine_target_map.control.kind            =   plugin
combine_target_map.control.type            =   createMapfile
combine_target_map.control.method          =   mapfile_all_to_one
combine_target_map.control.mapfile_dir     =   input.output.mapfile_dir
combine_target_map.control.filename        =   combine_tar_map.mapfile
combine_target_map.control.mapfile_in      =   ndppp_prep_target.output.mapfile

# sort the target data by frequency into groups so that NDPPP can concatenate them
sortmap_target.control.type               =  pythonplugin  
sortmap_target.control.executable         =  {{ sortmap_script }}
sortmap_target.argument.flags             =  [combine_target_map.output.mapfile]
sortmap_target.argument.filename          =  sortmap_target
sortmap_target.argument.mapfile_dir       =  input.output.mapfile_dir
sortmap_target.argument.target_path       =  input.output.working_directory/input.output.job_name
sortmap_target.argument.numSB             =  {{ num_SBs_per_group }}
sortmap_target.argument.NDPPPfill         =  True
sortmap_target.argument.stepname          =  dpppconcat
sortmap_target.argument.truncateLastSBs   =  True     # This means that a excess subbands that don't make a full group get discarded

# convert the output of sortmap_target into usable mapfiles
do_sortmap_maps.control.kind                      =  plugin
do_sortmap_maps.control.type                      =  mapfilenamesFromMapfiles
do_sortmap_maps.control.mapfile_groupmap          =  sortmap_target.output.groupmapfile.mapfile
do_sortmap_maps.control.mapfile_datamap           =  sortmap_target.output.mapfile.mapfile

# run NDPPP to concatenate the target
dpppconcat.control.type                   =  dppp
dpppconcat.control.max_per_node           =  {{ num_proc_per_node_limit }}   
dpppconcat.control.error_tolerance        =  {{ error_tolerance }}
dpppconcat.control.mapfile_out            =  do_sortmap_maps.output.groupmap # tell the pipeline to give the output useful names
dpppconcat.control.mapfiles_in            =  [do_sortmap_maps.output.datamap]
dpppconcat.control.inputkey               =  msin
dpppconcat.argument.msin.datacolumn       =  DATA
dpppconcat.argument.msin.missingdata      =  True    #\ these two lines will make NDPPP generate dummy data when
dpppconcat.argument.msin.orderms          =  False   #/ concatenating data
dpppconcat.argument.msout.datacolumn      =  DATA
dpppconcat.argumentmsout.writefullresflag =  False
dpppconcat.argument.msout.overwrite       = True
dpppconcat.argument.steps                 =  [flag]      # run the aoflagger (this used to be an extra step)
dpppconcat.argument.flag.type             =  aoflagger
dpppconcat.argument.flag.keepstatistics   =  false
dpppconcat.argument.flag.memoryperc       =  10
dpppconcat.argument.flag.strategy         =  {{ flagging_strategy }}

# check all files for minimum unflagged fraction
check_unflagged.control.type            =  pythonplugin
check_unflagged.control.executable      =  {{ check_flagged_script }}
check_unflagged.argument.flags          =  [dpppconcat.output.mapfile]
check_unflagged.argument.min_fraction   =  {{ min_unflagged_fraction }}
# this step writes hostnames into "check_unflagged.flagged.mapfile" due to a "feature" of the pythonplugin

# prune flagged files from mapfile
check_unflagged_map.control.kind               =  plugin
check_unflagged_map.control.type               =  pruneMapfile
check_unflagged_map.control.mapfile_in         =  check_unflagged.output.flagged.mapfile
check_unflagged_map.control.mapfile_dir        =  input.output.mapfile_dir
check_unflagged_map.control.filename           =  check_unflagged_map.mapfile
check_unflagged_map.control.prune_str          =  None

# if wished, download the tgss skymodel for the target
sky_tar.control.type               =  pythonplugin  
sky_tar.control.executable         =  {{ get_tgss_skymodel_script }}
sky_tar.argument.flags             =  [combine_target_map.output.mapfile]
sky_tar.argument.DoDownload        =  {{ use_tgss_target }}
sky_tar.argument.SkymodelPath      =  {{ target_skymodel }}
sky_tar.argument.Radius            =  5. #in degrees

# create a mapfile with the target skymodel, length = 1
create_target_model_map.control.kind              =  plugin
create_target_model_map.control.type              =  addListMapfile
create_target_model_map.control.hosts             =  ['localhost']
create_target_model_map.control.files             =  [ {{ target_skymodel }} ]
create_target_model_map.control.mapfile_dir       =  input.output.mapfile_dir
create_target_model_map.control.filename          =  target_model_name.mapfile

# make sourcedbs from the target skymodel, length = 1
# outtype = blob, because NDPPP likes that 
make_sourcedb_target.control.kind                 =  recipe
make_sourcedb_target.control.type                 =  executable_args
make_sourcedb_target.control.executable           =  {{ makesourcedb }}
make_sourcedb_target.control.error_tolerance      =  {{ error_tolerance }}
make_sourcedb_target.control.args_format          =  lofar
make_sourcedb_target.control.outputkey            =  out
make_sourcedb_target.control.mapfile_in           =  create_target_model_map.output.mapfile
make_sourcedb_target.control.inputkey             =  in
make_sourcedb_target.argument.format              =  <
make_sourcedb_target.argument.outtype             =  blob

# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb_target.control.kind               =  plugin
expand_sourcedb_target.control.type               =  expandMapfile
expand_sourcedb_target.control.mapfile_in         =  make_sourcedb_target.output.mapfile
expand_sourcedb_target.control.mapfile_to_match   =  check_unflagged_map.output.mapfile
expand_sourcedb_target.control.mapfile_dir        =  input.output.mapfile_dir
expand_sourcedb_target.control.filename           =  expand_sourcedb_target.datamap

# generate mapfile with the parmDB names to be used in the gsmcal steps
gsmcal_parmmap.control.kind               =  plugin
gsmcal_parmmap.control.type               =  createMapfile
gsmcal_parmmap.control.method             =  add_suffix_to_file
gsmcal_parmmap.control.mapfile_in         =  check_unflagged_map.output.mapfile
gsmcal_parmmap.control.add_suffix_to_file =  /instrument_directionindependent
gsmcal_parmmap.control.mapfile_dir        =  input.output.mapfile_dir
gsmcal_parmmap.control.filename           =  gsmcal_parmdbs.mapfile

# solve for phase-only calibration solutions
# solve and apply are seperate to allow to solve on a subset of baselines but apply to all
gsmcal_solve.control.type                             = dppp
gsmcal_solve.control.environment                      = {OMP_NUM_THREADS: 4} 
gsmcal_solve.control.error_tolerance                  = {{ error_tolerance }}
gsmcal_solve.control.inplace                          = True
gsmcal_solve.control.max_per_node                     = {{ num_proc_per_node_limit }}
gsmcal_solve.argument.numthreads                      = {{ max_dppp_threads }}
gsmcal_solve.argument.msin                            = check_unflagged_map.output.mapfile
gsmcal_solve.argument.msin.datacolumn                 = DATA
gsmcal_solve.argument.msin.baseline                   = CS*&; RS*&; CS*&RS*
gsmcal_solve.argument.msout.datacolumn                = CORRECTED_DATA
gsmcal_solve.argument.steps                           = [filter,gaincal]
gsmcal_solve.filter.type                              = filter
gsmcal_solve.filter.blrange                           = [150, 999999]
gsmcal_solve.argument.gaincal.type                    = gaincal
gsmcal_solve.argument.gaincal.maxiter                 = 500
gsmcal_solve.argument.gaincal.caltype                 = phaseonly
gsmcal_solve.argument.gaincal.nchan                   = 0
gsmcal_solve.argument.gaincal.solint                  = 1
gsmcal_solve.argument.gaincal.sourcedb                = expand_sourcedb_target.output.mapfile
gsmcal_solve.argument.gaincal.parmdb                  = gsmcal_parmmap.output.mapfile
gsmcal_solve.argument.gaincal.usebeammodel            = True
gsmcal_solve.argument.gaincal.usechannelfreq          = True
gsmcal_solve.argument.gaincal.beammode                = array_factor

# apply the phase-only calibration solutions
# solve and apply are seperate to allow to solve on a subset of baselines but apply to all
gsmcal_apply.control.type                             = dppp
gsmcal_apply.control.error_tolerance                  = {{ error_tolerance }}
gsmcal_apply.control.inplace                          = True
gsmcal_apply.control.max_per_node                     = {{ num_proc_per_node_limit }}
gsmcal_apply.argument.numthreads                      = {{ max_dppp_threads }}
gsmcal_apply.argument.msin                            = check_unflagged_map.output.mapfile
gsmcal_apply.argument.msin.datacolumn                 = DATA
gsmcal_apply.argument.msout.datacolumn                = CORRECTED_DATA
gsmcal_apply.argument.msout.writefullresflag          = False
gsmcal_apply.argument.steps                           = [applycal]
gsmcal_apply.argument.applycal.type                   = applycal
gsmcal_apply.argument.applycal.correction             = gain
gsmcal_apply.argument.applycal.parmdb                 = gsmcal_parmmap.output.mapfile

# generate a mapfile with all files in a single entry
h5_imp_gsmsol_map.control.kind               =  plugin
h5_imp_gsmsol_map.control.type               =  MapfileToOne
h5_imp_gsmsol_map.control.method             =  mapfile_all_to_one
h5_imp_gsmsol_map.control.mapfile_in         =  check_unflagged_map.output.mapfile
h5_imp_gsmsol_map.control.mapfile_dir        =  input.output.mapfile_dir
h5_imp_gsmsol_map.control.filename           =  h5_imp_gsmsol_map.mapfile

# import all instrument tables into one LoSoTo file
h5imp_gsmsol.control.type            =  pythonplugin
h5imp_gsmsol.control.executable      =  {{ losoto_importer }}
h5imp_gsmsol.control.error_tolerance =  {{ error_tolerance }}
h5imp_gsmsol.argument.flags          =  [h5_imp_gsmsol_map.output.mapfile,h5imp_gsmsol_losoto.h5]
h5imp_gsmsol.argument.instrument     =  /instrument_directionindependent
h5imp_gsmsol.argument.solsetName     =  sol000
h5imp_gsmsol.argument.compression    =  7

# plot the phase solutions from the phase-only calibration of the target
plot_gsm_phases.control.kind                          =  recipe
plot_gsm_phases.control.type                          =  executable_args
plot_gsm_phases.control.executable                    =  {{ losoto_executable }}
plot_gsm_phases.control.max_per_node                  =  {{ num_proc_per_node }}   
plot_gsm_phases.control.parsetasfile                  =  True
plot_gsm_phases.control.args_format                   =  losoto
plot_gsm_phases.control.mapfiles_in                   =  [h5imp_gsmsol.output.h5parm.mapfile]
plot_gsm_phases.control.inputkeys                     =  [hdf5file]
plot_gsm_phases.argument.flags                        =  [hdf5file]
plot_gsm_phases.argument.LoSoTo.Steps                 =  [plot]
plot_gsm_phases.argument.LoSoTo.Solset                =  [sol000]
plot_gsm_phases.argument.LoSoTo.Soltab                =  [sol000/phase000]
plot_gsm_phases.argument.LoSoTo.SolType               =  [phase]
plot_gsm_phases.argument.LoSoTo.ant                   =  []
plot_gsm_phases.argument.LoSoTo.pol                   =  [XX,YY]
plot_gsm_phases.argument.LoSoTo.dir                   =  [pointing]
plot_gsm_phases.argument.LoSoTo.Steps.plot.Operation  =  PLOT
plot_gsm_phases.argument.LoSoTo.Steps.plot.PlotType   =  2D
plot_gsm_phases.argument.LoSoTo.Steps.plot.Axes       =  [time,freq]
plot_gsm_phases.argument.LoSoTo.Steps.plot.TableAxis  =  [ant]
plot_gsm_phases.argument.LoSoTo.Steps.plot.ColorAxis  =  [pol]
plot_gsm_phases.argument.LoSoTo.Steps.plot.Reference  =  {{ reference_station }}
plot_gsm_phases.argument.LoSoTo.Steps.plot.PlotFlag   =  False
plot_gsm_phases.argument.LoSoTo.Steps.plot.Prefix     =  {{ inspection_directory }}/gsm_phases_

# generate mapfile with the antenna tables of the concatenated target datafiles
gsmcal_antmap.control.kind               =  plugin
gsmcal_antmap.control.type               =  createMapfile
gsmcal_antmap.control.method             =  add_suffix_to_file
gsmcal_antmap.control.mapfile_in         =  dpppconcat.output.mapfile
gsmcal_antmap.control.add_suffix_to_file =  /ANTENNA
gsmcal_antmap.control.mapfile_dir        =  input.output.mapfile_dir
gsmcal_antmap.control.filename           =  gsmcal_antmaps.mapfile

# plot the phase solutions from the phase-only calibration of the target
make_structurefunction.control.kind               =  recipe
make_structurefunction.control.type               =  executable_args
make_structurefunction.control.executable         =  {{ structurefunction_script }}
make_structurefunction.control.max_per_node       =  {{ num_proc_per_node }}
make_structurefunction.control.mapfiles_in        =  [gsmcal_parmmap.output.mapfile,gsmcal_antmap.output.mapfile,check_unflagged_map.output.mapfile]
make_structurefunction.control.inputkeys          =  [inparmdb,inants,outbase]
make_structurefunction.control.arguments          =  [inparmdb,inants,outbase]

# plot the phase solutions from the phase-only calibration of the target
old_plot_gsmphases.control.kind               =  recipe
old_plot_gsmphases.control.type               =  executable_args
old_plot_gsmphases.control.executable         =  {{ plotphases_script }}
old_plot_gsmphases.control.max_per_node       =  {{ num_proc_per_node }}   
old_plot_gsmphases.control.mapfiles_in        =  [gsmcal_parmmap.output.mapfile,check_unflagged_map.output.mapfile]
old_plot_gsmphases.control.inputkeys          =  [infile,outbase]
old_plot_gsmphases.control.arguments          =  [-p,infile,outbase]

# generate a mapfile of all the diagnostic plots
createmap_plots.control.kind            =   plugin
createmap_plots.control.type            =   createMapfile
createmap_plots.control.method          =   mapfile_from_folder
createmap_plots.control.mapfile_dir     =   input.output.mapfile_dir
createmap_plots.control.filename        =   diagnostic_plots.mapfile
createmap_plots.control.folder          =   input.output.working_directory/input.output.job_name
createmap_plots.control.pattern         =   *.png

# copy the diagnostic plots to the results_directory
copy_plots.control.kind               =  recipe
copy_plots.control.type               =  executable_args
copy_plots.control.executable         =  /bin/cp
copy_plots.control.max_per_node       =  {{ num_proc_per_node_limit }}
copy_plots.control.mapfile_in         =  createmap_plots.output.mapfile
copy_plots.control.inputkey           =  source
copy_plots.control.arguments          =  [source,{{ inspection_directory }}]

# create the results directory if needed
mk_results_dir.control.kind               =  plugin
mk_results_dir.control.type               =  makeDirectory
mk_results_dir.control.directory          =  {{ results_directory }}

# make mapfile with the filenames of the results that we want
make_results_mapfile.control.kind            =  plugin
make_results_mapfile.control.type            =  makeResultsMapfile
make_results_mapfile.control.mapfile_dir     =  input.output.mapfile_dir
make_results_mapfile.control.filename        =  make_results_mapfile.mapfile
make_results_mapfile.control.mapfile_in      =  check_unflagged_map.output.mapfile
make_results_mapfile.control.target_dir      =  {{ results_directory }}
make_results_mapfile.control.make_target_dir =  True
make_results_mapfile.control.new_suffix      =  .pre-cal.ms

# move the results to where we want them
move_results.control.kind               =  recipe
move_results.control.type               =  executable_args
move_results.control.executable         =  /bin/mv
move_results.control.max_per_node       =  {{ num_proc_per_node_limit }}
move_results.control.mapfiles_in        =  [check_unflagged_map.output.mapfile,make_results_mapfile.output.mapfile]
move_results.control.inputkeys          =  [source,destination]
move_results.control.arguments          =  [source,destination]