forked from lofar-astron/prefactor
-
Notifications
You must be signed in to change notification settings - Fork 0
/
convert_all_to_h5parm.parset
147 lines (121 loc) · 9.2 KB
/
convert_all_to_h5parm.parset
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
##########################################################################
# Convert_all_to_h5parm Pipeline #
# #
# Run once to match prefactor3.0 compatible h5parm solutions: #
# - requires LOFAR software version >= 3.1.0 #
# - requires losoto software version >= 2.0.0 #
# - expects shared filesystem, that all nodes can reach all files! #
# (E.g. a single workstation or compute cluster with shared filesystem #
# doesn't work on multiple nodes on CEP3.) #
##########################################################################
##########################################
### parameters you will need to adjust. ##
##########################################
## information about the calibrator data
! target_input_path = /data/scratch/drabent/prefactor/results ## specify the directory where your target data is stored
! target_input_pattern = L401323*.pre-cal.ms ## regular expression pattern of all your target files
! npys_directory = /data/scratch/drabent/prefactor/cal_values ## directory where the final h5parm solution set will be stored
## location of the software
! prefactor_directory = /home/drabent/prefactor/ ## path to your conversion pipeline copy
! losoto_directory = /home/drabent/losoto ## path to your local LoSoTo installation
##########################################
### parameters you may need to adjust ##
##########################################
! num_proc_per_node = input.output.max_per_node ## number of processes to use per step per node (usually max_per_node from pipeline.cfg)
! error_tolerance = False ## set this to True if you want the pipeline run to continue if single bands fail
##########################################
### parameters you may want to adjust ##
##########################################
## main directories
! lofar_directory = $LOFARROOT ## base directory of your LOFAR installation
! job_directory = input.output.job_directory ## directory of the prefactor outputs
## script and plugin directories
! scripts = {{ prefactor_directory }}/scripts
pipeline.pluginpath = {{ prefactor_directory }}/plugins
## result directories
! results_directory = {{ job_directory }}/results ## location of the results
! inspection_directory = {{ results_directory }}/inspection ## directory where the inspection plots will be stored
! cal_values_directory = {{ results_directory }}/cal_values ## directory where the final h5parm solution set will be stored
## calibrator solutions
! cal_solutions = {{ cal_values_directory }}/cal_solutions.h5
## RMextract settings
! ionex_server = "ftp://ftp.aiub.unibe.ch/CODE/" ## to download from the "standard" server
! ionex_prefix = CODG ## the prefix of the IONEX files
! ionex_path = {{ job_directory }}/IONEX/ ## path where the IONEX files can be stored or are already stored
########################################################
## ##
## BEGIN PIPELINE: DO NOT UPDATE BELOW THIS LINE! ##
## ##
########################################################
# which steps to run
pipeline.steps = [mk_targ_values_dir, map_target, combine_data_target_map, convert, h5imp_gsmcal, h5_gsmsol_map, h5exp_gsmcal]
#############################
## Tasks ##
#############################
# create the cal_values_directory if needed
mk_targ_values_dir.control.kind = plugin
mk_targ_values_dir.control.type = makeDirectory
mk_targ_values_dir.control.directory = {{ cal_values_directory }}
# --- Create mapfile for target data
map_target.control.kind = plugin # plugin -> short, non-parallel step
map_target.control.type = createMapfile # generate a new mapfile
map_target.control.method = mapfile_from_folder # look for all files in a given directory
map_target.control.mapfile_dir = input.output.mapfile_dir # put the mapfile into the runtime directory
map_target.control.filename = map_target.mapfile # name of the generated mapfile
map_target.control.folder = {{ target_input_path }} # directory in which to look for the data
map_target.control.pattern = {{ target_input_pattern }} # use only files that match this pattern
# combine all entries into one mapfile, for the sortmap script
combine_data_target_map.control.kind = plugin
combine_data_target_map.control.type = createMapfile
combine_data_target_map.control.method = mapfile_all_to_one
combine_data_target_map.control.mapfile_dir = input.output.mapfile_dir
combine_data_target_map.control.filename = combine_data_target_map.mapfile
combine_data_target_map.control.mapfile_in = map_target.output.mapfile
# --- convert npys to parmdbs for the target
convert.control.type = pythonplugin
convert.control.executable = {{ scripts }}/convert_npys_to_h5parm.py
convert.control.max_per_node = {{ num_proc_per_node }}
convert.control.error_tolerance = {{ error_tolerance }}
convert.control.mapfile_in = combine_data_target_map.output.mapfile
convert.control.inputkey = indata
convert.argument.flags = [indata, {{ cal_solutions }}]
convert.argument.store_basename = caldata_transfer
convert.argument.store_directory = {{ npys_directory }}
# export all instrument tables to h5parms
h5imp_gsmcal.control.kind = recipe
h5imp_gsmcal.control.type = executable_args
h5imp_gsmcal.control.executable = {{ losoto_directory }}/bin/parmdb2H5parm.py
h5imp_gsmcal.control.error_tolerance = {{ error_tolerance }}
h5imp_gsmcal.argument.instrument = instrument_directionindependent
h5imp_gsmcal.argument.flags = [-V, outputkey, map_target.output.mapfile]
# generate a mapfile with all files in a single entry
h5_gsmsol_map.control.kind = plugin
h5_gsmsol_map.control.type = createMapfile
h5_gsmsol_map.control.method = mapfile_all_to_one
h5_gsmsol_map.control.mapfile_in = h5imp_gsmcal.output.mapfile
h5_gsmsol_map.control.mapfile_dir = input.output.mapfile_dir
h5_gsmsol_map.control.filename = h5_imp_gsmsol_map.mapfile
# output the final soltab into an external h5parm
h5exp_gsmcal.control.kind = recipe
h5exp_gsmcal.control.type = executable_args
h5exp_gsmcal.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py
h5exp_gsmcal.control.error_tolerance = {{ error_tolerance }}
h5exp_gsmcal.argument.flags = [h5_gsmsol_map.output.mapfile]
h5exp_gsmcal.argument.insoltab = phase000
h5exp_gsmcal.argument.outsolset = target
h5exp_gsmcal.argument.outh5parm = {{ cal_solutions }}
# # --- add RMextract
# h5imp_RMextract.control.type = pythonplugin
# h5imp_RMextract.control.executable = {{ scripts }}/createRMh5parm.py
# h5imp_RMextract.control.error_tolerance = {{ error_tolerance }}
# h5imp_RMextract.argument.flags = [combine_data_target_map.output.mapfile, {{ cal_solutions }}]
# h5imp_RMextract.argument.ionex_server = {{ ionex_server }}
# h5imp_RMextract.argument.ionex_prefix = {{ ionex_prefix }}
# h5imp_RMextract.argument.ionexPath = {{ ionex_path }}
# h5imp_RMextract.argument.solset_name = target
# h5imp_RMextract.argument.all_stations = True
########################################################
## ##
## END PIPELINE ##
## ##
########################################################