-
Notifications
You must be signed in to change notification settings - Fork 27
/
Pre-Facet-Calibrator.parset
1137 lines (1053 loc) · 84.3 KB
/
Pre-Facet-Calibrator.parset
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
##########################################################################
# Pre-Facet Calibrator Calibration Pipeline v3.1 (08/10/2021) #
# #
# Calibrator part of the basic Pre-Facet calibration pipeline: #
# - requires LOFAR software version >= 3.1.0 #
# - requires losoto software version >= 2.0.0 #
# - expects shared filesystem, that all nodes can reach all files! #
# (E.g. a single workstation or compute cluster with shared filesystem #
# doesn't work on multiple nodes on CEP3.) #
##########################################################################
##########################################
### parameters you will need to adjust. ##
##########################################
## information about the calibrator data
! cal_input_path = /data/scratch/drabent ## specify the directory where your calibrator data is stored
! cal_input_pattern = L228161*.MS ## regular expression pattern of all your calibrator files
## location of the software
! prefactor_directory = /home/drabent/prefactor/ ## path to your prefactor copy
! losoto_directory = /home/drabent/losoto ## path to your local LoSoTo installation
! aoflagger = /home/drabent/aoflagger/bin/aoflagger ## path to your aoflagger executable
##########################################
### parameters you may need to adjust ##
##########################################
! refant = 'CS00.*' ## regular expression of reference antennas from which to choose.
! flag_baselines = [] ## NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: [] )
! process_baselines_cal = *& ## performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
! filter_baselines = {{ process_baselines_cal }} ## selects only this set of baselines to be processed. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
! do_smooth = False ## enable or disable baseline-based smoothing
! rfistrategy = HBAdefault.rfis ## strategy to be applied with the statistical flagger (AOFlagger)
! max2interpolate = 30 ## amount of channels in which interpolation should be performed for deriving the bandpass
! ampRange = [0, 0] ## range of median amplitudes accepted per station. Use [0, 0] to use typical ranges of [50, 225] for the current correlator or [0.0004, 0.0018] for the old one.
! skip_international = True ## skip fitting the bandpass for international stations (this avoids flagging them in many cases)
! raw_data = False ## use autoweight, set to True in case you are using raw data
! propagatesolutions = True ## use already derived solutions as initial guess for the upcoming time slot
! flagunconverged = False ## flag solutions for solves that did not converge (if they were also detected to diverge)
! maxStddev = -1.0 ## Maximum allowable standard deviation when outlier clipping is done. For phases, this should value should be in radians, for amplitudes in log(amp). If None (or negative), a value of 0.1 rad is used for phases and 0.01 for amplitudes
! fit_offset_PA = False ## Assume that together with a delay each station has also a differential phase offset (important for old LBA observations).
# options for solution transfer (for non-trusted calibrator observations)
! do_transfer = False ## enable or disable transfer of bandpass solutions from a reference to the final solutions. WARNING: Bad calibrator results may have strong impacts on further calibration results.
! solutions2transfer = {{ prefactor_directory }}/solutions/3C48.h5 ## location of h5parm reference solutions to transfer
! antennas2transfer = [FUSPID].* ## regular expression of antennas which solutions should be transferred to the final solution set, if none of the trusted calibrators have been observed (by default: all international stations)
! trusted_sources = 3C48,3C147,3C196,3C295,3C380
# demixing options (only used if demix step is added to the prep_cal_strategy variable)
! demix_sources = [CasA,CygA] ## choose sources to demix (provided as list)
! demix_target = "" ## if given, the target source model (its patch in the SourceDB) is taken into account when solving
! demix_freqstep = 16 ## number of channels to average when demixing.
! demix_timestep = 10 ## number of time slots to average when demixing
# definitions for pipeline options -- do not change!
! default_flagging = flagbaseline,flagelev,flagamp ## regular flagging after pre-processing by the observatory pipelines
! raw_flagging = flagedge,aoflag,{{ default_flagging }} ## full flagging (usually only necessary for raw data)
! 1st_order = ct,duplicateCbkp,plotTEC,residuals ## Do not change! Only cal_ion should be edited if needed
! 3rd_order = ct3,duplicateCbkp,plotTEC,plotTEC3,residuals3 ## Do not change! Only cal_ion should be edited if needed
! demix = demix, ## Do not change! Only demix_step should be edited if needed
! full_apply = ,apply_PA,apply_bandpass,apply_clock,apply_beam,apply_FR,apply_TEC ## Do not change!
! none = ## Do not change!
# pipeline options
! initial_flagging = {{ default_flagging }} ## choose {{ raw_flagging }} if you process raw data
! demix_step = {{ none }} ## choose {{ demix }} if you want to demix
! cal_ion = {{ 1st_order }},smooth ## choose {{ 3rd_order }} if you want to include 3rd order ionospheric effects (may be useful for LBA < 35 MHz), add smooth if you want to use the median of the clock in time (suggested for HBA+LB)
! tables2export = clock ## comma-separated list of tables to export from the ionospheric calibration step (cal_ion)
! final_apply = {{ none }} ## choose {{ full_apply }} if you want to apply all extracted solutions to the calibrator field
##########################################
### parameters for pipeline performance ##
##########################################
! num_proc_per_node = input.output.max_per_node ## number of processes to use per step per node (usually max_per_node from pipeline.cfg)
! num_proc_per_node_limit = 4 ## number of processes to use per step per node for tasks with high i/o (dppp or cp) or memory (eg calibration)
! max_dppp_threads = 10 ## number of threads per process for NDPPP
! memoryperc = 20 ## maximum of memory used for aoflagger in raw_flagging mode in percent
! min_length = 50 ## minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read.
! overhead = 0.8 ## Only use this fraction of the available memory for deriving the amount of data to be concatenated.
! min_separation = 30 ## minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING)
! max_separation_arcmin = 1.0 ## maximum distance to the phase center for which a skymodel will be still accepted. Change with caution!
! error_tolerance = False ## set this to True if you want the pipeline run to continue if single bands fail
##########################################
### parameters you may want to adjust ##
##########################################
## main directories
! lofar_directory = $LOFARROOT ## base directory of your LOFAR installation
! job_directory = input.output.job_directory ## directory of the prefactor outputs
! working_directory = input.output.working_directory/input.output.job_name ## specify the working_directory (intermediate data products)
! log_file = input.output.log_file ## location of the logfile
! mapfile_dir = input.output.mapfile_dir ## specify mapfile directory
## script and plugin directories
! scripts = {{ prefactor_directory }}/scripts
pipeline.pluginpath = {{ prefactor_directory }}/plugins
## skymodel directory
! calibrator_path_skymodel = {{ prefactor_directory }}/skymodels
! A-team_skymodel = {{ calibrator_path_skymodel }}/Ateam_LBA_CC.skymodel
## result directories
! results_directory = {{ job_directory }}/results ## location of the results
! inspection_directory = {{ results_directory }}/inspection ## directory where the inspection plots will be stored
! cal_values_directory = {{ results_directory }}/cal_values ## directory where the final h5parm solution set will be stored
## calibrator solutions
! cal_solutions = {{ cal_values_directory }}/cal_solutions.h5
## averaging for the calibrator data
! avg_timeresolution = 4. ## average to 4 sec/timeslot, units here are seconds
! avg_freqresolution = 48.82kHz ## average to 48.82 kHz/ch (= 4 ch/SB)
! bandpass_freqresolution = 195.3125kHz ## resolution of the bandpass table is 195.3125kHz (= 1 ch/SB)
########################################################
## ##
## BEGIN PIPELINE: DO NOT UPDATE BELOW THIS LINE! ##
## ##
########################################################
# which steps to run
pipeline.steps = [prep, PA, FR, bandpass, ion, finalize]
# pipeline substeps
pipeline.steps.prep = [createmap_cal, combine_data_map, check_Ateam_separation, mk_cal_values_dir, createmap_prepcal, createmap_instcal, create_ateam_model_map, make_sourcedb_ateam, expand_sourcedb_ateam, ndppp_prep_cal, combine_data_cal_map, ms_concat, ms_concat_map, expand_memory_map, aoflag, sky_cal, make_sourcedb, expand_sourcedb, expand_skymodel, calib_cal_parmmap, h5imp_cal_map, smooth_data, predict_cal, find_refant]
pipeline.steps.PA = [calib_cal, h5imp_cal_PA, prepare_losoto_PA, process_losoto_PA, h5exp_cal_PA, apply_PA, apply_beam]
pipeline.steps.FR = [smooth_corrected, calib_cal, h5imp_cal_FR, prepare_losoto_FR, process_losoto_FR, h5exp_cal_FR, apply_FR]
pipeline.steps.bandpass = [smooth_corrected, calib_cal2, h5imp_cal_bandpass, prepare_losoto_bandpass, prepare_losoto_bandpasstrans, process_losoto_bandpass, h5exp_cal_bandpass, h5parm_name, transfer_solutions, apply_PA, apply_bandpass, apply_beam, apply_FR]
pipeline.steps.ion = [smooth_corrected, calib_cal3, h5imp_cal_ion, prepare_losoto_ion, process_losoto_ion, h5exp_cal_ion]
pipeline.steps.finalize = [h5parm_name {{ final_apply }}, make_summary]
###############################
## Mapping calibrator files ##
###############################
# generate a mapfile of all the calibrator data
createmap_cal.control.kind = plugin
createmap_cal.control.type = createMapfile
createmap_cal.control.method = mapfile_from_folder
createmap_cal.control.mapfile_dir = {{ mapfile_dir }}
createmap_cal.control.filename = createmap_cal.mapfile
createmap_cal.control.folder = {{ cal_input_path }}
createmap_cal.control.pattern = {{ cal_input_pattern }}
# combine all entries into one mapfile, for the sortmap script
combine_data_map.control.kind = plugin
combine_data_map.control.type = createMapfile
combine_data_map.control.method = mapfile_all_to_one
combine_data_map.control.mapfile_dir = {{ mapfile_dir }}
combine_data_map.control.filename = combine_data_map.mapfile
combine_data_map.control.mapfile_in = createmap_cal.output.mapfile
# warn for potential nearby A-Team sources
check_Ateam_separation.control.type = pythonplugin
check_Ateam_separation.control.executable = {{ scripts }}/check_Ateam_separation.py
check_Ateam_separation.control.mapfile_in = combine_data_map.output.mapfile
check_Ateam_separation.control.inputkey = MSfile
check_Ateam_separation.argument.min_separation = {{ min_separation }}
check_Ateam_separation.argument.outputimage = {{ inspection_directory }}/A-Team_elevation_calibrator.png
check_Ateam_separation.argument.flags = [MSfile]
#############################
## Prepare for demixing ##
#############################
# generate a mapfile of the calibrator
createmap_prepcal.control.kind = plugin
createmap_prepcal.control.type = makeResultsMapfile
createmap_prepcal.control.mapfile_dir = {{ mapfile_dir }}
createmap_prepcal.control.filename = createmap_prepcal.mapfile
createmap_prepcal.control.mapfile_in = createmap_cal.output.mapfile
createmap_prepcal.control.target_dir = {{ working_directory }}
createmap_prepcal.control.make_target_dir = False
createmap_prepcal.control.new_suffix = .ndppp_prep_cal
# generate a mapfile for the instrument table of the calibrator
createmap_instcal.control.kind = plugin
createmap_instcal.control.type = changeMapfile
createmap_instcal.control.mapfile_in = createmap_prepcal.output.mapfile
createmap_instcal.control.join_files = instrument
createmap_instcal.control.newname = createmap_instcal.mapfile
# create a mapfile with the A-Team skymodel, length = 1
create_ateam_model_map.control.kind = plugin
create_ateam_model_map.control.type = addListMapfile
create_ateam_model_map.control.hosts = ['localhost']
create_ateam_model_map.control.files = [ {{ A-team_skymodel }} ]
create_ateam_model_map.control.mapfile_dir = {{ mapfile_dir }}
create_ateam_model_map.control.filename = ateam_model_name.mapfile
# make sourcedbs from the A-Team skymodel, length = 1
make_sourcedb_ateam.control.kind = recipe
make_sourcedb_ateam.control.type = executable_args
make_sourcedb_ateam.control.executable = {{ lofar_directory }}/bin/makesourcedb
make_sourcedb_ateam.control.error_tolerance = {{ error_tolerance }}
make_sourcedb_ateam.control.args_format = lofar
make_sourcedb_ateam.control.outputkey = out
make_sourcedb_ateam.control.mapfile_in = create_ateam_model_map.output.mapfile
make_sourcedb_ateam.control.inputkey = in
make_sourcedb_ateam.argument.format = <
make_sourcedb_ateam.argument.outtype = blob
# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb_ateam.control.kind = plugin
expand_sourcedb_ateam.control.type = expandMapfile
expand_sourcedb_ateam.control.mapfile_in = make_sourcedb_ateam.output.mapfile
expand_sourcedb_ateam.control.mapfile_to_match = createmap_cal.output.mapfile
expand_sourcedb_ateam.control.mapfile_dir = {{ mapfile_dir }}
expand_sourcedb_ateam.control.filename = expand_sourcedb_ateam.datamap
###############################
## Prepare calibrator ##
###############################
# create the cal_values_directory if needed
mk_cal_values_dir.control.kind = plugin
mk_cal_values_dir.control.type = makeDirectory
mk_cal_values_dir.control.directory = {{ cal_values_directory }}
# run NDPPP on the calibrator data
ndppp_prep_cal.control.type = dppp
ndppp_prep_cal.control.max_per_node = {{ num_proc_per_node_limit }}
ndppp_prep_cal.control.error_tolerance = {{ error_tolerance }}
ndppp_prep_cal.argument.numthreads = {{ max_dppp_threads }}
ndppp_prep_cal.argument.msin = createmap_cal.output.mapfile # The input data.
ndppp_prep_cal.argument.msin.datacolumn = DATA
ndppp_prep_cal.argument.msin.baseline = {{ filter_baselines }}
ndppp_prep_cal.argument.msin.autoweight = {{ raw_data }}
ndppp_prep_cal.argument.msout.datacolumn = DATA
ndppp_prep_cal.argument.msout.writefullresflag = False
ndppp_prep_cal.argument.msout.overwrite = True
ndppp_prep_cal.argument.msout.storagemanager = "Dysco"
ndppp_prep_cal.argument.msout.storagemanager.databitrate = 0
ndppp_prep_cal.argument.steps = [{{ initial_flagging }},{{ demix_step }}filter,avg]
ndppp_prep_cal.argument.flagedge.type = preflagger
ndppp_prep_cal.argument.flagedge.chan = [0..nchan/32-1,31*nchan/32..nchan-1] # we are running on a single subband
ndppp_prep_cal.argument.aoflag.type = aoflagger
ndppp_prep_cal.argument.aoflag.memoryperc = {{ memoryperc }}
ndppp_prep_cal.argument.aoflag.keepstatistics = false
ndppp_prep_cal.argument.flagbaseline.type = preflagger
ndppp_prep_cal.argument.flagbaseline.baseline = {{ flag_baselines }}
ndppp_prep_cal.argument.flagelev.type = preflagger
ndppp_prep_cal.argument.flagelev.elevation = 0deg..15deg
ndppp_prep_cal.argument.flagamp.type = preflagger
ndppp_prep_cal.argument.flagamp.amplmin = 1e-30
ndppp_prep_cal.argument.avg.type = average
ndppp_prep_cal.argument.avg.timeresolution = {{ avg_timeresolution }}
ndppp_prep_cal.argument.avg.freqresolution = {{ avg_freqresolution }}
ndppp_prep_cal.argument.filter.type = filter
ndppp_prep_cal.argument.filter.baseline = {{ filter_baselines }}
ndppp_prep_cal.argument.filter.remove = true
ndppp_prep_cal.argument.demix.type = demixer
ndppp_prep_cal.argument.demix.baseline = {{ process_baselines_cal }}
ndppp_prep_cal.argument.demix.demixfreqstep = {{ demix_freqstep }}
ndppp_prep_cal.argument.demix.demixtimestep = {{ demix_timestep }}
ndppp_prep_cal.argument.demix.ignoretarget = False
ndppp_prep_cal.argument.demix.targetsource = {{ demix_target }}
ndppp_prep_cal.argument.demix.subtractsources = {{ demix_sources }}
ndppp_prep_cal.argument.demix.ntimechunk = {{ max_dppp_threads }}
ndppp_prep_cal.argument.demix.skymodel = expand_sourcedb_ateam.output.mapfile
ndppp_prep_cal.argument.demix.freqstep = 1
ndppp_prep_cal.argument.demix.timestep = 1
ndppp_prep_cal.argument.demix.instrumentmodel = createmap_instcal.output.mapfile
# combine all entries into one mapfile (just for the find_skymodel_cal_auto script)
combine_data_cal_map.control.kind = plugin
combine_data_cal_map.control.type = createMapfile
combine_data_cal_map.control.method = mapfile_all_to_one
combine_data_cal_map.control.mapfile_dir = {{ mapfile_dir }}
combine_data_cal_map.control.filename = combine_data_cal_map.mapfile
combine_data_cal_map.control.mapfile_in = ndppp_prep_cal.output.mapfile
# virtually concatenate calibrator subbands
ms_concat.control.type = pythonplugin
ms_concat.control.executable = {{ scripts }}/concat_MS.py
ms_concat.control.error_tolerance = {{ error_tolerance }}
ms_concat.argument.filename = concatmapfile.mapfile
ms_concat.argument.mapfile_dir = {{ mapfile_dir }}
ms_concat.argument.min_length = {{ min_length }}
ms_concat.argument.overhead = {{ overhead }}
ms_concat.argument.flags = [combine_data_cal_map.output.mapfile,outputkey]
# convert the output of ms_concat into usable mapfiles
ms_concat_map.control.kind = plugin
ms_concat_map.control.type = mapfilenamesFromMapfiles
ms_concat_map.control.mapfile_concatmap = ms_concat.output.concatmapfile.mapfile
# convert the output of ms_concat_target into usable mapfiles
expand_memory_map.control.kind = plugin
expand_memory_map.control.type = expandMapfile
expand_memory_map.control.mapfile_in = ms_concat.output.memory.mapfile
expand_memory_map.control.mapfile_to_match = ms_concat_map.output.concatmap
expand_memory_map.control.mapfile_dir = {{ mapfile_dir }}
expand_memory_map.control.filename = expand_memory_map.mapfile
# run aoflagger on the concatenated data
aoflag.control.kind = recipe
aoflag.control.type = executable_args
aoflag.control.inplace = True
aoflag.control.executable = {{ aoflagger }}
aoflag.control.max_per_node = 1
aoflag.control.error_tolerance = {{ error_tolerance }}
aoflag.control.mapfiles_in = [ms_concat_map.output.concatmap,expand_memory_map.output.mapfile]
aoflag.control.inputkeys = [msin,memory]
aoflag.control.args_format = wsclean
aoflag.argument.strategy = {{ prefactor_directory }}/rfistrategies/{{ rfistrategy }}
aoflag.argument.flags = [-v,memory,-combine-spws,msin]
# find automatically the calibrator sky model
sky_cal.control.type = pythonplugin
sky_cal.control.executable = {{ scripts }}/find_skymodel_cal.py
sky_cal.control.error_tolerance = {{ error_tolerance }}
sky_cal.argument.flags = [combine_data_cal_map.output.mapfile]
sky_cal.argument.DirSkymodelCal = {{ calibrator_path_skymodel }}
sky_cal.argument.max_separation_arcmin = {{ max_separation_arcmin }}
# make the sourcedb
make_sourcedb.control.kind = recipe
make_sourcedb.control.type = executable_args
make_sourcedb.control.executable = {{ lofar_directory }}/bin/makesourcedb
make_sourcedb.control.error_tolerance = {{ error_tolerance }}
make_sourcedb.control.args_format = lofar
make_sourcedb.control.outputkey = out
make_sourcedb.control.mapfile_in = sky_cal.output.SkymodelCal.mapfile
make_sourcedb.control.inputkey = in
make_sourcedb.argument.format = <
make_sourcedb.argument.outtype = blob
# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb.control.kind = plugin
expand_sourcedb.control.type = expandMapfile
expand_sourcedb.control.mapfile_in = make_sourcedb.output.mapfile
expand_sourcedb.control.mapfile_to_match = ndppp_prep_cal.output.mapfile
expand_sourcedb.control.mapfile_dir = {{ mapfile_dir }}
expand_sourcedb.control.filename = expand_sourcedb.mapfile
# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_skymodel.control.kind = plugin
expand_skymodel.control.type = expandMapfile
expand_skymodel.control.mapfile_in = sky_cal.output.SkymodelName.mapfile
expand_skymodel.control.mapfile_to_match = ndppp_prep_cal.output.mapfile
expand_skymodel.control.mapfile_dir = {{ mapfile_dir }}
expand_skymodel.control.filename = expand_skymodel.mapfile
# generate mapfile with the h5parm names to be used in the calib_cal steps
calib_cal_parmmap.control.kind = plugin
calib_cal_parmmap.control.type = createMapfile
calib_cal_parmmap.control.method = add_suffix_to_file
calib_cal_parmmap.control.mapfile_in = ndppp_prep_cal.output.mapfile
calib_cal_parmmap.control.add_suffix_to_file = /instrument.h5
calib_cal_parmmap.control.mapfile_dir = {{ mapfile_dir }}
calib_cal_parmmap.control.filename = calib_cal_h5parms.mapfile
# generate a mapfile with all files in a single entry
h5imp_cal_map.control.kind = plugin
h5imp_cal_map.control.type = compressMapfile
h5imp_cal_map.control.mapfile_in = calib_cal_parmmap.output.mapfile
h5imp_cal_map.control.mapfile_dir = {{ mapfile_dir }}
h5imp_cal_map.control.filename = h5imp_cal_map.mapfile
# baseline-dependent smoothing
smooth_data.control.type = executable_args
smooth_data.control.inplace = True
smooth_data.control.max_per_node = {{ num_proc_per_node }}
smooth_data.control.error_tolerance = {{ error_tolerance }}
smooth_data.control.executable = {{ scripts }}/BLsmooth.py
smooth_data.control.mapfile_in = ndppp_prep_cal.output.mapfile
smooth_data.control.inputkey = msin
smooth_data.argument.flags = [-S,{{ do_smooth }},-r,-i,DATA,-o,SMOOTHED_DATA,msin]
# baseline-dependent smoothing
smooth_corrected.control.type = executable_args
smooth_corrected.control.inplace = True
smooth_corrected.control.max_per_node = {{ num_proc_per_node }}
smooth_corrected.control.error_tolerance = {{ error_tolerance }}
smooth_corrected.control.executable = {{ scripts }}/BLsmooth.py
smooth_corrected.control.mapfile_in = ndppp_prep_cal.output.mapfile
smooth_corrected.control.inputkey = msin
smooth_corrected.argument.flags = [-S,{{ do_smooth }},-r,-i,CORRECTED_DATA,-o,SMOOTHED_DATA,msin]
# predict to save time
predict_cal.control.type = dppp
predict_cal.control.inplace = True
predict_cal.control.max_per_node = {{ num_proc_per_node_limit }}
predict_cal.control.error_tolerance = {{ error_tolerance }}
predict_cal.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,expand_sourcedb.output.mapfile,expand_skymodel.output.mapfile]
predict_cal.control.inputkeys = [msfile,sourcedb,skymodel]
predict_cal.argument.msin = msfile
predict_cal.argument.numthreads = {{ max_dppp_threads }}
predict_cal.argument.msin.datacolumn = SMOOTHED_DATA
predict_cal.argument.msout.datacolumn = MODEL_DATA
predict_cal.argument.msout.storagemanager = "Dysco"
predict_cal.argument.msout.storagemanager.databitrate = 0
predict_cal.argument.steps = [predict]
predict_cal.argument.predict.type = predict
predict_cal.argument.predict.sourcedb = sourcedb
predict_cal.argument.predict.sources = skymodel
predict_cal.argument.predict.usebeammodel = False # TODO: put to true for skymodels with apparent fluxes
predict_cal.argument.predict.usechannelfreq = False
predict_cal.argument.predict.beammode = array_factor
# now run NDPPP on the averaged calibrator data (rotation+diagonal)
calib_cal.control.type = dppp
calib_cal.control.inplace = True
calib_cal.control.max_per_node = {{ num_proc_per_node_limit }}
calib_cal.control.error_tolerance = {{ error_tolerance }}
calib_cal.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,calib_cal_parmmap.output.mapfile]
calib_cal.control.inputkeys = [msfile,h5parm]
calib_cal.argument.msin = msfile
calib_cal.argument.numthreads = {{ max_dppp_threads }}
calib_cal.argument.msin.datacolumn = SMOOTHED_DATA
calib_cal.argument.steps = [solve]
calib_cal.argument.solve.type = ddecal
calib_cal.argument.solve.mode = rotation+diagonal
calib_cal.argument.solve.h5parm = h5parm
calib_cal.argument.solve.usemodelcolumn = True
calib_cal.argument.solve.uvlambdamin = 300
calib_cal.argument.solve.maxiter = 50
calib_cal.argument.solve.nchan = 1
calib_cal.argument.solve.solint = 1
calib_cal.argument.solve.propagatesolutions = {{ propagatesolutions }}
calib_cal.argument.solve.propagateconvergedonly = True
calib_cal.argument.solve.flagunconverged = {{ flagunconverged }}
calib_cal.argument.solve.flagdivergedonly = True
calib_cal.argument.solve.tolerance = 1e-3
# now run NDPPP on the averaged calibrator data (diagonal only)
calib_cal2.control.type = dppp
calib_cal2.control.inplace = True
calib_cal2.control.max_per_node = {{ num_proc_per_node_limit }}
calib_cal2.control.error_tolerance = {{ error_tolerance }}
calib_cal2.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,calib_cal_parmmap.output.mapfile]
calib_cal2.control.inputkeys = [msfile,h5parm]
calib_cal2.argument.msin = msfile
calib_cal2.argument.numthreads = {{ max_dppp_threads }}
calib_cal2.argument.msin.datacolumn = SMOOTHED_DATA
calib_cal2.argument.steps = [solve]
calib_cal2.argument.solve.type = ddecal
calib_cal2.argument.solve.mode = diagonal
calib_cal2.argument.solve.h5parm = h5parm
calib_cal2.argument.solve.usemodelcolumn = True
calib_cal2.argument.solve.uvlambdamin = 300
calib_cal2.argument.solve.maxiter = 50
calib_cal2.argument.solve.nchan = 1
calib_cal2.argument.solve.solint = 1
calib_cal2.argument.solve.propagatesolutions = {{ propagatesolutions }}
calib_cal2.argument.solve.propagateconvergedonly = True
calib_cal2.argument.solve.flagunconverged = {{ flagunconverged }}
calib_cal2.argument.solve.flagdivergedonly = True
calib_cal2.argument.solve.tolerance = 1e-3
# now run NDPPP on the averaged calibrator data (scalarphase only)
calib_cal3.control.type = dppp
calib_cal3.control.inplace = True
calib_cal3.control.max_per_node = {{ num_proc_per_node_limit }}
calib_cal3.control.error_tolerance = {{ error_tolerance }}
calib_cal3.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,calib_cal_parmmap.output.mapfile]
calib_cal3.control.inputkeys = [msfile,h5parm]
calib_cal3.argument.msin = msfile
calib_cal3.argument.numthreads = {{ max_dppp_threads }}
calib_cal3.argument.msin.datacolumn = SMOOTHED_DATA
calib_cal3.argument.steps = [solve]
calib_cal3.argument.solve.type = ddecal
calib_cal3.argument.solve.mode = scalarphase
calib_cal3.argument.solve.h5parm = h5parm
calib_cal3.argument.solve.usemodelcolumn = True
calib_cal3.argument.solve.uvlambdamin = 300
calib_cal3.argument.solve.maxiter = 50
calib_cal3.argument.solve.nchan = 1
calib_cal3.argument.solve.solint = 1
calib_cal3.argument.solve.propagatesolutions = {{ propagatesolutions }}
calib_cal3.argument.solve.propagateconvergedonly = True
calib_cal3.argument.solve.flagunconverged = {{ flagunconverged }}
calib_cal3.argument.solve.flagdivergedonly = True
calib_cal3.argument.solve.tolerance = 1e-3
# get a decent reference antenna
find_refant.control.kind = plugin
find_refant.control.type = findRefAnt
find_refant.control.mapfile_in = ndppp_prep_cal.output.mapfile
find_refant.control.station_filter = {{ refant }}
###########################
## calibrate Pol. Align ##
###########################
# collect all instrument tables into one h5parm
h5imp_cal_PA.control.kind = recipe
h5imp_cal_PA.control.type = executable_args
h5imp_cal_PA.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py
h5imp_cal_PA.control.error_tolerance = {{ error_tolerance }}
h5imp_cal_PA.control.mapfile_in = h5imp_cal_map.output.mapfile
h5imp_cal_PA.control.inputkey = h5parm
h5imp_cal_PA.control.outputkey = outh5parm
h5imp_cal_PA.argument.flags = [-q,-v,-c,h5parm]
h5imp_cal_PA.argument.outh5parm = outh5parm
# create losoto v2 parset file
prepare_losoto_PA.control.kind = plugin
prepare_losoto_PA.control.type = makeLosotoParset
prepare_losoto_PA.control.steps = [plotP3,plotPd,plotRot3,plotA3,bkp,align,plotAlign,residual,plotPr,plotPr2]
prepare_losoto_PA.control.filename = {{ job_directory }}/losoto_PA.parset
prepare_losoto_PA.control.global.ncpu = {{ num_proc_per_node }}
prepare_losoto_PA.control.plotP3.operation = PLOT
prepare_losoto_PA.control.plotP3.soltab = sol000/phase000
prepare_losoto_PA.control.plotP3.axesInPlot = [time,freq]
prepare_losoto_PA.control.plotP3.axisInTable = ant
prepare_losoto_PA.control.plotP3.plotFlag = True
prepare_losoto_PA.control.plotP3.prefix = {{ inspection_directory }}/polalign_ph_
prepare_losoto_PA.control.plotP3.refAnt = find_refant.output.refant
prepare_losoto_PA.control.plotP3.minmax = [-3.14,3.14]
prepare_losoto_PA.control.plotPd.operation = PLOT
prepare_losoto_PA.control.plotPd.soltab = sol000/phase000
prepare_losoto_PA.control.plotPd.axesInPlot = [time,freq]
prepare_losoto_PA.control.plotPd.axisInTable = ant
prepare_losoto_PA.control.plotPd.axisDiff = pol
prepare_losoto_PA.control.plotPd.plotFlag = True
prepare_losoto_PA.control.plotPd.prefix = {{ inspection_directory }}/polalign_ph_poldif
prepare_losoto_PA.control.plotPd.refAnt = find_refant.output.refant
prepare_losoto_PA.control.plotPd.minmax = [-3.14,3.14]
prepare_losoto_PA.control.plotRot3.operation = PLOT
prepare_losoto_PA.control.plotRot3.soltab = sol000/rotation000
prepare_losoto_PA.control.plotRot3.axesInPlot = [time,freq]
prepare_losoto_PA.control.plotRot3.axisInTable = ant
prepare_losoto_PA.control.plotRot3.plotFlag = True
prepare_losoto_PA.control.plotRot3.prefix = {{ inspection_directory }}/polalign_rotangle
prepare_losoto_PA.control.plotRot3.refAnt = find_refant.output.refant
prepare_losoto_PA.control.plotA3.operation = PLOT
prepare_losoto_PA.control.plotA3.soltab = sol000/amplitude000
prepare_losoto_PA.control.plotA3.axesInPlot = [time,freq]
prepare_losoto_PA.control.plotA3.axisInTable = ant
prepare_losoto_PA.control.plotA3.plotFlag = True
prepare_losoto_PA.control.plotA3.prefix = {{ inspection_directory }}/polalign_amp_
prepare_losoto_PA.control.bkp.operation = DUPLICATE
prepare_losoto_PA.control.bkp.soltab = sol000/phase000
prepare_losoto_PA.control.bkp.soltabOut = phaseOrig
prepare_losoto_PA.control.align.soltab = sol000/phase000
prepare_losoto_PA.control.align.operation = POLALIGN
prepare_losoto_PA.control.align.soltabOut = polalign
prepare_losoto_PA.control.align.average = True
prepare_losoto_PA.control.align.replace = True
prepare_losoto_PA.control.align.maxResidual = 1.0
prepare_losoto_PA.control.align.fitOffset = {{ fit_offset_PA }}
prepare_losoto_PA.control.align.refAnt = find_refant.output.refant
prepare_losoto_PA.control.plotAlign.operation = PLOT
prepare_losoto_PA.control.plotAlign.soltab = sol000/polalign
prepare_losoto_PA.control.plotAlign.axesInPlot = [time,freq]
prepare_losoto_PA.control.plotAlign.axisInTable = ant
prepare_losoto_PA.control.plotAlign.axisDiff = pol
prepare_losoto_PA.control.plotAlign.plotFlag = True
prepare_losoto_PA.control.plotAlign.prefix = {{ inspection_directory }}/polalign
prepare_losoto_PA.control.plotAlign.refAnt = find_refant.output.refant
prepare_losoto_PA.control.plotAlign.minmax = [-3.14,3.14]
prepare_losoto_PA.control.residual.operation = RESIDUALS
prepare_losoto_PA.control.residual.soltab = sol000/phase000
prepare_losoto_PA.control.residual.soltabsToSub = polalign
prepare_losoto_PA.control.plotPr.operation = PLOT
prepare_losoto_PA.control.plotPr.soltab = sol000/phase000
prepare_losoto_PA.control.plotPr.axesInPlot = [time,freq]
prepare_losoto_PA.control.plotPr.axisInTable = ant
prepare_losoto_PA.control.plotPr.axisDiff = pol
prepare_losoto_PA.control.plotPr.plotFlag = True
prepare_losoto_PA.control.plotPr.prefix = {{ inspection_directory }}/polalign_ph-res_poldif
prepare_losoto_PA.control.plotPr.refAnt = find_refant.output.refant
prepare_losoto_PA.control.plotPr.minmax = [-3.14,3.14]
prepare_losoto_PA.control.plotPr2.operation = PLOT
prepare_losoto_PA.control.plotPr2.soltab = sol000/phase000
prepare_losoto_PA.control.plotPr2.axesInPlot = [time,freq]
prepare_losoto_PA.control.plotPr2.axisInTable = ant
prepare_losoto_PA.control.plotPr2.axisInCol = pol
prepare_losoto_PA.control.plotPr2.plotFlag = True
prepare_losoto_PA.control.plotPr2.prefix = {{ inspection_directory }}/polalign_ph-res_
prepare_losoto_PA.control.plotPr2.refAnt = find_refant.output.refant
prepare_losoto_PA.control.plotPr2.minmax = [-3.14,3.14]
# do the processing on the LoSoTo file
process_losoto_PA.control.kind = recipe
process_losoto_PA.control.type = executable_args
process_losoto_PA.control.inplace = True
process_losoto_PA.control.executable = {{ losoto_directory }}/bin/losoto
process_losoto_PA.control.max_per_node = {{ num_proc_per_node }}
process_losoto_PA.control.mapfile_in = h5imp_cal_PA.output.mapfile
process_losoto_PA.control.inputkey = h5in
process_losoto_PA.argument.flags = [-v,h5in,{{ job_directory }}/losoto_PA.parset]
# output the final soltab into an external h5parm
h5exp_cal_PA.control.kind = recipe
h5exp_cal_PA.control.type = executable_args
h5exp_cal_PA.control.inplace = True
h5exp_cal_PA.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py
h5exp_cal_PA.control.error_tolerance = {{ error_tolerance }}
h5exp_cal_PA.control.mapfile_in = h5imp_cal_PA.output.mapfile
h5exp_cal_PA.control.inputkey = h5in
h5exp_cal_PA.argument.flags = [-q,-v,h5in]
h5exp_cal_PA.argument.insoltab = polalign
h5exp_cal_PA.argument.outh5parm = {{ cal_solutions }}
h5exp_cal_PA.argument.outsolset = calibrator
################################
## calibrate Farady Rotation ##
################################
# collect all instrument tables into one h5parm
h5imp_cal_FR.control.kind = recipe
h5imp_cal_FR.control.type = executable_args
h5imp_cal_FR.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py
h5imp_cal_FR.control.error_tolerance = {{ error_tolerance }}
h5imp_cal_FR.control.mapfile_in = h5imp_cal_map.output.mapfile
h5imp_cal_FR.control.inputkey = h5parm
h5imp_cal_FR.control.outputkey = outh5parm
h5imp_cal_FR.argument.flags = [-q,-v,-c,h5parm]
h5imp_cal_FR.argument.outh5parm = outh5parm
# create losoto v2 parset file
prepare_losoto_FR.control.kind = plugin
prepare_losoto_FR.control.type = makeLosotoParset
prepare_losoto_FR.control.steps = [plotP3,plotPd,plotRot3,plotA3,faraday,plotFR]
prepare_losoto_FR.control.filename = {{ job_directory }}/losoto_FR.parset
prepare_losoto_FR.control.global.ncpu = {{ num_proc_per_node }}
prepare_losoto_FR.control.plotP3.operation = PLOT
prepare_losoto_FR.control.plotP3.soltab = sol000/phase000
prepare_losoto_FR.control.plotP3.axesInPlot = [time,freq]
prepare_losoto_FR.control.plotP3.axisInTable = ant
prepare_losoto_FR.control.plotP3.plotFlag = True
prepare_losoto_FR.control.plotP3.prefix = {{ inspection_directory }}/fr_ph_
prepare_losoto_FR.control.plotP3.refAnt = find_refant.output.refant
prepare_losoto_FR.control.plotP3.minmax = [-3.14,3.14]
prepare_losoto_FR.control.plotPd.operation = PLOT
prepare_losoto_FR.control.plotPd.soltab = sol000/phase000
prepare_losoto_FR.control.plotPd.axesInPlot = [time,freq]
prepare_losoto_FR.control.plotPd.axisInTable = ant
prepare_losoto_FR.control.plotPd.axisDiff = pol
prepare_losoto_FR.control.plotPd.plotFlag = True
prepare_losoto_FR.control.plotPd.prefix = {{ inspection_directory }}/fr_ph_poldif
prepare_losoto_FR.control.plotPd.refAnt = find_refant.output.refant
prepare_losoto_FR.control.plotPd.minmax = [-3.14,3.14]
prepare_losoto_FR.control.plotRot3.operation = PLOT
prepare_losoto_FR.control.plotRot3.soltab = sol000/rotation000
prepare_losoto_FR.control.plotRot3.axesInPlot = [time,freq]
prepare_losoto_FR.control.plotRot3.axisInTable = ant
prepare_losoto_FR.control.plotRot3.plotFlag = True
prepare_losoto_FR.control.plotRot3.prefix = {{ inspection_directory }}/fr_rotangle
prepare_losoto_FR.control.plotRot3.refAnt = find_refant.output.refant
prepare_losoto_FR.control.plotA3.operation = PLOT
prepare_losoto_FR.control.plotA3.soltab = sol000/amplitude000
prepare_losoto_FR.control.plotA3.axesInPlot = [time,freq]
prepare_losoto_FR.control.plotA3.axisInTable = ant
prepare_losoto_FR.control.plotA3.plotFlag = True
prepare_losoto_FR.control.plotA3.prefix = {{ inspection_directory }}/fr_amp_
prepare_losoto_FR.control.faraday.operation = FARADAY
prepare_losoto_FR.control.faraday.soltab = sol000/rotation000
prepare_losoto_FR.control.faraday.soltabOut = faraday
prepare_losoto_FR.control.faraday.refAnt = find_refant.output.refant
prepare_losoto_FR.control.faraday.maxResidual = 1.
prepare_losoto_FR.control.plotFR.operation = PLOT
prepare_losoto_FR.control.plotFR.soltab = sol000/faraday
prepare_losoto_FR.control.plotFR.axesInPlot = [time]
prepare_losoto_FR.control.plotFR.axisInTable = ant
prepare_losoto_FR.control.plotFR.prefix = {{ inspection_directory }}/fr
# do the processing on the LoSoTo file
process_losoto_FR.control.kind = recipe
process_losoto_FR.control.type = executable_args
process_losoto_FR.control.inplace = True
process_losoto_FR.control.executable = {{ losoto_directory }}/bin/losoto
process_losoto_FR.control.max_per_node = {{ num_proc_per_node }}
process_losoto_FR.control.mapfile_in = h5imp_cal_FR.output.mapfile
process_losoto_FR.control.inputkey = h5in
process_losoto_FR.argument.flags = [-v,h5in,{{ job_directory }}/losoto_FR.parset]
# output the final soltab into an external h5parm
h5exp_cal_FR.control.kind = recipe
h5exp_cal_FR.control.type = executable_args
h5exp_cal_FR.control.inplace = True
h5exp_cal_FR.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py
h5exp_cal_FR.control.error_tolerance = {{ error_tolerance }}
h5exp_cal_FR.control.mapfile_in = h5imp_cal_FR.output.mapfile
h5exp_cal_FR.control.inputkey = h5in
h5exp_cal_FR.argument.flags = [-q,-v,h5in]
h5exp_cal_FR.argument.insoltab = faraday
h5exp_cal_FR.argument.outh5parm = {{ cal_solutions }}
h5exp_cal_FR.argument.outsolset = calibrator
################################
## calibrate bandpass ##
################################
# collect all instrument tables into one h5parm
h5imp_cal_bandpass.control.kind = recipe
h5imp_cal_bandpass.control.type = executable_args
h5imp_cal_bandpass.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py
h5imp_cal_bandpass.control.error_tolerance = {{ error_tolerance }}
h5imp_cal_bandpass.control.mapfile_in = h5imp_cal_map.output.mapfile
h5imp_cal_bandpass.control.inputkey = h5parm
h5imp_cal_bandpass.control.outputkey = outh5parm
h5imp_cal_bandpass.argument.flags = [-q,-v,-c,h5parm]
h5imp_cal_bandpass.argument.outh5parm = outh5parm
# create losoto v2 parset file
prepare_losoto_bandpass.control.kind = plugin
prepare_losoto_bandpass.control.type = makeLosotoParset
prepare_losoto_bandpass.control.steps = [duplicateAbkp,plotA1,flag,flagbp,flagextend,plotA2,merge,smooth,bandpass,interp,smoothb,plotB1,plotB2,plotB3]
prepare_losoto_bandpass.control.filename = {{ job_directory }}/losoto_bandpass.parset
prepare_losoto_bandpass.control.global.ncpu = {{ num_proc_per_node }}
prepare_losoto_bandpass.control.duplicateAbkp.operation = DUPLICATE
prepare_losoto_bandpass.control.duplicateAbkp.soltab = sol000/amplitude000
prepare_losoto_bandpass.control.duplicateAbkp.soltabOut = amplitudeOrig000
prepare_losoto_bandpass.control.plotA1.operation = PLOT
prepare_losoto_bandpass.control.plotA1.soltab = sol000/amplitude000
prepare_losoto_bandpass.control.plotA1.axesInPlot = [time,freq]
prepare_losoto_bandpass.control.plotA1.axisInTable = ant
prepare_losoto_bandpass.control.plotA1.plotFlag = True
prepare_losoto_bandpass.control.plotA1.prefix = {{ inspection_directory }}/ampBFlag_
prepare_losoto_bandpass.control.flag.operation = FLAG
prepare_losoto_bandpass.control.flag.soltab = sol000/amplitude000
prepare_losoto_bandpass.control.flag.axesToFlag = [time,freq]
prepare_losoto_bandpass.control.flag.order = [100,40]
prepare_losoto_bandpass.control.flag.maxCycles = 1
prepare_losoto_bandpass.control.flag.maxRms = 5
prepare_losoto_bandpass.control.flag.replace = False
prepare_losoto_bandpass.control.flag.preFlagZeros = False
prepare_losoto_bandpass.control.flag.mode = smooth
prepare_losoto_bandpass.control.flagbp.operation = FLAGSTATION
prepare_losoto_bandpass.control.flagbp.soltab = sol000/amplitude000
prepare_losoto_bandpass.control.flagbp.mode = bandpass
prepare_losoto_bandpass.control.flagbp.ampRange = {{ ampRange }}
prepare_losoto_bandpass.control.flagbp.skipInternational = {{ skip_international }}
prepare_losoto_bandpass.control.flagextend.operation = FLAGEXTEND
prepare_losoto_bandpass.control.flagextend.soltab = sol000/amplitude000
prepare_losoto_bandpass.control.flagextend.axesToExt = [time,freq]
prepare_losoto_bandpass.control.flagextend.size = [200,80]
prepare_losoto_bandpass.control.flagextend.percent = 50
prepare_losoto_bandpass.control.flagextend.maxCycles = 2
prepare_losoto_bandpass.control.plotA2.operation = PLOT
prepare_losoto_bandpass.control.plotA2.soltab = sol000/amplitude000
prepare_losoto_bandpass.control.plotA2.axesInPlot = [time,freq]
prepare_losoto_bandpass.control.plotA2.axisInTable = ant
prepare_losoto_bandpass.control.plotA2.plotFlag = True
prepare_losoto_bandpass.control.plotA2.prefix = {{ inspection_directory }}/ampAFlag_
prepare_losoto_bandpass.control.merge.operation = REWEIGHT
prepare_losoto_bandpass.control.merge.mode = copy
prepare_losoto_bandpass.control.merge.soltab = sol000/phase000
prepare_losoto_bandpass.control.merge.soltabImport = amplitude000
prepare_losoto_bandpass.control.smooth.operation = SMOOTH
prepare_losoto_bandpass.control.smooth.soltab = sol000/amplitude000
prepare_losoto_bandpass.control.smooth.pol = XX,YY
prepare_losoto_bandpass.control.smooth.axesToSmooth = [time]
prepare_losoto_bandpass.control.smooth.mode = median
prepare_losoto_bandpass.control.smooth.log = True
prepare_losoto_bandpass.control.smooth.replace = False
prepare_losoto_bandpass.control.bandpass.operation = SMOOTH
prepare_losoto_bandpass.control.bandpass.soltab = sol000/amplitude000
prepare_losoto_bandpass.control.bandpass.pol = XX,YY
prepare_losoto_bandpass.control.bandpass.axesToSmooth = [freq]
prepare_losoto_bandpass.control.bandpass.mode = savitzky-golay
prepare_losoto_bandpass.control.bandpass.size = [17]
prepare_losoto_bandpass.control.bandpass.degree = 2
prepare_losoto_bandpass.control.bandpass.log = True
prepare_losoto_bandpass.control.interp.operation = INTERPOLATE
prepare_losoto_bandpass.control.interp.soltab = sol000/amplitude000
prepare_losoto_bandpass.control.interp.outSoltab = bandpass
prepare_losoto_bandpass.control.interp.axisToRegrid = freq
prepare_losoto_bandpass.control.interp.maxFlaggedWidth = {{ max2interpolate }}
prepare_losoto_bandpass.control.interp.newDelta = {{ bandpass_freqresolution }}
prepare_losoto_bandpass.control.interp.delta = {{ avg_freqresolution }}
prepare_losoto_bandpass.control.interp.log = True
prepare_losoto_bandpass.control.smoothb.operation = SMOOTH
prepare_losoto_bandpass.control.smoothb.soltab = sol000/bandpass
prepare_losoto_bandpass.control.smoothb.pol = XX,YY
prepare_losoto_bandpass.control.smoothb.axesToSmooth = [time]
prepare_losoto_bandpass.control.smoothb.mode = median
prepare_losoto_bandpass.control.smoothb.log = True
prepare_losoto_bandpass.control.smoothb.replace = True
prepare_losoto_bandpass.control.plotB1.operation = PLOT
prepare_losoto_bandpass.control.plotB1.soltab = sol000/bandpass
prepare_losoto_bandpass.control.plotB1.axesInPlot = [time,freq]
prepare_losoto_bandpass.control.plotB1.axisInTable = ant
prepare_losoto_bandpass.control.plotB1.plotFlag = True
prepare_losoto_bandpass.control.plotB1.prefix = {{ inspection_directory }}/bandpass_
prepare_losoto_bandpass.control.plotB2.operation = PLOT
prepare_losoto_bandpass.control.plotB2.soltab = sol000/bandpass
prepare_losoto_bandpass.control.plotB2.axesInPlot = freq
prepare_losoto_bandpass.control.plotB2.axisInTable = ant
prepare_losoto_bandpass.control.plotB2.axisInCol = pol
prepare_losoto_bandpass.control.plotB2.plotFlag = True
prepare_losoto_bandpass.control.plotB2.prefix = {{ inspection_directory }}/bandpass_
prepare_losoto_bandpass.control.plotB2.time.minmaxstep = [0,1e20,500000]
prepare_losoto_bandpass.control.plotB3.operation = PLOT
prepare_losoto_bandpass.control.plotB3.soltab = sol000/bandpass
prepare_losoto_bandpass.control.plotB3.axesInPlot = freq
prepare_losoto_bandpass.control.plotB3.axisInCol = ant
prepare_losoto_bandpass.control.plotB3.plotFlag = True
prepare_losoto_bandpass.control.plotB3.prefix = {{ inspection_directory }}/bandpass_
prepare_losoto_bandpass.control.plotB3.time.minmaxstep = [0,1e20,500000]
# create losoto v2 parset file
prepare_losoto_bandpasstrans.control.kind = plugin
prepare_losoto_bandpasstrans.control.type = makeLosotoParset
prepare_losoto_bandpasstrans.control.steps = [plotB1,plotB2,plotB3]
prepare_losoto_bandpasstrans.control.filename = {{ job_directory }}/losoto_transfer.parset
prepare_losoto_bandpasstrans.control.global.ncpu = {{ num_proc_per_node }}
prepare_losoto_bandpasstrans.control.plotB1.operation = PLOT
prepare_losoto_bandpasstrans.control.plotB1.soltab = calibrator/bandpass
prepare_losoto_bandpasstrans.control.plotB1.axesInPlot = [time,freq]
prepare_losoto_bandpasstrans.control.plotB1.axisInTable = ant
prepare_losoto_bandpasstrans.control.plotB1.plotFlag = True
prepare_losoto_bandpasstrans.control.plotB1.prefix = {{ inspection_directory }}/bandpass_transfer_
prepare_losoto_bandpasstrans.control.plotB2.operation = PLOT
prepare_losoto_bandpasstrans.control.plotB2.soltab = calibrator/bandpass
prepare_losoto_bandpasstrans.control.plotB2.axesInPlot = freq
prepare_losoto_bandpasstrans.control.plotB2.axisInTable = ant
prepare_losoto_bandpasstrans.control.plotB2.axisInCol = pol
prepare_losoto_bandpasstrans.control.plotB2.plotFlag = True
prepare_losoto_bandpasstrans.control.plotB2.prefix = {{ inspection_directory }}/bandpass_transfer_
prepare_losoto_bandpasstrans.control.plotB2.time.minmaxstep = [0,1e20,500000]
prepare_losoto_bandpasstrans.control.plotB3.operation = PLOT
prepare_losoto_bandpasstrans.control.plotB3.soltab = calibrator/bandpass
prepare_losoto_bandpasstrans.control.plotB3.axesInPlot = freq
prepare_losoto_bandpasstrans.control.plotB3.axisInCol = ant
prepare_losoto_bandpasstrans.control.plotB3.plotFlag = True
prepare_losoto_bandpasstrans.control.plotB3.prefix = {{ inspection_directory }}/bandpass_transfer_
prepare_losoto_bandpasstrans.control.plotB3.time.minmaxstep = [0,1e20,500000]
# do the processing on the LoSoTo file
process_losoto_bandpass.control.kind = recipe
process_losoto_bandpass.control.type = executable_args
process_losoto_bandpass.control.inplace = True
process_losoto_bandpass.control.executable = {{ losoto_directory }}/bin/losoto
process_losoto_bandpass.control.max_per_node = {{ num_proc_per_node }}
process_losoto_bandpass.control.mapfile_in = h5imp_cal_bandpass.output.mapfile
process_losoto_bandpass.control.inputkey = h5in
process_losoto_bandpass.argument.flags = [-v,h5in,{{ job_directory }}/losoto_bandpass.parset]
# output the final soltab into an external h5parm
h5exp_cal_bandpass.control.kind = recipe
h5exp_cal_bandpass.control.type = executable_args
h5exp_cal_bandpass.control.inplace = True
h5exp_cal_bandpass.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py
h5exp_cal_bandpass.control.error_tolerance = {{ error_tolerance }}
h5exp_cal_bandpass.control.mapfile_in = h5imp_cal_bandpass.output.mapfile
h5exp_cal_bandpass.control.inputkey = h5in
h5exp_cal_bandpass.argument.flags = [-q,-v,h5in]
h5exp_cal_bandpass.argument.insoltab = bandpass
h5exp_cal_bandpass.argument.outh5parm = {{ cal_solutions }}
h5exp_cal_bandpass.argument.outsolset = calibrator
# transfer solutions if necessary
transfer_solutions.control.type = pythonplugin
transfer_solutions.control.executable = {{ scripts }}/transfer_solutions.py
transfer_solutions.control.error_tolerance = {{ error_tolerance }}
transfer_solutions.control.skip_infile = True
transfer_solutions.control.mapfile_in = sky_cal.output.SkymodelName.mapfile
transfer_solutions.argument.flags = [{{ cal_solutions }}, {{ solutions2transfer }}]
transfer_solutions.argument.insolset = calibrator
transfer_solutions.argument.outsolset = calibrator
transfer_solutions.argument.insoltab = bandpass
transfer_solutions.argument.outsoltab = bandpass
transfer_solutions.argument.antenna = {{ antennas2transfer }}
transfer_solutions.argument.parset = {{ job_directory }}/losoto_transfer.parset
transfer_solutions.argument.trusted_sources = {{ trusted_sources }}
transfer_solutions.argument.max_separation_arcmin = {{ max_separation_arcmin }}
transfer_solutions.argument.do_transfer = {{ do_transfer }}
################################
## calibrate ionosphere ##
################################
# collect all instrument tables into one h5parm
h5imp_cal_ion.control.kind = recipe
h5imp_cal_ion.control.type = executable_args
h5imp_cal_ion.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py
h5imp_cal_ion.control.error_tolerance = {{ error_tolerance }}
h5imp_cal_ion.control.mapfile_in = h5imp_cal_map.output.mapfile
h5imp_cal_ion.control.inputkey = h5parm
h5imp_cal_ion.control.outputkey = outh5parm
h5imp_cal_ion.argument.flags = [-q,-v,-c,h5parm]
h5imp_cal_ion.argument.outh5parm = outh5parm
# create losoto v2 parset file
prepare_losoto_ion.control.kind = plugin
prepare_losoto_ion.control.type = makeLosotoParset
prepare_losoto_ion.control.steps = [plotP3,duplicatePbkp,{{ cal_ion }},plotClock,plotPr,flagstation]
prepare_losoto_ion.control.filename = {{ job_directory }}/losoto_ion.parset
prepare_losoto_ion.control.global.ncpu = {{ num_proc_per_node }}
prepare_losoto_ion.control.plotP3.operation = PLOT
prepare_losoto_ion.control.plotP3.soltab = sol000/phase000
prepare_losoto_ion.control.plotP3.axesInPlot = [time,freq]
prepare_losoto_ion.control.plotP3.axisInTable = ant
prepare_losoto_ion.control.plotP3.plotFlag = True
prepare_losoto_ion.control.plotP3.prefix = {{ inspection_directory }}/ion_ph
prepare_losoto_ion.control.plotP3.refAnt = find_refant.output.refant
prepare_losoto_ion.control.plotP3.minmax = [-3.14,3.14]
prepare_losoto_ion.control.duplicatePbkp.operation = DUPLICATE
prepare_losoto_ion.control.duplicatePbkp.soltab = sol000/phase000
prepare_losoto_ion.control.duplicatePbkp.soltabOut = phaseOrig
prepare_losoto_ion.control.ct.operation = CLOCKTEC
prepare_losoto_ion.control.ct.soltab = sol000/phase000
prepare_losoto_ion.control.ct.clocksoltabOut = clock
prepare_losoto_ion.control.ct.tecsoltabOut = tec
prepare_losoto_ion.control.ct.tec3rdsoltabOut = tec3rd
prepare_losoto_ion.control.ct.offsetsoltabOut = phase_offset
prepare_losoto_ion.control.ct.CombinePol = False
prepare_losoto_ion.control.ct.FlagBadChannels = False
prepare_losoto_ion.control.ct.Fit3rdOrder = False
prepare_losoto_ion.control.ct.Circular = False
prepare_losoto_ion.control.ct3.operation = CLOCKTEC
prepare_losoto_ion.control.ct3.soltab = sol000/phase000
prepare_losoto_ion.control.ct3.clocksoltabOut = clock
prepare_losoto_ion.control.ct3.tecsoltabOut = tec
prepare_losoto_ion.control.ct3.tec3rdsoltabOut = tec3rd
prepare_losoto_ion.control.ct3.offsetsoltabOut = phase_offset
prepare_losoto_ion.control.ct3.CombinePol = False
prepare_losoto_ion.control.ct3.FlagBadChannels = False
prepare_losoto_ion.control.ct3.Fit3rdOrder = True
prepare_losoto_ion.control.ct3.Circular = False
prepare_losoto_ion.control.plotClock.operation = PLOT
prepare_losoto_ion.control.plotClock.soltab = sol000/clock
prepare_losoto_ion.control.plotClock.axesInPlot = [time]
prepare_losoto_ion.control.plotClock.axisInTable = ant
prepare_losoto_ion.control.plotClock.prefix = {{ inspection_directory }}/clock
prepare_losoto_ion.control.plotClock.plotFlag = False
prepare_losoto_ion.control.plotClock.refAnt = find_refant.output.refant
prepare_losoto_ion.control.plotTEC.operation = PLOT
prepare_losoto_ion.control.plotTEC.soltab = sol000/tec
prepare_losoto_ion.control.plotTEC.axesInPlot = [time]
prepare_losoto_ion.control.plotTEC.axisInTable = ant
prepare_losoto_ion.control.plotTEC.prefix = {{ inspection_directory }}/tec
prepare_losoto_ion.control.plotTEC.plotFlag = False
prepare_losoto_ion.control.plotTEC.refAnt = find_refant.output.refant
prepare_losoto_ion.control.plotTEC.minmax = [-0.5,0.5]
prepare_losoto_ion.control.plotTEC3.operation = PLOT
prepare_losoto_ion.control.plotTEC3.soltab = sol000/tec3rd
prepare_losoto_ion.control.plotTEC3.axesInPlot = [time]
prepare_losoto_ion.control.plotTEC3.axisInTable = ant
prepare_losoto_ion.control.plotTEC3.prefix = {{ inspection_directory }}/tec3rd
prepare_losoto_ion.control.plotTEC3.plotFlag = False
prepare_losoto_ion.control.plotTEC3.refAnt = find_refant.output.refant
prepare_losoto_ion.control.plotTEC3.minmax = [-0.5,0.5]
prepare_losoto_ion.control.duplicateCbkp.operation = DUPLICATE
prepare_losoto_ion.control.duplicateCbkp.soltab = sol000/clock
prepare_losoto_ion.control.duplicateCbkp.soltabOut = OrigClock
prepare_losoto_ion.control.smooth.operation = SMOOTH
prepare_losoto_ion.control.smooth.soltab = sol000/clock
prepare_losoto_ion.control.smooth.axesToSmooth = [time]
prepare_losoto_ion.control.smooth.mode = median
prepare_losoto_ion.control.smooth.log = False
prepare_losoto_ion.control.smooth.replace = True
prepare_losoto_ion.control.residuals.operation = RESIDUALS
prepare_losoto_ion.control.residuals.soltab = sol000/phase000
prepare_losoto_ion.control.residuals.soltabsToSub = [tec,OrigClock]
prepare_losoto_ion.control.residuals3.operation = RESIDUALS
prepare_losoto_ion.control.residuals3.soltab = sol000/phase000
prepare_losoto_ion.control.residuals3.soltabsToSub = [tec,OrigClock,tec3rd] # only for very low-freq dataset
prepare_losoto_ion.control.plotPr.operation = PLOT
prepare_losoto_ion.control.plotPr.soltab = sol000/phase000
prepare_losoto_ion.control.plotPr.axesInPlot = [time,freq]
prepare_losoto_ion.control.plotPr.axisInTable = ant
prepare_losoto_ion.control.plotPr.plotFlag = True
prepare_losoto_ion.control.plotPr.prefix = {{ inspection_directory }}/ion_ph-res
prepare_losoto_ion.control.plotPr.refAnt = find_refant.output.refant
prepare_losoto_ion.control.plotPr.minmax = [-3.14,3.14]
prepare_losoto_ion.control.flagstation.operation = FLAGSTATION
prepare_losoto_ion.control.flagstation.soltab = sol000/phase000
prepare_losoto_ion.control.flagstation.maxStddev = {{ maxStddev }}
prepare_losoto_ion.control.flagstation.mode = resid
prepare_losoto_ion.control.flagstation.refAnt = find_refant.output.refant
prepare_losoto_ion.control.flagstation.soltabExport = clock
# do the processing on the LoSoTo file
process_losoto_ion.control.kind = recipe
process_losoto_ion.control.type = executable_args
process_losoto_ion.control.inplace = True
process_losoto_ion.control.executable = {{ losoto_directory }}/bin/losoto
process_losoto_ion.control.max_per_node = {{ num_proc_per_node }}
process_losoto_ion.control.mapfile_in = h5imp_cal_ion.output.mapfile
process_losoto_ion.control.inputkey = h5in
process_losoto_ion.argument.flags = [-v,h5in,{{ job_directory }}/losoto_ion.parset]
# output the final soltab into an external h5parm
h5exp_cal_ion.control.kind = recipe
h5exp_cal_ion.control.type = executable_args
h5exp_cal_ion.control.inplace = True
h5exp_cal_ion.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py
h5exp_cal_ion.control.error_tolerance = {{ error_tolerance }}
h5exp_cal_ion.control.mapfile_in = h5imp_cal_ion.output.mapfile
h5exp_cal_ion.control.inputkey = h5in
h5exp_cal_ion.argument.flags = [-q,-v,h5in]
h5exp_cal_ion.argument.insoltab = {{ tables2export }}
h5exp_cal_ion.argument.outh5parm = {{ cal_solutions }}
h5exp_cal_ion.argument.outsolset = calibrator
################################
## applying the results ##
################################
# apply the PA solutions
apply_PA.control.type = dppp
apply_PA.control.error_tolerance = {{ error_tolerance }}