Merge branch 'master' of gh:revoltek/losoto

.github/workflows/python.yml

@@ -0,0 +1,27 @@
+name: Python package
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+      - name: Install
+        run: pip install .
+      - name: Test
+        # We cannot yet use `pytest`; requires different test file names
+        run: python tools/losoto_test.py

README.md

@@ -1,43 +1,51 @@
 LoSoTo: LOFAR solutions tool
 ============================
 
-Main developer:
+### Main developer:
 * Francesco de Gasperin
 
-Contributed code:
+### Contributed code:
 * David Rafferty
+* Henrik Edler
 * Maaijke Mevius
 * Jose Sabater Montes
 * Martin Hardcastle
 * Andreas Horneffer
 * and the LOFAR Survey Key Science Project
 
-Cite:
-* If you use LoSoTo for your data reduction, please acknowledge it with "This work has made use of the Lofar Solution Tool (LoSoTo), developed by F. de Gasperin."
+### Cite:
+* If you use LoSoTo for your data reduction, please cite: de Gasperin et al. 2019, Astronomy & Astrophysics, Volume 622, id.A5
+(https://ui.adsabs.harvard.edu/abs/2019A&A...622A...5D/abstract)
 
-Install:
-* Get LoSoTo from https://github.com/revoltek/losoto 
-* Install losoto by simply: python setup.py install --prefix=~/mydir/
-* Alternatively: pip install --upgrade --user https://github.com/revoltek/losoto/archive/master.zip 
-* In cep3 use the copy of the code in ~fdg/scripts/losoto/ (source the tool/lofarinit.[c]sh file which is shipped with the code)
-* Prepare a parset starting from the parset/losoto2.parset
-* in case of problems write to Francesco de Gasperin: [email protected]
+### Install:
+* Install latest official release of LoSoTo from PyPI:
+  `pip install --upgrade losoto`
+* Or, install latest development version of LoSoTo from GitHub:
+  `pip install --upgrade --user https://github.com/revoltek/losoto/archive/master.zip`
+* Or clone the LoSoTo repository from https://github.com/revoltek/losoto, and install LoSoTo using:
+  `python setup.py install --prefix=~/mydir/`
+* Prepare a `parset` file, which can be based on the file `parset/losoto2.parset`
+* In case of problems write to Francesco de Gasperin: [email protected]
 
-Documentation:
+### H5parm plotter (GUI):
+This is an external project maintained by Frits Sweijen: https://github.com/tikk3r/lofar-h5plot
+
+### Documentation:
 * Documentation of LoSoTo API is at: __http://revoltek.github.io/losoto/losoto.html__
 * A detailed explanation of all the parameters is at: __http://revoltek.github.io/losoto/losoto.operations.html__
 * A few important articles/how-to are on the github wiki: __https://github.com/revoltek/losoto/wiki__
 
-Contents:
+### Contents:
 * __bin/__: the losoto executable and some manipulators for H5parms
 * __docs/__: documentation
 * __examples/__: some examples h5parm to use with validation/test parsets
-* __losoto/operations/__: containts all the modules for operations
+* __losoto/operations/__: contains all the modules for operations
 * __parsets/__: some examples parsets
 * __tools/__: contains some external tools
 
-Required packages:
-* PyTables version:  >3.0.0
+### Required packages:
+* Python 3
+* PyTables version:  >3.4.0
 * HDF5 version:      >1.8.4
 * NumPy version:     >1.9.0
-* configparser (backport from python 3.0 if using python 2.7)
+* Scipy              >1.4 (for interpolatedirections)

bin/H5parm2parmdb.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # This tool is used to convert an H5parm file to parmdb format by writing to
@@ -10,16 +10,12 @@
 import sys, os, glob, re, time
 import numpy as np
 import shutil
-import logging
 import pyrap.tables as pt
 import lofar.parmdb
 from losoto import _version
 from losoto import _logging
 from losoto.h5parm import h5parm
-try:
-    import progressbar
-except ImportError:
-    import losoto.progressbar as progressbar
+import losoto.progressbar as progressbar
 
 
 def parmdbToAxes(solEntry):
@@ -342,11 +338,14 @@ def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth):
     (options, args) = opt.parse_args()
     global ipbar, pbar
 
+    logger = _logging.Logger('info')
+    logging = _logging.logger
+
     # Check options
     if len(args) != 2:
         opt.print_help()
         sys.exit()
-    if options.verbose: _logging.setLevel("debug")
+    if options.verbose: logger.set_level("debug")
 
     # Check input H5parm file
     h5parmFile = args[0]
@@ -550,13 +549,13 @@ def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth):
                     # check whether this is clock or tec; if so, reshape properly to account for all freqs in the parmdb
                     # anyway these tables are freq-indep
                     if solType == "Clock":# or solType == "TEC" or solType == "RotationMeasure":
-                        # find freq-dimensionality 
+                        # find freq-dimensionality
                         nfreq = freqs.shape[0]
                         #print val.shape
                         # reshape such that all freq arrays are filled properly
-                        val = np.tile( val, np.append([nfreq], np.ones(len(val.shape),dtype=np.int) ) )
+                        val = np.tile( val, np.append([nfreq], np.ones(len(val.shape),dtype=int) ) )
                         #print val.shape
-                        weights = np.tile( weights, np.append([nfreq], np.ones(len(weights.shape),dtype=np.int) ) )
+                        weights = np.tile( weights, np.append([nfreq], np.ones(len(weights.shape),dtype=int) ) )
 
                     flags = np.zeros(shape=weights.shape, dtype=bool)
                     flags[np.where(weights == 0)] = True

bin/H5parm_benchmark.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # This tool comapre the performances of H5parm with parmdb.
@@ -9,7 +9,6 @@
 
 import sys, os, time
 import numpy as np
-import logging
 import lofar.parmdb
 from losoto import _version
 from losoto import _logging
@@ -25,7 +24,8 @@
 opt.add_option('-n', '--numiter', help='Number of iterations (default=100)', type=int, default=100)
 (options, args) = opt.parse_args()
 
-_logging.setLevel('debug')
+logger = _logging.Logger('debug')
+logging = _logging.logger
 
 n = options.numiter
 

bin/H5parm_collector.py

@@ -1,9 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from __future__ import print_function
-
-import sys, os, logging
+import sys, os
 import codecs
 import numpy as np
 from itertools import chain
@@ -22,14 +20,18 @@
 parser.add_argument('--outh5parm', '-o', default='output.h5', dest='outh5parm', help='Output h5parm name [default: output.h5]')
 parser.add_argument('--verbose', '-V', '-v', default=False, action='store_true', help='Go Vebose! (default=False)')
 parser.add_argument('--squeeze', '-q', default=False, action='store_true', help='Remove all axes with a length of 1 (default=False)')
+parser.add_argument('--history', '-H', default=False, action='store_true', help='Keep history of the export soltabs (default=False)')
 parser.add_argument('--clobber', '-c', default=False, action='store_true', help='Replace exising outh5parm file instead of appending to it (default=False)')
 args = parser.parse_args()
 
 if len(args.h5parmFiles) < 1:
     parser.print_help()
     sys.exit(1)
 
-if args.verbose: _logging.setLevel("debug")
+# set logs
+logger = _logging.Logger('info')
+logging = _logging.logger
+if args.verbose: logger.set_level("debug")
 
 ################################
 
@@ -68,13 +70,16 @@
 
 for insoltab in insoltabs:
     soltabs = []
+    history = ''
     pointingNames = []; antennaNames = []
     pointingDirections = []; antennaPositions = []
 
     for h5 in h5s:
         solset = h5.getSolset(insolset)
         soltab = solset.getSoltab(insoltab)
         soltabs.append( soltab )
+        history += soltab.getHistory()
+
         # collect pointings
         sous = solset.getSou()
         for k,v in list(sous.items()):
@@ -88,12 +93,17 @@
             if k not in antennaNames:
                 antennaNames.append(k)
                 antennaPositions.append(v)
-
+                
     # create output axes
     logging.info("Sorting output axes...")
     axes = soltabs[0].getAxesNames()
     if args.squeeze:
-        axes = [axis for axis in axes if soltabs[0].getAxisLen(axis) > 1]
+        axes = [axis for axis in axes if soltabs[0].getAxisLen(axis) > 1 or axis == 'freq' ]
+        removed_axes = list(set(soltabs[0].getAxesNames()) - set(axes))
+        if len(removed_axes) == 0:
+            args.squeeze = False
+        else:
+            axes_squeeze = tuple([soltabs[0].getAxesNames().index(removed_axis) for removed_axis in removed_axes ])
     typ = soltabs[0].getType()
     allAxesVals = {axis:[] for axis in axes}
     allShape = []
@@ -111,30 +121,28 @@
     logging.debug("Shape:"+str(allShape))
     allVals = np.empty( shape=allShape )
     allVals[:] = np.nan
-    allWeights = np.zeros( shape=allShape )#, dtype=np.float16 )
+    allWeights = np.zeros( shape=allShape )#, dtype=float )
 
     # fill arrays
     logging.info("Filling new table...")
     for soltab in soltabs:
         coords = []
         for axis in axes:
             coords.append( np.searchsorted( allAxesVals[axis], soltab.getAxisValues(axis) ) )
-        if args.squeeze:    
-            allVals[np.ix_(*coords)] = np.squeeze(soltab.obj.val)
-            allWeights[np.ix_(*coords)] = np.squeeze(soltab.obj.weight)
+        if args.squeeze:
+            allVals[np.ix_(*coords)] = np.squeeze(np.array(soltab.obj.val), axis = axes_squeeze)
+            allWeights[np.ix_(*coords)] = np.squeeze(np.array(soltab.obj.weight), axis = axes_squeeze)
         else:
             allVals[np.ix_(*coords)] = soltab.obj.val
             allWeights[np.ix_(*coords)] = soltab.obj.weight
-
+            
 
     # TODO: leave correct weights - this is a workaround for h5parm with weight not in float16
     allWeights[ allWeights != 0 ] = 1.
 
     # TODO: flag nans waiting for DPPP to do it
     allWeights[ np.isnan(allVals) ] = 0.
 
-    # TODO: interpolate on selected axes
-
     logging.info('Writing output...')
     solsetOutName = args.outsolset
     soltabOutName = insoltab
@@ -146,9 +154,13 @@
         solsetOut = h5Out.makeSolset(solsetOutName)
 
     # create soltab
-    solsetOut.makeSoltab(typ, soltabOutName, axesNames=axes, \
+    soltabOut = solsetOut.makeSoltab(typ, soltabOutName, axesNames=axes, \
                      axesVals=[ allAxesVals[axis] for axis in axes ], \
                      vals=allVals, weights=allWeights)
+
+    # add history table if requested
+    if args.history:
+        soltabOut.addHistory(history, date = False)
 
 sourceTable = solsetOut.obj._f_get_child('source')
 antennaTable = solsetOut.obj._f_get_child('antenna')

bin/H5parm_copy.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # This tool is used to merge 2 H5parms.
@@ -8,9 +8,7 @@
 
 import sys, os, glob
 import numpy as np
-import logging
 import pyrap.tables as pt
-import lofar.parmdb
 from losoto import _version
 from losoto import _logging
 import losoto.h5parm
@@ -27,7 +25,11 @@
     if len(args) != 2:
         opt.print_help()
         sys.exit()
-    if options.verbose: _logging.setLevel("debug")
+
+    # log
+    logger = _logging.Logger('info')
+    logging = _logging.logger
+    if options.verbose: logger.set_level("debug")
 
     h5parmFrom = args[0]
     h5parmTo = args[1]
@@ -50,20 +52,19 @@
     # write table
     ht = losoto.h5parm.h5parm(h5parmToFile, readonly=False)
     # check if the solset exists
-    if solsetTo in ht.getSolsets():
+    if solsetTo in ht.getSolsetNames():
         logging.critical('Destination solset already exists, quitting.')
         sys.exit(1)
     ssT = ht.makeSolset(solsetName = solsetTo, addTables=False)
     # write the soltabs
-    ssF._f_copy_children(ssT, recursive=True)
+    ssF.obj._f_copy_children(ssT.obj, recursive=True)
 
     del hf
 
     # Add entry to history
-    soltabs = ht.getSoltabs(solset=solsetTo)
+    soltabs = ht.getSolset(solsetTo).getSoltabs()
     for st in soltabs:
-        sw = losoto.h5parm.solWriter(soltabs[st])
-        sw.addHistory('Copied (from {0}:{1})'.format(h5parmFromFile, solsetFrom))
+        st.addHistory('Copied (from {0}:{1})'.format(h5parmFromFile, solsetFrom))
     del ht
 
     logging.info("Done.")