From 2ed8514ff5629e61412dca72b6b435b90ace3e6f Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 6 Dec 2023 11:22:09 -0500 Subject: [PATCH 1/9] Adding functionality to slice utils, adding pol index sorter --- pyuvdata/utils.py | 136 +++++++++++++++++++++++++++++++++------------- 1 file changed, 98 insertions(+), 38 deletions(-) diff --git a/pyuvdata/utils.py b/pyuvdata/utils.py index 0b4edeb0e9..6ade5ae9ed 100644 --- a/pyuvdata/utils.py +++ b/pyuvdata/utils.py @@ -1250,6 +1250,42 @@ def reorder_conj_pols(pols): return conj_order +def determine_pol_order(pols, order="AIPS"): + """ + Determine order of input polarization numbers. + + Determines the order by which to sort a given list of polarizations, according to + the ordering scheme. Two orders are currently supported: "AIPS" and "CASA". The + main difference between the two is the grouping of same-handed polarizations for + AIPS (whereas CASA orders the polarizations such that same-handed pols are on the + ends of the array). + + Parameters + ---------- + pols : array_like of str or int + Polarization array (strings or ints). + order : str + Polarization ordering scheme, either "CASA" or "AIPS". + + Returns + ------- + index_array : ndarray of int + Indices to reorder polarization array. + """ + if order == "AIPS": + index_array = np.argsort(np.abs(pols)) + elif order == "CASA": + casa_order = np.array([1, 2, 3, 4, -1, -3, -4, -2, -5, -7, -8, -6, 0]) + pol_inds = [] + for pol in pols: + pol_inds.append(np.where(casa_order == pol)[0][0]) + index_array = np.argsort(pol_inds) + else: + raise ValueError('order must be either "AIPS" or "CASA".') + + return index_array + + def LatLonAlt_from_XYZ(xyz, frame="ITRS", check_acceptability=True): """ Calculate lat/lon/alt from ECEF x,y,z. @@ -5706,27 +5742,39 @@ def _get_dset_shape(dset, indices): return dset_shape, indices -def _convert_to_slices(indices, max_nslice_frac=0.1): +def _convert_to_slices( + indices, max_nslice_frac=0.1, max_nslice=None, return_index_on_fail=False +): """ Convert list of indices to a list of slices. Parameters ---------- indices : list - A 1D list of integers for array indexing. + A 1D list of integers for array indexing (boolean ndarrays are also supported). max_nslice_frac : float A float from 0 -- 1. If the number of slices needed to represent input 'indices' divided by len(indices) exceeds this fraction, then we determine that we cannot easily represent 'indices' with a list of slices. + max_nslice : int + Optional argument, defines the maximum number of slices for determining if + `indices` can be easily represented with a list of slices. If set, then + the argument supplied to `max_nslice_frac` is ignored. + return_index_on_fail : bool + If set to True and the list of input indexes cannot easily be respresented by + a list of slices (as defined by `max_nslice` or `max_nslice_frac`), then return + the input list of index values instead of a list of suboptimal slices. Returns ------- - list - list of slice objects used to represent indices - bool + slice_list : list + Nominally the list of slice objects used to represent indices. However, if + `return_index_on_fail=True` and input indexes cannot easily be respresented, + return a 1-element list containing the input for `indices`. + check : bool If True, indices is easily represented by slices - (max_nslice_frac condition met), otherwise False + (`max_nslice_frac` or `max_nslice` conditions met), otherwise False. Notes ----- @@ -5734,60 +5782,72 @@ def _convert_to_slices(indices, max_nslice_frac=0.1): if: indices = [1, 2, 3, 4, 10, 11, 12, 13, 14] then: slices = [slice(1, 5, 1), slice(11, 15, 1)] """ - # check for integer index - if isinstance(indices, (int, np.integer)): - indices = [indices] - # check for already a slice + # check for already a slice or a single index position if isinstance(indices, slice): return [indices], True + if isinstance(indices, (int, np.integer)): + return [slice(indices, indices + 1, 1)], True + + # check for boolean index + if isinstance(indices, np.ndarray) and (indices.dtype == bool): + eval_ind = np.where(indices)[0] + else: + eval_ind = indices # assert indices is longer than 2, or return trivial solutions - if len(indices) == 0: + if len(eval_ind) == 0: return [slice(0, 0, 0)], False - elif len(indices) == 1: - return [slice(indices[0], indices[0] + 1, 1)], True - elif len(indices) == 2: - return [slice(indices[0], indices[1] + 1, indices[1] - indices[0])], True + if len(eval_ind) <= 2: + return [ + slice(eval_ind[0], eval_ind[-1] + 1, max(eval_ind[-1] - eval_ind[0], 1)) + ], True + + # Catch the simplest case of "give me a single slice or exit" + if (max_nslice == 1) and return_index_on_fail: + step = eval_ind[1] - eval_ind[0] + if all(np.diff(eval_ind) == step): + return [slice(eval_ind[0], eval_ind[-1] + 1, step)], True + return [indices], False # setup empty slices list - Ninds = len(indices) + Ninds = len(eval_ind) slices = [] # iterate over indices - for i, ind in enumerate(indices): - if i == 0: - # start the first slice object - start = ind - last_step = indices[i + 1] - ind + start = last_step = None + for ind in eval_ind: + if last_step is None: + # Check if this is the first slice, in which case start is None + if start is None: + start = ind + continue + last_step = ind - start + last_ind = ind continue # calculate step from previous index - step = ind - indices[i - 1] + step = ind - last_ind # if step != last_step, this ends the slice if step != last_step: # append to list - slices.append(slice(start, indices[i - 1] + 1, last_step)) - - # check if this is the last element - if i == Ninds - 1: - # append last element - slices.append(slice(ind, ind + 1, 1)) - continue + slices.append(slice(start, last_ind + 1, last_step)) # setup next step start = ind - last_step = indices[i + 1] - ind + last_step = None + + last_ind = ind - # check if this is the last element - elif i == Ninds - 1: - # end slice and append - slices.append(slice(start, ind + 1, step)) + # Append the last slice + slices.append(slice(start, ind + 1, last_step)) - # determine whether slices are a reasonable representation - Nslices = len(slices) - passed = (float(Nslices) / len(indices)) < max_nslice_frac + # determine whether slices are a reasonable representation, and determine max_nslice + # if only max_nslice_frac was supplied. + if max_nslice is None: + max_nslice = max_nslice_frac * Ninds + check = len(slices) <= max_nslice - return slices, passed + return [indices] if (not check and return_index_on_fail) else slices, check def _get_slice_len(s, axlen): From 88b1f9934afd0786b01119d1db62370e167ae2b8 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 6 Dec 2023 11:23:35 -0500 Subject: [PATCH 2/9] Adding pol_order util, updates to args for read/write MS --- pyuvdata/uvdata/uvdata.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index 89f60338b1..6b7b183183 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -4826,14 +4826,10 @@ def reorder_pols( "contain integers and be length Npols." ) index_array = order - elif order == "AIPS": - index_array = np.argsort(np.abs(self.polarization_array)) - elif order == "CASA": - casa_order = np.array([1, 2, 3, 4, -1, -3, -4, -2, -5, -7, -8, -6]) - pol_inds = [] - for pol in self.polarization_array: - pol_inds.append(np.where(casa_order == pol)[0][0]) - index_array = np.argsort(pol_inds) + elif (order == "AIPS") or (order == "CASA"): + index_array = uvutils.determine_pol_order( + self.polarization_array, order=order + ) else: raise ValueError( "order must be one of: 'AIPS', 'CASA', or an " @@ -10755,9 +10751,9 @@ def read_ms(self, filepath, **kwargs): data_column : str name of CASA data column to read into data_array. Options are: 'DATA', 'MODEL', or 'CORRECTED_DATA' - pol_order : str + pol_order : str or None Option to specify polarizations order convention, options are - 'CASA' or 'AIPS'. + 'CASA', 'AIPS', or None (no reordering). Default is 'AIPS'. background_lsts : bool When set to True, the lst_array is calculated in a background thread. run_check : bool @@ -11639,9 +11635,9 @@ def read( data_column : str name of CASA data column to read into data_array. Options are: 'DATA', 'MODEL', or 'CORRECTED_DATA'. - pol_order : str + pol_order : str or None Option to specify polarizations order convention, options are - 'CASA' or 'AIPS'. + 'CASA', 'AIPS', or None (no reordering). Default is 'AIPS'. ignore_single_chan : bool Option to ignore single channel spectral windows in measurement sets to limit object size. Some measurement sets (e.g., those from ALMA) use single @@ -12673,6 +12669,7 @@ def write_ms( self, filename, force_phase=False, + flip_conj=False, clobber=False, run_check=True, check_extra=True, @@ -12692,6 +12689,13 @@ def write_ms( force_phase : bool Option to automatically phase unprojected data to zenith of the first timestamp. + flip_conj : bool + If set to True, and the UVW coordinates are flipped (i.e., multiplied by + -1) and the visibilities are complex conjugated prior to write, such that + the data are written with the "opposite" conjugation scheme to what UVData + normally uses. Note that this is only needed for specific subset of + applications that read MS-formated data, and should only be used by expert + users. Default is False. clobber : bool Option to overwrite the file if it already exists. run_check : bool @@ -12727,6 +12731,7 @@ def write_ms( ms_obj.write_ms( filename, force_phase=force_phase, + flip_conj=flip_conj, clobber=clobber, run_check=run_check, check_extra=check_extra, From 928c20f5016a8c718d140e25bb5374c0128e17ba Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 6 Dec 2023 11:24:57 -0500 Subject: [PATCH 3/9] Sorting polarizations by CASA standards for ms-write. Adding better indexing handling via utils. --- pyuvdata/uvdata/ms.py | 83 +++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/pyuvdata/uvdata/ms.py b/pyuvdata/uvdata/ms.py index 1d33952a53..327408c6c4 100644 --- a/pyuvdata/uvdata/ms.py +++ b/pyuvdata/uvdata/ms.py @@ -331,7 +331,7 @@ def _write_ms_data_description(self, filepath): data_descrip_table.done() - def _write_ms_feed(self, filepath): + def _write_ms_feed(self, filepath, pol_order): """ Write out the feed information into a CASA table. @@ -339,7 +339,9 @@ def _write_ms_feed(self, filepath): ---------- filepath : str path to MS (without FEED suffix) - + pol_order : slice or list of int + Ordering of the polarization axis on write, only used if not writing a + flex-pol dataset. """ if not casa_present: # pragma: no cover raise ImportError(no_casa_message) from casa_error @@ -353,7 +355,7 @@ def _write_ms_feed(self, filepath): spectral_window_id_table = -1 * np.ones(nfeeds_table, dtype=np.int32) # we want "x" or "y", *not* "e" or "n", so as not to confuse CASA - pol_str = uvutils.polnum2str(self.polarization_array) + pol_str = uvutils.polnum2str(self.polarization_array[pol_order]) else: nfeeds_table *= self.Nspws spectral_window_id_table = np.repeat( @@ -778,7 +780,7 @@ def _write_ms_observation(self, filepath): observation_table.done() - def _write_ms_polarization(self, filepath): + def _write_ms_polarization(self, filepath, pol_order): """ Write out the polarization information into a CASA table. @@ -786,7 +788,9 @@ def _write_ms_polarization(self, filepath): ---------- filepath : str path to MS (without POLARIZATION suffix) - + pol_order : slice or list of int + Ordering of the polarization axis on write, only used if not writing a + flex-pol dataset. """ if not casa_present: # pragma: no cover raise ImportError(no_casa_message) from casa_error @@ -794,7 +798,7 @@ def _write_ms_polarization(self, filepath): pol_table = tables.table(filepath + "::POLARIZATION", ack=False, readonly=False) if self.flex_spw_polarization_array is None: - pol_str = uvutils.polnum2str(self.polarization_array) + pol_str = uvutils.polnum2str(self.polarization_array[pol_order]) feed_pols = { feed for pol in pol_str for feed in uvutils.POL_TO_FEED_DICT[pol] } @@ -809,7 +813,10 @@ def _write_ms_polarization(self, filepath): "CORR_TYPE", 0, np.array( - [POL_AIPS2CASA_DICT[aipspol] for aipspol in self.polarization_array] + [ + POL_AIPS2CASA_DICT[pol] + for pol in self.polarization_array[pol_order] + ] ), ) pol_table.putcell("CORR_PRODUCT", 0, pol_tuples) @@ -1075,6 +1082,7 @@ def write_ms( self, filepath, force_phase=False, + flip_conj=False, clobber=False, run_check=True, check_extra=True, @@ -1095,6 +1103,13 @@ def write_ms( timestamp. clobber : bool Option to overwrite the file if it already exists. + flip_conj : bool + If set to True, and the UVW coordinates are flipped (i.e., multiplied by + -1) and the visibilities are complex conjugated prior to write, such that + the data are written with the "opposite" conjugation scheme to what UVData + normally uses. Note that this is only needed for specific subset of + applications that read MS-formated data, and should only be used by expert + users. Default is False. run_check : bool Option to check for the existence and proper shapes of parameters before writing the file. @@ -1131,6 +1146,13 @@ def write_ms( else: raise IOError("File exists; skipping") + # Determine polarization order for writing out in CASA standard order, check + # if this order can be represented by a single slice. + pol_order = uvutils.determine_pol_order(self.polarization_array, order="CASA") + [pol_order], _ = uvutils._convert_to_slices( + pol_order, max_nslice=1, return_index_on_fail=True + ) + # CASA does not have a way to handle "unprojected" data in the way that UVData # objects can, so we need to check here whether or not any such data exists # (and if need be, fix it). @@ -1187,9 +1209,14 @@ def write_ms( # about ordering, so just write the data-related arrays as is to disk for attr, col in zip(attr_list, col_list): if self.future_array_shapes: - ms.putcol(col, getattr(self, attr)) + temp_vals = getattr(self, attr)[:, :, pol_order] else: - ms.putcol(col, np.squeeze(getattr(self, attr), axis=1)) + temp_vals = getattr(self, attr)[:, 0, :, pol_order] + + if flip_conj and (attr == "data_array"): + temp_vals = np.conj(temp_vals) + + ms.putcol(col, temp_vals) # Band-averaged weights are used for some things in CASA - calculate them # here using median nsamples. @@ -1202,7 +1229,7 @@ def write_ms( ant_1_array = self.ant_1_array ant_2_array = self.ant_2_array integration_time = self.integration_time - uvw_array = self.uvw_array + uvw_array = self.uvw_array * (-1 if flip_conj else 1) scan_number_array = self.scan_number_array else: # If we have _more_ than one spectral window, then we need to handle each @@ -1227,11 +1254,17 @@ def write_ms( last_row = 0 for scan_num in sorted(np.unique(self.scan_number_array)): # Select all data from the scan - scan_screen = self.scan_number_array == scan_num + scan_screen = np.where(self.scan_number_array == scan_num)[0] + + # See if we can represent scan_screen with a single slice, which + # reduces overhead of copying a new array. + [scan_slice], _ = uvutils._convert_to_slices( + scan_screen, max_nslice=1, return_index_on_fail=True + ) # Get the number of records inside the scan, where 1 record = 1 spw in # 1 baseline at 1 time - Nrecs = np.sum(scan_screen) + Nrecs = len(scan_screen) # Record which SPW/"Data Description" this data is matched to data_desc_array[last_row : last_row + (Nrecs * self.Nspws)] = np.repeat( @@ -1240,7 +1273,7 @@ def write_ms( # Record index positions blt_map_array[last_row : last_row + (Nrecs * self.Nspws)] = np.tile( - np.where(scan_screen)[0], self.Nspws + scan_screen, self.Nspws ) # Extract out the relevant data out of our data-like arrays that @@ -1248,12 +1281,19 @@ def write_ms( val_dict = {} for attr, col in zip(attr_list, col_list): if self.future_array_shapes: - val_dict[col] = getattr(self, attr)[scan_screen] + val_dict[col] = getattr(self, attr)[scan_slice] else: val_dict[col] = np.squeeze( - getattr(self, attr)[scan_screen], axis=1 + getattr(self, attr)[scan_slice], axis=1 ) + # Have to do this separately since uou can't supply multiple index + # arrays at once. + val_dict[col] = val_dict[col][:, :, pol_order] + + if flip_conj: + val_dict["DATA"] = np.conj(val_dict["DATA"]) + # This is where the bulk of the heavy lifting is - use the per-spw # channel masks to record one spectral window at a time. for spw_num in self.spw_array: @@ -1278,7 +1318,7 @@ def write_ms( ant_2_array = self.ant_2_array[blt_map_array] integration_time = self.integration_time[blt_map_array] time_array = time_array[blt_map_array] - uvw_array = self.uvw_array[blt_map_array] + uvw_array = self.uvw_array[blt_map_array] * (-1 if flip_conj else 1) scan_number_array = self.scan_number_array[blt_map_array] # Write out the units of the visibilities, post a warning if its not in Jy since @@ -1341,12 +1381,12 @@ def write_ms( self._write_ms_antenna(filepath) self._write_ms_data_description(filepath) - self._write_ms_feed(filepath) + self._write_ms_feed(filepath, pol_order=pol_order) self._write_ms_field(filepath) self._write_ms_source(filepath) self._write_ms_spectralwindow(filepath) self._write_ms_pointing(filepath) - self._write_ms_polarization(filepath) + self._write_ms_polarization(filepath, pol_order=pol_order) self._write_ms_observation(filepath) self._write_ms_history(filepath) @@ -1743,6 +1783,9 @@ def _read_ms_main( pol_list = [POL_CASA2AIPS_DICT[key] for key in pol_list] flex_pol = None + # Check to see if we want to allow flex pol, in which case each data_desc will + # get assigned it's own spectral window with a potentially different + # polarization per window (which we separately record). if ( allow_flex_pol and all_single_pol @@ -1754,6 +1797,7 @@ def _read_ms_main( ] data_dict[key]["POL_IDX"] = np.array([0]) pol_list = np.array([0]) + npols = 1 flex_pol = np.array( [spw_dict[key]["POL"] for key in sorted(spw_dict.keys())], dtype=int ) @@ -2329,7 +2373,8 @@ def read_ms( self.freq_array = np.expand_dims(self.freq_array, 0) # order polarizations - self.reorder_pols(order=pol_order, run_check=False) + if pol_order is not None: + self.reorder_pols(order=pol_order, run_check=False) if use_future_array_shapes: self.use_future_array_shapes() From 8a254eb762dad564bdd636347d622a7651473442 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 6 Dec 2023 11:33:07 -0500 Subject: [PATCH 4/9] Adding test coverage for flip_conj --- pyuvdata/uvdata/tests/test_ms.py | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/pyuvdata/uvdata/tests/test_ms.py b/pyuvdata/uvdata/tests/test_ms.py index 91bea489f2..83230d593d 100644 --- a/pyuvdata/uvdata/tests/test_ms.py +++ b/pyuvdata/uvdata/tests/test_ms.py @@ -1010,3 +1010,56 @@ def test_ms_bad_history(sma_mir, tmp_path): # Make sure the history is actually preserved correctly. sma_ms = UVData.from_file(filename, use_future_array_shapes=True) assert sma_mir.history in sma_ms.history + + +@pytest.mark.filterwarnings("ignore:Telescope EVLA is not in known_telescopes.") +@pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") +def test_flip_conj(nrao_uv, tmp_path): + filename = os.path.join(tmp_path, "flip_conj.ms") + nrao_uv.set_uvws_from_antenna_positions() + + with uvtest.check_warnings( + UserWarning, match="Writing in the MS file that the units of the data are unca" + ): + nrao_uv.write_ms(filename, flip_conj=True) + + with uvtest.check_warnings( + UserWarning, match="UVW orientation appears to be flipped," + ): + uv = UVData.from_file(filename, use_future_array_shapes=True) + + assert nrao_uv == uv + + +@pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") +def test_flip_conj_multispw(sma_mir, tmp_path): + sma_mir._set_app_coords_helper() + filename = os.path.join(tmp_path, "flip_conj_multispw.ms") + + sma_mir.write_ms(filename, flip_conj=True) + with uvtest.check_warnings( + UserWarning, match="UVW orientation appears to be flipped," + ): + ms_uv = UVData.from_file(filename, use_future_array_shapes=True) + + # MS doesn't have the concept of an "instrument" name like FITS does, and instead + # defaults to the telescope name. Make sure that checks out here. + assert sma_mir.instrument == "SWARM" + assert ms_uv.instrument == "SMA" + sma_mir.instrument = ms_uv.instrument + + # Quick check for history here + assert ms_uv.history != sma_mir.history + ms_uv.history = sma_mir.history + + # Only MS has extra keywords, verify those look as expected. + assert ms_uv.extra_keywords == {"DATA_COL": "DATA", "observer": "SMA"} + assert sma_mir.extra_keywords == {} + sma_mir.extra_keywords = ms_uv.extra_keywords + + # Make sure the filenames line up as expected. + assert sma_mir.filename == ["sma_test.mir"] + assert ms_uv.filename == ["flip_conj_multispw.ms"] + sma_mir.filename = ms_uv.filename = None + + assert sma_mir == ms_uv From a76ac17e6637ec58a8ef2ad960369fc103198355 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 6 Dec 2023 12:15:47 -0500 Subject: [PATCH 5/9] Adding test coverage for new utils --- pyuvdata/tests/test_utils.py | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/pyuvdata/tests/test_utils.py b/pyuvdata/tests/test_utils.py index 8848886556..76e87d716d 100644 --- a/pyuvdata/tests/test_utils.py +++ b/pyuvdata/tests/test_utils.py @@ -4026,6 +4026,24 @@ def test_read_slicing(): data = uvutils._index_dset(dset, slices) assert data.shape == tuple(shape) + # Handling bool arrays + bool_arr = np.zeros((100,), dtype=bool) + index_arr = np.arange(1, 100, 2) + bool_arr[index_arr] = True + assert uvutils._convert_to_slices(bool_arr) == uvutils._convert_to_slices(index_arr) + + # Index return on fail + index_arr[0] = 0 + bool_arr[0:2] = [True, False] + + for item in [index_arr, bool_arr]: + result, check = uvutils._convert_to_slices( + item, max_nslice=1, return_index_on_fail=True + ) + assert not check + assert len(result) == 1 + assert result[0] is item + @pytest.mark.parametrize( "blt_order", @@ -4336,3 +4354,26 @@ def test_check_surface_based_positions_earthmoonloc(tel_loc, check_frame): uvutils.check_surface_based_positions( telescope_loc=loc, telescope_frame=frame ) + + +def test_determine_pol_order_err(): + with pytest.raises(ValueError, match='order must be either "AIPS" or "CASA".'): + uvutils.determine_pol_order([], "ABC") + + +@pytest.mark.parametrize( + "pols,aips_order,casa_order", + [ + [[-8, -7, -6, -5], [3, 2, 1, 0], [3, 1, 0, 2]], + [[-5, -6, -7, -8], [0, 1, 2, 3], [0, 2, 3, 1]], + [[1, 2, 3, 4], [0, 1, 2, 3], [0, 1, 2, 3]], + ], +) +@pytest.mark.parametrize("order", ["CASA", "AIPS"]) +def test_pol_order(pols, aips_order, casa_order, order): + check = uvutils.determine_pol_order(pols, order=order) + + if order == "CASA": + assert all(check == casa_order) + if order == "AIPS": + assert all(check == aips_order) From 07e4af71d1f662e35db90f458e5fdb2a24a68993 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 6 Dec 2023 12:31:14 -0500 Subject: [PATCH 6/9] Updating CHANGELOG --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05a752893b..b5c95cd61e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ All notable changes to this project will be documented in this file. - Added a `check_surface_based_positions` positions method for verifying antenna positions are near surface of whatever celestial body their positions are referenced to (either the Earth or Moon, currently). +- Added a switch to `UVData.write_ms` called `flip_conj`, which allows a user to write +out data with the baseline conjugation scheme flipped from the standard `UVData` +convention. +- Added the `utils.determine_pol_order` method for determining polarization +order based on a specified scheme ("AIPS" or "CASA"). ### Changed - Increased the tolerance to 75 mas (equivalent to 5 ms time error) for a warning about @@ -16,6 +21,9 @@ tolerance value to be user-specified. - Changed the behavior of checking of telescope location to look at the combination of `antenna_positions` and `telescope_location` together for `UVData`, `UVCal`, and `UVFlag`. Additionally, failing this check results in a warning (was an error). +- Changed `UVData.write_ms` to sort polarizations based on CASA-preferred ordering. +- Added some functionality to the `utils._convert_to_slices` method to enable quick +assessment of whether an indexing array can be replaced by a single slice. ## [2.4.1] - 2023-10-13 From 3da63df74e49b2e2f78e6b50d3ce0033134bf96c Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 6 Dec 2023 16:16:38 -0500 Subject: [PATCH 7/9] Cleaning up flake8 complaints --- pyuvdata/tests/test_telescopes.py | 2 +- pyuvdata/tests/test_utils.py | 2 +- pyuvdata/uvbeam/tests/test_cst_beam.py | 2 +- pyuvdata/uvdata/tests/test_mir_parser.py | 4 ++-- pyuvdata/uvdata/tests/test_uvdata.py | 2 +- pyuvdata/uvdata/tests/test_uvfits.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyuvdata/tests/test_telescopes.py b/pyuvdata/tests/test_telescopes.py index e2969359b4..a15fad53ef 100644 --- a/pyuvdata/tests/test_telescopes.py +++ b/pyuvdata/tests/test_telescopes.py @@ -79,7 +79,7 @@ def test_extra_parameter_iter(): for prop in telescope_obj.extra(): extra.append(prop) for a in extra_parameters: - a in extra, "expected attribute " + a + " not returned in extra iterator" + assert a in extra, "expected attribute " + a + " not returned in extra iterator" def test_unexpected_parameters(): diff --git a/pyuvdata/tests/test_utils.py b/pyuvdata/tests/test_utils.py index 76e87d716d..232b00ee6d 100644 --- a/pyuvdata/tests/test_utils.py +++ b/pyuvdata/tests/test_utils.py @@ -2819,7 +2819,7 @@ def test_strict_cliques(): adj_link[-1] = frozenset({5, 6, 7, 8, 1}) with pytest.raises(ValueError, match="Non-isolated cliques found in graph."): - uvutils._find_cliques(adj_link, strict=True), + uvutils._find_cliques(adj_link, strict=True) def test_reorder_conj_pols_non_list(): diff --git a/pyuvdata/uvbeam/tests/test_cst_beam.py b/pyuvdata/uvbeam/tests/test_cst_beam.py index d401805a94..34933fed10 100644 --- a/pyuvdata/uvbeam/tests/test_cst_beam.py +++ b/pyuvdata/uvbeam/tests/test_cst_beam.py @@ -209,7 +209,7 @@ def test_read_yaml_override(cst_efield_2freq_mod): beam_type="efield", telescope_name="test", use_future_array_shapes=True, - ), + ) assert beam1 == beam2 diff --git a/pyuvdata/uvdata/tests/test_mir_parser.py b/pyuvdata/uvdata/tests/test_mir_parser.py index fb45fb44b8..e8b48b17e1 100644 --- a/pyuvdata/uvdata/tests/test_mir_parser.py +++ b/pyuvdata/uvdata/tests/test_mir_parser.py @@ -176,7 +176,7 @@ def test_mir_raw_data(mir_data, tmp_path): mir_data._write_cross_data(filepath) # Sub out the file we need to read from - mir_data._file_dict = {filepath: item for item in mir_data._file_dict.values()} + mir_data._file_dict = {filepath: list(mir_data._file_dict.values())[0]} raw_data = mir_data._read_data("cross", return_vis=False) assert raw_data.keys() == mir_data.raw_data.keys() @@ -201,7 +201,7 @@ def test_mir_auto_data(mir_data, tmp_path): mir_data._write_auto_data(filepath) # Sub out the file we need to read from, and fix a couple of attributes that changed # since we are no longer spoofing values (after reading in data from old-style file) - mir_data._file_dict = {filepath: item for item in mir_data._file_dict.values()} + mir_data._file_dict = {filepath: list(mir_data._file_dict.values())[0]} mir_data._file_dict[filepath]["auto"]["filetype"] = "ach_read" int_dict, mir_data._ac_dict = mir_data.ac_data._generate_recpos_dict(reindex=True) mir_data._file_dict[filepath]["auto"]["int_dict"] = int_dict diff --git a/pyuvdata/uvdata/tests/test_uvdata.py b/pyuvdata/uvdata/tests/test_uvdata.py index 0aa1736992..13fa61e01b 100644 --- a/pyuvdata/uvdata/tests/test_uvdata.py +++ b/pyuvdata/uvdata/tests/test_uvdata.py @@ -9539,7 +9539,7 @@ def test_frequency_average_nsample_precision(casa_uvfits): uvobj.nsample_array = uvobj.nsample_array.astype(np.float16) with uvtest.check_warnings(UserWarning, "eq_coeffs vary by frequency"): - uvobj.frequency_average(n_chan_to_avg=2), + uvobj.frequency_average(n_chan_to_avg=2) assert uvobj.Nfreqs == (uvobj2.Nfreqs / 2) diff --git a/pyuvdata/uvdata/tests/test_uvfits.py b/pyuvdata/uvdata/tests/test_uvfits.py index 37e574d5a9..07f31f8c48 100644 --- a/pyuvdata/uvdata/tests/test_uvfits.py +++ b/pyuvdata/uvdata/tests/test_uvfits.py @@ -926,7 +926,7 @@ def test_readwriteread_error_single_time(tmp_path, casa_uvfits): "The integration time is not specified and only one time", ], ): - uv_out.read(write_file2, use_future_array_shapes=True), + uv_out.read(write_file2, use_future_array_shapes=True) return From 9c862da2a8a33c79417921ad529fe68ea5122a69 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 6 Dec 2023 20:16:26 -0500 Subject: [PATCH 8/9] Addressing review comments --- CHANGELOG.md | 12 ++++++------ pyuvdata/utils.py | 5 ++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5c95cd61e..5437de52ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,16 +4,19 @@ All notable changes to this project will be documented in this file. ## [Unreleased] ### Added -- Added a `check_surface_based_positions` positions method for verifying antenna -positions are near surface of whatever celestial body their positions are referenced to -(either the Earth or Moon, currently). - Added a switch to `UVData.write_ms` called `flip_conj`, which allows a user to write out data with the baseline conjugation scheme flipped from the standard `UVData` convention. - Added the `utils.determine_pol_order` method for determining polarization order based on a specified scheme ("AIPS" or "CASA"). +- Added a `check_surface_based_positions` positions method for verifying antenna +positions are near surface of whatever celestial body their positions are referenced to +(either the Earth or Moon, currently). ### Changed +- Changed `UVData.write_ms` to sort polarizations based on CASA-preferred ordering. +- Added some functionality to the `utils._convert_to_slices` method to enable quick +assessment of whether an indexing array can be replaced by a single slice. - Increased the tolerance to 75 mas (equivalent to 5 ms time error) for a warning about values in `lst_array` not conforming to expectations for `UVData`, `UVCal`, and `UVFlag` (was 1 mas) inside of `check`. Additionally, added a keyword to `check` enable the @@ -21,9 +24,6 @@ tolerance value to be user-specified. - Changed the behavior of checking of telescope location to look at the combination of `antenna_positions` and `telescope_location` together for `UVData`, `UVCal`, and `UVFlag`. Additionally, failing this check results in a warning (was an error). -- Changed `UVData.write_ms` to sort polarizations based on CASA-preferred ordering. -- Added some functionality to the `utils._convert_to_slices` method to enable quick -assessment of whether an indexing array can be replaced by a single slice. ## [2.4.1] - 2023-10-13 diff --git a/pyuvdata/utils.py b/pyuvdata/utils.py index 6ade5ae9ed..b5bc6534d9 100644 --- a/pyuvdata/utils.py +++ b/pyuvdata/utils.py @@ -5847,7 +5847,10 @@ def _convert_to_slices( max_nslice = max_nslice_frac * Ninds check = len(slices) <= max_nslice - return [indices] if (not check and return_index_on_fail) else slices, check + if return_index_on_fail and not check: + return [indices], check + else: + return slices, check def _get_slice_len(s, axlen): From eb3c56c3a99c8c4d22d138888ae11f814faa6843 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 6 Dec 2023 20:44:13 -0500 Subject: [PATCH 9/9] Adding a smidgen more of test coverage --- pyuvdata/tests/test_utils.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pyuvdata/tests/test_utils.py b/pyuvdata/tests/test_utils.py index 232b00ee6d..efedf7384f 100644 --- a/pyuvdata/tests/test_utils.py +++ b/pyuvdata/tests/test_utils.py @@ -4027,12 +4027,15 @@ def test_read_slicing(): assert data.shape == tuple(shape) # Handling bool arrays - bool_arr = np.zeros((100,), dtype=bool) - index_arr = np.arange(1, 100, 2) + bool_arr = np.zeros((10000,), dtype=bool) + index_arr = np.arange(1, 10000, 2) bool_arr[index_arr] = True assert uvutils._convert_to_slices(bool_arr) == uvutils._convert_to_slices(index_arr) + assert uvutils._convert_to_slices(bool_arr, return_index_on_fail=True) == ( + uvutils._convert_to_slices(index_arr, return_index_on_fail=True) + ) - # Index return on fail + # Index return on fail with two slices index_arr[0] = 0 bool_arr[0:2] = [True, False] @@ -4044,6 +4047,17 @@ def test_read_slicing(): assert len(result) == 1 assert result[0] is item + # Check a more complicated pattern w/ just the max_slice_frac defined + index_arr = np.arange(0, 100) ** 2 + bool_arr[:] = False + bool_arr[index_arr] = True + + for item in [index_arr, bool_arr]: + result, check = uvutils._convert_to_slices(item, return_index_on_fail=True) + assert not check + assert len(result) == 1 + assert result[0] is item + @pytest.mark.parametrize( "blt_order",