From 30447f0715b44b65aec55d34ab32d62d3ece3cbe Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 10:45:47 -0400 Subject: [PATCH 01/20] BUG: fixed load without data Fixed the JHU APL load routines to handle an empty file list. --- pysatNASA/instruments/methods/jhuapl.py | 73 +++++++++++++++---------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 1f92c802..15f541ad 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -6,6 +6,7 @@ import pandas as pds import xarray as xr +import pysat from pysat.utils.coords import expand_xarray_dims from pysat.utils.io import load_netcdf @@ -100,6 +101,10 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, inst.load(2003, 1) """ + # Initialize the output + mdata = pysat.Meta() + data = xr.Dataset() + # Define the input variables labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} @@ -140,12 +145,13 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, # Update the fill value, using information from the global header mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} - # After loading all the data, determine which dimensions need to be - # expanded. Pad the data so that all dimensions are the same shape. - single_data = expand_xarray_dims(single_data, mdata, dims_equal=False) + if len(single_data) > 0: + # After loading all the data, determine which dimensions need to be + # expanded. Pad the data so that all dimensions are the same shape. + single_data = expand_xarray_dims(single_data, mdata, dims_equal=False) - # Combine all the data, indexing along time - data = xr.combine_by_coords(single_data) + # Combine all the data, indexing along time + data = xr.combine_by_coords(single_data) return data, mdata @@ -193,6 +199,10 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, inst.load(2003, 1) """ + # Initialize the output + mdata = pysat.Meta() + data = xr.Dataset() + # Define the input variables and working variables labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} load_time = 'TIME_DAY' @@ -270,7 +280,7 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, for i, ntime in enumerate(build_dtimes(sdata, '_NIGHT')): if abs(ntime - ftime[i]).total_seconds() > 1.0: raise ValueError('Day and night times differ') - + # Remove redundant time variables and rname the 'nAlong' dimension sdata = sdata.drop_vars(time_vars).swap_dims({'nAlong': 'time'}) @@ -305,31 +315,34 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} # Combine all time dimensions - if combine_times: - data_list = expand_xarray_dims([inners[dim] if dim == 'time' else - inners[dim].rename_dims({dim: 'time'}) - for dim in time_dims], mdata, - dims_equal=False) - else: - data_list = [inners[dim] for dim in time_dims] - - # Combine all the data, indexing along time - data = xr.merge(data_list) + if inners is not None: + if combine_times: + data_list = expand_xarray_dims( + [inners[dim] if dim == 'time' else + inners[dim].rename_dims({dim: 'time'}) + for dim in time_dims], mdata, dims_equal=False) + else: + data_list = [inners[dim] for dim in time_dims] - # Set additional coordinates - data = data.set_coords(coords).assign_coords({'time': data['time']}) - if tag == 'sdr-imaging': - data = data.assign_coords( - {'nchan': ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong"], - "nchanAur": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", - "LBHlong"], - "nCross": sdata.nCross.data, - "nCrossDayAur": sdata.nCrossDayAur.data}) - elif tag == 'sdr-spectrograph': - data = data.assign_coords({"nchan": ["121.6nm", "130.4nm", "135.6nm", - "LBHshort", "LBHlong", "?"]}) + # Combine all the data, indexing along time + data = xr.merge(data_list) - # Ensure the data is ordered correctly - data = data.sortby('time') + # Set additional coordinates + data = data.set_coords(coords).assign_coords({'time': data['time']}) + if tag == 'sdr-imaging': + data = data.assign_coords( + {'nchan': ["121.6nm", "130.4nm", "135.6nm", "LBHshort", + "LBHlong"], + "nchanAur": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", + "LBHlong"], + "nCross": sdata.nCross.data, + "nCrossDayAur": sdata.nCrossDayAur.data}) + elif tag == 'sdr-spectrograph': + data = data.assign_coords({"nchan": ["121.6nm", "130.4nm", + "135.6nm", "LBHshort", + "LBHlong", "?"]}) + + # Ensure the data is ordered correctly + data = data.sortby('time') return data, mdata From 2958226c682a1309d39db7ee05c47e2eaf1f3acd Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 11:30:00 -0400 Subject: [PATCH 02/20] STY: updated labels Updated labels definition to use meta labels attributes instead of strings and keys. --- pysatNASA/instruments/methods/jhuapl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 15f541ad..fecf77fe 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -106,7 +106,8 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, data = xr.Dataset() # Define the input variables - labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} + labels = {mdata.labels.units: ('UNITS', str), + mdata.labels.desc: ('TITLE', str)} # CDAWeb stores these files in the NetCDF format instead of the CDF format single_data = list() @@ -204,7 +205,8 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, data = xr.Dataset() # Define the input variables and working variables - labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} + labels = {mdata.labels.units: ('UNITS', str), + mdata.labels.desc: ('TITLE', str)} load_time = 'TIME_DAY' time_vars = ['YEAR_DAY', 'DOY_DAY', 'TIME_EPOCH_DAY', 'YEAR_NIGHT', 'DOY_NIGHT', 'TIME_NIGHT', 'TIME_EPOCH_NIGHT'] From f72e2ab033578d2578f9e0d1955da330e0f2180f Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 13:50:34 -0400 Subject: [PATCH 03/20] ENH: updated clean method Updated the TIMED GUVI clean method for imaging data. --- pysatNASA/instruments/timed_guvi.py | 51 +++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 291d57ea..1d265614 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -92,8 +92,11 @@ for tag in inst_ids[iid]} for iid in ['high_res', 'low_res']} _clean_warn = {inst_id: {tag: mm_nasa.clean_warnings - for tag in inst_ids[inst_id]} + for tag in inst_ids[inst_id] if tag != 'sdr-imaging'} for inst_id in inst_ids.keys()} +for inst_id in ['high_res', 'low_res']: + _clean_warn[inst_id]['sdr-imaging']['dirty'] = mm_nasa.clean_warnings[ + 'dirty'] # ---------------------------------------------------------------------------- # Instrument methods @@ -101,8 +104,50 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_timed, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn + +def clean(self): + """Clean TIMED GUVI imaging data. + + Note + ---- + Supports 'clean', 'dusty', 'dirty', 'none'. Method is + not called by pysat if clean_level is None or 'none'. + + """ + + if self.tag == "sdr-imaging" and self.clean_level in ['clean', 'dusty']: + # Find the flag variables + dqi_vars = [var for var in self.variables if var.find('DQI') == 0] + + # Find the variables affected by each flag + dat_vars = {dqi: [var for var in self.variables if var.find(dqi) > 0] + if dqi.find('AURORAL') >= 0 else + [var for var in self.variables if var.find('AURORAL') < 0 + and var.find(dqi) > 0] for dqi in dqi_vars} + + for dqi in dqi_vars: + if self.clean_level == 'clean': + # For clean, require DQI of zero (MeV noise only) + dqi_bad = self.data[dqi].values > 0 + else: + # For dusty, allow the SAA region as well + dqi_bad = self.data[dqi].values > 1 + + # Apply the DQI mask to the data, replacing bad values with + # appropriate fill values + for dat_var in dat_vars[dqi]: + if self.data[dat_var].shape == dqi_bad.shape: + # Only apply to data with the correct dimensions + fill_val = self.meta[dat_var, self.meta.labels.fill_val] + self.data[dat_var].values[dqi_bad] = fill_val + else: + # Follow the same warning format as the general clean warning, but + # with additional information. + pysat.logger.warning(' '.join(['No cleaning routines available for', + self.platform, self.name, self.tag, + self.inst_id, 'at clean level', + self.clean_level])) + return # ---------------------------------------------------------------------------- # Instrument functions From d5053c2c55f8a381175bb391a6db5c8af0618672 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 13:56:31 -0400 Subject: [PATCH 04/20] BUG: fixed dict format Fixed the dictionary formatting for `_clean_warn`. --- pysatNASA/instruments/timed_guvi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 1d265614..ba2a1272 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -95,8 +95,8 @@ for tag in inst_ids[inst_id] if tag != 'sdr-imaging'} for inst_id in inst_ids.keys()} for inst_id in ['high_res', 'low_res']: - _clean_warn[inst_id]['sdr-imaging']['dirty'] = mm_nasa.clean_warnings[ - 'dirty'] + _clean_warn[inst_id]['sdr-imaging'] = {'dirty': mm_nasa.clean_warnings[ + 'dirty']} # ---------------------------------------------------------------------------- # Instrument methods From 44c6a36c4a617102e9e364377e67b69952a77318 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 14:25:53 -0400 Subject: [PATCH 05/20] BUG: expanded masked dimensions Allow for channels when masking for the clean routine. --- pysatNASA/instruments/timed_guvi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index ba2a1272..58d2e5ec 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -136,7 +136,8 @@ def clean(self): # Apply the DQI mask to the data, replacing bad values with # appropriate fill values for dat_var in dat_vars[dqi]: - if self.data[dat_var].shape == dqi_bad.shape: + if(self.data[dat_var].shape == dqi_bad.shape or + self.data[dat_var].shape[:-1] == dqi_bad.shape): # Only apply to data with the correct dimensions fill_val = self.meta[dat_var, self.meta.labels.fill_val] self.data[dat_var].values[dqi_bad] = fill_val From fcedfedb67fd79dcc259692e2683cbfa6a230444 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 16:14:00 -0400 Subject: [PATCH 06/20] STY: removed whitespace Removed extra whitespace and some typo characters in the docstring. --- pysatNASA/instruments/methods/jhuapl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index fecf77fe..27ef2b77 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -175,7 +175,7 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, strict_dim_check : bool Used for xarray data (`pandas_format` is False). If True, warn the user that the desired epoch, 'TIME_DAY', is not present as a dimension in the - NetCDF file. If False, no warning is raised. (default=True)``` + NetCDF file. If False, no warning is raised. (default=True) combine_times : bool For SDR data, optionally combine the different datetime coordinates into a single time coordinate (default=False) @@ -203,7 +203,7 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, # Initialize the output mdata = pysat.Meta() data = xr.Dataset() - + # Define the input variables and working variables labels = {mdata.labels.units: ('UNITS', str), mdata.labels.desc: ('TITLE', str)} @@ -282,7 +282,7 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, for i, ntime in enumerate(build_dtimes(sdata, '_NIGHT')): if abs(ntime - ftime[i]).total_seconds() > 1.0: raise ValueError('Day and night times differ') - + # Remove redundant time variables and rname the 'nAlong' dimension sdata = sdata.drop_vars(time_vars).swap_dims({'nAlong': 'time'}) From 068d54a68984852c12b1b65c28df667e0892126d Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 16:14:31 -0400 Subject: [PATCH 07/20] ENH: added concat_data method Added a concat_data method to fix issues loading data. --- pysatNASA/instruments/timed_guvi.py | 90 +++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 58d2e5ec..dcd3b6d5 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -60,7 +60,9 @@ import datetime as dt import functools +import xarray as xr +import pysat from pysat.instruments.methods import general as mm_gen from pysatNASA.instruments.methods import cdaweb as cdw @@ -114,16 +116,15 @@ def clean(self): not called by pysat if clean_level is None or 'none'. """ - if self.tag == "sdr-imaging" and self.clean_level in ['clean', 'dusty']: # Find the flag variables - dqi_vars = [var for var in self.variables if var.find('DQI') == 0] + dqi_vars = [var for var in self.variables if var.find('DQI') == 0] # Find the variables affected by each flag dat_vars = {dqi: [var for var in self.variables if var.find(dqi) > 0] if dqi.find('AURORAL') >= 0 else [var for var in self.variables if var.find('AURORAL') < 0 - and var.find(dqi) > 0] for dqi in dqi_vars} + and var.find(dqi) > 0] for dqi in dqi_vars} for dqi in dqi_vars: if self.clean_level == 'clean': @@ -136,8 +137,8 @@ def clean(self): # Apply the DQI mask to the data, replacing bad values with # appropriate fill values for dat_var in dat_vars[dqi]: - if(self.data[dat_var].shape == dqi_bad.shape or - self.data[dat_var].shape[:-1] == dqi_bad.shape): + if(self.data[dat_var].shape == dqi_bad.shape + or self.data[dat_var].shape[:-1] == dqi_bad.shape): # Only apply to data with the correct dimensions fill_val = self.meta[dat_var, self.meta.labels.fill_val] self.data[dat_var].values[dqi_bad] = fill_val @@ -150,6 +151,85 @@ def clean(self): self.clean_level])) return + +def concat_data(self, new_data, combine_times=False, **kwargs): + """Concatonate data to self.data for TIMED GUVI data. + + Parameters + ---------- + new_data : xarray.Dataset or list of such objects + New data objects to be concatonated + combine_times : bool + For SDR data, optionally combine the different datetime coordinates + into a single time coordinate (default=False) + **kwargs : dict + Optional keyword arguments passed to xr.concat + + Note + ---- + For xarray, `dim=Instrument.index.name` is passed along to xarray.concat + except if the user includes a value for dim as a keyword argument. + + """ + # Establish the time dimensions by data type + time_dims = [self.index.name] + + if self.tag == 'sdr-imaging': + time_dims.append('time_auroral') + elif self.tag == 'sdr-spectrograph': + time_dims.extend(['time_gaim_day', 'time_gaim_night']) + + # Concatonate using the appropriate method for the number of time + # dimensions + if len(time_dims) == 1: + # There is only one time dimensions, but other dimensions may + # need to be adjusted + new_data = pysat.utils.coords.expand_xarray_dims( + new_data, self.meta, exclude_dims=time_dims) + + # Combine the data + self.data = xr.combine_by_coords(new_data, **kwargs) + else: + inners = None + for ndata in new_data: + # Separate into inner datasets + inner_keys = {dim: [key for key in ndata.keys() + if dim in ndata[key].dims] for dim in time_dims} + inner_dat = {dim: ndata.get(inner_keys[dim]) for dim in time_dims} + + # Add 'single_var's into 'time' dataset to keep track + sv_keys = [val.name for val in ndata.values() + if 'single_var' in val.dims] + singlevar_set = ndata.get(sv_keys) + inner_dat['time'] = xr.merge([inner_dat['time'], singlevar_set]) + + # Concatenate along desired dimension with previous data + if inners is None: + # No previous data, assign the data separated by dimension + inners = dict(inner_dat) + else: + # Concatenate with existing data + inners = {dim: xr.concat([inners[dim], inner_dat[dim]], + dim=dim) for dim in time_dims} + + # Combine all time dimensions + if inners is not None: + if combine_times: + data_list = pysat.utils.coords.expand_xarray_dims( + [inners[dim] if dim == self.index.name else + inners[dim].rename_dims({dim: self.index.name}) + for dim in time_dims], self.meta, dims_equal=False) + else: + data_list = [inners[dim] for dim in time_dims] + + # Combine all the data, indexing along time + self.data = xr.merge(data_list) + + # Ensure the data is ordered correctly + self.data = self.data.sortby(self.index.name) + return + + # ---------------------------------------------------------------------------- # Instrument functions # From acc33958d659f1e993ee5eb68517cb539b504753 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 16:30:09 -0400 Subject: [PATCH 08/20] BUG: remove time sort in `concat_data` Remove the time sorting in `concat_data`, as data should be concatenated in the desired order. --- pysatNASA/instruments/timed_guvi.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index dcd3b6d5..b0c5f921 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -224,9 +224,6 @@ def concat_data(self, new_data, combine_times=False, **kwargs): # Combine all the data, indexing along time self.data = xr.merge(data_list) - - # Ensure the data is ordered correctly - self.data = self.data.sortby(self.index.name) return From 77668d2970964ef6a233cdc27b781f95fb9352d9 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 16:30:27 -0400 Subject: [PATCH 09/20] DOC: update changelog Updated the changelog with a summary of the changes in this branch. --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d579816..48cf0e3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). ## [0.X.X] - 2023-XX-XX +* Bug Fixes + * Allow graceful failure with no files in jhuapl load functions +* Enhancements + * Added custom `concat_data` method to TIMED-GUVI data + * Added cleaning to TIMED-GUVI SDR imaging data * Maintenance * Implemented unit tests for cleaning warnings From c2f72334c322370054f4e17e75a966b324e6cb9d Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 25 Jul 2023 12:12:24 -0400 Subject: [PATCH 10/20] STY: updated spacing Added a whitespace after the `if` keyword. Also replaced hardcoded key with the appropriate variable. --- pysatNASA/instruments/timed_guvi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index b0c5f921..599dc437 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -137,8 +137,8 @@ def clean(self): # Apply the DQI mask to the data, replacing bad values with # appropriate fill values for dat_var in dat_vars[dqi]: - if(self.data[dat_var].shape == dqi_bad.shape - or self.data[dat_var].shape[:-1] == dqi_bad.shape): + if self.data[dat_var].shape == dqi_bad.shape or self.data[ + dat_var].shape[:-1] == dqi_bad.shape: # Only apply to data with the correct dimensions fill_val = self.meta[dat_var, self.meta.labels.fill_val] self.data[dat_var].values[dqi_bad] = fill_val @@ -201,7 +201,8 @@ def concat_data(self, new_data, combine_times=False, **kwargs): sv_keys = [val.name for val in ndata.values() if 'single_var' in val.dims] singlevar_set = ndata.get(sv_keys) - inner_dat['time'] = xr.merge([inner_dat['time'], singlevar_set]) + inner_dat[self.index.name] = xr.merge([inner_dat[self.index.name], + singlevar_set]) # Concatenate along desired dimension with previous data if inners is None: From 819d8e1225d828673d8a29a83e55ad7f61ff1e8e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 25 Jul 2023 12:42:20 -0400 Subject: [PATCH 11/20] ENH: added metadata Added metadata for the 'time_auroral' and 'nCross' variables. --- pysatNASA/instruments/methods/jhuapl.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 27ef2b77..321c4dc4 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -316,6 +316,10 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, # Update the fill value, using information from the global header mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} + # Add metadata for 'time_auroral' and 'nCross' variables + mdata['time_auroral'] = {'desc': 'Auroral time index'} + mdata['nCross'] = {'desc': 'Number of cross-track observations'} + # Combine all time dimensions if inners is not None: if combine_times: From 499fe07e80c7bd6dd01a2e62c316449adb41b11e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Nov 2023 11:39:47 -0500 Subject: [PATCH 12/20] MAINT: update rc tests --- .github/workflows/pysat_rc.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index e0ab0742..85afb492 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -31,10 +31,8 @@ jobs: - name: Install pysat RC run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat - - name: Install standard dependencies - run: | - pip install -r requirements.txt - pip install -r test_requirements.txt + - name: Install with standard dependencies + run: pip install .[test] - name: Set up pysat run: | @@ -42,9 +40,22 @@ jobs: python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" - name: Test with pytest - run: pytest -vs --cov=pysatNASA/ + run: pytest - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls --rcfile=setup.cfg --service=github + COVERALLS_PARALLEL: true + run: coveralls --rcfile=pyproject.toml --service=github + + finish: + name: Finish Coverage Analysis + needs: build + runs-on: ubuntu-latest + steps: + - name: Coveralls Finished + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip install --upgrade coveralls + coveralls --service=github --finish From 959481e35c0cb2702e1950d4568d50da1b0c5251 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 27 Feb 2024 10:31:13 -0500 Subject: [PATCH 13/20] BUG: fix JHUAPL time Fix time where microseconds can become too big from rounding. --- pysatNASA/instruments/methods/jhuapl.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 321c4dc4..2b20cc53 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -43,14 +43,15 @@ def build_dtimes(data, var, epoch=None, epoch_var='time'): for i, sec in enumerate(data[skey].values)] secs = [int(np.floor((sec - hours[i] * 3600 - mins[i] * 60))) for i, sec in enumerate(data[skey].values)] + microsecs = [int(np.floor((sec - hours[i] * 3600 - mins[i] * 60 + - secs[i]) * 1.0e6)) + for i, sec in enumerate(data[skey].values)] dtimes = [ dt.datetime.strptime( - "{:4d}-{:03d}-{:02d}-{:02d}-{:02d}-{:06.0f}".format( + "{:4d}-{:03d}-{:02d}-{:02d}-{:02d}-{:06d}".format( int(data[ykey].values[i]), int(data[dkey].values[i]), - hours[i], mins[i], secs[i], - (sec - hours[i] * 3600 - mins[i] * 60 - secs[i]) * 1.0e6), - '%Y-%j-%H-%M-%S-%f') - for i, sec in enumerate(data[skey].values)] + hours[i], mins[i], secs[i], microsec), '%Y-%j-%H-%M-%S-%f') + for i, microsec in enumerate(microsecs)] else: dtimes = [ dt.datetime.strptime("{:4d}-{:03d}".format( From 7c80f0d26c331863cf85148b6f5d1deb118ef503 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 27 Feb 2024 10:38:12 -0500 Subject: [PATCH 14/20] STY: removed type equality Updated type equalities to use `isinstance`. --- pysatNASA/instruments/methods/_cdf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/_cdf.py b/pysatNASA/instruments/methods/_cdf.py index c02dc55d..06d4869e 100644 --- a/pysatNASA/instruments/methods/_cdf.py +++ b/pysatNASA/instruments/methods/_cdf.py @@ -437,13 +437,14 @@ def to_pysat(self, flatten_twod=True, index = None for varname, df in cdata.items(): if varname not in ('Epoch', 'DATE'): - if type(df) == pds.Series: + if isinstance(df, pds.Series): data[varname] = df # CDF data Series are saved using a mix of Range and # Datetime Indexes. This requires that the user specify # the desired index when creating a DataFrame - if type(df.index) == pds.DatetimeIndex and index is None: + if isinstance(df.index, + pds.DatetimeIndex) and index is None: index = df.index if index is None: From 566244ec08f86db23b03f3c91610bade7e1fe54a Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 27 Feb 2024 10:46:24 -0500 Subject: [PATCH 15/20] TST: added an ignore list for links Added a `linkcheck_ignore` statement, as the GATS page checks to see if the link request is human or machine. --- docs/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 43245c9d..8c23a0b8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -175,3 +175,6 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} + +# Links to ignore, as they require human interaction +linkcheck_ignore = [r'https://saber.gats-inc.com/temp_errors.php'] From d175a358719ad99b62a2d9e860dda0b25c926bfa Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 4 Mar 2024 11:01:18 -0500 Subject: [PATCH 16/20] BUG: fixed inners assignment Ensured that only dimensions with data are renamed. Also updated test date to highlight a time with better data coverage across all instruments. --- pysatNASA/instruments/timed_guvi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 599dc437..b6011b62 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -88,7 +88,7 @@ # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {iid: {tag: dt.datetime(2005, 6, 28) for tag in inst_ids[iid]} +_test_dates = {iid: {tag: dt.datetime(2007, 12, 13) for tag in inst_ids[iid]} for iid in inst_ids.keys()} _test_load_opt = {iid: {tag: {'combine_times': True} for tag in inst_ids[iid]} for iid in ['high_res', @@ -219,7 +219,8 @@ def concat_data(self, new_data, combine_times=False, **kwargs): data_list = pysat.utils.coords.expand_xarray_dims( [inners[dim] if dim == self.index.name else inners[dim].rename_dims({dim: self.index.name}) - for dim in time_dims], self.meta, dims_equal=False) + for dim in time_dims if len(inners[dim].dims) > 0], + self.meta, dims_equal=False) else: data_list = [inners[dim] for dim in time_dims] From eaaef3b66c133a7180c4875c9e2ad5c7e98fee98 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 4 Mar 2024 11:09:32 -0500 Subject: [PATCH 17/20] TST: update GUVI test dates Update TIMED GUVI test dates, as coverage needs are different for different instruments. --- pysatNASA/instruments/timed_guvi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index b6011b62..32935188 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -88,8 +88,10 @@ # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {iid: {tag: dt.datetime(2007, 12, 13) for tag in inst_ids[iid]} - for iid in inst_ids.keys()} +_test_dates = { + iid: {tag: dt.datetime(2007 if tag.find('spectrograph') > 0 else 2005, 12, + 13) for tag in inst_ids[iid]} + for iid in inst_ids.keys()} _test_load_opt = {iid: {tag: {'combine_times': True} for tag in inst_ids[iid]} for iid in ['high_res', 'low_res']} From 33ed75f64f384f0c6035cdd302dc9d703f6838ea Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 5 Mar 2024 16:54:12 -0500 Subject: [PATCH 18/20] MAINT: remove new test flag from timed_guvi --- pysatNASA/instruments/timed_guvi.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 66b4a476..c49d81d9 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -95,9 +95,6 @@ _test_load_opt = {iid: {tag: {'combine_times': True} for tag in inst_ids[iid]} for iid in ['high_res', 'low_res']} -# TODO(#218): Remove when compliant with multi-day load tests -_new_tests = {iid: {tag: False for tag in inst_ids[iid]} - for iid in ['high_res', 'low_res']} _clean_warn = {inst_id: {tag: mm_nasa.clean_warnings for tag in inst_ids[inst_id] if tag != 'sdr-imaging'} for inst_id in inst_ids.keys()} From a6a61fb3b740920c3f706d053364dc83a3754cc7 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 6 Mar 2024 18:26:19 -0500 Subject: [PATCH 19/20] Update pysatNASA/instruments/timed_guvi.py --- pysatNASA/instruments/timed_guvi.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index c49d81d9..66b4a476 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -95,6 +95,9 @@ _test_load_opt = {iid: {tag: {'combine_times': True} for tag in inst_ids[iid]} for iid in ['high_res', 'low_res']} +# TODO(#218): Remove when compliant with multi-day load tests +_new_tests = {iid: {tag: False for tag in inst_ids[iid]} + for iid in ['high_res', 'low_res']} _clean_warn = {inst_id: {tag: mm_nasa.clean_warnings for tag in inst_ids[inst_id] if tag != 'sdr-imaging'} for inst_id in inst_ids.keys()} From 483c8e1d9bd03471881058179711803f4548667c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:34:53 -0500 Subject: [PATCH 20/20] MAINT: only skip high res data --- pysatNASA/instruments/timed_guvi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 66b4a476..cdaa28d4 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -96,8 +96,7 @@ for tag in inst_ids[iid]} for iid in ['high_res', 'low_res']} # TODO(#218): Remove when compliant with multi-day load tests -_new_tests = {iid: {tag: False for tag in inst_ids[iid]} - for iid in ['high_res', 'low_res']} +_new_tests = {'high_res': {tag: False for tag in inst_ids['high_res']}} _clean_warn = {inst_id: {tag: mm_nasa.clean_warnings for tag in inst_ids[inst_id] if tag != 'sdr-imaging'} for inst_id in inst_ids.keys()}