Merge pull request #223 from pysat/bug/cdaweb_clean

BUG/MAINT: cdaweb downloads, ICON IVM-B tests, new flag to skip new tests

.github/workflows/pysat_rc.yml

@@ -31,10 +31,8 @@ jobs:
     - name: Install pysat RC
       run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat
 
-    - name: Install standard dependencies
-      run: |
-        pip install -r requirements.txt
-        pip install -r test_requirements.txt
+    - name: Install with standard dependencies
+      run: pip install .[test]
 
     - name: Set up pysat
       run: |

CHANGELOG.md

@@ -16,6 +16,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * New window needs to be integer for calculate_imf_steadiness
   * Fixed version import
   * Fixed a bug when data fails to load for CDF pandas objects
+  * Fixed a bug where cdas_download may drop the requested end date file
 * Documentation
   * Added example of how to export data for archival
   * Updated documentation refs
@@ -28,6 +29,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * Added tests for OMNI HRO routines
   * Use standard clean routine for C/NOFS VEFI mag data
   * Added version cap for sphinx_rtd_theme
+  * Include standard tests for ICON IVM-B
 
 ## [0.0.5] - 2023-06-27
 * New Instruments

pyproject.toml

@@ -78,7 +78,7 @@ Source = "https://github.com/pysat/pysatNASA"
 omit = ["*/instruments/templates/"]
 
 [tool.pytest.ini_options]
-addopts = "-vs --cov=pysatNASA"
+addopts = "--cov=pysatNASA"
 markers = [
   "all_inst",
   "download",

pysatNASA/instruments/cnofs_plp.py

@@ -79,6 +79,8 @@
 # Instrument test attributes
 
 _test_dates = {'': {'': dt.datetime(2009, 1, 1)}}
+# TODO(#222): Remove when compliant with multi-day load tests
+_new_tests = {'': {'': False}}
 
 # ----------------------------------------------------------------------------
 # Instrument methods

pysatNASA/instruments/icon_euv.py

@@ -65,6 +65,8 @@
 # Instrument test attributes
 
 _test_dates = {'': {'': dt.datetime(2020, 1, 1)}}
+# TODO(#218, #222): Remove when compliant with multi-day load tests
+_new_tests = {'': {'': False}}
 _test_load_opt = {'': {'': {'keep_original_names': True}}}
 
 # ----------------------------------------------------------------------------

pysatNASA/instruments/icon_ivm.py

@@ -68,9 +68,7 @@
 # Instrument test attributes
 
 _test_dates = {'a': {'': dt.datetime(2020, 1, 1)},
-               'b': {'': dt.datetime(2020, 1, 1)}}  # IVM-B not yet engaged
-_test_download = {'b': {kk: False for kk in tags.keys()}}
-_password_req = {'b': {kk: True for kk in tags.keys()}}
+               'b': {'': dt.datetime(2021, 6, 15)}}
 _test_load_opt = {jj: {'': {'keep_original_names': True}}
                   for jj in inst_ids.keys()}
 

pysatNASA/instruments/maven_insitu_kp.py

@@ -52,6 +52,8 @@
 # Instrument test attributes
 
 _test_dates = {'': {'': dt.datetime(2020, 1, 1)}}
+# TODO(#218, #222): Remove when compliant with multi-day load tests
+_new_tests = {'': {'': False}}
 
 # ----------------------------------------------------------------------------
 # Instrument methods

pysatNASA/instruments/methods/_cdf.py

@@ -437,13 +437,14 @@ def to_pysat(self, flatten_twod=True,
         index = None
         for varname, df in cdata.items():
             if varname not in ('Epoch', 'DATE'):
-                if type(df) == pds.Series:
+                if isinstance(df, pds.Series):
                     data[varname] = df
 
                     # CDF data Series are saved using a mix of Range and
                     # Datetime Indexes. This requires that the user specify
                     # the desired index when creating a DataFrame
-                    if type(df.index) == pds.DatetimeIndex and index is None:
+                    if isinstance(df.index,
+                                  pds.DatetimeIndex) and index is None:
                         index = df.index
 
         if index is None:

pysatNASA/instruments/methods/cdaweb.py

@@ -964,9 +964,12 @@ def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None,
     elif start == stop:
         stop = start + dt.timedelta(days=1)
 
-    if isinstance(start, pds._libs.tslibs.timestamps.Timestamp):
-        start = start.tz_localize('utc')
-        stop = stop.tz_localize('utc')
+    # Ensure that valid date types are used.
+    start = pysat.utils.time.filter_datetime_input(start)
+    stop = pysat.utils.time.filter_datetime_input(stop)
+
+    # cdasws needs a time for the stop date.
+    stop += dt.timedelta(seconds=86399)
 
     og_files = cdas.get_original_files(dataset=dataset, start=start, end=stop)
 

pysatNASA/instruments/timed_guvi.py

@@ -89,8 +89,11 @@
 _test_dates = {iid: {tag: dt.datetime(2005, 6, 28) for tag in inst_ids[iid]}
                for iid in inst_ids.keys()}
 _test_load_opt = {iid: {tag: {'combine_times': True}
-                        for tag in inst_ids[iid]} for iid in ['high_res',
-                                                              'low_res']}
+                        for tag in inst_ids[iid]}
+                  for iid in ['high_res', 'low_res']}
+# TODO(#218): Remove when compliant with multi-day load tests
+_new_tests = {iid: {tag: False for tag in inst_ids[iid]}
+              for iid in ['high_res', 'low_res']}
 _clean_warn = {inst_id: {tag: mm_nasa.clean_warnings
                          for tag in inst_ids[inst_id]}
                for inst_id in inst_ids.keys()}