Merge pull request #217 from pysat/bug/de2_vefi

BUG: de2 vefi load

CHANGELOG.md

@@ -4,6 +4,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 
 ## [0.0.6] - 2023-XX-XX
 * New Instruments
+  * DE2 VEFIMAGB - electric and magnetic field on the same cadence
   * MAVEN mag
   * MAVEN SEP
   * MAVEN in situ key parameters
@@ -18,6 +19,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 * Documentation
   * Added example of how to export data for archival
   * Updated documentation refs
+* Deprecations
+  * Deprecated '' tag for de2_vefi module, support moved to de2_vefimagb
 * Maintenance
   * Implemented unit tests for cleaning warnings
   * Use pip install for readthedocs

docs/supported_instruments.rst

@@ -97,6 +97,14 @@ DE2 VEFI
 .. automodule:: pysatNASA.instruments.de2_vefi
    :members:
 
+.. _de2_vefi:
+
+DE2 VEFIMAGB
+------------
+
+.. automodule:: pysatNASA.instruments.de2_vefimagb
+   :members:
+
 .. _de2_wats:
 
 DE2 WATS

pysatNASA/instruments/__init__.py

@@ -7,8 +7,8 @@
 from pysatNASA.instruments import methods  # noqa F401
 
 __all__ = ['ace_epam_l2', 'ace_mag_l2', 'ace_sis_l2', 'ace_swepam_l2',
-           'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_fpi',
-           'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_vefi', 'de2_wats',
+           'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_fpi', 'de2_lang',
+           'de2_nacs', 'de2_rpa', 'de2_vefi', 'de2_vefimagb', 'de2_wats',
            'dmsp_ssusi', 'formosat1_ivm',
            'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti',
            'igs_gps', 'iss_fpmu', 'jpl_gps', 'maven_insitu_kp',

pysatNASA/instruments/de2_vefi.py

@@ -1,5 +1,9 @@
 """Module for the DE2 VEFI instrument.
 
+.. deprecated:: 0.0.6
+    The '' tag is deprecated. This data set is replaced by the de2_vefimagb
+    instrument. The '' tag will be removed in 0.1.0+ to reduce redundancy.
+
 From CDAWeb (adpated):
 This directory gathers data for the VEFI instrument that flew on the DE 2
 spacecraft which was launched on 3 August 1981 into an elliptical orbit with
@@ -28,9 +32,9 @@
 name
     'vefi'
 tag
-    'dca' or 'ac'
+    '', 'dca', 'ac'
 inst_id
-    None Supported
+    none supported
 
 
 Warnings
@@ -42,7 +46,9 @@
 
 import datetime as dt
 import functools
+import warnings
 
+import pysat
 from pysat.instruments.methods import general as mm_gen
 from pysatNASA.instruments.methods import cdaweb as cdw
 from pysatNASA.instruments.methods import de2 as mm_de2
@@ -67,8 +73,18 @@
 # ----------------------------------------------------------------------------
 # Instrument methods
 
-# Use standard init routine
-init = functools.partial(mm_nasa.init, module=mm_de2, name=name)
+def init(self):
+    """Initialize the Instrument object with instrument specific values."""
+
+    if self.tag == '':
+        warnings.warn(" ".join(["The '' tag for `de2_vefi` has been",
+                                "deprecated and will be removed in 0.1.0+."]),
+                      DeprecationWarning, stacklevel=2)
+
+    mm_nasa.init(self, module=mm_de2, name=self.name)
+
+    return
+
 
 # Use default clean
 clean = mm_nasa.clean
@@ -89,10 +105,61 @@
 list_files = functools.partial(mm_gen.list_files,
                                supported_tags=supported_tags)
 
+
 # Set the load routine
-# Forcing use of cdflib as default since pysatCDF has a known issue with vefi
-# data.  See pysat/pysatCDF#48
-load = functools.partial(cdw.load, use_cdflib=True)
+def load(fnames, tag='', inst_id='', **kwargs):
+    """Load DE2 VEFI data.
+
+    This routine is called as needed by pysat. It is not intended
+    for direct user interaction.
+
+    Parameters
+    ----------
+    fnames : array-like
+        Iterable of filename strings, full path, to data files to be loaded.
+        This input is nominally provided by pysat itself.
+    tag : str
+        Tag name used to identify particular data set to be loaded.
+        This input is nominally provided by pysat itself. (default='')
+    inst_id : str
+        Instrument ID used to identify particular data set to be loaded.
+        This input is nominally provided by pysat itself. (default='')
+
+    Returns
+    -------
+    data : pds.DataFrame
+        A pandas DataFrame with data prepared for the `pysat.Instrument`.
+    meta : pysat.Meta
+        Metadata formatted for a pysat.Instrument object.
+
+    Note
+    ----
+    Several variables relating to time stored in different formats are dropped.
+    These are redundant and complicate the load procedure.
+
+    """
+
+    if tag == '':
+        # Warn user that e-field data is dropped.
+        estr = 'E-field data dropped'
+        pysat.logger.warn(estr)
+
+        # Drop E-field data
+        if 'use_cdflib' in kwargs.keys():
+            kwargs.pop('use_cdflib')
+        data, meta = cdw.load_xarray(fnames, tag=tag, inst_id=inst_id,
+                                     epoch_name='mtimeEpoch',
+                                     drop_dims='vtimeEpoch', **kwargs)
+        if hasattr(data, 'to_pandas'):
+            data = data.to_pandas()
+        else:
+            # xarray 0.16 support required for operational server
+            data = data.to_dataframe()
+    else:
+        data, meta = cdw.load_pandas(fnames, tag=tag, inst_id=inst_id, **kwargs)
+
+    return data, meta
+
 
 # Set the download routine
 download_tags = {'': {'': 'DE2_62MS_VEFIMAGB',

pysatNASA/instruments/de2_vefimagb.py

@@ -0,0 +1,168 @@
+"""Module for the DE2 VEFI instrument.
+
+From CDAWeb (adpated):
+This directory gathers data for the VEFI and Magnetometer instruments that flew
+on the DE 2 spacecraft which was launched on 3 August 1981 into an elliptical
+orbit with an altitude range of 300 km to 1000 km and re-entered the atmosphere
+on 19 February 1983.
+
+
+References
+----------
+Maynard, N. C., E. A. Bielecki, H. G. Burdick, Instrumentation for vector
+electric field measurements from DE-B, Space Sci. Instrum., 5, 523, 1981.
+
+Properties
+----------
+platform
+    'de2'
+name
+    'vefimagb'
+tag
+    'e', 'b'
+inst_id
+    none supported
+
+Note
+----
+Electric and Magnetic fields have the same cadence, but different time indices.
+Currently loads one index per instrument. Files kept in the same directory to
+prevent duplication of downloads.
+
+Examples
+--------
+::
+
+    import datetime as dt
+    import pysat
+
+    # Set electric field instrument
+    vefi = pysat.Instrument(platform='de2', name='vefimagb', tag='e')
+    vefi.download(dt.datetime(1983, 1, 1))
+
+    # Set magnetic field instrument
+    mag = pysat.Instrument(platform='de2', name='vefimagb', tag='b')
+
+    # Both instruments can be loaded from the same download
+    vefi.load(1983, 1)
+    mag.load(1983, 1)
+
+
+"""
+
+import datetime as dt
+import functools
+import os
+
+from pysat.instruments.methods import general as mm_gen
+from pysatNASA.instruments.methods import cdaweb as cdw
+from pysatNASA.instruments.methods import de2 as mm_de2
+from pysatNASA.instruments.methods import general as mm_nasa
+
+# ----------------------------------------------------------------------------
+# Instrument attributes
+
+platform = 'de2'
+name = 'vefimagb'
+tags = {'e': '62 ms cadence Electric Field data',
+        'b': '62 ms cadence Magnetometer data'}
+inst_ids = {'': [tag for tag in tags.keys()]}
+
+# Because both data products are stored in one file, tag not used
+directory_format = os.path.join('{platform}', '{name}', '{inst_id}')
+
+# ----------------------------------------------------------------------------
+# Instrument test attributes
+
+_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}}
+
+
+# ----------------------------------------------------------------------------
+# Instrument methods
+
+# Use standard init routine
+init = functools.partial(mm_nasa.init, module=mm_de2, name='vefi')
+
+# No cleaning, use standard warning function instead
+clean = mm_nasa.clean_warn
+
+# ----------------------------------------------------------------------------
+# Instrument functions
+#
+# Use the default CDAWeb and pysat methods
+
+# Set the list_files routine
+datestr = '{year:04d}{month:02d}{day:02d}_v{version:02d}'
+fname = 'de2_62ms_vefimagb_{datestr:s}.cdf'
+supported_tags = {'': {tag: fname.format(datestr=datestr)
+                       for tag in tags.keys()}}
+list_files = functools.partial(mm_gen.list_files,
+                               supported_tags=supported_tags)
+
+
+# Set the load routine
+def load(fnames, tag='', inst_id='', **kwargs):
+    """Load DE2 VEFI data.
+
+    This routine is called as needed by pysat. It is not intended
+    for direct user interaction.
+
+    Parameters
+    ----------
+    fnames : array-like
+        Iterable of filename strings, full path, to data files to be loaded.
+        This input is nominally provided by pysat itself.
+    tag : str
+        Tag name used to identify particular data set to be loaded.
+        This input is nominally provided by pysat itself. (default='')
+    inst_id : str
+        Instrument ID used to identify particular data set to be loaded.
+        This input is nominally provided by pysat itself. (default='')
+    kwargs : dict
+        Additional kwargs that may be supplied to the instrument during the
+        course of unit tests. Not implemented for this instrument.
+
+    Returns
+    -------
+    data : pds.DataFrame
+        A pandas DataFrame with data prepared for the `pysat.Instrument`.
+    meta : pysat.Meta
+        Metadata formatted for a pysat.Instrument object.
+
+    Note
+    ----
+    Several variables relating to time stored in different formats are dropped.
+    These are redundant and complicate the load procedure.
+
+    """
+
+    # Select which epoch to use, drop the rest.
+    if tag == 'b':
+        epoch_name = 'mtimeEpoch'
+        drop_dims = 'vtimeEpoch'
+    elif tag == 'e':
+        epoch_name = 'vtimeEpoch'
+        drop_dims = 'mtimeEpoch'
+
+    # Load and drop appropriate data.
+    data, meta = cdw.load_xarray(fnames, tag=tag, inst_id=inst_id,
+                                 epoch_name=epoch_name,
+                                 drop_dims=drop_dims)
+    # Convert to pandas.
+    if hasattr(data, 'to_pandas'):
+        data = data.to_pandas()
+    else:
+        # xarray 0.16 support required for operational server
+        data = data.to_dataframe()
+
+    return data, meta
+
+
+# Set the download routine
+download_tags = {'': {'e': 'DE2_62MS_VEFIMAGB',
+                      'b': 'DE2_62MS_VEFIMAGB'}}
+download = functools.partial(cdw.cdas_download, supported_tags=download_tags)
+
+# Set the list_remote_files routine
+list_remote_files = functools.partial(cdw.cdas_list_remote_files,
+                                      supported_tags=download_tags)

pysatNASA/instruments/methods/cdaweb.py

@@ -35,9 +35,11 @@
     from cdflib import cdf_to_xarray
 
 try:
+    # Use pysatCDF as default for pandas data sets
     import pysatCDF
     auto_CDF = pysatCDF.CDF
 except ImportError:
+    # Use cdflib as default for pandas data sets
     auto_CDF = libCDF
 
 

pysatNASA/tests/test_instruments.py

@@ -33,13 +33,17 @@
 instruments = clslib.InstLibTests.initialize_test_package(
     clslib.InstLibTests, inst_loc=pysatNASA.instruments)
 
+# Additional tests required for pandas instruments if pysatCDF installed
 # Create a new list of instruments with the option of forcing cdflib
 instruments['cdf'] = []
+# Create list of pandas instruments where this is not needed
+skip_cdf_list = ['de2_vefimagb']
+
 for inst in instruments['download']:
     fname = inst['inst_module'].supported_tags[inst['inst_id']][inst['tag']]
     if '.cdf' in fname:
         temp_inst, _ = clslib.initialize_test_inst_and_date(inst)
-        if temp_inst.pandas_format:
+        if temp_inst.pandas_format and temp_inst.name not in skip_cdf_list:
             instruments['cdf'].append(inst)
 
 
@@ -142,7 +146,8 @@ def teardown_method(self):
 
         return
 
-    @pytest.mark.parametrize("inst_module,tag", [('jpl_gps', 'roti')])
+    @pytest.mark.parametrize("inst_module,tag", [('jpl_gps', 'roti'),
+                                                 ('de2_vefi', '')])
     def test_deprecated_instruments(self, inst_module, tag):
         """Check that instantiating old instruments raises a DeprecationWarning.
 
@@ -160,9 +165,7 @@ def test_deprecated_instruments(self, inst_module, tag):
                                                  inst_module),
                              tag=tag, use_header=True)
 
-        warn_msgs = [" ".join(["The instrument module",
-                               "`{:}`".format(inst_module),
-                               "has been deprecated and will be removed",
+        warn_msgs = [" ".join(["has been deprecated and will be removed",
                                "in 0.1.0+."])]
 
         # Ensure the minimum number of warnings were raised.