Merge branch 'develop' into guvi_imaging_fix

.readthedocs.yml

@@ -19,4 +19,7 @@ sphinx:
 # Optionally declare the Python requirements required to build your docs
 python:
    install:
-   - requirements: docs/requirements.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - doc

CHANGELOG.md

@@ -5,11 +5,17 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 ## [0.X.X] - 2023-XX-XX
 * Bug Fixes
   * Allow graceful failure with no files in jhuapl load functions
+  * New window needs to be integer for calculate_imf_steadiness
 * Enhancements
   * Added custom `concat_data` method to TIMED-GUVI data
   * Added cleaning to TIMED-GUVI SDR imaging data
+* Documentation
+  * Added example of how to export data for archival
 * Maintenance
   * Implemented unit tests for cleaning warnings
+  * Use pip install for readthedocs
+  * Moved references and acknowledgements to methods files
+  * Added tests for OMNI HRO routines
 
 ## [0.0.5] - 2023-06-27
 * New Instruments

README.md

@@ -30,6 +30,7 @@ Python 3.6+.
 | numpy            |                   |                  |
 | pandas           |                   |                  |
 | requests         |                   |                  |
+| scipy>=1.4.0     |                   |                  |
 | xarray           |                   |                  |
 
 ## PyPi Installation

docs/archival.rst

@@ -0,0 +1,69 @@
+Building data files for archival at NASA SPDF
+=============================================
+
+The codes and routines at :py:mod:`pysatNASA` are designed for end-users of NASA data
+products. However, pysat in general has also been used to build operational
+instruments for generating archival data to be uploaded to the Space Physics
+Data Facility (SPDF) at NASA.
+
+In general, such instruments should include separate naming conventions. An
+example of this is the REACH data, where netCDF4 files are generated for
+archival purposes as part of the :py:mod:`ops_reach` package, but can be accessed by
+the end user through :py:mod:`pysatNASA`.
+
+In general, a :py:class:`pysat.Instrument` object can be constructed for any
+dataset. Full instructions and conventions can be found
+`here <https://pysat.readthedocs.io/en/latest/new_instrument.html>`_.  In the
+case of the REACH data, the operational code reads in a series of csv files and
+updates the metadata according to user specifications. Once the file is loaded,
+it can be exported to a netCDF4 file via pysat. In the simplest case, this is
+
+::
+
+  reach = pysat.Instrument(inst_module=aero_reach, tag='l1b', inst_id=inst_id)
+  pysat.utils.io.inst_to_netcdf(reach, 'output_file.nc', epoch_name='Epoch')
+
+
+However, there are additional options when translating pysat metadata to SPDF
+preferred formats.  An example of this is
+
+::
+
+  # Use meta translation table to include SPDF preferred format.
+  # Note that multiple names are output for compliance with pysat.
+  # Using the most generalized form for labels for future compatibility.
+  meta_dict = {reach.meta.labels.min_val: ['VALIDMIN'],
+               reach.meta.labels.max_val: ['VALIDMAX'],
+               reach.meta.labels.units: ['UNITS'],
+               reach.meta.labels.name: ['CATDESC', 'LABLAXIS', 'FIELDNAM'],
+               reach.meta.labels.notes: ['VAR_NOTES'],
+               reach.meta.labels.fill_val: ['_FillValue'],
+               'Depend_0': ['DEPEND_0'],
+               'Format': ['FORMAT'],
+               'Monoton': ['MONOTON'],
+               'Var_Type': ['VAR_TYPE']}
+
+  pysat.utils.io.inst_to_netcdf(reach, 'output_file.nc', epoch_name='Epoch',
+                                meta_translation=meta_dict,
+                                export_pysat_info=False)
+
+
+In this case, note that the pysat 'name' label is output to three different
+metadata values required by the ITSP standards. Additionally, the
+:py:attr:`export_pysat_info` option is set to false here. This drops several
+internal :py:mod:`pysat` metadata values before writing to file.
+
+A full guide to SPDF metadata standards can be found 
+`here <https://spdf.gsfc.nasa.gov/istp_guide/istp_guide.html>`_.
+
+Other best practices for archival include adding the operational software version
+to the metadata header before writing. The pysat version will be automatically
+written to the metadata.
+
+::
+
+  reach.meta.header.Software_version = ops_reach.__version__
+
+
+A full example script to generate output files can be found at
+https://github.com/jklenzing/ops_reach/blob/main/scripts/netcdf_gen.py

docs/index.rst

@@ -18,6 +18,7 @@ CDAWeb interface.
    supported_constellations.rst
    examples.rst
    develop_guide.rst
+   archival.rst
    migration_guide.rst
    history.rst
 

docs/installation.rst

@@ -25,6 +25,7 @@ Python 3.6+ and pysat 3.1.0+.
   numpy
   pandas
   requests
+  scipy>=1.4.0
   xarray
  ================== =================
 

pyproject.toml

@@ -46,6 +46,7 @@ dependencies = [
   "pandas",
   "pysat >= 3.1",
   "requests",
+  "scipy >= 1.4",
   "xarray"
 ]
 
@@ -66,7 +67,7 @@ doc = [
   "m2r2",
   "numpydoc",
   "sphinx",
-  "sphinx_rtd_theme"
+  "sphinx_rtd_theme >= 1.2.2"
 ]
 
 [project.urls]

pysatNASA/instruments/formosat1_ivm.py

@@ -26,9 +26,9 @@
 import functools
 
 from pysat.instruments.methods import general as mm_gen
-from pysat import logger
 
 from pysatNASA.instruments.methods import cdaweb as cdw
+from pysatNASA.instruments.methods import formosat as mm_formosat
 from pysatNASA.instruments.methods import general as mm_nasa
 
 # ----------------------------------------------------------------------------
@@ -47,31 +47,7 @@
 # ----------------------------------------------------------------------------
 # Instrument methods
 
-
-def init(self):
-    """Initialize the Instrument object with instrument specific values.
-
-    Runs once upon instantiation.
-
-    """
-    self.acknowledgements = ' '.join(('Data provided through NASA CDAWeb',
-                                      'Key Parameters - Shin-Yi Su',
-                                      '(Institute of Space Science,',
-                                      'National Central University,',
-                                      'Taiwan, R.O.C.)'))
-    self.references = ' '.join(('Yeh, H.C., S.‐Y. Su, Y.C. Yeh, J.M. Wu,',
-                                'R. A. Heelis, and B. J. Holt, Scientific',
-                                'mission of the IPEI payload on board',
-                                'ROCSAT‐1, Terr. Atmos. Ocean. Sci., 9,',
-                                'suppl., 1999a.\n',
-                                'Yeh, H.C., S.‐Y. Su, R.A. Heelis, and',
-                                'J.M. Wu, The ROCSAT‐1 IPEI preliminary',
-                                'results, Vertical ion drift statistics,',
-                                'Terr. Atmos. Ocean. Sci., 10, 805,',
-                                '1999b.'))
-    logger.info(self.acknowledgements)
-
-    return
+init = functools.partial(mm_nasa.init, module=mm_formosat, name=name)
 
 
 # Use default clean

pysatNASA/instruments/iss_fpmu.py

@@ -28,10 +28,10 @@
 import functools
 
 from pysat.instruments.methods import general as mm_gen
-from pysat import logger
 
 from pysatNASA.instruments.methods import cdaweb as cdw
 from pysatNASA.instruments.methods import general as mm_nasa
+from pysatNASA.instruments.methods import iss as mm_iss
 
 # ----------------------------------------------------------------------------
 # Instrument attributes
@@ -49,34 +49,7 @@
 # ----------------------------------------------------------------------------
 # Instrument methods
 
-
-def init(self):
-    """Initialize the Instrument object with instrument specific values.
-
-    Runs once upon instantiation.
-
-    """
-
-    ackn_str = ' '.join(('Data provided through NASA CDAWeb.  Contact',
-                         '[email protected] for support and use.'))
-    logger.info(ackn_str)
-    self.acknowledgements = ackn_str
-    self.references = ' '.join(('V. N. Coffey et al., "Validation of the',
-                                'Plasma Densities and Temperatures From',
-                                'the ISS Floating Potential Measurement',
-                                'Unit," in IEEE Transactions on Plasma',
-                                'Science, vol. 36, no. 5, pp. 2301-2308,',
-                                'Oct. 2008,',
-                                'doi: 10.1109/TPS.2008.2004271.\n',
-                                'A. Barjatya, C.M. Swenson, D.C.',
-                                'Thompson, and K.H. Wright Jr., Data',
-                                'analysis of the Floating Potential',
-                                'Measurement Unit aboard the',
-                                'International Space Station, Rev. Sci.',
-                                'Instrum. 80, 041301 (2009),',
-                                'https://doi.org/10.1063/1.3116085'))
-
-    return
+init = functools.partial(mm_nasa.init, module=mm_iss, name=name)
 
 
 # Use default clean

pysatNASA/instruments/methods/__init__.py

@@ -5,9 +5,11 @@
 from pysatNASA.instruments.methods import cnofs  # noqa F401
 from pysatNASA.instruments.methods import de2  # noqa F401
 from pysatNASA.instruments.methods import dmsp  # noqa F401
+from pysatNASA.instruments.methods import formosat  # noqa F401
 from pysatNASA.instruments.methods import general  # noqa F401
 from pysatNASA.instruments.methods import gps  # noqa F401
 from pysatNASA.instruments.methods import icon  # noqa F401
+from pysatNASA.instruments.methods import iss  # noqa F401
 from pysatNASA.instruments.methods import jhuapl  # noqa F401
 from pysatNASA.instruments.methods import omni  # noqa F401
 from pysatNASA.instruments.methods import ses14  # noqa F401

pysatNASA/instruments/methods/formosat.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+"""Provides non-instrument specific routines for C/NOFS data."""
+
+ackn_str = ' '.join(('Data provided through NASA CDAWeb Key Parameters -',
+                     'Shin-Yi Su (Institute of Space Science, National Central',
+                     'University, Taiwan, R.O.C.)'))
+refs = {'ivm': ' '.join(('Yeh, H.C., S.‐Y. Su, Y.C. Yeh, J.M. Wu, R. A.',
+                         'Heelis, and B. J. Holt, Scientific mission of the',
+                         'IPEI payload on board ROCSAT‐1, Terr. Atmos. Ocean.',
+                         'Sci., 9, suppl., 1999a.\n',
+                         'Yeh, H.C., S.‐Y. Su, R.A. Heelis, and J.M. Wu, The',
+                         'ROCSAT‐1 IPEI preliminary results, Vertical ion',
+                         'drift statistics, Terr. Atmos. Ocean. Sci., 10, 805,',
+                         '1999b.'))}

pysatNASA/instruments/methods/iss.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""Provides non-instrument specific routines for C/NOFS data."""
+
+ackn_str = ' '.join(("R.M. Suggs, S.L. Koontz, NASA Johnson Space Center",
+                     "Contact Rob Suggs for support and use.",
+                     "[email protected]. Please acknowledge the data",
+                     "providers and CDAWeb when using these data."))
+
+refs = {'fpmu': ' '.join(('V. N. Coffey et al., "Validation of the Plasma',
+                          'Densities and Temperatures From the ISS Floating',
+                          'Potential Measurement Unit," in IEEE Transactions',
+                          'on Plasma Science, vol. 36, no. 5, pp. 2301-2308,',
+                          'Oct. 2008, doi: 10.1109/TPS.2008.2004271.',
+                          '\n',
+                          'A. Barjatya, C.M. Swenson, D.C. Thompson, and K.H.',
+                          'Wright Jr., Data analysis of the Floating Potential',
+                          'Measurement Unit aboard the International Space',
+                          'Station, Rev. Sci. Instrum. 80, 041301 (2009),',
+                          'https://doi.org/10.1063/1.3116085',
+                          '\n',
+                          'Debchoudhury, S., Barjatya, A., Minow, J. I.,',
+                          'Coffey, V. N., & Chandler, M. O. (2021).',
+                          'Observations and validation of plasma density,',
+                          'temperature, and O+ abundance from a Langmuir',
+                          'probe onboard the International Space Station.',
+                          'Journal of Geophysical Research: Space',
+                          'Physics, 126, e2021JA029393.',
+                          'https://doi.org/10.1029/2021JA029393'))
+        }

pysatNASA/instruments/methods/omni.py

@@ -9,6 +9,17 @@
 import pysat
 
 
+ackn_str = ' '.join(('For full acknowledgement info, please see:',
+                     'https://omniweb.gsfc.nasa.gov/html/citing.html'))
+
+refs = {'hro': ' '.join(('J.H. King and N.E. Papitashvili, Solar',
+                         'wind spatial scales in and comparisons',
+                         'of hourly Wind and ACE plasma and',
+                         'magnetic field data, J. Geophys. Res.,',
+                         'Vol. 110, No. A2, A02209,',
+                         '10.1029/2004JA010649.'))}
+
+
 def time_shift_to_magnetic_poles(inst):
     """Shift OMNI times to intersection with the magnetic pole.
 
@@ -113,7 +124,7 @@ def calculate_imf_steadiness(inst, steady_window=15, min_window_frac=0.75,
     sample_rate = int(rates[inst.tag])
     max_wnum = np.floor(steady_window / sample_rate)
     if max_wnum != steady_window / sample_rate:
-        steady_window = max_wnum * sample_rate
+        steady_window = int(max_wnum * sample_rate)
         pysat.logger.warning(" ".join(("sample rate is not a factor of the",
                                        "statistical window")))
         pysat.logger.warning(" ".join(("new statistical window is",
@@ -130,24 +141,11 @@ def calculate_imf_steadiness(inst, steady_window=15, min_window_frac=0.75,
 
     # Calculate the running circular standard deviation of the clock angle
     circ_kwargs = {'high': 360.0, 'low': 0.0, 'nan_policy': 'omit'}
-    try:
-        ca_std = \
-            inst['clock_angle'].rolling(min_periods=min_wnum,
-                                        window=steady_window,
-                                        center=True).apply(stats.circstd,
-                                                           kwargs=circ_kwargs,
-                                                           raw=True)
-    except TypeError:
-        pysat.logger.warn(' '.join(['To automatically remove NaNs from the',
-                                    'calculation, please upgrade to scipy 1.4',
-                                    'or newer.']))
-        circ_kwargs.pop('nan_policy')
-        ca_std = \
-            inst['clock_angle'].rolling(min_periods=min_wnum,
-                                        window=steady_window,
-                                        center=True).apply(stats.circstd,
-                                                           kwargs=circ_kwargs,
-                                                           raw=True)
+    ca_std = inst['clock_angle'].rolling(min_periods=min_wnum,
+                                         window=steady_window,
+                                         center=True).apply(stats.circstd,
+                                                            kwargs=circ_kwargs,
+                                                            raw=True)
     inst['clock_angle_std'] = pds.Series(ca_std, index=inst.data.index)
 
     # Determine how long the clock angle and IMF magnitude are steady

pysatNASA/instruments/omni_hro.py

@@ -44,10 +44,10 @@
 import pandas as pds
 import warnings
 
-import pysat
 from pysat.instruments.methods import general as mm_gen
 
 from pysatNASA.instruments.methods import cdaweb as cdw
+from pysatNASA.instruments.methods import general as mm_nasa
 from pysatNASA.instruments.methods import omni as mm_omni
 
 # ----------------------------------------------------------------------------
@@ -68,25 +68,7 @@
 # ----------------------------------------------------------------------------
 # Instrument methods
 
-
-def init(self):
-    """Initialize the Instrument object with instrument specific values.
-
-    Runs once upon instantiation.
-
-    """
-
-    ackn_str = ''.join(('For full acknowledgement info, please see: ',
-                        'https://omniweb.gsfc.nasa.gov/html/citing.html'))
-    self.acknowledgements = ackn_str
-    self.references = ' '.join(('J.H. King and N.E. Papitashvili, Solar',
-                                'wind spatial scales in and comparisons',
-                                'of hourly Wind and ACE plasma and',
-                                'magnetic field data, J. Geophys. Res.,',
-                                'Vol. 110, No. A2, A02209,',
-                                '10.1029/2004JA010649.'))
-    pysat.logger.info(ackn_str)
-    return
+init = functools.partial(mm_nasa.init, module=mm_omni, name=name)
 
 
 def clean(self):