diff --git a/.readthedocs.yml b/.readthedocs.yml index 23fa5ade..d0bd564d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -19,4 +19,7 @@ sphinx: # Optionally declare the Python requirements required to build your docs python: install: - - requirements: docs/requirements.txt + - method: pip + path: . + extra_requirements: + - doc diff --git a/CHANGELOG.md b/CHANGELOG.md index 48cf0e3a..055c0e6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,17 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [0.X.X] - 2023-XX-XX * Bug Fixes * Allow graceful failure with no files in jhuapl load functions + * New window needs to be integer for calculate_imf_steadiness * Enhancements * Added custom `concat_data` method to TIMED-GUVI data * Added cleaning to TIMED-GUVI SDR imaging data +* Documentation + * Added example of how to export data for archival * Maintenance * Implemented unit tests for cleaning warnings + * Use pip install for readthedocs + * Moved references and acknowledgements to methods files + * Added tests for OMNI HRO routines ## [0.0.5] - 2023-06-27 * New Instruments diff --git a/README.md b/README.md index 9c96b121..86d45ad5 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Python 3.6+. | numpy | | | | pandas | | | | requests | | | +| scipy>=1.4.0 | | | | xarray | | | ## PyPi Installation diff --git a/docs/archival.rst b/docs/archival.rst new file mode 100644 index 00000000..ad1b1d5e --- /dev/null +++ b/docs/archival.rst @@ -0,0 +1,69 @@ +Building data files for archival at NASA SPDF +============================================= + +The codes and routines at :py:mod:`pysatNASA` are designed for end-users of NASA data +products. However, pysat in general has also been used to build operational +instruments for generating archival data to be uploaded to the Space Physics +Data Facility (SPDF) at NASA. + +In general, such instruments should include separate naming conventions. An +example of this is the REACH data, where netCDF4 files are generated for +archival purposes as part of the :py:mod:`ops_reach` package, but can be accessed by +the end user through :py:mod:`pysatNASA`. + +In general, a :py:class:`pysat.Instrument` object can be constructed for any +dataset. Full instructions and conventions can be found +`here `_. In the +case of the REACH data, the operational code reads in a series of csv files and +updates the metadata according to user specifications. Once the file is loaded, +it can be exported to a netCDF4 file via pysat. In the simplest case, this is + +:: + + reach = pysat.Instrument(inst_module=aero_reach, tag='l1b', inst_id=inst_id) + pysat.utils.io.inst_to_netcdf(reach, 'output_file.nc', epoch_name='Epoch') + + +However, there are additional options when translating pysat metadata to SPDF +preferred formats. An example of this is + +:: + + # Use meta translation table to include SPDF preferred format. + # Note that multiple names are output for compliance with pysat. + # Using the most generalized form for labels for future compatibility. + meta_dict = {reach.meta.labels.min_val: ['VALIDMIN'], + reach.meta.labels.max_val: ['VALIDMAX'], + reach.meta.labels.units: ['UNITS'], + reach.meta.labels.name: ['CATDESC', 'LABLAXIS', 'FIELDNAM'], + reach.meta.labels.notes: ['VAR_NOTES'], + reach.meta.labels.fill_val: ['_FillValue'], + 'Depend_0': ['DEPEND_0'], + 'Format': ['FORMAT'], + 'Monoton': ['MONOTON'], + 'Var_Type': ['VAR_TYPE']} + + pysat.utils.io.inst_to_netcdf(reach, 'output_file.nc', epoch_name='Epoch', + meta_translation=meta_dict, + export_pysat_info=False) + + +In this case, note that the pysat 'name' label is output to three different +metadata values required by the ITSP standards. Additionally, the +:py:attr:`export_pysat_info` option is set to false here. This drops several +internal :py:mod:`pysat` metadata values before writing to file. + +A full guide to SPDF metadata standards can be found +`here `_. + +Other best practices for archival include adding the operational software version +to the metadata header before writing. The pysat version will be automatically +written to the metadata. + +:: + + reach.meta.header.Software_version = ops_reach.__version__ + + +A full example script to generate output files can be found at +https://github.com/jklenzing/ops_reach/blob/main/scripts/netcdf_gen.py diff --git a/docs/index.rst b/docs/index.rst index cda7cf4f..926c4f76 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,7 @@ CDAWeb interface. supported_constellations.rst examples.rst develop_guide.rst + archival.rst migration_guide.rst history.rst diff --git a/docs/installation.rst b/docs/installation.rst index 7b5858b7..2722700f 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -25,6 +25,7 @@ Python 3.6+ and pysat 3.1.0+. numpy pandas requests + scipy>=1.4.0 xarray ================== ================= diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 3d71207e..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -extras_require -m2r2 -numpydoc -pysat -pysatNASA diff --git a/pyproject.toml b/pyproject.toml index b35d7492..a2f0fc20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dependencies = [ "pandas", "pysat >= 3.1", "requests", + "scipy >= 1.4", "xarray" ] @@ -66,7 +67,7 @@ doc = [ "m2r2", "numpydoc", "sphinx", - "sphinx_rtd_theme" + "sphinx_rtd_theme >= 1.2.2" ] [project.urls] diff --git a/pysatNASA/instruments/formosat1_ivm.py b/pysatNASA/instruments/formosat1_ivm.py index 3ac177d9..ceccc95c 100644 --- a/pysatNASA/instruments/formosat1_ivm.py +++ b/pysatNASA/instruments/formosat1_ivm.py @@ -26,9 +26,9 @@ import functools from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import formosat as mm_formosat from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- @@ -47,31 +47,7 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - self.acknowledgements = ' '.join(('Data provided through NASA CDAWeb', - 'Key Parameters - Shin-Yi Su', - '(Institute of Space Science,', - 'National Central University,', - 'Taiwan, R.O.C.)')) - self.references = ' '.join(('Yeh, H.C., S.‐Y. Su, Y.C. Yeh, J.M. Wu,', - 'R. A. Heelis, and B. J. Holt, Scientific', - 'mission of the IPEI payload on board', - 'ROCSAT‐1, Terr. Atmos. Ocean. Sci., 9,', - 'suppl., 1999a.\n', - 'Yeh, H.C., S.‐Y. Su, R.A. Heelis, and', - 'J.M. Wu, The ROCSAT‐1 IPEI preliminary', - 'results, Vertical ion drift statistics,', - 'Terr. Atmos. Ocean. Sci., 10, 805,', - '1999b.')) - logger.info(self.acknowledgements) - - return +init = functools.partial(mm_nasa.init, module=mm_formosat, name=name) # Use default clean diff --git a/pysatNASA/instruments/iss_fpmu.py b/pysatNASA/instruments/iss_fpmu.py index c98f907c..d65272e3 100644 --- a/pysatNASA/instruments/iss_fpmu.py +++ b/pysatNASA/instruments/iss_fpmu.py @@ -28,10 +28,10 @@ import functools from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import iss as mm_iss # ---------------------------------------------------------------------------- # Instrument attributes @@ -49,34 +49,7 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - ackn_str = ' '.join(('Data provided through NASA CDAWeb. Contact', - 'Rob.Suggs@nasa.gov for support and use.')) - logger.info(ackn_str) - self.acknowledgements = ackn_str - self.references = ' '.join(('V. N. Coffey et al., "Validation of the', - 'Plasma Densities and Temperatures From', - 'the ISS Floating Potential Measurement', - 'Unit," in IEEE Transactions on Plasma', - 'Science, vol. 36, no. 5, pp. 2301-2308,', - 'Oct. 2008,', - 'doi: 10.1109/TPS.2008.2004271.\n', - 'A. Barjatya, C.M. Swenson, D.C.', - 'Thompson, and K.H. Wright Jr., Data', - 'analysis of the Floating Potential', - 'Measurement Unit aboard the', - 'International Space Station, Rev. Sci.', - 'Instrum. 80, 041301 (2009),', - 'https://doi.org/10.1063/1.3116085')) - - return +init = functools.partial(mm_nasa.init, module=mm_iss, name=name) # Use default clean diff --git a/pysatNASA/instruments/methods/__init__.py b/pysatNASA/instruments/methods/__init__.py index 8542fcbc..4459d353 100644 --- a/pysatNASA/instruments/methods/__init__.py +++ b/pysatNASA/instruments/methods/__init__.py @@ -5,9 +5,11 @@ from pysatNASA.instruments.methods import cnofs # noqa F401 from pysatNASA.instruments.methods import de2 # noqa F401 from pysatNASA.instruments.methods import dmsp # noqa F401 +from pysatNASA.instruments.methods import formosat # noqa F401 from pysatNASA.instruments.methods import general # noqa F401 from pysatNASA.instruments.methods import gps # noqa F401 from pysatNASA.instruments.methods import icon # noqa F401 +from pysatNASA.instruments.methods import iss # noqa F401 from pysatNASA.instruments.methods import jhuapl # noqa F401 from pysatNASA.instruments.methods import omni # noqa F401 from pysatNASA.instruments.methods import ses14 # noqa F401 diff --git a/pysatNASA/instruments/methods/formosat.py b/pysatNASA/instruments/methods/formosat.py new file mode 100644 index 00000000..41ee0f22 --- /dev/null +++ b/pysatNASA/instruments/methods/formosat.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for C/NOFS data.""" + +ackn_str = ' '.join(('Data provided through NASA CDAWeb Key Parameters -', + 'Shin-Yi Su (Institute of Space Science, National Central', + 'University, Taiwan, R.O.C.)')) +refs = {'ivm': ' '.join(('Yeh, H.C., S.‐Y. Su, Y.C. Yeh, J.M. Wu, R. A.', + 'Heelis, and B. J. Holt, Scientific mission of the', + 'IPEI payload on board ROCSAT‐1, Terr. Atmos. Ocean.', + 'Sci., 9, suppl., 1999a.\n', + 'Yeh, H.C., S.‐Y. Su, R.A. Heelis, and J.M. Wu, The', + 'ROCSAT‐1 IPEI preliminary results, Vertical ion', + 'drift statistics, Terr. Atmos. Ocean. Sci., 10, 805,', + '1999b.'))} diff --git a/pysatNASA/instruments/methods/iss.py b/pysatNASA/instruments/methods/iss.py new file mode 100644 index 00000000..24ed41ce --- /dev/null +++ b/pysatNASA/instruments/methods/iss.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for C/NOFS data.""" + +ackn_str = ' '.join(("R.M. Suggs, S.L. Koontz, NASA Johnson Space Center", + "Contact Rob Suggs for support and use.", + "Rob.Suggs@nasa.gov. Please acknowledge the data", + "providers and CDAWeb when using these data.")) + +refs = {'fpmu': ' '.join(('V. N. Coffey et al., "Validation of the Plasma', + 'Densities and Temperatures From the ISS Floating', + 'Potential Measurement Unit," in IEEE Transactions', + 'on Plasma Science, vol. 36, no. 5, pp. 2301-2308,', + 'Oct. 2008, doi: 10.1109/TPS.2008.2004271.', + '\n', + 'A. Barjatya, C.M. Swenson, D.C. Thompson, and K.H.', + 'Wright Jr., Data analysis of the Floating Potential', + 'Measurement Unit aboard the International Space', + 'Station, Rev. Sci. Instrum. 80, 041301 (2009),', + 'https://doi.org/10.1063/1.3116085', + '\n', + 'Debchoudhury, S., Barjatya, A., Minow, J. I.,', + 'Coffey, V. N., & Chandler, M. O. (2021).', + 'Observations and validation of plasma density,', + 'temperature, and O+ abundance from a Langmuir', + 'probe onboard the International Space Station.', + 'Journal of Geophysical Research: Space', + 'Physics, 126, e2021JA029393.', + 'https://doi.org/10.1029/2021JA029393')) + } diff --git a/pysatNASA/instruments/methods/omni.py b/pysatNASA/instruments/methods/omni.py index 2844b0cb..178e91cb 100644 --- a/pysatNASA/instruments/methods/omni.py +++ b/pysatNASA/instruments/methods/omni.py @@ -9,6 +9,17 @@ import pysat +ackn_str = ' '.join(('For full acknowledgement info, please see:', + 'https://omniweb.gsfc.nasa.gov/html/citing.html')) + +refs = {'hro': ' '.join(('J.H. King and N.E. Papitashvili, Solar', + 'wind spatial scales in and comparisons', + 'of hourly Wind and ACE plasma and', + 'magnetic field data, J. Geophys. Res.,', + 'Vol. 110, No. A2, A02209,', + '10.1029/2004JA010649.'))} + + def time_shift_to_magnetic_poles(inst): """Shift OMNI times to intersection with the magnetic pole. @@ -113,7 +124,7 @@ def calculate_imf_steadiness(inst, steady_window=15, min_window_frac=0.75, sample_rate = int(rates[inst.tag]) max_wnum = np.floor(steady_window / sample_rate) if max_wnum != steady_window / sample_rate: - steady_window = max_wnum * sample_rate + steady_window = int(max_wnum * sample_rate) pysat.logger.warning(" ".join(("sample rate is not a factor of the", "statistical window"))) pysat.logger.warning(" ".join(("new statistical window is", @@ -130,24 +141,11 @@ def calculate_imf_steadiness(inst, steady_window=15, min_window_frac=0.75, # Calculate the running circular standard deviation of the clock angle circ_kwargs = {'high': 360.0, 'low': 0.0, 'nan_policy': 'omit'} - try: - ca_std = \ - inst['clock_angle'].rolling(min_periods=min_wnum, - window=steady_window, - center=True).apply(stats.circstd, - kwargs=circ_kwargs, - raw=True) - except TypeError: - pysat.logger.warn(' '.join(['To automatically remove NaNs from the', - 'calculation, please upgrade to scipy 1.4', - 'or newer.'])) - circ_kwargs.pop('nan_policy') - ca_std = \ - inst['clock_angle'].rolling(min_periods=min_wnum, - window=steady_window, - center=True).apply(stats.circstd, - kwargs=circ_kwargs, - raw=True) + ca_std = inst['clock_angle'].rolling(min_periods=min_wnum, + window=steady_window, + center=True).apply(stats.circstd, + kwargs=circ_kwargs, + raw=True) inst['clock_angle_std'] = pds.Series(ca_std, index=inst.data.index) # Determine how long the clock angle and IMF magnitude are steady diff --git a/pysatNASA/instruments/omni_hro.py b/pysatNASA/instruments/omni_hro.py index d7a205cf..b37f2954 100644 --- a/pysatNASA/instruments/omni_hro.py +++ b/pysatNASA/instruments/omni_hro.py @@ -44,10 +44,10 @@ import pandas as pds import warnings -import pysat from pysat.instruments.methods import general as mm_gen from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa from pysatNASA.instruments.methods import omni as mm_omni # ---------------------------------------------------------------------------- @@ -68,25 +68,7 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - ackn_str = ''.join(('For full acknowledgement info, please see: ', - 'https://omniweb.gsfc.nasa.gov/html/citing.html')) - self.acknowledgements = ackn_str - self.references = ' '.join(('J.H. King and N.E. Papitashvili, Solar', - 'wind spatial scales in and comparisons', - 'of hourly Wind and ACE plasma and', - 'magnetic field data, J. Geophys. Res.,', - 'Vol. 110, No. A2, A02209,', - '10.1029/2004JA010649.')) - pysat.logger.info(ackn_str) - return +init = functools.partial(mm_nasa.init, module=mm_omni, name=name) def clean(self): diff --git a/pysatNASA/tests/test_omni_hro.py b/pysatNASA/tests/test_omni_hro.py index b3032024..1bd95fc4 100644 --- a/pysatNASA/tests/test_omni_hro.py +++ b/pysatNASA/tests/test_omni_hro.py @@ -1,6 +1,7 @@ """Unit tests for OMNI HRO special functions.""" import datetime as dt +import logging import numpy as np import warnings @@ -145,7 +146,7 @@ def test_clock_angle_std(self): return def test_dayside_recon(self): - """Test the IMF steadiness standard deviation calculation.""" + """Test the dayside reconnection calculation.""" # Run the clock angle and steadiness routines omni.calculate_clock_angle(self.test_inst) @@ -163,6 +164,36 @@ def test_dayside_recon(self): assert np.all(test_diff < 1.0e-6) return + def test_time_shift_to_magnetic_poles(self): + """Test the time shift routines.""" + + # Choose values to result in 1 hour shift + self.test_inst['Vx'] = 6371.2 + self.test_inst['BSN_x'] = 3600.0 + + old_index = self.test_inst.index.copy() + omni.time_shift_to_magnetic_poles(self.test_inst) + + # Check shifted index + assert (old_index[0] - self.test_inst.index[0]).seconds == 3600 + # Check new cadence + assert (self.test_inst.index[1] - self.test_inst.index[0]).seconds == 60 + return + + def test_calculate_imf_steadiness_warnings(self, caplog): + """Test imf steadiness routine.""" + + omni.calculate_clock_angle(self.test_inst) + with caplog.at_level(logging.INFO, logger='pysat'): + omni.calculate_imf_steadiness(self.test_inst, steady_window=5.1, + min_window_frac=0.8) + captured = caplog.text + warn_msgs = ["sample rate is not a factor", + "new statistical window"] + for msg in warn_msgs: + assert msg in captured + return + class TestDeprecation(object): """Unit tests for deprecation warnings in `pysat.instrument.omni_hro`."""