Merge branch 'develop' into guvi_imaging_fix

.github/workflows/main.yml

@@ -74,4 +74,17 @@ jobs:
     - name: Publish results to coveralls
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        COVERALLS_PARALLEL: true
       run: coveralls --rcfile=pyproject.toml --service=github
+
+  finish:
+    name: Finish Coverage Analysis
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Coveralls Finished
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        pip install --upgrade coveralls
+        coveralls --service=github --finish

.zenodo.json

@@ -21,7 +21,7 @@
         "orcid": "0000-0002-8191-4765"
       },
       {
-        "affilitation":"University of Colorado at Boulder",
+        "affiliation":"University of Colorado at Boulder",
         "name": "Navarro, Luis",
         "orcid": "0000-0002-6362-6575"
       },
@@ -32,6 +32,11 @@
       {
         "name": "Spence, Carey",
         "orcid": "0000-0001-8340-5625"
+      },
+      {
+        "affiliation": "NASA Postdoctoral Program, Goddard Space Flight Center",
+        "name": "Esman, Teresa",
+        "orcid": "0000-0003-0382-6281"
       }
     ]
 }

CHANGELOG.md

@@ -2,20 +2,33 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/).
 
-## [0.X.X] - 2023-XX-XX
+## [0.0.6] - 2023-XX-XX
+* New Instruments
+  * MAVEN mag
+  * MAVEN SEP
+  * MAVEN in situ key parameters
+  * REACH Dosimeter
+* New Features
+  * Allow files to be unzipped after download
+  * Added custom `concat_data` method to TIMED-GUVI data
+  * Added cleaning to TIMED-GUVI SDR imaging data
 * Bug Fixes
+  * Fix general clean routine to skip transformation matrices
+  * New window needs to be integer for calculate_imf_steadiness
+  * Fixed version import
+  * Fixed a bug when data fails to load for CDF pandas objects
   * Allow graceful failure with no files in jhuapl load functions
   * New window needs to be integer for calculate_imf_steadiness
-* Enhancements
-  * Added custom `concat_data` method to TIMED-GUVI data
-  * Added cleaning to TIMED-GUVI SDR imaging data
 * Documentation
   * Added example of how to export data for archival
+  * Updated documentation refs
 * Maintenance
   * Implemented unit tests for cleaning warnings
   * Use pip install for readthedocs
   * Moved references and acknowledgements to methods files
   * Added tests for OMNI HRO routines
+  * Use standard clean routine for C/NOFS VEFI mag data
+  * Added version cap for sphinx_rtd_theme
 
 ## [0.0.5] - 2023-06-27
 * New Instruments

docs/supported_instruments.rst

@@ -178,6 +178,30 @@ JPL GPS
 .. automodule:: pysatNASA.instruments.jpl_gps
    :members:
 
+.. _maven_insitu_kp:
+
+MAVEN INSITU KP
+---------------
+
+.. automodule:: pysatNASA.instruments.maven_insitu_kp
+   :members:
+
+.. _maven_mag:
+
+MAVEN MAG
+---------
+
+.. automodule:: pysatNASA.instruments.maven_mag
+   :members:
+
+.. _maven_sep:
+
+MAVEN SEP
+---------
+
+.. automodules:: pysatNASA.instruments.maven_sep
+   :members:
+
 .. _omni_hro:
 
 OMNI HRO
@@ -186,6 +210,14 @@ OMNI HRO
 .. automodule:: pysatNASA.instruments.omni_hro
    :members: calculate_clock_angle, calculate_imf_steadiness, time_shift_to_magnetic_poles
 
+.. _reach_dosimeter:
+
+REACH DOSIMETER
+----------
+
+.. automodule:: pysatNASA.instruments.reach_dosimeter
+  :members:
+
 .. _ses14_gold:
 
 SES14 GOLD

pyproject.toml

@@ -53,7 +53,7 @@ dependencies = [
 [project.optional-dependencies]
 pysatcdf = ["pysatCDF"]
 test = [
-  "coveralls < 3.3",
+  "coveralls",
   "flake8",
   "flake8-docstrings",
   "hacking >= 1.0",
@@ -67,7 +67,7 @@ doc = [
   "m2r2",
   "numpydoc",
   "sphinx",
-  "sphinx_rtd_theme >= 1.2.2"
+  "sphinx_rtd_theme >= 1.2.2, < 2.0.0"
 ]
 
 [project.urls]

pysatNASA/__init__.py

@@ -6,15 +6,18 @@
 
 """
 
-import importlib
-import importlib_metadata
+try:
+    from importlib import metadata
+except ImportError:
+    import importlib_metadata as metadata
+
+import os
 
 from pysatNASA import constellations  # noqa F401
 from pysatNASA import instruments  # noqa F401
 
-# set version
-try:
-    __version__ = importlib.metadata.version('pysatNASA')
-except AttributeError:
-    # Python 3.6 requires a different version
-    __version__ = importlib_metadata.version('pysatNASA')
+__version__ = metadata.version('pysatNASA')
+
+# Set directory for test data
+here = os.path.abspath(os.path.dirname(__file__))
+test_data_path = os.path.join(here, 'tests', 'test_data')

pysatNASA/instruments/__init__.py

@@ -11,8 +11,9 @@
            'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_vefi', 'de2_wats',
            'dmsp_ssusi', 'formosat1_ivm',
            'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti',
-           'igs_gps', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold',
-           'timed_guvi', 'timed_saber', 'timed_see']
+           'igs_gps', 'iss_fpmu', 'jpl_gps', 'maven_insitu_kp',
+           'maven_mag', 'maven_sep', 'omni_hro', 'reach_dosimeter',
+           'ses14_gold', 'timed_guvi', 'timed_saber', 'timed_see']
 
 for inst in __all__:
     exec("from pysatNASA.instruments import {x}".format(x=inst))

pysatNASA/instruments/cnofs_vefi.py

@@ -101,9 +101,13 @@ def clean(self):
 
     """
 
-    if (self.clean_level == 'dusty') | (self.clean_level == 'clean'):
-        idx, = np.where(self['B_flag'] == 0)
-        self.data = self[idx, :]
+    mm_nasa.clean(self)
+
+    # Apply legacy clean routine for version 5 data
+    if 'B_flag' in self.variables:
+        if (self.clean_level == 'dusty') | (self.clean_level == 'clean'):
+            idx, = np.where(self['B_flag'] == 0)
+            self.data = self[idx, :]
 
     return
 

pysatNASA/instruments/maven_insitu_kp.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+"""Module for the MAVEN insitu instruments.
+
+Supports the in situ Key Parameter (kp) data from multiple instruments
+onboard the Mars Atmosphere and Volatile Evolution (MAVEN) satellite.
+
+Accesses local data in CDF format.
+Downloads from CDAWeb.
+
+Properties
+----------
+platform
+    'maven'
+name
+    'insitu_kp'
+tag
+    None supported
+inst_id
+    None supported
+
+
+Examples
+--------
+::
+    import pysat
+
+    insitu = pysat.Instrument(platform='maven', name='insitu_kp')
+    insitu.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31))
+    insitu.load(2020, 1, use_header=True)
+
+"""
+
+import datetime as dt
+import functools
+
+from pysat.instruments.methods import general as mm_gen
+from pysatNASA.instruments.methods import cdaweb as cdw
+from pysatNASA.instruments.methods import general as mm_nasa
+from pysatNASA.instruments.methods import maven as mm_mvn
+
+# ----------------------------------------------------------------------------
+# Instrument attributes
+
+platform = 'maven'
+name = 'insitu_kp'
+tags = {'': 'in situ Key Parameter data'}
+inst_ids = {'': ['']}
+
+pandas_format = False
+
+# ----------------------------------------------------------------------------
+# Instrument test attributes
+
+_test_dates = {'': {'': dt.datetime(2020, 1, 1)}}
+
+# ----------------------------------------------------------------------------
+# Instrument methods
+
+# Use standard init routine
+init = functools.partial(mm_nasa.init, module=mm_mvn, name=name)
+
+
+# Use default clean
+clean = functools.partial(mm_nasa.clean,
+                          skip_names=['Rotation_matrix_IAU_MARS_MAVEN_MSO',
+                                      'Rotation_matrix_SPACECRAFT_MAVEN_MSO'])
+
+
+# ----------------------------------------------------------------------------
+# Instrument functions
+#
+# Use the MAVEN and pysat methods
+
+# Set the list_files routine
+fname = ''.join(('mvn_insitu_kp-4sec_{year:04d}{month:02d}{day:02d}_',
+                 'v{version:02d}_r{revision:02d}.cdf'))
+supported_tags = {'': {'': fname}}
+list_files = functools.partial(mm_gen.list_files,
+                               supported_tags=supported_tags)
+# Set the download routine
+basic_tag = {'remote_dir': ''.join(('/pub/data/maven/insitu/kp-4sec/',
+                                    'cdfs/{year:04d}/{month:02d}')),
+             'fname': fname}
+download_tags = {'': {'': basic_tag}}
+download = functools.partial(cdw.download, supported_tags=download_tags)
+
+# Set the list_remote_files routine
+list_remote_files = functools.partial(cdw.list_remote_files,
+                                      supported_tags=download_tags)
+
+
+# Set the load routine
+load = functools.partial(cdw.load, epoch_name='epoch',
+                         pandas_format=pandas_format, use_cdflib=True)

pysatNASA/instruments/maven_mag.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+"""Module for the MAVEN mag instrument.
+
+Supports the Magnetometer (MAG) onboard the Mars Atmosphere
+and Volatile Evolution (MAVEN) satellite.
+Accesses local data in CDF format.
+Downloads from CDAWeb.
+
+Properties
+----------
+platform
+    'maven'
+name
+    'mag'
+tag
+    None supported
+inst_id
+    None supported
+
+Warnings
+--------
+
+- Only supports level-2 sunstate 1 second data.
+
+Examples
+--------
+::
+    import pysat
+
+    mag = pysat.Instrument(platform='maven', name='mag')
+    mag.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31))
+    mag.load(2020, 1, use_header = True)
+
+"""
+
+import datetime as dt
+import functools
+
+from pysat.instruments.methods import general as mm_gen
+from pysatNASA.instruments.methods import cdaweb as cdw
+from pysatNASA.instruments.methods import general as mm_nasa
+from pysatNASA.instruments.methods import maven as mm_mvn
+
+# ----------------------------------------------------------------------------
+# Instrument attributes
+
+platform = 'maven'
+name = 'mag'
+tags = {'': 'Level 2 magnetometer data'}
+inst_ids = {'': ['']}
+
+pandas_format = False
+# ----------------------------------------------------------------------------
+# Instrument test attributes
+
+_test_dates = {'': {'': dt.datetime(2020, 1, 1)}}
+
+# ----------------------------------------------------------------------------
+# Instrument methods
+
+# Use standard init routine
+init = functools.partial(mm_nasa.init, module=mm_mvn, name=name)
+
+
+# Use default clean
+clean = mm_nasa.clean
+
+
+# ----------------------------------------------------------------------------
+# Instrument functions
+#
+# Use the CDAWeb and pysat methods
+
+# Set the list_files routine
+fname = ''.join(('mvn_mag_l2-sunstate-1sec_{year:04d}{month:02d}{day:02d}_',
+                 'v{version:02d}_r{revision:02d}.cdf'))
+supported_tags = {'': {'': fname}}
+list_files = functools.partial(mm_gen.list_files,
+                               supported_tags=supported_tags)
+# Set the download routine
+basic_tag = {'remote_dir': ''.join(('/pub/data/maven/mag/l2/sunstate-1sec',
+                                    '/cdfs/{year:04d}/{month:02d}')),
+             'fname': fname}
+download_tags = {'': {'': basic_tag}}
+download = functools.partial(cdw.download, supported_tags=download_tags)
+
+# Set the list_remote_files routine
+list_remote_files = functools.partial(cdw.list_remote_files,
+                                      supported_tags=download_tags)
+
+# Set the load routine
+load = functools.partial(cdw.load, epoch_name='epoch',
+                         pandas_format=pandas_format, use_cdflib=True)