diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 61125636..aeccb359 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,4 +74,17 @@ jobs: - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_PARALLEL: true run: coveralls --rcfile=pyproject.toml --service=github + + finish: + name: Finish Coverage Analysis + needs: build + runs-on: ubuntu-latest + steps: + - name: Coveralls Finished + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip install --upgrade coveralls + coveralls --service=github --finish diff --git a/.zenodo.json b/.zenodo.json index 1b5b3db1..6d54b11f 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -21,7 +21,7 @@ "orcid": "0000-0002-8191-4765" }, { - "affilitation":"University of Colorado at Boulder", + "affiliation":"University of Colorado at Boulder", "name": "Navarro, Luis", "orcid": "0000-0002-6362-6575" }, @@ -32,6 +32,11 @@ { "name": "Spence, Carey", "orcid": "0000-0001-8340-5625" + }, + { + "affiliation": "NASA Postdoctoral Program, Goddard Space Flight Center", + "name": "Esman, Teresa", + "orcid": "0000-0003-0382-6281" } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 055c0e6c..44410a33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,20 +2,33 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -## [0.X.X] - 2023-XX-XX +## [0.0.6] - 2023-XX-XX +* New Instruments + * MAVEN mag + * MAVEN SEP + * MAVEN in situ key parameters + * REACH Dosimeter +* New Features + * Allow files to be unzipped after download + * Added custom `concat_data` method to TIMED-GUVI data + * Added cleaning to TIMED-GUVI SDR imaging data * Bug Fixes + * Fix general clean routine to skip transformation matrices + * New window needs to be integer for calculate_imf_steadiness + * Fixed version import + * Fixed a bug when data fails to load for CDF pandas objects * Allow graceful failure with no files in jhuapl load functions * New window needs to be integer for calculate_imf_steadiness -* Enhancements - * Added custom `concat_data` method to TIMED-GUVI data - * Added cleaning to TIMED-GUVI SDR imaging data * Documentation * Added example of how to export data for archival + * Updated documentation refs * Maintenance * Implemented unit tests for cleaning warnings * Use pip install for readthedocs * Moved references and acknowledgements to methods files * Added tests for OMNI HRO routines + * Use standard clean routine for C/NOFS VEFI mag data + * Added version cap for sphinx_rtd_theme ## [0.0.5] - 2023-06-27 * New Instruments diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 0980414e..d01cf43a 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -178,6 +178,30 @@ JPL GPS .. automodule:: pysatNASA.instruments.jpl_gps :members: +.. _maven_insitu_kp: + +MAVEN INSITU KP +--------------- + +.. automodule:: pysatNASA.instruments.maven_insitu_kp + :members: + +.. _maven_mag: + +MAVEN MAG +--------- + +.. automodule:: pysatNASA.instruments.maven_mag + :members: + +.. _maven_sep: + +MAVEN SEP +--------- + +.. automodules:: pysatNASA.instruments.maven_sep + :members: + .. _omni_hro: OMNI HRO @@ -186,6 +210,14 @@ OMNI HRO .. automodule:: pysatNASA.instruments.omni_hro :members: calculate_clock_angle, calculate_imf_steadiness, time_shift_to_magnetic_poles +.. _reach_dosimeter: + +REACH DOSIMETER +---------- + +.. automodule:: pysatNASA.instruments.reach_dosimeter + :members: + .. _ses14_gold: SES14 GOLD diff --git a/pyproject.toml b/pyproject.toml index a2f0fc20..ae7cbd36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ [project.optional-dependencies] pysatcdf = ["pysatCDF"] test = [ - "coveralls < 3.3", + "coveralls", "flake8", "flake8-docstrings", "hacking >= 1.0", @@ -67,7 +67,7 @@ doc = [ "m2r2", "numpydoc", "sphinx", - "sphinx_rtd_theme >= 1.2.2" + "sphinx_rtd_theme >= 1.2.2, < 2.0.0" ] [project.urls] diff --git a/pysatNASA/__init__.py b/pysatNASA/__init__.py index ea9f8ff2..650c6162 100644 --- a/pysatNASA/__init__.py +++ b/pysatNASA/__init__.py @@ -6,15 +6,18 @@ """ -import importlib -import importlib_metadata +try: + from importlib import metadata +except ImportError: + import importlib_metadata as metadata + +import os from pysatNASA import constellations # noqa F401 from pysatNASA import instruments # noqa F401 -# set version -try: - __version__ = importlib.metadata.version('pysatNASA') -except AttributeError: - # Python 3.6 requires a different version - __version__ = importlib_metadata.version('pysatNASA') +__version__ = metadata.version('pysatNASA') + +# Set directory for test data +here = os.path.abspath(os.path.dirname(__file__)) +test_data_path = os.path.join(here, 'tests', 'test_data') diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index 5407b031..dfeb8391 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -11,8 +11,9 @@ 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_vefi', 'de2_wats', 'dmsp_ssusi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', - 'igs_gps', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', - 'timed_guvi', 'timed_saber', 'timed_see'] + 'igs_gps', 'iss_fpmu', 'jpl_gps', 'maven_insitu_kp', + 'maven_mag', 'maven_sep', 'omni_hro', 'reach_dosimeter', + 'ses14_gold', 'timed_guvi', 'timed_saber', 'timed_see'] for inst in __all__: exec("from pysatNASA.instruments import {x}".format(x=inst)) diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py index f2e4bf24..334b7565 100644 --- a/pysatNASA/instruments/cnofs_vefi.py +++ b/pysatNASA/instruments/cnofs_vefi.py @@ -101,9 +101,13 @@ def clean(self): """ - if (self.clean_level == 'dusty') | (self.clean_level == 'clean'): - idx, = np.where(self['B_flag'] == 0) - self.data = self[idx, :] + mm_nasa.clean(self) + + # Apply legacy clean routine for version 5 data + if 'B_flag' in self.variables: + if (self.clean_level == 'dusty') | (self.clean_level == 'clean'): + idx, = np.where(self['B_flag'] == 0) + self.data = self[idx, :] return diff --git a/pysatNASA/instruments/maven_insitu_kp.py b/pysatNASA/instruments/maven_insitu_kp.py new file mode 100644 index 00000000..660a3ad3 --- /dev/null +++ b/pysatNASA/instruments/maven_insitu_kp.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +"""Module for the MAVEN insitu instruments. + +Supports the in situ Key Parameter (kp) data from multiple instruments +onboard the Mars Atmosphere and Volatile Evolution (MAVEN) satellite. + +Accesses local data in CDF format. +Downloads from CDAWeb. + +Properties +---------- +platform + 'maven' +name + 'insitu_kp' +tag + None supported +inst_id + None supported + + +Examples +-------- +:: + import pysat + + insitu = pysat.Instrument(platform='maven', name='insitu_kp') + insitu.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) + insitu.load(2020, 1, use_header=True) + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import maven as mm_mvn + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'maven' +name = 'insitu_kp' +tags = {'': 'in situ Key Parameter data'} +inst_ids = {'': ['']} + +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'': dt.datetime(2020, 1, 1)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_mvn, name=name) + + +# Use default clean +clean = functools.partial(mm_nasa.clean, + skip_names=['Rotation_matrix_IAU_MARS_MAVEN_MSO', + 'Rotation_matrix_SPACECRAFT_MAVEN_MSO']) + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the MAVEN and pysat methods + +# Set the list_files routine +fname = ''.join(('mvn_insitu_kp-4sec_{year:04d}{month:02d}{day:02d}_', + 'v{version:02d}_r{revision:02d}.cdf')) +supported_tags = {'': {'': fname}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) +# Set the download routine +basic_tag = {'remote_dir': ''.join(('/pub/data/maven/insitu/kp-4sec/', + 'cdfs/{year:04d}/{month:02d}')), + 'fname': fname} +download_tags = {'': {'': basic_tag}} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) + + +# Set the load routine +load = functools.partial(cdw.load, epoch_name='epoch', + pandas_format=pandas_format, use_cdflib=True) diff --git a/pysatNASA/instruments/maven_mag.py b/pysatNASA/instruments/maven_mag.py new file mode 100644 index 00000000..c1125bd7 --- /dev/null +++ b/pysatNASA/instruments/maven_mag.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +"""Module for the MAVEN mag instrument. + +Supports the Magnetometer (MAG) onboard the Mars Atmosphere +and Volatile Evolution (MAVEN) satellite. +Accesses local data in CDF format. +Downloads from CDAWeb. + +Properties +---------- +platform + 'maven' +name + 'mag' +tag + None supported +inst_id + None supported + +Warnings +-------- + +- Only supports level-2 sunstate 1 second data. + +Examples +-------- +:: + import pysat + + mag = pysat.Instrument(platform='maven', name='mag') + mag.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) + mag.load(2020, 1, use_header = True) + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import maven as mm_mvn + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'maven' +name = 'mag' +tags = {'': 'Level 2 magnetometer data'} +inst_ids = {'': ['']} + +pandas_format = False +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'': dt.datetime(2020, 1, 1)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_mvn, name=name) + + +# Use default clean +clean = mm_nasa.clean + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(('mvn_mag_l2-sunstate-1sec_{year:04d}{month:02d}{day:02d}_', + 'v{version:02d}_r{revision:02d}.cdf')) +supported_tags = {'': {'': fname}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) +# Set the download routine +basic_tag = {'remote_dir': ''.join(('/pub/data/maven/mag/l2/sunstate-1sec', + '/cdfs/{year:04d}/{month:02d}')), + 'fname': fname} +download_tags = {'': {'': basic_tag}} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) + +# Set the load routine +load = functools.partial(cdw.load, epoch_name='epoch', + pandas_format=pandas_format, use_cdflib=True) diff --git a/pysatNASA/instruments/maven_sep.py b/pysatNASA/instruments/maven_sep.py new file mode 100644 index 00000000..b1fe80b2 --- /dev/null +++ b/pysatNASA/instruments/maven_sep.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +"""Module for the MAVEN sep instrument. + +Supports the Solar Energetic Particle (SEP) data from +onboard the Mars Atmosphere and Volatile Evolution (MAVEN) satellite. + +Accesses local data in CDF format. +Downloads from CDAWeb. + +Properties +---------- +platform + 'maven' +name + 'sep' +tag + None supported +inst_id + ['s1', 's2'] + +Examples +-------- +:: + import pysat + + insitu = pysat.Instrument(platform='maven', name='sep', inst_id='s1') + insitu.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) + insitu.load(2020, 1, use_header=True) + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import maven as mm_mvn + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'maven' +name = 'sep' +tags = {'': 'Level 2 Solar Energetic Particle data'} +inst_ids = {'s1': [''], 's2': ['']} + +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {'': dt.datetime(2020, 1, 1)} for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_mvn, name=name) + + +# Use default clean +clean = mm_nasa.clean + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the MAVEN and pysat methods + +# Set the list_files routine +fname = ''.join(('mvn_sep_l2_s1-cal-svy-full_{year:04d}{month:02d}{day:02d}_', + 'v{version:02d}_r{revision:02d}.cdf')) + +fname2 = ''.join(('mvn_sep_l2_s2-cal-svy-full_{year:04d}{month:02d}{day:02d}_', + 'v{version:02d}_r{revision:02d}.cdf')) + +supported_tags = {'s1': {'': fname}, + 's2': {'': fname2}} + +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the download routine +basic_tag = {'remote_dir': ''.join(('/pub/data/maven/sep/l2/s1-cal-svy-full', + '/{year:04d}/{month:02d}')), + 'fname': fname} + +basic_tag2 = {'remote_dir': ''.join(('/pub/data/maven/sep/l2/s2-cal-svy-full', + '/{year:04d}/{month:02d}')), + 'fname': fname2} + +download_tags = {'s1': {'': basic_tag}, + 's2': {'': basic_tag2}} + +# Set the download routine +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) + +# Set the load routine +load = functools.partial(cdw.load, epoch_name='epoch', + pandas_format=pandas_format, use_cdflib=True) diff --git a/pysatNASA/instruments/methods/__init__.py b/pysatNASA/instruments/methods/__init__.py index 4459d353..a1910352 100644 --- a/pysatNASA/instruments/methods/__init__.py +++ b/pysatNASA/instruments/methods/__init__.py @@ -12,5 +12,6 @@ from pysatNASA.instruments.methods import iss # noqa F401 from pysatNASA.instruments.methods import jhuapl # noqa F401 from pysatNASA.instruments.methods import omni # noqa F401 +from pysatNASA.instruments.methods import reach # noqa F401 from pysatNASA.instruments.methods import ses14 # noqa F401 from pysatNASA.instruments.methods import timed # noqa F401 diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 235090eb..d3a63e84 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -12,8 +12,10 @@ import os import pandas as pds import requests +import tempfile from time import sleep import xarray as xr +import zipfile from bs4 import BeautifulSoup from cdasws import CdasWs @@ -218,7 +220,7 @@ def load_pandas(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), # Load data from any files provided if len(fnames) <= 0: - return pds.DataFrame(None), None + return pds.DataFrame(None), pysat.Meta() else: if use_cdflib is not None: if use_cdflib: @@ -268,6 +270,8 @@ def load_pandas(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), # Combine individual files together if len(ldata) > 0: data = pds.concat(ldata) + else: + data, meta = pds.DataFrame(None), pysat.Meta() return data, meta @@ -362,7 +366,7 @@ def load_xarray(fnames, tag='', inst_id='', # Load data from any files provided if len(fnames) <= 0: - return xr.Dataset() + return xr.Dataset(), pysat.Meta() else: # Using cdflib wrapper to load the CDF and format data and # metadata for pysat using some assumptions. Depending upon your needs @@ -386,9 +390,11 @@ def load_xarray(fnames, tag='', inst_id='', ldata.append(temp_data) # Combine individual files together, concat along epoch - if len(ldata) > 0: + if len(ldata) > 1: data = xr.combine_nested(ldata, epoch_name, combine_attrs='override') + else: + data = ldata[0] all_vars = io.xarray_all_vars(data) @@ -469,7 +475,6 @@ def load_xarray(fnames, tag='', inst_id='', return data, meta -# TODO(#103): Include support to unzip / untar files after download. def download(date_array, data_path, tag='', inst_id='', supported_tags=None, remote_url='https://cdaweb.gsfc.nasa.gov'): """Download NASA CDAWeb data. @@ -512,6 +517,7 @@ def download(date_array, data_path, tag='', inst_id='', supported_tags=None, """ + # Get information about remote data product location inst_dict = try_inst_dict(inst_id, tag, supported_tags) # Naming scheme for files on the CDAWeb server @@ -524,6 +530,13 @@ def download(date_array, data_path, tag='', inst_id='', supported_tags=None, start=date_array[0], stop=date_array[-1]) + # Create temproary directory if files need to be unzipped. + if 'zip_method' in inst_dict.keys(): + zip_method = inst_dict['zip_method'] + temp_dir = tempfile.TemporaryDirectory() + else: + zip_method = None + # Download only requested files that exist remotely for date, fname in remote_files.items(): # Format files for specific dates and download location @@ -546,18 +559,19 @@ def download(date_array, data_path, tag='', inst_id='', supported_tags=None, formatted_remote_dir.strip('/'), fname)) - saved_local_fname = os.path.join(data_path, fname) - # Perform download logger.info(' '.join(('Attempting to download file for', date.strftime('%d %B %Y')))) try: with requests.get(remote_path) as req: if req.status_code != 404: - with open(saved_local_fname, 'wb') as open_f: - open_f.write(req.content) - logger.info('Successfully downloaded {:}.'.format( - saved_local_fname)) + if zip_method: + get_file(req.content, data_path, fname, + temp_path=temp_dir.name, zip_method=zip_method) + else: + get_file(req.content, data_path, fname) + logger.info(''.join(('Successfully downloaded ', + fname, '.'))) else: logger.info(' '.join(('File not available for', date.strftime('%d %B %Y')))) @@ -566,6 +580,52 @@ def download(date_array, data_path, tag='', inst_id='', supported_tags=None, date.strftime('%d %B %Y')))) # Pause to avoid excessive pings to server sleep(0.2) + + if zip_method: + # Cleanup temporary directory + temp_dir.cleanup() + + return + + +def get_file(remote_file, data_path, fname, temp_path=None, zip_method=None): + """Retrieve a file, unzipping if necessary. + + Parameters + ---------- + remote_file : file content + File content retireved via requests. + data_path : str + Path to pysat archival directory. + fname : str + Name of file on the remote server. + temp_path : str + Path to temporary directory. (Default=None) + zip_method : str + The method used to zip the file. Supports 'zip' and None. + If None, downloads files directly. (default=None) + + """ + + if zip_method: + # Use a temporary location. + dl_fname = os.path.join(temp_path, fname) + else: + # Use the pysat data directory. + dl_fname = os.path.join(data_path, fname) + + # Download the file to desired destination. + with open(dl_fname, 'wb') as open_f: + open_f.write(remote_file) + + # Unzip and move the files from the temporary directory. + if zip_method == 'zip': + with zipfile.ZipFile(dl_fname, 'r') as open_zip: + open_zip.extractall(data_path) + + elif zip_method is not None: + logger.warning('{:} is not a recognized zip method'.format(zip_method)) + return diff --git a/pysatNASA/instruments/methods/formosat.py b/pysatNASA/instruments/methods/formosat.py index 41ee0f22..840acd12 100644 --- a/pysatNASA/instruments/methods/formosat.py +++ b/pysatNASA/instruments/methods/formosat.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Provides non-instrument specific routines for C/NOFS data.""" +"""Provides non-instrument specific routines for FORMOSAT data.""" ackn_str = ' '.join(('Data provided through NASA CDAWeb Key Parameters -', 'Shin-Yi Su (Institute of Space Science, National Central', diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index 1d075c46..16326279 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -51,24 +51,36 @@ def init(self, module, name): return -def clean(self): +def clean(self, skip_names=None): """Clean data to the specified level. + Parameters + ---------- + skip_names : list of str + List of names to skip for cleaning. (default=None) + Note ---- Basic cleaning to replace fill values with NaN """ - # Get a list of coords for the data + # Get a list of coords for the data. These should be skipped for cleaning. if self.pandas_format: - coords = [self.data.index.name] + skip_key = [self.data.index.name] else: - coords = [key for key in self.data.coords.keys()] + skip_key = [key for key in self.data.coords.keys()] + + if skip_names: + # Add additional variable names to skip + for key in skip_names: + skip_key.append(key) for key in self.variables: + # Check for symmetric dims + # Indicates transformation matrix, xarray cannot broadcast # Skip over the coordinates when cleaning - if key not in coords: + if key not in skip_key: fill = self.meta[key, self.meta.labels.fill_val] # Replace fill with nan diff --git a/pysatNASA/instruments/methods/igs.py b/pysatNASA/instruments/methods/igs.py index 63ad4a5d..c050e447 100644 --- a/pysatNASA/instruments/methods/igs.py +++ b/pysatNASA/instruments/methods/igs.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Provides non-instrument specific routines for JPL ROTI data.""" +"""Provides non-instrument specific routines for IGS GPS data.""" ackn_str = ' '.join(("The GPS Total Electron Content (TEC) data", "produced by the International Global Navigation", diff --git a/pysatNASA/instruments/methods/iss.py b/pysatNASA/instruments/methods/iss.py index 24ed41ce..dd7553e4 100644 --- a/pysatNASA/instruments/methods/iss.py +++ b/pysatNASA/instruments/methods/iss.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Provides non-instrument specific routines for C/NOFS data.""" +"""Provides non-instrument specific routines for ISS data.""" ackn_str = ' '.join(("R.M. Suggs, S.L. Koontz, NASA Johnson Space Center", "Contact Rob Suggs for support and use.", diff --git a/pysatNASA/instruments/methods/maven.py b/pysatNASA/instruments/methods/maven.py new file mode 100644 index 00000000..d6a5aa6f --- /dev/null +++ b/pysatNASA/instruments/methods/maven.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for MAVEN data. + +Created on Thu Jul 13 11:21:01 2023 + +@author: tesman +""" + +ackn_str = ''.join(('Jakosky, B.M., Lin, R.P., Grebowsky, J.M. et al.', + ' The Mars Atmosphere and Volatile Evolution', + '(MAVEN) Mission. Space Sci Rev 195, 3–48 (2015).', + ' https://doi.org/10.1007/s11214-015-0139-x')) +refs = {'mission': ''.join(('Jakosky, B.M., Lin, R.P., Grebowsky, J.M. et', + ' al. The Mars Atmosphere and Volatile Evolution', + '(MAVEN) Mission. Space Sci Rev', + ' 195, 3–48 (2015).', + ' https://doi.org/10.1007/s11214-015-0139-x')), + 'insitu_kp': '', + 'mag': ''.join(('Connerney, J., and P. Lawton, MAVEN MAG', + ' PDS Archive SIS - This document ', + 'describes the format and content of the MAVEN', + ' Magnetometer (MAG) Planetary Data System ', + '(PDS) data archive. ', + 'It includes descriptions of the Standard', + 'Data Products and associated metadata, ', + 'and the volume archive format,', + 'content, and generation pipeline. ', + 'Connerney, J.E.P.; Espley, J.; Lawton, P.;', + ' Murphy, S.; Odom, J.; Oliversen, R.;', + 'and Sheppard, D., The MAVEN Magnetic Field', + ' Investigation, Space Sci Rev,', + 'Vol 195, Iss 1-4, pp.257-291, 2015. ', + 'doi:10.1007/s11214-015-0169-4')), + 'sep': ''.join(('Larson, D.E., Lillis, R.J., Lee, C.O. et al.', + 'The MAVEN Solar Energetic Particle Investigation.', + ' Space Sci Rev 195, 153–172 (2015).', + ' https://doi.org/10.1007/s11214-015-0218-z'))} diff --git a/pysatNASA/instruments/methods/reach.py b/pysatNASA/instruments/methods/reach.py new file mode 100644 index 00000000..cc9b11e1 --- /dev/null +++ b/pysatNASA/instruments/methods/reach.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for REACH data.""" + +ackn_str = "Please load a file for full acknowledgments." + +refs = {'dosimeter': + '\n'.join((' '.join(("Guild, T., O'Brien, T.P., Boyd,", + "A.J., Mazur, J.E., Halford, A.J., (2019)", + "Intra-calibration of REACH Dosimeters,", + "AEROSPACE REPORT NO. TOR-2019-02361")), + ' '.join(("Halford, A.J., Guild, T., O'Brien, T.P., Boyd,", + "A.J., Mazur, J.E. (2019)", + "REACH Maps and Indices for UDL: Version 1,", + "AEROSPACE REPORT NO. TOR-2019-02650")), + ' '.join(("Guild, T., O'Brien, T.P., Boyd, A.J., Mazur,", + "J.E. (2021)", + "REACH Maps and Indices for UDL: Version 1", + "AEROSPACE REPORT NO. TOR-2021-01076"))))} diff --git a/pysatNASA/instruments/methods/timed.py b/pysatNASA/instruments/methods/timed.py index e6d4e2fa..726dc800 100644 --- a/pysatNASA/instruments/methods/timed.py +++ b/pysatNASA/instruments/methods/timed.py @@ -23,7 +23,18 @@ 'and Instrumentation for Atmospheric and Space ', 'Research III, (20 October 1999); ', 'doi:10.1117/12.366380']), - 'saber': '', + 'saber': ' '.join(['Esplin, R., Mlynczak, M. G., Russell, J., Gordley,', + 'L., & The SABER Team. (2023). Sounding of the', + 'Atmosphere using Broadband Emission Radiometry', + '(SABER): Instrument and science measurement', + 'description. Earth and Space Science, 10,', + 'e2023EA002999.', + 'https://doi.org/10.1029/2023EA002999.\n', + 'Overview of the SABER experiment and preliminary', + 'calibration results (1999). J. M. Russell III, M.', + 'G. Mlynczak, L. L. Gordley, J. J. Tansock, Jr.,', + 'and R. W. Esplin, Proc. SPIE 3756, 277,', + 'DOI:10.1117/12.366382']), 'see': ' '.join(('Woods, T. N., Eparvier, F. G., Bailey,', 'S. M., Chamberlin, P. C., Lean, J.,', 'Rottman, G. J., Solomon, S. C., Tobiska,', diff --git a/pysatNASA/instruments/reach_dosimeter.py b/pysatNASA/instruments/reach_dosimeter.py new file mode 100644 index 00000000..c7cebdcb --- /dev/null +++ b/pysatNASA/instruments/reach_dosimeter.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +"""The REACH dosimeter instrument. + +Supports the dosimeter instrument on the Responsive Environmental Assessment +Commercially Hosted (REACH) mission. + +The Responsive Environmental Assessment Commercially Hosted (REACH) +constellation is collection of 32 small sensors hosted on six orbital planes of +the Iridium-Next space vehicles in low earth orbit. Each sensor contains two +micro-dosimeters sensitive to the passage of charged particles from the Earth's +radiation belts. There are six distinct dosimeter types spread among the 64 +individual sensors, which are unique in shielding and electronic threshold. +When taken together, this effectively enables a high time-cadence measurement +of protons and electrons in six integral energy channels over the entire globe. + +Properties +---------- +platform + 'reach' +name + 'dosimeter' +tag + None Supported +inst_id + '101', '102', '105', '108', '113', '114', '115', '116', '133', '134', '135', + '136', '137', '138', '139', '140', '148', '149', '162', '163', '164', '165', + '166', '169', '170', '171', '172', '173', '175', '176', '180', '181' + + +""" + +import datetime as dt +import functools +import numpy as np + +from pysat._meta import MetaHeader +from pysat.instruments.methods import general as mm_gen +from pysat.utils.io import load_netcdf + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import reach as mm_reach + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'reach' +name = 'dosimeter' +tags = {'': 'Dosimeter data from the REACH mission'} +iids = ['101', '102', '105', '108', '113', '114', '115', '116', '133', '134', + '135', '136', '137', '138', '139', '140', '148', '149', '162', '163', + '164', '165', '166', '169', '170', '171', '172', '173', '175', '176', + '180', '181'] +inst_ids = {iid: [tag for tag in tags.keys()] for iid in iids} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {iid: {tag: dt.datetime(2019, 12, 1) for tag in tags.keys()} + for iid in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_reach, name=name) + +# Use default clean +clean = mm_nasa.clean + + +def preprocess(self): + """Update acknowledgement with info from file.""" + + self.acknowledgements = self.meta.header.Acknowledgement + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +datestr = '{year:04d}{month:02d}{day:02d}' +fname = 'reach-vid-{inst_id}_dosimeter-l1c_{datestr}_v{{version:01d}}.nc' +supported_tags = {iid: {'': fname.format(inst_id=iid, datestr=datestr)} + for iid in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + + +def load(fnames, tag=None, inst_id=None): + """Load REACH data into `pandas.DataFrame` and `pysat.Meta` objects. + + This routine is called as needed by pysat. It is not intended + for direct user interaction. + + Parameters + ---------- + fnames : array-like + iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + inst_id : str + Satellite ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + + Returns + ------- + data : pds.DataFrame + A pandas DataFrame with data prepared for the pysat.Instrument + meta : pysat.Meta + Metadata formatted for a pysat.Instrument object. + + Note + ---- + Any additional keyword arguments passed to pysat.Instrument + upon instantiation are passed along to this routine. + + Examples + -------- + :: + + inst = pysat.Instrument('reach', 'dosimeter', inst_id='101', tag='') + inst.load(2020, 1) + + """ + + # Use standard netcdf interface + labels = {'units': ('UNITS', str), 'name': ('LONG_NAME', str), + 'notes': ('VAR_NOTES', str), 'desc': ('CATDESC', str), + 'min_val': ('VALIDMIN', (int, float)), + 'max_val': ('VALIDMAX', (int, float)), + 'fill_val': ('_FillValue', (int, float))} + data, meta = load_netcdf(fnames, epoch_name='Epoch', + meta_kwargs={'labels': labels}) + + # Update header variables + header = meta.header.to_dict() + new_header = {} + for key in header.keys(): + new_key = key.replace('-', '_to_') + new_header[new_key] = header[key] + if np.isnan(new_header['Notes']): + new_header['Notes'] = '' + + meta.header = MetaHeader(new_header) + + return data, meta + + +# Support download routine +download_tags = {iid: {'': 'REACH-VID-{iid}_DOSIMETER-L1C'.format(iid=iid)} + for iid in inst_ids.keys()} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Support listing files currently on CDAWeb +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/tests/test_data/empty.cdf b/pysatNASA/tests/test_data/empty.cdf new file mode 100644 index 00000000..a14b4157 Binary files /dev/null and b/pysatNASA/tests/test_data/empty.cdf differ diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 8b8ffb13..dea3dacb 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -18,7 +18,6 @@ # Make sure to import your instrument library here import pysatNASA - try: import pysatCDF # noqa: F401 # If this successfully imports, tests need to be run with both pysatCDF diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index dd7870e6..640c2b46 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -1,8 +1,11 @@ """Unit tests for the cdaweb instrument methods.""" import datetime as dt +import logging +import os import pandas as pds import requests +import tempfile import pytest @@ -42,12 +45,69 @@ def test_remote_file_list_connection_error_append(self): assert str(excinfo.value).find('pysat -> Request potentially') > 0 return - def test_load_with_empty_file_list(self): - """Test that empty data is returned if no files are requested.""" + @pytest.mark.parametrize("pandas_format", [True, False]) + def test_load_with_empty_file_list(self, pandas_format): + """Test that empty data is returned if no files are requested. - data, meta = cdw.load(fnames=[]) + Parameters + ---------- + pandas_format : bool + If True, pandas. If False, xarray + """ + + data, meta = cdw.load(fnames=[], pandas_format=pandas_format) assert len(data) == 0 - assert meta is None + assert meta.empty + return + + def test_bad_load_cdf_warning(self, caplog): + """Test that warning when cdf file does not have expected params.""" + + fpath = os.path.join(pysatNASA.test_data_path, 'empty.cdf') + with caplog.at_level(logging.WARNING, logger='pysat'): + data, meta = cdw.load(fnames=[fpath], pandas_format=True) + + captured = caplog.text + + # Check for appropriate warning + warn_msg = "unable to load" + assert warn_msg in captured + + return + + def test_bad_xarray_kwarg_warning(self, caplog): + """Test that warning is raised when xarray is used outside of cdflib.""" + + with caplog.at_level(logging.WARNING, logger='pysat'): + data, meta = cdw.load(fnames=[], pandas_format=False, + use_cdflib=False) + captured = caplog.text + + # Check for appropriate warning + warn_msg = "`use_cdflib` option is not currently enabled" + assert warn_msg in captured + + return + + def test_bad_zip_warning_get_files(self, caplog): + """Test that warning is raised for unsupported zip method.""" + + # Specifiy small file to get + url = 'https://cdaweb.gsfc.nasa.gov/pub/000_readme.txt' + req = requests.get(url) + + # Download to temporary location + temp_dir = tempfile.TemporaryDirectory() + + with caplog.at_level(logging.WARNING, logger='pysat'): + cdw.get_file(req.content, '.', 'test.txt', temp_path=temp_dir.name, + zip_method='badzip') + captured = caplog.text + + # Check for appropriate warning + warn_msg = "not a recognized zip method" + assert warn_msg in captured + return @pytest.mark.parametrize("bad_key,bad_val,err_msg", diff --git a/test_requirements.txt b/test_requirements.txt index 6c5e6eff..2d727ddf 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -10,4 +10,4 @@ pytest pytest-cov pytest-ordering sphinx -sphinx_rtd_theme +sphinx_rtd_theme>=1.2.2,<2.0.0