Merge pull request #556 from emthompson-usgs/refactor

Some initial refactoring

docs/developer/readers.md

@@ -93,8 +93,8 @@ import numpy as np
 from obspy.core.utcdatetime import UTCDateTime
 
 # local imports
-from gmprocess.stationtrace import StationTrace
-from gmprocess.stationstream import StationStream
+from gmprocess.core.stationtrace import StationTrace
+from gmprocess.core.stationstream import StationStream
 from gmprocess.io.seedname import get_channel_name, is_channel_north
 
 TEXT_HDR_ROWS = 17

docs/scripting/data-structures/stream-collection.md

@@ -10,7 +10,7 @@ together.
 ```python
 import glob
 from gmprocess.io.read import read_data
-from gmprocess.streamcollection import StreamCollection
+from gmprocess.core.streamcollection import StreamCollection
 
 # these sample files can be found in the repository
 # under gmprocess/data/testdata/knet/us2000cnnl

docs/scripting/scripting.md

@@ -8,10 +8,10 @@ processing steps, and the make plots of the processed waveforms.
 import os
 import pkg_resources
 
-from gmprocess.streamcollection import StreamCollection
-from gmprocess.config import get_config
-from gmprocess.processing import process_streams
-from gmprocess.event import get_event_object
+from gmprocess.core.streamcollection import StreamCollection
+from gmprocess.utils.config import get_config
+from gmprocess.waveform_processing.processing import process_streams
+from gmprocess.utils.event import get_event_object
 
 # Path to example data
 datapath = os.path.join('data', 'testdata', 'demo', 'ci38457511', 'raw')

docs/scripting/waveform-metrics.md

@@ -7,10 +7,10 @@ metrics.
 import os
 import pkg_resources
 
-from gmprocess.streamcollection import StreamCollection
-from gmprocess.config import get_config
-from gmprocess.processing import process_streams
-from gmprocess.event import get_event_object
+from gmprocess.core.streamcollection import StreamCollection
+from gmprocess.utils.config import get_config
+from gmprocess.waveform_processing.processing import process_streams
+from gmprocess.utils.event import get_event_object
 from gmprocess.metrics.station_summary import StationSummary
 
 # Path to example data

gmprocess/bin/gmconvert.py

@@ -7,9 +7,9 @@
 import logging
 
 # local imports
-from gmprocess.logging import setup_logger
-from gmprocess.args import add_shared_args
-from gmprocess.streamcollection import StreamCollection
+from gmprocess.utils.logging import setup_logger
+from gmprocess.utils.args import add_shared_args
+from gmprocess.core.streamcollection import StreamCollection
 from gmprocess.io.read import read_data
 from gmprocess.io.read_directory import directory_to_streams
 

gmprocess/bin/gminfo.py

@@ -14,8 +14,8 @@
 
 # local imports
 from gmprocess.io.read import _get_format, read_data
-from gmprocess.args import add_shared_args
-from gmprocess.stationtrace import REV_PROCESS_LEVELS
+from gmprocess.utils.args import add_shared_args
+from gmprocess.core.stationtrace import REV_PROCESS_LEVELS
 
 COLUMNS = ['Filename', 'Format', 'Process Level',
  'Start Time', 'End Time',
@@ -179,6 +179,7 @@ def main():
  # Shared arguments
  parser = add_shared_args(parser)
  args = parser.parse_args()
+
  if not args.concise and args.save:
  msg = '''
  ****************************************************************

gmprocess/bin/gmprocess.py

@@ -17,18 +17,18 @@
 import numpy as np
 
 # local imports
-from gmprocess.args import add_shared_args
+from gmprocess.utils.args import add_shared_args
 from gmprocess.io.fetch_utils import (get_events, update_config,
  save_shakemap_amps, download,
  draw_stations_map)
-from gmprocess.logging import setup_logger
+from gmprocess.utils.logging import setup_logger
 from gmprocess.io.asdf.stream_workspace import StreamWorkspace
-from gmprocess.processing import process_streams
-from gmprocess.report import build_report_latex
-from gmprocess.plot import summary_plots, plot_regression, plot_moveout
-from gmprocess.config import get_config
-from gmprocess.tables import set_precisions
-from gmprocess.constants import DEFAULT_FLOAT_FORMAT, DEFAULT_NA_REP
+from gmprocess.waveform_processing.processing import process_streams
+from gmprocess.io.report import build_report_latex
+from gmprocess.utils.plot import summary_plots, plot_regression, plot_moveout
+from gmprocess.utils.config import get_config
+from gmprocess.utils.tables import set_precisions
+from gmprocess.utils.constants import DEFAULT_FLOAT_FORMAT, DEFAULT_NA_REP
 
 NON_IMT_COLS = set(['EarthquakeId',
  'EarthquakeTime',

gmprocess/bin/gmsetup.py

@@ -10,8 +10,8 @@
 import pkg_resources
 
 # local imports
-from gmprocess.constants import CONFIG_FILE_PRODUCTION
-from gmprocess.args import add_shared_args
+from gmprocess.utils.constants import CONFIG_FILE_PRODUCTION
+from gmprocess.utils.args import add_shared_args
 
 
 def main():

gmprocess/stationstream.py → gmprocess/core/stationstream.py

@@ -11,7 +11,7 @@
  Response, InstrumentSensitivity)
 # local imports
 from .stationtrace import StationTrace
-from gmprocess.exception import GMProcessException
+from gmprocess.utils.exception import GMProcessException
 
 UNITS = {
  'acc': 'cm/s/s',

gmprocess/stationtrace.py → gmprocess/core/stationtrace.py

@@ -17,7 +17,7 @@
 
 # local imports
 from gmprocess._version import get_versions
-from gmprocess.config import get_config
+from gmprocess.utils.config import get_config
 from gmprocess.io.seedname import get_units_type
 
 UNITS = {'acc': 'cm/s^2',

gmprocess/streamcollection.py → gmprocess/core/streamcollection.py

@@ -16,12 +16,12 @@
 import pandas as pd
 import numpy as np
 
-from gmprocess.exception import GMProcessException
+from gmprocess.utils.exception import GMProcessException
 from gmprocess.metrics.station_summary import StationSummary
-from gmprocess.stationtrace import REV_PROCESS_LEVELS
-from gmprocess.stationstream import StationStream
+from gmprocess.core.stationtrace import REV_PROCESS_LEVELS
+from gmprocess.core.stationstream import StationStream
 from gmprocess.io.read_directory import directory_to_streams
-from gmprocess.config import get_config
+from gmprocess.utils.config import get_config
 
 
 INDENT = 2

gmprocess/io/asdf/core.py

@@ -27,7 +27,8 @@ def is_asdf(filename):
 
 
 def read_asdf(filename, eventid=None, stations=None, label=None):
- """Read Streams of data (complete with processing metadata) from an ASDF file.
+ """Read Streams of data (complete with processing metadata) from an ASDF
+ file.
 
  Args:
  filename (str):

gmprocess/io/asdf/stream_workspace.py

@@ -18,13 +18,13 @@
 from mapio.gmt import GMTGrid
 
 # local imports
-from gmprocess.stationtrace import (StationTrace, TIMEFMT_MS, NS_SEIS,
-   _get_person_agent, _get_software_agent)
-from gmprocess.stationstream import StationStream
-from gmprocess.streamcollection import StreamCollection
+from gmprocess.core.stationtrace import (
+ StationTrace, TIMEFMT_MS, NS_SEIS, _get_person_agent, _get_software_agent)
+from gmprocess.core.stationstream import StationStream
+from gmprocess.core.streamcollection import StreamCollection
 from gmprocess.metrics.station_summary import StationSummary, XML_UNITS
-from gmprocess.exception import GMProcessException
-from gmprocess.event import ScalarEvent
+from gmprocess.utils.exception import GMProcessException
+from gmprocess.utils.event import ScalarEvent
 
 TIMEPAT = '[0-9]{4}-[0-9]{2}-[0-9]{2}T'
 EVENT_TABLE_COLUMNS = ['id', 'time', 'latitude',
@@ -139,7 +139,7 @@ def format_netsta(stats):
 
 
 def format_nslc(stats):
- #loc = '' if stats.location == '--' else stats.location
+ # loc = '' if stats.location == '--' else stats.location
  return '{st.network}.{st.station}.{st.location}.{st.channel}'.format(
  st=stats)
 
@@ -149,7 +149,7 @@ def format_nslct(stats, tag):
 
 
 def format_nslit(stats, inst, tag):
- #loc = '' if stats.location == '--' else stats.location
+ # loc = '' if stats.location == '--' else stats.location
  return '{st.network}.{st.station}.{st.location}.{inst}_{tag}'.format(
  st=stats, inst=inst, tag=tag)
 
@@ -362,15 +362,16 @@ def addStreams(self, event, streams, label=None):
  base_dtype = ''.join([part.capitalize()
  for part in name_parts])
  for array_name, array in spectrum.items():
- path = base_dtype + array_name.capitalize() + "/" + procname
+ path = base_dtype + array_name.capitalize() \
+ + "/" + procname
  try:
  self.dataset.add_auxiliary_data(
  array,
  data_type='Cache',
  path=path,
  parameters={}
  )
- except Exception as e:
+ except Exception:
  pass
 
  inventory = stream.getInventory()
@@ -418,10 +419,11 @@ def getStreams(self, eventid, stations=None, labels=None):
  """
  trace_auxholder = []
  stream_auxholder = []
- if 'TraceProcessingParameters' in self.dataset.auxiliary_data:
- trace_auxholder = self.dataset.auxiliary_data.TraceProcessingParameters
- if 'StreamProcessingParameters' in self.dataset.auxiliary_data:
- stream_auxholder = self.dataset.auxiliary_data.StreamProcessingParameters
+ auxdata = self.dataset.auxiliary_data
+ if 'TraceProcessingParameters' in auxdata:
+ trace_auxholder = auxdata.TraceProcessingParameters
+ if 'StreamProcessingParameters' in auxdata:
+ stream_auxholder = auxdata.StreamProcessingParameters
  streams = []
 
  if stations is None:
@@ -477,9 +479,10 @@ def getStreams(self, eventid, stations=None, labels=None):
  # get the trace spectra arrays from auxiliary,
  # repack into stationtrace object
  spectra = {}
- if 'Cache' in self.dataset.auxiliary_data:
- for aux in self.dataset.auxiliary_data['Cache'].list():
- auxarray = self.dataset.auxiliary_data['Cache'][aux]
+
+ if 'Cache' in auxdata:
+ for aux in auxdata['Cache'].list():
+ auxarray = auxdata['Cache'][aux]
  if top not in auxarray.list():
  continue
  auxarray_top = auxarray[top]
@@ -650,7 +653,8 @@ def calcMetrics(self, eventid, stations=None, labels=None, config=None,
  self.insert_aux(xmlstr, 'StationMetrics', metricpath)
 
  def getTables(self, label, streams=None, stream_label=None):
- '''Retrieve dataframes containing event information and IMC/IMT metrics.
+ '''Retrieve dataframes containing event information and IMC/IMT
+ metrics.
 
  Args:
  label (str):
@@ -850,7 +854,8 @@ def getFitSpectraTable(self, eventid, label, streams):
 
  def getStreamMetrics(self, eventid, network, station, label, streams=None,
  stream_label=None):
- """Extract a StationSummary object from the ASDF file for a given input Stream.
+ """Extract a StationSummary object from the ASDF file for a given
+ input Stream.
 
  Args:
  eventid (str):
@@ -906,7 +911,8 @@ def getStreamMetrics(self, eventid, network, station, label, streams=None,
  if top in auxholder:
  tauxholder = auxholder[top]
  if metricpath not in tauxholder:
- fmt = 'Stream metrics path (%s) not in WaveFormMetrics auxiliary_data.'
+ fmt = ('Stream metrics path (%s) not in WaveFormMetrics '
+ 'auxiliary_data.')
  logging.warning(fmt % metricpath)
  return None
 

gmprocess/io/asdf/utils.py

@@ -4,6 +4,7 @@
 # third party imports
 import h5py
 
+
 class TallyStorage(object):
  """Tally storage used within each group.
  """
@@ -72,7 +73,8 @@ def compute_storage(self, items, store_subtotals=False):
  'groups': {}
  }
  else:
- raise ValueError("Group item '{}' is of type '{}', expected "
- "'h5.Dataset' or 'h5.Group'".format(name, type(item)))
+ raise ValueError(
+ "Group item '{}' is of type '{}', expected "
+ "'h5.Dataset' or 'h5.Group'".format(name, type(item)))
  subtotal_bytes += item_bytes
  return (subtotal_bytes, storage)

gmprocess/io/bhrc/core.py

@@ -6,12 +6,12 @@
 
 # third party imports
 import numpy as np
-from gmprocess.constants import UNIT_CONVERSIONS
 from obspy.core.utcdatetime import UTCDateTime
 
 # local
-from gmprocess.stationstream import StationStream
-from gmprocess.stationtrace import StationTrace, PROCESS_LEVELS
+from gmprocess.utils.constants import UNIT_CONVERSIONS
+from gmprocess.core.stationstream import StationStream
+from gmprocess.core.stationtrace import StationTrace, PROCESS_LEVELS
 from gmprocess.io.seedname import get_channel_name, get_units_type
 
 

gmprocess/io/cosmos/cesmd_fetcher.py

@@ -11,11 +11,12 @@
 # local imports
 from gmprocess.io.fetcher import DataFetcher, _get_first_value
 from gmprocess.io.read import read_data
-from gmprocess.streamcollection import StreamCollection
-from gmprocess.config import get_config
-from gmprocess.exception import GMProcessException
-from gmprocess.io.cosmos.cesmd_search import (get_records, get_metadata,
- get_stations_dataframe)
+from gmprocess.core.streamcollection import StreamCollection
+from gmprocess.utils.config import get_config
+from gmprocess.utils.exception import GMProcessException
+from gmprocess.io.cosmos.cesmd_search import (
+ get_records, get_metadata,
+ get_stations_dataframe)
 
 # default values for this fetcher
 # if None specified in constructor, AND no parameters specified in
@@ -271,8 +272,8 @@ def retrieveData(self, event_dict):
  else:
  # web service has a maximum number of stations you're allowed to
  # fetch (note that this may not be the same as the number of files)
- # so we're splitting up the stations by distance and downloading them
- # in chunks.
+ # so we're splitting up the stations by distance and downloading
+ # them in chunks.
  dataframe = get_stations_dataframe(event)
  distances = dataframe['epidist'].to_numpy()
  distances.sort()
@@ -300,14 +301,18 @@ def retrieveData(self, event_dict):
  except GMProcessException as gpe:
  eqfmt = 'M%.1f %s'
  eqdesc = eqfmt % (
- self.magnitude, self.time.strftime('%Y-%m-%d %H:%M:%S'))
+ self.magnitude,
+ self.time.strftime('%Y-%m-%d %H:%M:%S')
+ )
  if '404' in str(gpe):
  fmt = ('Could not find data records for %s '
  'between %.1f km and %.1f km')
  logging.info(fmt % (eqdesc, mindist, maxdist))
  else:
  logging.warning(
- 'Unplanned exception getting records for %s' % eqdesc)
+ 'Unplanned exception getting records for %s'
+ % eqdesc
+ )
  continue
  datafiles += tfiles