Merge branch 'develop' into feature/spack-stack-1.7

test/marine/CMakeLists.txt

@@ -88,7 +88,7 @@ CHECK_AND_SET_PATH(
  BUFR_TEST_DIR
 )
 if (NOT BUFR_TEST_DIR_EXISTS)
- message(STATUS "BUFR test file directory not found -- bufr to ioda tests not generted.")
+ message(WARNING "BUFR test file directory not found -- bufr to ioda tests not generated.")
  set(GENERATE_BUFR2IODA_TESTS FALSE)
 else()
  # message(STATUS "Found bufr test directory: ${BUFR_TEST_DIR}")
@@ -105,7 +105,7 @@ else()
  OCEAN_BASIN_FILE
  )
  if (NOT OCEAN_BASIN_FILE_EXISTS)
- message("Ocean basin data file not found -- bufr to ioda tests not generated.")
+ message(WARNING "Ocean basin data file not found -- bufr to ioda tests not generated.")
  set(GENERATE_BUFR2IODA_TESTS FALSE)
  endif()
  # message(STATUS "Found ocean basin data in ${OCEAN_BASIN_FILE}")
@@ -116,17 +116,26 @@ endif()
 function(ADD_INSITU_TEST testname testbufr)
  # set(CONFIG_TYPE "json")
  set(CONFIG_TYPE "yaml")
- set(DATE "2021063006")
+
+ if (testbufr STREQUAL "dbuoy")
+ set(DATE "2019010700")
+ set(CYCLE "00")
+ else()
+ set(DATE "2021063006")
+ set(CYCLE "06")
+ endif()
+
  set(TEST "bufr2ioda_insitu_${testname}")
 
  set(TESTREF_FILE "${TEST}_${DATE}.ref")
 
  # stage the input file to directory ${BUFR_INPUT_DIR}
  set(BUFR_INPUT_DIR ${TEST_WORKING_DIR})
- set(BUFR_TEST_FILE "${DATE}-gdas.t06z.${testbufr}.tm00.bufr_d")
+ set(BUFR_TEST_FILE "${DATE}-gdas.t${CYCLE}z.${testbufr}.tm00.bufr_d")
  set(BUFR_FILE "${BUFR_TEST_DIR}/${BUFR_TEST_FILE}")
  if (NOT EXISTS ${BUFR_FILE})
- message(FATAL_ERROR "BUFR file ${BUFR_FILE} not found")
+ message(WARNING "BUFR file ${BUFR_FILE} not found, test not generated")
+ return()
  endif()
  file(COPY ${BUFR_FILE} DESTINATION ${BUFR_INPUT_DIR})
 
@@ -144,7 +153,7 @@ function(ADD_INSITU_TEST testname testbufr)
  )
 
  add_test(
- NAME test_${TEST}
+ NAME test_gdasapp_${TEST}
  COMMAND ${MARINE_BUFR2IODA_DIR}/${TEST}.py -c ${CONFIG_FILE} -t ${TESTREF_DIR}/${TESTREF_FILE}
  WORKING_DIRECTORY ${TEST_WORKING_DIR}
  )
@@ -156,6 +165,8 @@ if (GENERATE_BUFR2IODA_TESTS)
  ADD_INSITU_TEST("profile_bathy" "bathy")
  ADD_INSITU_TEST("profile_glider" "subpfl")
  ADD_INSITU_TEST("profile_tesac" "tesac")
+ ADD_INSITU_TEST("profile_tropical" "dbuoy")
  ADD_INSITU_TEST("profile_xbtctd" "xbtctd")
+ ADD_INSITU_TEST("surface_drifter" "dbuoy")
  ADD_INSITU_TEST("surface_trkob" "trkob")
 endif()

test/marine/testinput/bufr2ioda_insitu_profile_tropical_2019010700.yaml.in

@@ -0,0 +1,13 @@
+---
+data_format: dbuoy
+subsets: dbuoy
+source: NCEP data tank
+data_type: drifter
+cycle_type: gdas
+cycle_datetime: '2019010700'
+dump_directory: __BUFRINPUTDIR__
+ioda_directory: __IODAOUTPUTDIR__
+ocean_basin: __OCEANBASIN__
+data_description: 6-hrly in situ drifter profiles
+data_provider: U.S. NOAA
+

test/marine/testinput/bufr2ioda_insitu_surface_drifter_2019010700.yaml.in

@@ -0,0 +1,13 @@
+---
+data_format: dbuoy
+subsets: dbuoy
+source: NCEP data tank
+data_type: drifter
+cycle_type: gdas
+cycle_datetime: '2019010700'
+dump_directory: __BUFRINPUTDIR__
+ioda_directory: __IODAOUTPUTDIR__
+ocean_basin: __OCEANBASIN__
+data_description: 6-hrly in situ drifter profiles
+data_provider: U.S. NOAA
+

test/marine/testref/bufr2ioda_insitu_profile_tropical_2019010700.ref

@@ -0,0 +1,25 @@
+dateTime: 570, int64 min, max = 1546808400, 1546830000
+dateTime hash = cda368f6c8f06e883db7100f29073084ee92dc0a0021d995a6fe7ce477004da6
+rcptdateTime: 570, int64 min, max = -1, 1546871640
+rcptdateTime hash = d9545d679013eb8fcbc8fda146d184b83378faccf55c710416f4ad72d2819f68
+lon: 570, float32 min, max = -179.89999389648438, 165.10000610351562
+lon hash = c4a1e37e42a02279ec33a69b950b10327d2185adf99d5c0c6f5ffbe1f5c66292
+lat: 570, float32 min, max = -8.300000190734863, 8.0
+lat hash = d748eb8c0e55e7a2c684ee35d73a849ef945944967f8cb73ab31cf91bafdbcb2
+depth: 570, float32 min, max = 0.0, 500.0
+depth hash = 5b20e83513245f3f37e6c43a46cacd2c71d504da1ce175fbfe2eeffab487b20b
+stationID: 570, <U5
+stationID hash = 5f8b99db791b6ff6d036afa526a3fdb93fbf406f27edb433bf059eb79dcc7c83
+buoy type: 570, int32 min, max = 21, 22
+buoy type hash = d3272b603b672422fc5143d64b5cc4f1eabe28d503d27e5a48f0e81a00839175
+temp: 570, float32 min, max = 7.129998683929443, 30.68999671936035
+temp hash = e6717249bfabf10cc3fca1d4185a8140a9161c8879cbfd99372c95cc49d38318
+saln: 570, float32 min, max = 33.4900016784668, 35.9900016784668
+saln hash = 5c9743af2319b7739072b75852a8f93351bf206e52bd97001c951e1ed11f765b
+seqNum: 570, int32 min, max = 0, 42
+seqNum hash = fc43f0b71432d0fc8db56729b9326f2e355024bd6130a27a26c974bb08fd1cd8
+PreQC: 570, int32 min, max = 0, 0
+ObsError_temp: 570, float32 min, max = 0.019999999552965164, 0.019999999552965164
+ObsError_saln: 570, float32 min, max = 0.009999999776482582, 0.009999999776482582
+OceanBasin: 570, int32 min, max = 2, 3
+OceanBasin hash = 80657c516cdb0791966161edd120f40f431b0fd6cd7a41adedf6f30340464daf

test/marine/testref/bufr2ioda_insitu_surface_drifter_2019010700.ref

@@ -0,0 +1,20 @@
+dateTime: 1141, int64 min, max = 1546808400, 1546832700
+dateTime hash = 02f5fae41328ff1ec131013ab8128eba63cc5bee37fedce0b389757d46b6d5aa
+rcptdateTime: 1141, int64 min, max = 1546808820, 1546871640
+rcptdateTime hash = 74fbeddd35e6a8f43499935edc14e78d0b119adcfa834573308d675a6fe6b011
+lon: 1141, float32 min, max = -176.77999877929688, 179.33999633789062
+lon hash = 48c658ba33511e38da5708e36c34b0dc60fbd57b1e8bb4c902d40c3ce10acce0
+lat: 1141, float32 min, max = -67.37000274658203, 66.22000122070312
+lat hash = 27d8fe2bde6d871658c815bae17ff2b1d88ea0b20d61a794ad2ae95da6fb3ba5
+stationID: 1141, <U5
+stationID hash = c927c5aa32db8a15062353125ad9c03b5f1500fa342b4e1eaaa6ac5126edde4e
+buoy type: 1141, int32 min, max = 1, 5
+buoy type hash = 8f9d69eb266ef4a997eeac74b10b128e8e4cdf87effad4e9006d8a5bc9d54d50
+temp: 1141, float32 min, max = -5.000006198883057, 28.800012588500977
+temp hash = cf8f297695b65ff5d3eab9180448abb64745394b58ecd1032d4bf887d7b0e20b
+seqNum: 1141, int32 min, max = 0, 572
+seqNum hash = 260a2544b205c6aab0cbed526f157592d14bdb9c4a4e8cc26f32fc38aa42a54e
+PreQC: 1141, int32 min, max = 0, 0
+ObsError_temp: 1141, float32 min, max = 0.019999999552965164, 0.019999999552965164
+OceanBasin: 1141, int32 min, max = 0, 5
+OceanBasin hash = ed90140a73edc2d0e0946729d1268b51ab7472e7cb2c8c2890b0c0c6e4a65405

ush/ioda/bufr2ioda/marine/b2i/bufr2ioda_insitu_profile_tropical.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import sys
+from b2iconverter.util import parse_arguments
+from b2iconverter.bufr2ioda_config import Bufr2iodaConfig
+from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter
+from tropical_ioda_variables import TropicalIODAVariables
+
+
+platform_description = 'Tropical mooring profiles from dbuoy: temperature and salinity'
+
+
+class TropicalConfig(Bufr2iodaConfig):
+
+ def ioda_filename(self):
+ return f"{self.cycle_type}.t{self.hh}z.insitu_profile_tropical.{self.cycle_datetime}.nc4"
+
+
+if __name__ == '__main__':
+
+ script_name, config_file, log_file, test_file = parse_arguments()
+
+ bufr2ioda_config = TropicalConfig(
+ script_name,
+ config_file,
+ platform_description)
+
+ ioda_vars = TropicalIODAVariables()
+ ioda_vars.set_temperature_var_name("waterTemperature")
+ ioda_vars.set_temperature_error(0.02)
+ ioda_vars.set_salinity_var_name("salinity")
+ ioda_vars.set_salinity_error(0.01)
+
+ tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file)
+
+ tropical.run()
+
+ if test_file:
+ result = tropical.test(test_file)
+ sys.exit(result)

ush/ioda/bufr2ioda/marine/b2i/bufr2ioda_insitu_surface_drifter.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+import sys
+from b2iconverter.util import parse_arguments
+from b2iconverter.bufr2ioda_config import Bufr2iodaConfig
+from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter
+from drifter_ioda_variables import DrifterIODAVariables
+
+
+platform_description = 'Lagrangian drifter drogue profiles from dbuoy: temperature'
+
+
+class DrifterConfig(Bufr2iodaConfig):
+
+ def ioda_filename(self):
+ return f"{self.cycle_type}.t{self.hh}z.insitu_surface_drifter.{self.cycle_datetime}.nc4"
+
+
+if __name__ == '__main__':
+
+ script_name, config_file, log_file, test_file = parse_arguments()
+
+ bufr2ioda_config = DrifterConfig(
+ script_name,
+ config_file,
+ platform_description)
+
+ ioda_vars = DrifterIODAVariables()
+ ioda_vars.set_temperature_var_name("waterTemperature")
+ ioda_vars.set_temperature_error(0.02)
+
+ drifter = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file)
+
+ drifter.run()
+
+ if test_file:
+ result = drifter.test(test_file)
+ sys.exit(result)

ush/ioda/bufr2ioda/marine/b2i/drifter_ioda_variables.py

@@ -0,0 +1,132 @@
+import numpy as np
+from pyiodaconv import bufr
+from b2iconverter.ioda_variables import IODAVariables
+from b2iconverter.ioda_metadata import IODAMetadata
+from b2iconverter.ioda_addl_vars import IODAAdditionalVariables, compute_seq_num
+from b2iconverter.util import *
+
+
+class DrifterIODAVariables(IODAVariables):
+
+ def __init__(self):
+ self.construct()
+ self.metadata = DrifterMetadata()
+ self.additional_vars = DrifterAdditionalVariables(self)
+
+ def build_query(self):
+ q = bufr.QuerySet()
+ q.add('year', '*/YEAR')
+ q.add('month', '*/MNTH')
+ q.add('day', '*/DAYS')
+ q.add('hour', '*/HOUR')
+ q.add('minute', '*/MINU')
+ q.add('ryear', '*/RCPTIM/RCYR')
+ q.add('rmonth', '*/RCPTIM/RCMO')
+ q.add('rday', '*/RCPTIM/RCDY')
+ q.add('rhour', '*/RCPTIM/RCHR')
+ q.add('rminute', '*/RCPTIM/RCMI')
+ q.add('stationID', '*/RPID')
+ q.add('latitude', '*/CLAT')
+ q.add('longitude', '*/CLON')
+ q.add('depth', '*/DTSCUR/DBSS')
+ q.add('temp', '*/DTSCUR/STMP')
+ q.add('buoy_type', '*/RPSEC4/BUYT')
+ return q
+
+ def set_obs_from_query_result(self, r):
+ self.temp = r.get('temp', group_by='depth')
+ self.temp -= 273.15
+
+ def filter(self):
+ T_mask = self.TemperatureFilter()
+
+ # Separate Drifter profiles from dbuoy tank
+ # buoy_type:
+ # 1 - Standard Lagrangian drifter (Global Drifter Programme)
+ # 4 - Ice drifter
+ # 5 - SVPG Standard Lagrangian drifter with GPS
+ values_to_select = [1, 4, 5]
+ buoy_mask = np.isin(self.metadata.buoy_type, values_to_select)
+
+ mask = T_mask & buoy_mask
+
+ self.metadata.filter(mask)
+ self.temp = self.temp[mask]
+
+ def write_to_ioda_file(self, obsspace):
+ self.metadata.write_to_ioda_file(obsspace)
+ self.additional_vars.write_to_ioda_file(obsspace)
+ self.write_obs_value_t(obsspace)
+
+ def log_obs(self, logger):
+ self.log_temperature(logger)
+
+
+class DrifterMetadata(IODAMetadata):
+ def set_from_query_result(self, r):
+ self.set_date_time_from_query_result(r)
+ self.set_rcpt_date_time_from_query_result(r)
+ self.set_lon_from_query_result(r)
+ self.set_lat_from_query_result(r)
+ self.set_station_id_from_query_result(r)
+ self.buoy_type = r.get('buoy_type', group_by='depth')
+
+ def set_rcpt_date_time_from_query_result(self, r):
+ self.rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute')
+ self.rcptdateTime = self.rcptdateTime[:, 0]
+ self.rcptdateTime = self.rcptdateTime.astype(np.int64)
+ k = int(len(self.dateTime) / len(self.rcptdateTime))
+ self.rcptdateTime = np.tile(self.rcptdateTime, k)
+
+ def filter(self, mask):
+ self.dateTime = self.dateTime[mask]
+ self.rcptdateTime = self.rcptdateTime[mask]
+ self.lat = self.lat[mask]
+ self.lon = self.lon[mask]
+ self.stationID = self.stationID[mask]
+ self.buoy_type = self.buoy_type[mask]
+
+ def write_to_ioda_file(self, obsspace):
+ write_date_time(obsspace, self.dateTime)
+ write_rcpt_date_time(obsspace, self.rcptdateTime)
+ write_longitude(obsspace, self.lon)
+ write_latitude(obsspace, self.lat)
+ write_station_id(obsspace, self.stationID)
+ obsspace.create_var(
+ 'MetaData/BuoyType',
+ dtype=self.buoy_type.dtype, fillval=self.buoy_type.fill_value
+ ) \
+ .write_attr('long_name', 'Buoy Type') \
+ .write_data(self.buoy_type)
+
+ def log(self, logger):
+ self.log_date_time(logger)
+ self.log_rcpt_date_time(logger)
+ self.log_longitude(logger)
+ self.log_latitude(logger)
+ self.log_station_id(logger)
+ log_variable(logger, "buoy type", self.buoy_type)
+ logger.debug(f"buoy type hash = {compute_hash(self.buoy_type)}")
+
+
+class DrifterAdditionalVariables(IODAAdditionalVariables):
+
+ def construct(self):
+ self.seqNum = compute_seq_num(self.ioda_vars.metadata.lon, self.ioda_vars.metadata.lat)
+ n = len(self.seqNum)
+ self.PreQC = (np.ma.masked_array(np.full(n, 0))).astype(np.int32)
+ self.ObsError_temp = \
+ np.float32(np.ma.masked_array(np.full(n, self.ioda_vars.T_error)))
+ self.compute_ocean_basin()
+
+ def write_to_ioda_file(self, obsspace):
+ self.write_seq_num(obsspace)
+ self.write_preqc(obsspace, self.ioda_vars.T_name)
+ self.write_obs_errorT(obsspace)
+ self.write_ocean_basin(obsspace)
+
+ def log(self, logger):
+ self.log_seq_num(logger)
+ self.log_preqc(logger)
+ self.log_obs_error_temp(logger)
+ self.log_ocean_basin(logger)

ush/ioda/bufr2ioda/marine/b2i/trkob_ioda_variables.py

@@ -56,7 +56,6 @@ def write_to_ioda_file(self, obsspace):
  def log(self, logger):
  self.log_date_time(logger)
  self.log_rcpt_date_time(logger)
- # self.logLonLat(logger)
  self.log_longitude(logger)
  self.log_latitude(logger)
  self.log_station_id(logger)