From 050f1f5c297a83dbf803ac6639e8b5f80a973905 Mon Sep 17 00:00:00 2001 From: Nathan Wendt Date: Mon, 11 Mar 2024 14:01:47 -0500 Subject: [PATCH 1/2] Performance updates Use `set` instead of `list` to hold possible elements within the GEMPAK file object. `set` has better performance when checking for element membership. This change also allowed for modifications to the looping done to unpack elements such that performace would be further increased. Adds climate surface files for testing. This also updates the surface file type detection that was producing incorrect results. String data was removed from unmerged sounding unpacking routine as this should not appear in those file types per GEMPAK documentation. Add additional tests to improve coverage. --- .codespellignore | 7 +- src/metpy/io/gempak.py | 557 ++++++++++++------------ src/metpy/static-data-manifest.txt | 7 + staticdata/gem_big_endian.grd | Bin 0 -> 38912 bytes staticdata/gem_climate.csv | 2 + staticdata/gem_climate.sfc | Bin 0 -> 3584 bytes staticdata/gem_little_endian.grd | Bin 0 -> 38912 bytes staticdata/gem_merged_nopack.csv | 93 ++++ staticdata/gem_merged_nopack.snd | Bin 0 -> 12288 bytes staticdata/gem_multilevel_multidate.grd | Bin 0 -> 38912 bytes tests/io/test_gempak.py | 104 ++++- 11 files changed, 487 insertions(+), 283 deletions(-) create mode 100644 staticdata/gem_big_endian.grd create mode 100644 staticdata/gem_climate.csv create mode 100644 staticdata/gem_climate.sfc create mode 100644 staticdata/gem_little_endian.grd create mode 100644 staticdata/gem_merged_nopack.csv create mode 100644 staticdata/gem_merged_nopack.snd create mode 100644 staticdata/gem_multilevel_multidate.grd diff --git a/.codespellignore b/.codespellignore index 0fd217cf2af..62442481238 100644 --- a/.codespellignore +++ b/.codespellignore @@ -6,12 +6,11 @@ thta = 2 lambda grid: grid if grid.PARM in parameter else False, col_head.SELV, - 'SELV': col_head.SELV, + 'SELV': col_head.SELV, col_head.SELV, row_head.SELV, - 'SELV': row_head.SELV, - 'SELV': col_head.SELV, - 'SELV': col_head.SELV, + 'SELV': row_head.SELV, + 'SELV': col_head.SELV, Klystron Warmup Integer*2 N/A 0 to 1 1 0=Normal, 1=Preheat 146 # GFS, NAM, RAP, or other gridded dataset (e.g., NARR). # This attribute can be set to False if the vector components are grid relative (e.g., for NAM diff --git a/src/metpy/io/gempak.py b/src/metpy/io/gempak.py index 32cb3d1082f..53f91213b17 100644 --- a/src/metpy/io/gempak.py +++ b/src/metpy/io/gempak.py @@ -645,7 +645,7 @@ def __init__(self, file): 'NavigationBlock') if navb_size != nav_stuct.size // BYTES_PER_WORD: - raise ValueError('Navigation block size does not match GEMPAK specification') + raise ValueError('Navigation block size does not match GEMPAK specification.') else: self.navigation_block = ( self._buffer.read_struct(nav_stuct) @@ -665,7 +665,7 @@ def __init__(self, file): if anlb_size not in [anlb1_struct.size // BYTES_PER_WORD, anlb2_struct.size // BYTES_PER_WORD]: - raise ValueError('Analysis block size does not match GEMPAK specification') + raise ValueError('Analysis block size does not match GEMPAK specification.') else: anlb_type = self._buffer.read_struct(struct.Struct(self.prefmt + 'f'))[0] self._buffer.jump_to(anlb_start) @@ -741,9 +741,9 @@ def __init__(self, file): def _swap_bytes(self, binary): """Swap between little and big endian.""" - self.swaped_bytes = (struct.pack('@i', 1) != binary) + self.swapped_bytes = (struct.pack('@i', 1) != binary) - if self.swaped_bytes: + if self.swapped_bytes: if sys.byteorder == 'little': self.prefmt = '>' self.endian = 'big' @@ -932,9 +932,9 @@ def __init__(self, file, *args, **kwargs): if self._buffer.read_int(4, self.endian, False) == USED_FLAG: self.column_headers.append(self._buffer.read_struct(column_headers_fmt)) - self._gdinfo = [] + self._gdinfo = set() for n, head in enumerate(self.column_headers): - self._gdinfo.append( + self._gdinfo.add( Grid( n, head.GTM1[0], @@ -954,7 +954,7 @@ def __init__(self, file, *args, **kwargs): def gdinfo(self): """Return grid information.""" - return self._gdinfo + return sorted(self._gdinfo) def _get_crs(self): """Create CRS from GEMPAK navigation block.""" @@ -1244,7 +1244,7 @@ def gdxarray(self, parameter=None, date_time=None, coordinate=None, level2 = [level2] # Figure out which columns to extract from the file - matched = self._gdinfo.copy() + matched = sorted(self._gdinfo) if parameter is not None: matched = filter( @@ -1291,9 +1291,8 @@ def gdxarray(self, parameter=None, date_time=None, coordinate=None, grids = [] irow = 0 # Only one row for grids - for icol, col_head in enumerate(self.column_headers): - if icol not in gridno: - continue + for icol in gridno: + col_head = self.column_headers[icol] for iprt, part in enumerate(self.parts): pointer = (self.prod_desc.data_block_ptr + (irow * self.prod_desc.columns * self.prod_desc.parts) @@ -1391,7 +1390,7 @@ def __init__(self, file, *args, **kwargs): self.merged = 'SNDT' in (part.name for part in self.parts) - self._sninfo = [] + self._sninfo = set() for irow, row_head in enumerate(self.row_headers): for icol, col_head in enumerate(self.column_headers): pointer = (self.prod_desc.data_block_ptr @@ -1402,7 +1401,7 @@ def __init__(self, file, *args, **kwargs): data_ptr = self._buffer.read_int(4, self.endian, False) if data_ptr: - self._sninfo.append( + self._sninfo.add( Sounding( irow, icol, @@ -1419,144 +1418,140 @@ def __init__(self, file, *args, **kwargs): def sninfo(self): """Return sounding information.""" - return self._sninfo + return sorted(self._sninfo) def _unpack_merged(self, sndno): """Unpack merged sounding data.""" soundings = [] - for irow, row_head in enumerate(self.row_headers): - for icol, col_head in enumerate(self.column_headers): - if (irow, icol) not in sndno: + for irow, icol in sndno: + row_head = self.row_headers[irow] + col_head = self.column_headers[icol] + sounding = { + 'STID': col_head.STID, + 'STNM': col_head.STNM, + 'SLAT': col_head.SLAT, + 'SLON': col_head.SLON, + 'SELV': col_head.SELV, + 'STAT': col_head.STAT, + 'COUN': col_head.COUN, + 'DATE': row_head.DATE, + 'TIME': row_head.TIME, + } + for iprt, part in enumerate(self.parts): + pointer = (self.prod_desc.data_block_ptr + + (irow * self.prod_desc.columns * self.prod_desc.parts) + + (icol * self.prod_desc.parts + iprt)) + self._buffer.jump_to(self._start, _word_to_position(pointer)) + self.data_ptr = self._buffer.read_int(4, self.endian, False) + if not self.data_ptr: continue - sounding = {'STID': col_head.STID, - 'STNM': col_head.STNM, - 'SLAT': col_head.SLAT, - 'SLON': col_head.SLON, - 'SELV': col_head.SELV, - 'STAT': col_head.STAT, - 'COUN': col_head.COUN, - 'DATE': row_head.DATE, - 'TIME': row_head.TIME, - } - for iprt, part in enumerate(self.parts): - pointer = (self.prod_desc.data_block_ptr - + (irow * self.prod_desc.columns * self.prod_desc.parts) - + (icol * self.prod_desc.parts + iprt)) - self._buffer.jump_to(self._start, _word_to_position(pointer)) - self.data_ptr = self._buffer.read_int(4, self.endian, False) - if not self.data_ptr: - continue - self._buffer.jump_to(self._start, _word_to_position(self.data_ptr)) - self.data_header_length = self._buffer.read_int(4, self.endian, False) - data_header = self._buffer.set_mark() - self._buffer.jump_to(data_header, - _word_to_position(part.header_length + 1)) - lendat = self.data_header_length - part.header_length - - fmt_code = { - DataTypes.real: 'f', - DataTypes.realpack: 'i', - DataTypes.character: 's', - }.get(part.data_type) - - if fmt_code is None: - raise NotImplementedError(f'No methods for data type {part.data_type}') - - if fmt_code == 's': - lendat *= BYTES_PER_WORD - - packed_buffer = ( - self._buffer.read_struct( - struct.Struct(f'{self.prefmt}{lendat}{fmt_code}') - ) + self._buffer.jump_to(self._start, _word_to_position(self.data_ptr)) + self.data_header_length = self._buffer.read_int(4, self.endian, False) + data_header = self._buffer.set_mark() + self._buffer.jump_to(data_header, + _word_to_position(part.header_length + 1)) + lendat = self.data_header_length - part.header_length + + fmt_code = { + DataTypes.real: 'f', + DataTypes.realpack: 'i', + }.get(part.data_type) + + if fmt_code is None: + raise NotImplementedError(f'No methods for data type {part.data_type}') + + packed_buffer = ( + self._buffer.read_struct( + struct.Struct(f'{self.prefmt}{lendat}{fmt_code}') ) + ) - parameters = self.parameters[iprt] - nparms = len(parameters['name']) + parameters = self.parameters[iprt] + nparms = len(parameters['name']) - if part.data_type == DataTypes.realpack: - unpacked = self._unpack_real(packed_buffer, parameters, lendat) - for iprm, param in enumerate(parameters['name']): - sounding[param] = unpacked[iprm::nparms] - else: - for iprm, param in enumerate(parameters['name']): - sounding[param] = np.array( - packed_buffer[iprm::nparms], dtype=np.float32 - ) + if part.data_type == DataTypes.realpack: + unpacked = self._unpack_real(packed_buffer, parameters, lendat) + for iprm, param in enumerate(parameters['name']): + sounding[param] = unpacked[iprm::nparms] + else: + for iprm, param in enumerate(parameters['name']): + sounding[param] = np.array( + packed_buffer[iprm::nparms], dtype=np.float32 + ) - soundings.append(sounding) + soundings.append(sounding) return soundings def _unpack_unmerged(self, sndno): """Unpack unmerged sounding data.""" soundings = [] - for irow, row_head in enumerate(self.row_headers): - for icol, col_head in enumerate(self.column_headers): - if (irow, icol) not in sndno: + for irow, icol in sndno: + row_head = self.row_headers[irow] + col_head = self.column_headers[icol] + sounding = { + 'STID': col_head.STID, + 'STNM': col_head.STNM, + 'SLAT': col_head.SLAT, + 'SLON': col_head.SLON, + 'SELV': col_head.SELV, + 'STAT': col_head.STAT, + 'COUN': col_head.COUN, + 'DATE': row_head.DATE, + 'TIME': row_head.TIME, + } + for iprt, part in enumerate(self.parts): + pointer = (self.prod_desc.data_block_ptr + + (irow * self.prod_desc.columns * self.prod_desc.parts) + + (icol * self.prod_desc.parts + iprt)) + self._buffer.jump_to(self._start, _word_to_position(pointer)) + self.data_ptr = self._buffer.read_int(4, self.endian, False) + if not self.data_ptr: continue - sounding = {'STID': col_head.STID, - 'STNM': col_head.STNM, - 'SLAT': col_head.SLAT, - 'SLON': col_head.SLON, - 'SELV': col_head.SELV, - 'STAT': col_head.STAT, - 'COUN': col_head.COUN, - 'DATE': row_head.DATE, - 'TIME': row_head.TIME, - } - for iprt, part in enumerate(self.parts): - pointer = (self.prod_desc.data_block_ptr - + (irow * self.prod_desc.columns * self.prod_desc.parts) - + (icol * self.prod_desc.parts + iprt)) - self._buffer.jump_to(self._start, _word_to_position(pointer)) - self.data_ptr = self._buffer.read_int(4, self.endian, False) - if not self.data_ptr: - continue - self._buffer.jump_to(self._start, _word_to_position(self.data_ptr)) - self.data_header_length = self._buffer.read_int(4, self.endian, False) - data_header = self._buffer.set_mark() - self._buffer.jump_to(data_header, - _word_to_position(part.header_length + 1)) - lendat = self.data_header_length - part.header_length - - fmt_code = { - DataTypes.real: 'f', - DataTypes.realpack: 'i', - DataTypes.character: 's', - }.get(part.data_type) - - if fmt_code is None: - raise NotImplementedError(f'No methods for data type {part.data_type}') - - if fmt_code == 's': - lendat *= BYTES_PER_WORD - - packed_buffer = ( - self._buffer.read_struct( - struct.Struct(f'{self.prefmt}{lendat}{fmt_code}') - ) + self._buffer.jump_to(self._start, _word_to_position(self.data_ptr)) + self.data_header_length = self._buffer.read_int(4, self.endian, False) + data_header = self._buffer.set_mark() + self._buffer.jump_to(data_header, + _word_to_position(part.header_length + 1)) + lendat = self.data_header_length - part.header_length + + fmt_code = { + DataTypes.real: 'f', + DataTypes.realpack: 'i', + DataTypes.character: 's', + }.get(part.data_type) + + if fmt_code is None: + raise NotImplementedError(f'No methods for data type {part.data_type}') + + if fmt_code == 's': + lendat *= BYTES_PER_WORD + + packed_buffer = ( + self._buffer.read_struct( + struct.Struct(f'{self.prefmt}{lendat}{fmt_code}') ) + ) - parameters = self.parameters[iprt] - nparms = len(parameters['name']) - sounding[part.name] = {} - - if part.data_type == DataTypes.realpack: - unpacked = self._unpack_real(packed_buffer, parameters, lendat) - for iprm, param in enumerate(parameters['name']): - sounding[part.name][param] = unpacked[iprm::nparms] - elif part.data_type == DataTypes.character: - for iprm, param in enumerate(parameters['name']): - sounding[part.name][param] = ( - self._decode_strip(packed_buffer[iprm]) - ) - else: - for iprm, param in enumerate(parameters['name']): - sounding[part.name][param] = ( - np.array(packed_buffer[iprm::nparms], dtype=np.float32) - ) + parameters = self.parameters[iprt] + nparms = len(parameters['name']) + sounding[part.name] = {} - soundings.append(self._merge_sounding(sounding)) + if part.data_type == DataTypes.realpack: + unpacked = self._unpack_real(packed_buffer, parameters, lendat) + for iprm, param in enumerate(parameters['name']): + sounding[part.name][param] = unpacked[iprm::nparms] + elif part.data_type == DataTypes.character: + for iprm, param in enumerate(parameters['name']): + sounding[part.name][param] = ( + self._decode_strip(packed_buffer[iprm]) + ) + else: + for iprm, param in enumerate(parameters['name']): + sounding[part.name][param] = ( + np.array(packed_buffer[iprm::nparms], dtype=np.float32) + ) + + soundings.append(self._merge_sounding(sounding)) return soundings def _merge_significant_temps(self, merged, parts, section, pbot): @@ -2159,7 +2154,7 @@ def snxarray(self, station_id=None, station_number=None, country = [c.upper() for c in country] # Figure out which columns to extract from the file - matched = self._sninfo.copy() + matched = sorted(self._sninfo) if station_id is not None: matched = filter( @@ -2278,7 +2273,7 @@ def __init__(self, file, *args, **kwargs): self._get_surface_type() - self._sfinfo = [] + self._sfinfo = set() if self.surface_type == 'standard': for irow, row_head in enumerate(self.row_headers): for icol, col_head in enumerate(self.column_headers): @@ -2291,7 +2286,7 @@ def __init__(self, file, *args, **kwargs): data_ptr = self._buffer.read_int(4, self.endian, False) if data_ptr: - self._sfinfo.append( + self._sfinfo.add( Surface( irow, icol, @@ -2317,7 +2312,7 @@ def __init__(self, file, *args, **kwargs): data_ptr = self._buffer.read_int(4, self.endian, False) if data_ptr: - self._sfinfo.append( + self._sfinfo.add( Surface( irow, icol, @@ -2343,7 +2338,7 @@ def __init__(self, file, *args, **kwargs): data_ptr = self._buffer.read_int(4, self.endian, False) if data_ptr: - self._sfinfo.append( + self._sfinfo.add( Surface( irow, icol, @@ -2362,15 +2357,22 @@ def __init__(self, file, *args, **kwargs): def sfinfo(self): """Return station information.""" - return self._sfinfo + return sorted(self._sfinfo) def _get_surface_type(self): - """Determine type of surface file.""" - if len(self.row_headers) == 1: + """Determine type of surface file. + + Notes + ----- + See GEMPAK SFLIB documentation for type definitions. + """ + if (len(self.row_headers) == 1 + and 'DATE' in self.column_keys + and 'STID' in self.column_keys): self.surface_type = 'ship' - elif 'DATE' in self.row_keys: + elif 'DATE' in self.row_keys and 'STID' in self.column_keys: self.surface_type = 'standard' - elif 'DATE' in self.column_keys: + elif 'DATE' in self.column_keys and 'STID' in self.row_keys: self.surface_type = 'climate' else: raise TypeError('Unknown surface data type') @@ -2393,92 +2395,91 @@ def _key_types(self, keys): def _unpack_climate(self, sfcno): """Unpack a climate surface data file.""" stations = [] - for icol, col_head in enumerate(self.column_headers): - for irow, row_head in enumerate(self.row_headers): - if (irow, icol) not in sfcno: + for irow, icol in sfcno: + col_head = self.column_headers[icol] + row_head = self.row_headers[irow] + station = { + 'STID': row_head.STID, + 'STNM': row_head.STNM, + 'SLAT': row_head.SLAT, + 'SLON': row_head.SLON, + 'SELV': row_head.SELV, + 'STAT': row_head.STAT, + 'COUN': row_head.COUN, + 'STD2': row_head.STD2, + 'SPRI': row_head.SPRI, + 'DATE': col_head.DATE, + 'TIME': col_head.TIME, + } + for iprt, part in enumerate(self.parts): + pointer = (self.prod_desc.data_block_ptr + + (irow * self.prod_desc.columns * self.prod_desc.parts) + + (icol * self.prod_desc.parts + iprt)) + self._buffer.jump_to(self._start, _word_to_position(pointer)) + self.data_ptr = self._buffer.read_int(4, self.endian, False) + if not self.data_ptr: continue - station = {'STID': row_head.STID, - 'STNM': row_head.STNM, - 'SLAT': row_head.SLAT, - 'SLON': row_head.SLON, - 'SELV': row_head.SELV, - 'STAT': row_head.STAT, - 'COUN': row_head.COUN, - 'STD2': row_head.STD2, - 'SPRI': row_head.SPRI, - 'DATE': col_head.DATE, - 'TIME': col_head.TIME, - } - for iprt, part in enumerate(self.parts): - pointer = (self.prod_desc.data_block_ptr - + (irow * self.prod_desc.columns * self.prod_desc.parts) - + (icol * self.prod_desc.parts + iprt)) - self._buffer.jump_to(self._start, _word_to_position(pointer)) - self.data_ptr = self._buffer.read_int(4, self.endian, False) - if not self.data_ptr: - continue - self._buffer.jump_to(self._start, _word_to_position(self.data_ptr)) - self.data_header_length = self._buffer.read_int(4, self.endian, False) - data_header = self._buffer.set_mark() - self._buffer.jump_to(data_header, - _word_to_position(part.header_length + 1)) - lendat = self.data_header_length - part.header_length - - fmt_code = { - DataTypes.real: 'f', - DataTypes.realpack: 'i', - DataTypes.character: 's', - }.get(part.data_type) - - if fmt_code is None: - raise NotImplementedError(f'No methods for data type {part.data_type}') - - if fmt_code == 's': - lendat *= BYTES_PER_WORD - - packed_buffer = ( - self._buffer.read_struct( - struct.Struct(f'{self.prefmt}{lendat}{fmt_code}') - ) + self._buffer.jump_to(self._start, _word_to_position(self.data_ptr)) + self.data_header_length = self._buffer.read_int(4, self.endian, False) + data_header = self._buffer.set_mark() + self._buffer.jump_to(data_header, + _word_to_position(part.header_length + 1)) + lendat = self.data_header_length - part.header_length + + fmt_code = { + DataTypes.real: 'f', + DataTypes.realpack: 'i', + DataTypes.character: 's', + }.get(part.data_type) + + if fmt_code is None: + raise NotImplementedError(f'No methods for data type {part.data_type}') + + if fmt_code == 's': + lendat *= BYTES_PER_WORD + + packed_buffer = ( + self._buffer.read_struct( + struct.Struct(f'{self.prefmt}{lendat}{fmt_code}') ) + ) - parameters = self.parameters[iprt] + parameters = self.parameters[iprt] - if part.data_type == DataTypes.realpack: - unpacked = self._unpack_real(packed_buffer, parameters, lendat) - for iprm, param in enumerate(parameters['name']): - station[param] = unpacked[iprm] - elif part.data_type == DataTypes.character: - for iprm, param in enumerate(parameters['name']): - station[param] = self._decode_strip(packed_buffer[iprm]) - else: - for iprm, param in enumerate(parameters['name']): - station[param] = np.array( - packed_buffer[iprm], dtype=np.float32 - ) + if part.data_type == DataTypes.realpack: + unpacked = self._unpack_real(packed_buffer, parameters, lendat) + for iprm, param in enumerate(parameters['name']): + station[param] = unpacked[iprm] + elif part.data_type == DataTypes.character: + for iprm, param in enumerate(parameters['name']): + station[param] = self._decode_strip(packed_buffer[iprm]) + else: + for iprm, param in enumerate(parameters['name']): + station[param] = np.array( + packed_buffer[iprm], dtype=np.float32 + ) - stations.append(station) + stations.append(station) return stations def _unpack_ship(self, sfcno): """Unpack ship (moving observation) surface data file.""" stations = [] - irow = 0 - for icol, col_head in enumerate(self.column_headers): - if (irow, icol) not in sfcno: - continue - station = {'STID': col_head.STID, - 'STNM': col_head.STNM, - 'SLAT': col_head.SLAT, - 'SLON': col_head.SLON, - 'SELV': col_head.SELV, - 'STAT': col_head.STAT, - 'COUN': col_head.COUN, - 'STD2': col_head.STD2, - 'SPRI': col_head.SPRI, - 'DATE': col_head.DATE, - 'TIME': col_head.TIME, - } + for irow, icol in sfcno: # irow should always be zero + col_head = self.column_headers[icol] + station = { + 'STID': col_head.STID, + 'STNM': col_head.STNM, + 'SLAT': col_head.SLAT, + 'SLON': col_head.SLON, + 'SELV': col_head.SELV, + 'STAT': col_head.STAT, + 'COUN': col_head.COUN, + 'STD2': col_head.STD2, + 'SPRI': col_head.SPRI, + 'DATE': col_head.DATE, + 'TIME': col_head.TIME, + } for iprt, part in enumerate(self.parts): pointer = (self.prod_desc.data_block_ptr + (irow * self.prod_desc.columns * self.prod_desc.parts) @@ -2533,69 +2534,69 @@ def _unpack_ship(self, sfcno): def _unpack_standard(self, sfcno): """Unpack a standard surface data file.""" stations = [] - for irow, row_head in enumerate(self.row_headers): - for icol, col_head in enumerate(self.column_headers): - if (irow, icol) not in sfcno: + for irow, icol in sfcno: + row_head = self.row_headers[irow] + col_head = self.column_headers[icol] + station = { + 'STID': col_head.STID, + 'STNM': col_head.STNM, + 'SLAT': col_head.SLAT, + 'SLON': col_head.SLON, + 'SELV': col_head.SELV, + 'STAT': col_head.STAT, + 'COUN': col_head.COUN, + 'STD2': col_head.STD2, + 'SPRI': col_head.SPRI, + 'DATE': row_head.DATE, + 'TIME': row_head.TIME, + } + for iprt, part in enumerate(self.parts): + pointer = (self.prod_desc.data_block_ptr + + (irow * self.prod_desc.columns * self.prod_desc.parts) + + (icol * self.prod_desc.parts + iprt)) + self._buffer.jump_to(self._start, _word_to_position(pointer)) + self.data_ptr = self._buffer.read_int(4, self.endian, False) + if not self.data_ptr: continue - station = {'STID': col_head.STID, - 'STNM': col_head.STNM, - 'SLAT': col_head.SLAT, - 'SLON': col_head.SLON, - 'SELV': col_head.SELV, - 'STAT': col_head.STAT, - 'COUN': col_head.COUN, - 'STD2': col_head.STD2, - 'SPRI': col_head.SPRI, - 'DATE': row_head.DATE, - 'TIME': row_head.TIME, - } - for iprt, part in enumerate(self.parts): - pointer = (self.prod_desc.data_block_ptr - + (irow * self.prod_desc.columns * self.prod_desc.parts) - + (icol * self.prod_desc.parts + iprt)) - self._buffer.jump_to(self._start, _word_to_position(pointer)) - self.data_ptr = self._buffer.read_int(4, self.endian, False) - if not self.data_ptr: - continue - self._buffer.jump_to(self._start, _word_to_position(self.data_ptr)) - self.data_header_length = self._buffer.read_int(4, self.endian, False) - data_header = self._buffer.set_mark() - self._buffer.jump_to(data_header, - _word_to_position(part.header_length + 1)) - lendat = self.data_header_length - part.header_length - - fmt_code = { - DataTypes.real: 'f', - DataTypes.realpack: 'i', - DataTypes.character: 's', - }.get(part.data_type) - - if fmt_code is None: - raise NotImplementedError(f'No methods for data type {part.data_type}') - - if fmt_code == 's': - lendat *= BYTES_PER_WORD - - packed_buffer = ( - self._buffer.read_struct( - struct.Struct(f'{self.prefmt}{lendat}{fmt_code}') - ) + self._buffer.jump_to(self._start, _word_to_position(self.data_ptr)) + self.data_header_length = self._buffer.read_int(4, self.endian, False) + data_header = self._buffer.set_mark() + self._buffer.jump_to(data_header, + _word_to_position(part.header_length + 1)) + lendat = self.data_header_length - part.header_length + + fmt_code = { + DataTypes.real: 'f', + DataTypes.realpack: 'i', + DataTypes.character: 's', + }.get(part.data_type) + + if fmt_code is None: + raise NotImplementedError(f'No methods for data type {part.data_type}') + + if fmt_code == 's': + lendat *= BYTES_PER_WORD + + packed_buffer = ( + self._buffer.read_struct( + struct.Struct(f'{self.prefmt}{lendat}{fmt_code}') ) + ) - parameters = self.parameters[iprt] + parameters = self.parameters[iprt] - if part.data_type == DataTypes.realpack: - unpacked = self._unpack_real(packed_buffer, parameters, lendat) - for iprm, param in enumerate(parameters['name']): - station[param] = unpacked[iprm] - elif part.data_type == DataTypes.character: - for iprm, param in enumerate(parameters['name']): - station[param] = self._decode_strip(packed_buffer[iprm]) - else: - for iprm, param in enumerate(parameters['name']): - station[param] = packed_buffer[iprm] + if part.data_type == DataTypes.realpack: + unpacked = self._unpack_real(packed_buffer, parameters, lendat) + for iprm, param in enumerate(parameters['name']): + station[param] = unpacked[iprm] + elif part.data_type == DataTypes.character: + for iprm, param in enumerate(parameters['name']): + station[param] = self._decode_strip(packed_buffer[iprm]) + else: + for iprm, param in enumerate(parameters['name']): + station[param] = packed_buffer[iprm] - stations.append(station) + stations.append(station) return stations @staticmethod @@ -2772,7 +2773,7 @@ def sfjson(self, station_id=None, station_number=None, country = [c.upper() for c in country] # Figure out which columns to extract from the file - matched = self._sfinfo.copy() + matched = sorted(self._sfinfo) if station_id is not None: matched = filter( diff --git a/src/metpy/static-data-manifest.txt b/src/metpy/static-data-manifest.txt index c08f0338959..fda9b6eac9a 100644 --- a/src/metpy/static-data-manifest.txt +++ b/src/metpy/static-data-manifest.txt @@ -35,13 +35,20 @@ cubic_test.npz 8b9d01c2177a057b3352bb6c5b775dae6b796d37ba35b4775fcb65300dc06ccf dec9_sounding.txt 4f60955bee4a59e2da0c225d778b9a04a149e9a17b4dce6bfefc111240b3b165 gem_azimuthal.grd 2b1cb2a358135035dbf6926f0e8ba8122f8b3a0477da5183943a1299ee602093 gem_azimuthal.npz 57d9f69cfa08893ae2d388bda7fb15a1b306ba205e3f15b9e40e963e7db79b0f +gem_big_endian.grd cf2828181311677081e86253e2b7eb5778450ee92cf6b8161dc0635687b1fd9d +gem_climate.csv 24c562fac9d5b81c08092ce5cf5522b57c74a63a184864e211016110fbc983c6 +gem_climate.sfc 92f7fc278e16a9f2fbdb054b6c2a0f9f2aea420658841687e34ad2ad9a6552a7 gem_conical.grd 0184d05a596d623171135fc3243060aef69da8601c5ef8448bb6f39914474c3d gem_conical.npz fd77f4e4fb2884087bc022c957277b4ec11bb21a35c9d3e261b64df425b1da28 gem_cylindrical.grd 1b2ee56e4ee50d8d3012aa319f24a7c09e092db8e8a5a8f58e2d857a8fd1520f gem_cylindrical.npz 68ea33a58b24b651254dd4d2c935c33fe4f414ffd404296c9098b6404a37e408 +gem_little_endian.grd 12c5c094c6505697c89426f362aa9bd6ba7de12f25c690ea73fb6d0159925536 +gem_merged_nopack.csv 57ac3d9dc6b2461f0d0b8739452dceaa88f11fc8d55f75d8f05ed0981af0bf97 +gem_merged_nopack.snd 141a1122e9638b0dde5d2ec424c03748d93577b84f10bbe7c29d9d3228200a88 gem_model_mrg.csv 304afb3bb4b662f7adfc8803e8661fc217a691d0b1790d5e0e6d085f198c4257 gem_model_mrg.snd 6d229a7af8d3e86d8da02403023cdaf0f86b6c982eb8b6f7f84c3836811df014 gem_multi_time.grd a11746f162f2ea3944573246a477db94026f76e72ce7ecb98bd08c8475e2ca3f +gem_multilevel_multidate.grd a5642ad733e88e9512d979b7c95049e9572591622d9ac94645622c1723742909 gem_packing_dec.grd 547cda1ffb5e143623a33afce76a6cfa8c3c543eed04cf388f3005ccf1ba677d gem_packing_dec.npz b4b3b838b245c0e0d0d803c049a9169c34fe1c09f267d98df5a0d0586297d18d gem_packing_diff.grd 35de37fb378fe1c8cb1be56962ab3496ee079a3c6df4ef838cf1e2e8fd86da3f diff --git a/staticdata/gem_big_endian.grd b/staticdata/gem_big_endian.grd new file mode 100644 index 0000000000000000000000000000000000000000..d30e285af6a5b20d9909ef18ed978b218efa2b87 GIT binary patch literal 38912 zcmeI$y^0f26ae6xoz+4G?QAT@LaY;8F+2I$VcARw5>sjE1NZ>KDp>gdK7wLpV`~vm ze1xqmtZlZi6XLlhZZTFC5_bJA9L_ywlFZx><|dPR+{~)Oa#YQ$vZ;ZQaxZePo-Dc%y?QYm)wAf&qBm>qOsmD9nob7m zTR-39zxIH*&pT-^ipg+T_M@9)cs-@T*OaQ~{in+x`)>x1zI|Ule(|yToYL|`^dlPt z2%H;%qrF&4fA;=mdqscj0qs4ZRkz=-CP3g{3EaLKpB&%ek&CzQ+GknC$i2~nrYSe0 z{TlVz70AE;;_7!IE<|)T_1RsJEB{M@-Tb_6{I>!(AI8#OMLdbf|Jhty@A-q*zn|rz zXAmGjfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ;M@uP1bBvD Ar2qf` literal 0 HcmV?d00001 diff --git a/staticdata/gem_climate.csv b/staticdata/gem_climate.csv new file mode 100644 index 00000000000..c21c9b0b81b --- /dev/null +++ b/staticdata/gem_climate.csv @@ -0,0 +1,2 @@ +STN,YYMMDD/HHMM,PMSL,ALTI,TMPC,DWPC,SKNT,DRCT,GUST,WNUM,CHC1,CHC2,CHC3,VSBY,P03D,P03I,MSUN,SNOW,WEQS,P24I,TDXC,TDNC,P03C,CTYL,CTYM,CTYH,P06I,T6XC,T6NC,CEIL,P01I,SNEW +LWC,210401/0000,1031.3,30.44,9.4,-9.4,12.0,340.0,-9999.0,-9999.0,1004.0,-9999.0,-9999.0,10.0,3005.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,0.5,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0 diff --git a/staticdata/gem_climate.sfc b/staticdata/gem_climate.sfc new file mode 100644 index 0000000000000000000000000000000000000000..88ad00a3d833ba89e8b73e052d734c7187ed5c6a GIT binary patch literal 3584 zcmeH}zi$&U6vrJv{3tCMSQt<_At5%DrjQ8*U2QkDa_5-(rAN0)42b`t*(wl?t(R=u6*lXr zb6JH!`wr;9n9cob@K<1eockRFW{4ev8Ek|VbMAtZ?{y7zDK8&>xpZ1yxm-(Ohgx$X1nbl?x$Kg|6A literal 0 HcmV?d00001 diff --git a/staticdata/gem_little_endian.grd b/staticdata/gem_little_endian.grd new file mode 100644 index 0000000000000000000000000000000000000000..a12cb66eace79edbcc74ca3c8876ada06111adbe GIT binary patch literal 38912 zcmeI#v5FH>6adhts}?F~XJauIi*;fv?#^Vg!(t``iK(>o1N;CX5v=?GKS8mwv9(xG z`~)itYa0u@g?QgiaEXmrNQj;bH}}3blg!M4nY?L+?WDY0)aA4++Hzbr!*)2H76*r; zVR0s`>LTXmdEav}AN4}yrO0oemdhxoar^p*(}8YN z0-twNy0fW+JMP8>ntO+JQ%`%%wCzRhN9{MGSue8R%m#HciTO#)P1;-Ia#oe&QI#JA z^EmS;C)s_Z{n21h#Fo7-uMJZAQa#ywTE03sKKlOkVfFsmqob5QmMOhk)%XEHfWX!Z z%)2YtfBtq7_y2nQ`V9Epc>UW(1PJ^sfh#w=|L|Pjyv{53^;kyanfiX)7OPm_!)TiY z^7=2o$yDM=Ie3peAMoSz)vwX<&;Rk?w|}bw+i-JS|EV z|2q|^gjZjK8~?R1bN^lc-&LS1?!S2dC3&$nHN}IZri?O4QifcuJVKVF_DqsOwHgg0 z8jS|rnI!S=!RN5Frn9w;58IM+FA{~G5m zl_WpJ(aVAQR%WY}zBD1xnoL<-2tE>dt^vQ}R=#@#9rgEfxqi2Uh(7}HQxJcYtT{50 z{yzQ>Icn8ma9{kH4Ctd}Ng7MlIyiq1=B;v#CPMyeNL5-S&y(x-Ig9yvs&z-EJZ`Q6jE+MaQW_WRxW z@(*+m^m^96+DlmqsG=Ll+_VMz7Rn3l$AM30Wu+}JUnkwAKE1TwdDZ1dV~X8`esSFq zL<(xJlMEWh|m=s(+jI*Up3AiwPDrIV!2;F1LXtt168wQfN==sOC2G&QBc(VlyP z*cQjZB(B&)C&?W|J&#I~>F7i{hYi&GkyoRKAWmD{-wq~?roGwm_*|y_X(5^T(+Hhp zYKeNrn#xLj%|63|S<#~e(l0PTH+6Uml^2(K;(mp&p#E!#lraf$Tq^Wh^Nt50tohP4 zq}`6`7`F-XS!u%kG$+kmz})C+65nxwu4+vuZNQTgrdOI7h!e_Qv{^|czbK4b3vnu$ z%SsLAq4`VL`qZ`L?CvDQu}7ZP=0JU_*;u-iywEzB^mkHV%%O}Yt8LihO;JLr4r-D&4_OW?gxI%Tsu30IXq1xnGd$1Ztrm~3>de_ z?0Gecjh&cIsx{jS{sw$GxXC>HdNg|)mO+MHEI|A>EKnbZd^lpou=1=7B6$_+>JNF% z?oWs}f4SnM%GG>c6(Y`a#94_rkIlXRieVcMY$3H)7lA)QoK&+$Q-$Y;#ImzRd&ynD zV%@QkkJx~MRCB~7Yc3gLSy7XHc)yAN)O3n|V=g>cbzA68CK`d+M;y_!C6TAp(mJOGrCcNR} z80L1>jyA9L26=w-iANU1unRlv==WFuhJHWtvEr#IJ^9S}G0bX#9p$^CVZZs>(U9Bk5howxR^b}WWFBfC!^SqVqqDx2ltkxzmK9WmX9Z2h*fDtT?5LD1 zDU!4k^{mFrN~iIGjicEBmnO7LVHL&SIgK4_+K`WTH(*|oc;5O_>4qKj>!EkSxKV1H zBtC#jUFs;3VKwq;gK?Mh8>gdKMx-yjn^zA!3A_XJQ+Viik?iOEarD6zM`fQWk@bJy zfu}bdr^as*HGxV~oS;uY{7%S+s(hZ+L~5%2CB}_M9omEUQ+dAMM0&MbL+GO`^cG&* ze-iDR(*$}W>e&Nz)2aHAGbYoe-CV#IgS#U45h{0y)YIQDv;+@Dd^hMvs=Pc)PbIk( z_#EW#j{L{+v;rfGY&eBZt=vvIvt}l{ziJR~xMDnF8Cl)DDO3t>5B*e@e{(1|B~JOo z3#N4h{~GbfApT;$?wXNlUkB0`rf%TF5q}cmFX2uLBUtCffi!kn4{&$f$2rhPf6gbl zA2>z+^AS51AKc%hNt7kJf%ifD1*lIvpW7*#Ww#$kb21tDR~2zq@{2i9>{8wsI?i1I z?}TxKalg{|np=@9C36IA-*g~&Yv^ZU+-&YzKa#nRA4U&3dxAS++)#|UpVw+;WN}Ni z^xlJ!;Pp}eP~=~r&bK^^;l1!vUNo)ELTy6%tUCKX^SWc8uU!#mA0JrN$Vz82`bAs8 zt72RO^2y^ZvLcyvy)N|GjtL4i{!J%rjpp~uMI15TCR7?b75Zm1P#;?{ZVVfGsT%d{ zHAC^L`-t{;T+aJD=Bs**=5Hc7&w~CTEh~*deJpX3tzMI13A2=ut$(FeJmPtNsztA^ z1Gncw{|l9*6^Ma(;raJ2x=&u%gn*w%oE3;uz;~oZu_O1b=p>It;3YIQB@T77#Cb5R zK9%+^Q#6_*;EA7jx#x8OCtbE8E2@g$^62NR)WWXi#qJ% z4;Too(a`Yp!9naZV6 zJ?NyB2Cn}z7y7N1JUz3OkdTh_$=z^%u2mlR9=r?Lva~k`aoQsvqxw8cUcFqW>C@!- znr%wg)%ED+>!Ex}_npvJ#=O3Flg{TN&&Tk#(=l!qc!sRk>t!jp2z7fz#M&IF&*Fwr z8)ReTWLer=q-p0r*&ljkOFMIH9BB(9UOZ$iI^NYY%y*ukC0O(Q2FvgE#3Pbe-! zbI3*4*}Ue19U?wa>pA5V#?63t7Vo>rrvzgm|G}lWj|t$D!L#}L^gXmj9ecTO%}>hA zSfk39R?UX~4IP_Q*-i4gpjPR`wVT^YeflDRgMdDkqVuR+!Od*M3jNl>40YJI#CkCHiQ^+b+3^G@k!wF!4*RNdmm@EA5M$k(FWIyp>@i^v=Y)=#Gs2 z8u>V5?%t@gg{%H~_e0dPdPSVg{87{iSU}SHO&(_8^ zKO!f;-3ooh^QE3+0^#puS(0vm3xAC9O##}%vwCgkupY9@oquqgRc!nEYbD;kcu~iv z&}Yc%{TJsC(^#AXG)DvVFH9$Yi;%g)PjZ@5I`}BnM+fR%&8bK0%51w0DK+gDo)W$_ z#Vt_(jXCG^7Pn+)C$jx~p?1;3;o8pUcDt4IDT5z``w{Q4p;#-)!x6s#zW%)V;*a_4 z*s1R1T%lP#ufWqI&U~|YepO#D_pt5f zC~;VSNn8ACi1yKi9JkZIP>j1rya({XmkqoYwZv&iF0!*4QvF!Qyexzt& z1KpvMJ+*bG_bZ+;IKm{>57qxxoMS{i#aUp%yeQ_MmPzbbPo3$07wv&jBaW8uyNvje zY{1R!sE;drdS|uHWu$+#)*#F_=NW+ zTC!Y?pB?nAdM>$ith%=GGgosvmt@pU%Blg*@Mg z9N=?v)i~Zom8j$(&J7RYN4A1r`^1k#315F2`~2_V@62I2%bAx)HEPgb2QPsid1BUR zYGOT$#W@oGS`xn7j5zP$$6dHhtr(TJz4xc?!H*`^*x|j(MV_i2zS}LUh-LcP;{9{M zdl`du+LlV&U!o2N@Qw?ggc`% z{=W2HUIRt=O-ULE?J{*A%NwwiN>WSZ`o$1-w|Ed=GSI@4xDh_0m2!8(ZM55^7S_J<-_S>X!0c|a%dkG?Wa&67Wmg@5l0 zPAdFhg39wwM9`4!uHd~5CrA$|hRyLT#lSsoSMx~w#v z*IXCLTmvo>ollffWXPjU_QmrzJYCIS^Fd=aZCX9m46z4v~Chhct{Ei}%#x zzczO9r?t|fc&&8g4?n1`WAIHH%`o_pXw-i*k665cE#2)%W@T9m8k?4P{Rs9k6KGffU`<}*Q`)_8e{7dAs!8?_|4>h7C zGs5^DKM`lE#oq<*QsS%F(XyxW`FR(yKTDFj&iB}he5{a95ayN2YaMlvn`X8rPk${y z{*Q^}oFw!aGQ0%(Pest@!gq*sRT@8ix(*dQrD+ND8Q7Bp)HssVUFA}XQt-9lV^x02 zeJb@hRE30FpHU`$Z6s|QjN{LL6M1G(weJNVq|pq5eiHPV+_2#~d9pD^_RqV8=b;09 z!ALwmTlh4)eQeU+w{8=Q?<*U_n-PbpqxtggS%|X}^^b7->_$;}1 zw$9LtJ}>;P=t~;hky8M2bTU78`J3+F@Y3yYoTTd;uW0R?hnWVC?hJiPi_aGOzd`K( zqEF{;Zn>-nt5)nUUs+TQoMOIa6VCv~7>49Nq7bj4||hzr}Osr1QJfQ|no!mudX8 zyWk;~^M>#Xrf%>@5r|*Q{NoqXSPhRq+ap{qFvCtjcMuj^268fR0FHyG(nAbX$N5R){SA#Ay&Qa?z;<3puEl=gD4-)-QxgO$(eyGXZ$2yYL z&)P>w%~-Nb{WzfS_@Tf3r9`ouh8SOgn1u$V+lN`&4H`ex=py~Hao zGqPOQGV&;Qt!|-x4Q-3Wj^?6Y@1mYY>`NCc{Zmsl#OV(GWtIQ*;WBBxGZk@c5r-h3 zB9)IyzCpCwO}c|?Y_t(x!_7Q5h{&kO%$_zrvQOX!>PZnuo^tq)1| zga^7o^~~&;<2!TJuUcXav2eBTY1j{aO{_N^Rld_;O`Dj|$29B)uLrJE`QiTdG+S?_ zbh6H5_T5`@`|iV4j`KN{8dXyyDINI;|25jevzyUxo77P5{+i0-cX#3e*CwmHT6vp) z_^<9*FN0P7yp0C^JNPf`MX4zotcwx+!}4(UY>FEV{a*AX#h%{>d=_7I+Q5#bc+=xr z^b^q^OidYt{b2#G+9-^@OrA%RlUsm`{-A|h{748DehqsJ<~0QKT8eSQ*v{}J)PHL$ z@R{Jlz~`uZz|Cm-c6K|(K5H7gljhB9)1XiMW8cu?*JkrL*DzL`8%=k#6+UhZ?r#9* zwUE~=G_bVaBI$vN-IOq|LF_{9ARhfJ262oy_rNC}=?(oL=*51xT;+FG8fn{K1$Re1 zMPESpI3v>(7^vhYe4h?=n2Y$L_hV$ou7%Q~A4Q*0Mw}^#lc@4k(`-7^+f8XZx0lM7 zC#_dGdysyt&u2Rb`$tm56;mW@F_=;Z+mPf2m6(#3Jm z+aOO%ALrIvBP(!iL}%Dd0Dn&{eoFMw!l0>5FEkGXe?=uJ3Tz+W5wMg^x#dKUOt$nZ z%SyxXyzSvFl_ku7R%3eq@ifK!`~mfSYvjIfMVwIFi&hnW_BP^2VqW+D zwE46J;Ag0%Z@Z5hNdlW#zY=9FLcvSHg>Nn3WvPkmZPE*Jy^_T*t7jF_55+sBu6yvy zWu?1lpgsor!~Bi&YBuk$tz=HiX!wq7LbWfMh&Vn*^+yk*DKE-PyNKj`9Sr{`ee*&)F)mtA(K^& zb|EdEZd4*leCQ^=m=C>q5aaH&)5O%|$Wt*uV@We_aoHAz&e&-la(8 zY8><{vDaa~;`amxc+WRmS@5V`a@7;rihQCY9o=~m&)on$F17!KKBlj}C0*WjF+a9e zr{d(;k$iF$_i1XG#-uBu;QuV+Y*)C;=awE7gwcQuYugo~0#!-DS{Cl9@33<-K zbF!ar@N_2DtNN2KY;%>MC-rE}eY1Fs-+t(W&=12o2K~^HKibipuV?XdodoYr)&4B} zzD84(O42N_{d{<>mh?@;JNfCYeAMABd~t|c&;6HbQ77x;a;stAC_8grko!UN`1~S~ zPYw83^pVFNfc|mCIp7=cVrn^;h<F=qvm_ z`mK0h;P=HkS;!ZbU!yl4T$005P0GlNr^rdKDST3!Lg?>U`k1#&N?gk#vhs$Wy9bH) zsN%~0UdapELt3@b^E!0}FSh*t-lha~&L!}3U>oIcMyGTcbso<9({OD`T1}J@v|AjbK(RaOV#^@bB|x)D01W5 z0KROlSjWbSd}IlI+d%yYj5{9V?&V!#6#AFDqdYg|2J&$S9|ev$mrjz~yJySg_P53J z`HAQJp2wWtACMO`x`#Nez&)Vf&fAU5VC_tQ$eo7#smwUkoP6J9AfM=jzBbOU>ho67 z&lUT1S*eAq&-36%xW5fiAJG@ysp@ZeoR;0nA1nI4cBJU;ApRoJqF2wm+UL+$CuOAr V@!W!cJXei*f&Blw{%=&^e*x*Qd&mF) literal 0 HcmV?d00001 diff --git a/staticdata/gem_multilevel_multidate.grd b/staticdata/gem_multilevel_multidate.grd new file mode 100644 index 0000000000000000000000000000000000000000..98b72ea1a8ac78b907174e682c75f8d5dfab9f99 GIT binary patch literal 38912 zcmeI0y^bSA5XT1&2mlE(5`rZ_#CN23_KrW+yWH+-1*?l7}_BssH@9RNd_vdpsV`db~BGnXdk>`gK)L+q3D#RsQs7l3(XX7x`sA zn_f&Wua8d7&ZkE&Wm)#JJa38sxp*Kyp!_=OCI8?y|6J1NQg(y3uSmJ1ugUX@hDCr|fAI;96J(TCs?AdrS zyOQ!NDZ9G(@G^fk$}i7H*a&V!b4iMauk84IJU)_|A)j}~S@zfH%i&k~_b30o`1kM6 zM!$aj`HL+3J#On6?S*o`4ub&BfB22}sytY067(0#3jQH~}Z%1e|~q z=pTW{^Tp!voNq|GvjDR+y`MDhnaBq#A|m zW182I+9lS+x73EN(&nt{@Z79TS2k}mPZM7Bs{7|#;9Tti)jS6C>bmOjR@ZE^=3Vtw z3(D;;XEGF4Pv*nczK77E5Brw-7O5OZ_JfGB3{!9&&e3*IS|7IEMDEHzwwXumIWE|k z{b2H>;1M%JE*DyZf|vH*KJ9~V=^hUTj79S=5E&HlW(+|km(qMb7RlOfPYast{}tK^ z4#)G6aIl-a%)hW_O@tMB#T>2I=2Aa=%j^P?kyMJ@1CV` z2JpGXS6g_9Nt@<#NcZ`IjM*P>3kndYlo z)2`KRP^)^?Su16QKG>JA$4`aNiTD;C>`l2m;KLmYtl(?R!QViNXAe+oe%H}=z4c*> z^?t+_al5iI9W}HLX9JMH`QW4Zsl2C>$1cTK#2e>QV}-a>QsidiZO%@0>NX8onf7IC zhnCs`)`8bZ-q^cNgqNNZcyhN0girA%6U1Rir^(J=rr)l)m6IZoDt(j}QgFeJ1Wq&g zI^sM5j~h!lyb9mRfjp$cGl#}wI8KwZ#je?ujUW0;tba%{?y2qIAMhZY4yHopRESN) z%NTv!y+#Eq#2<+vK6%E#DC%oEV*RvgEn`jF&8=cZ&6np3EkX=&C#Qd%2rHgf5uxxZ z?C*I&N@Eeb8P8oxu-U&k%bM-JB(!~%9Ij>94krRe-JsavNel7FGZ%abyuc2R-jL~S z91VFJH{8{%G?YSpc29{jtoJR`Q1wgufC}t}kMZ1&U2Ia6p@t_4ek(x?Em00`{EtV^ zZ~VV7q-L`(OIvFSQNK;9uCO5RW*6J(Y7SPW${FiJ0UKHuz#1HGDl0i;Tx` zOh&o+SgK2Rylkc_PFij)DTTZ=y5KLMpuElsV)i?d|MXx)eib1hD!~rlh+N(pkMt|j zL@V6tW2r7(=R@0-PnediCvBxADyx5CZ-<}ozd-zV%2R1LrpjQ9e=rckV69B`d`7ay zt4xTIiLvYz3bC=KVo3)wWQCxtqqN4N*$_7YO~gH6uzx2ZM`3P)o-(ux`Q`JVMo?4A7in6wA5m> z3i+eHtSs$!*Lk%rU12ScQ?-efDlfz&{mgtV?zg<}pq``LHG%r`Q{&h&B@&m$In1Da z`FjSOmRV`LzHzA{s5W!0tyNealVXk>DC_r-Ib(JxjEss(pc7x4@KT;674upiIi9C) zIocCjwT>0|z)t04sj*@k*7S|Zruv%w+ew?w+sF2Ge5(${ecQX@2x@O->aeKOX5V*wMxBB>ujERxkSul0Ua zc5WcglojWwT9N*juVbZtvaX$aWB-Y)wm|KyO4Hs|+|qXa&cn*(tIBp|r@oc;)Ow_0 zoAO!G-hId~Te6N7S@glSc6D4P#qmQQQCVlM>ZA4dEmc06YD{S!>3w6e%HEJm%7*!6 zUTM2k;~!ETNEk0zmXe5Bh3yZCLuyNGwbh#a#!chVTA8G5SGBR$hb0*+rBFq+iQR3E z)nxwm`xqsXYk!8r7%}!M*mQWl(B?{+UpcFFT0#3jQH~}Z%1e|~qZ~{)i2{-{K-~^n26L11fzzH}3C*TB}fD>>6PQVE`0Vm)D PoWMR1$g;&E+oS7$r3c7Z literal 0 HcmV?d00001 diff --git a/tests/io/test_gempak.py b/tests/io/test_gempak.py index 92de1a34456..f10d48e9880 100644 --- a/tests/io/test_gempak.py +++ b/tests/io/test_gempak.py @@ -7,7 +7,7 @@ import logging import numpy as np -from numpy.testing import assert_allclose, assert_almost_equal +from numpy.testing import assert_allclose, assert_almost_equal, assert_equal import pandas as pd import pytest @@ -17,6 +17,18 @@ logging.getLogger('metpy.io.gempak').setLevel(logging.ERROR) +@pytest.mark.parametrize('order', ['little', 'big']) +def test_byte_swap(order): + """"Test byte swapping.""" + g = get_test_data(f'gem_{order}_endian.grd') + + grid = GempakGrid(g).gdxarray()[0].squeeze() + + reference = np.ones((113, 151), dtype='int32') + + assert_equal(grid, reference) + + @pytest.mark.parametrize('grid_name', ['none', 'diff', 'dec', 'grib']) def test_grid_loading(grid_name): """Test reading grids with different packing.""" @@ -94,6 +106,35 @@ def test_merged_sounding(): np.testing.assert_allclose(gdtar, ddtar, rtol=1e-10, atol=1e-2) +def test_merged_sounding_no_packing(): + """Test loading a merged sounding without data packing.""" + gso = GempakSounding(get_test_data('gem_merged_nopack.snd')).snxarray( + station_id='OUN') + + gpres = gso[0].pressure.values + gtemp = gso[0].temp.values.squeeze() + gdwpt = gso[0].dwpt.values.squeeze() + gdrct = gso[0].drct.values.squeeze() + gsped = gso[0].sped.values.squeeze() + ghght = gso[0].hght.values.squeeze() + + gempak = pd.read_csv(get_test_data('gem_merged_nopack.csv', as_file_obj=False), + na_values=-9999) + dpres = gempak.PRES.values + dtemp = gempak.TEMP.values + ddwpt = gempak.DWPT.values + ddrct = gempak.DRCT.values + dsped = gempak.SPED.values + dhght = gempak.HGHT.values + + assert_allclose(gpres, dpres, rtol=1e-10, atol=1e-2) + assert_allclose(gtemp, dtemp, rtol=1e-10, atol=1e-2) + assert_allclose(gdwpt, ddwpt, rtol=1e-10, atol=1e-2) + assert_allclose(gdrct, ddrct, rtol=1e-10, atol=1e-2) + assert_allclose(gsped, dsped, rtol=1e-10, atol=1e-2) + assert_allclose(ghght, dhght, rtol=1e-10, atol=1e-1) + + @pytest.mark.parametrize('gem,gio,station', [ ('gem_sigw_hght_unmrg.csv', 'gem_sigw_hght_unmrg.snd', 'TOP'), ('gem_sigw_pres_unmrg.csv', 'gem_sigw_pres_unmrg.snd', 'WAML') @@ -160,6 +201,24 @@ def test_unmerged_sigw_pressure_sounding(): assert_allclose(ghght, dhght, rtol=1e-10, atol=1e-1) +def test_climate_surface(): + """Test to read a cliamte surface file.""" + gsf = GempakSurface(get_test_data('gem_climate.sfc')) + gstns = gsf.sfjson() + + gempak = pd.read_csv(get_test_data('gem_climate.csv', as_file_obj=False)) + gempak['YYMMDD/HHMM'] = pd.to_datetime(gempak['YYMMDD/HHMM'], format='%y%m%d/%H%M') + gempak = gempak.set_index(['STN', 'YYMMDD/HHMM']) + + for stn in gstns: + idx_key = (stn['properties']['station_id'], + stn['properties']['date_time']) + gemsfc = gempak.loc[idx_key, :] + + for param, val in stn['values'].items(): + assert val == pytest.approx(gemsfc[param.upper()]) + + def test_standard_surface(): """Test to read a standard surface file.""" skip = ['text', 'spcl'] @@ -261,6 +320,20 @@ def test_date_parsing(): assert dat == datetime(2000, 1, 2) +@pytest.mark.parametrize('access_type', ['STID', 'STNM']) +def test_surface_access(access_type): + """Test for proper surface retrieval with multi-parameter filter.""" + g = get_test_data('gem_surface_with_text.sfc') + gsf = GempakSurface(g) + + if access_type == 'STID': + gsf.sfjson(station_id='MSN', country='US', state='WI', + date_time='202109070000') + elif access_type == 'STNM': + gsf.sfjson(station_number=726410, country='US', state='WI', + date_time='202109070000') + + @pytest.mark.parametrize('text_type,date_time', [ ('text', '202109070000'), ('spcl', '202109071600') ]) @@ -276,6 +349,20 @@ def test_surface_text(text_type, date_time): assert text == gem_text +@pytest.mark.parametrize('access_type', ['STID', 'STNM']) +def test_sounding_access(access_type): + """Test for proper sounding retrieval with multi-parameter filter.""" + g = get_test_data('gem_merged_nopack.snd') + gso = GempakSounding(g) + + if access_type == 'STID': + gso.snxarray(station_id='OUN', country='US', state='OK', + date_time='202101200000') + elif access_type == 'STNM': + gso.snxarray(station_number=72357, country='US', state='OK', + date_time='202101200000') + + @pytest.mark.parametrize('text_type', ['txta', 'txtb', 'txtc', 'txpb']) def test_sounding_text(text_type): """Test for proper decoding of coded message text.""" @@ -313,6 +400,21 @@ def test_special_surface_observation(): assert stn['vsby'] == 2 +def test_multi_level_multi_time_access(): + """Test accessing data with multiple levels and times.""" + g = get_test_data('gem_multilevel_multidate.grd') + + grid = GempakGrid(g) + + grid.gdxarray( + parameter='STPC', + date_time='202403040000', + level=0, + date_time2='202403050000', + level2=1 + ) + + def test_multi_time_grid(): """Test files with multiple times on a single grid.""" g = get_test_data('gem_multi_time.grd') From d811c65c910a5c7cdae8704ff2e9c3eb3dbbd1b3 Mon Sep 17 00:00:00 2001 From: Nathan Wendt Date: Thu, 14 Mar 2024 08:40:26 -0500 Subject: [PATCH 2/2] Minor test update --- tests/io/test_gempak.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/io/test_gempak.py b/tests/io/test_gempak.py index f10d48e9880..8a389e04c20 100644 --- a/tests/io/test_gempak.py +++ b/tests/io/test_gempak.py @@ -409,6 +409,7 @@ def test_multi_level_multi_time_access(): grid.gdxarray( parameter='STPC', date_time='202403040000', + coordinate='HGHT', level=0, date_time2='202403050000', level2=1