Merge pull request #3431 from nawendt/gem-perf

GEMPAK performance updates

.codespellignore

@@ -6,12 +6,11 @@
     thta = 2
                 lambda grid: grid if grid.PARM in parameter else False,
                             col_head.SELV,
-                            'SELV': col_head.SELV,
+                'SELV': col_head.SELV,
                                 col_head.SELV,
                                 row_head.SELV,
-                           'SELV': row_head.SELV,
-                       'SELV': col_head.SELV,
-                           'SELV': col_head.SELV,
+                'SELV': row_head.SELV,
+                'SELV': col_head.SELV,
 Klystron Warmup    Integer*2  N/A  0 to 1  1  0=Normal, 1=Preheat  146
 # GFS, NAM, RAP, or other gridded dataset (e.g., NARR).
 # This attribute can be set to False if the vector components are grid relative (e.g., for NAM

src/metpy/static-data-manifest.txt

@@ -35,13 +35,20 @@ cubic_test.npz 8b9d01c2177a057b3352bb6c5b775dae6b796d37ba35b4775fcb65300dc06ccf
 dec9_sounding.txt 4f60955bee4a59e2da0c225d778b9a04a149e9a17b4dce6bfefc111240b3b165
 gem_azimuthal.grd 2b1cb2a358135035dbf6926f0e8ba8122f8b3a0477da5183943a1299ee602093
 gem_azimuthal.npz 57d9f69cfa08893ae2d388bda7fb15a1b306ba205e3f15b9e40e963e7db79b0f
+gem_big_endian.grd cf2828181311677081e86253e2b7eb5778450ee92cf6b8161dc0635687b1fd9d
+gem_climate.csv 24c562fac9d5b81c08092ce5cf5522b57c74a63a184864e211016110fbc983c6
+gem_climate.sfc 92f7fc278e16a9f2fbdb054b6c2a0f9f2aea420658841687e34ad2ad9a6552a7
 gem_conical.grd 0184d05a596d623171135fc3243060aef69da8601c5ef8448bb6f39914474c3d
 gem_conical.npz fd77f4e4fb2884087bc022c957277b4ec11bb21a35c9d3e261b64df425b1da28
 gem_cylindrical.grd 1b2ee56e4ee50d8d3012aa319f24a7c09e092db8e8a5a8f58e2d857a8fd1520f
 gem_cylindrical.npz 68ea33a58b24b651254dd4d2c935c33fe4f414ffd404296c9098b6404a37e408
+gem_little_endian.grd 12c5c094c6505697c89426f362aa9bd6ba7de12f25c690ea73fb6d0159925536
+gem_merged_nopack.csv 57ac3d9dc6b2461f0d0b8739452dceaa88f11fc8d55f75d8f05ed0981af0bf97
+gem_merged_nopack.snd 141a1122e9638b0dde5d2ec424c03748d93577b84f10bbe7c29d9d3228200a88
 gem_model_mrg.csv 304afb3bb4b662f7adfc8803e8661fc217a691d0b1790d5e0e6d085f198c4257
 gem_model_mrg.snd 6d229a7af8d3e86d8da02403023cdaf0f86b6c982eb8b6f7f84c3836811df014
 gem_multi_time.grd a11746f162f2ea3944573246a477db94026f76e72ce7ecb98bd08c8475e2ca3f
+gem_multilevel_multidate.grd a5642ad733e88e9512d979b7c95049e9572591622d9ac94645622c1723742909
 gem_packing_dec.grd 547cda1ffb5e143623a33afce76a6cfa8c3c543eed04cf388f3005ccf1ba677d
 gem_packing_dec.npz b4b3b838b245c0e0d0d803c049a9169c34fe1c09f267d98df5a0d0586297d18d
 gem_packing_diff.grd 35de37fb378fe1c8cb1be56962ab3496ee079a3c6df4ef838cf1e2e8fd86da3f

staticdata/gem_climate.csv

@@ -0,0 +1,2 @@
+STN,YYMMDD/HHMM,PMSL,ALTI,TMPC,DWPC,SKNT,DRCT,GUST,WNUM,CHC1,CHC2,CHC3,VSBY,P03D,P03I,MSUN,SNOW,WEQS,P24I,TDXC,TDNC,P03C,CTYL,CTYM,CTYH,P06I,T6XC,T6NC,CEIL,P01I,SNEW
+LWC,210401/0000,1031.3,30.44,9.4,-9.4,12.0,340.0,-9999.0,-9999.0,1004.0,-9999.0,-9999.0,10.0,3005.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,0.5,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0

staticdata/gem_merged_nopack.csv

@@ -0,0 +1,93 @@
+PRES,TEMP,DWPT,DRCT,SPED,HGHT
+1000.0,-9999.0,-9999.0,-9999.0,-9999.0,261.0
+990.0,8.4,-3.6,10.0,3.6,357.0
+959.11,5.88,-5.19,350.0,5.15,610.0
+925.0,3.0,-7.0,0.0,5.15,899.0
+923.28,2.85,-7.03,0.0,5.15,914.0
+888.92,-0.17,-7.6,5.0,5.66,1219.0
+883.0,-0.7,-7.7,4.12,6.56,1272.77
+871.0,-0.9,-11.9,2.32,8.42,1382.02
+864.0,1.2,-22.8,1.26,9.51,1446.65
+855.73,1.35,-25.65,0.0,10.81,1524.0
+853.0,1.4,-26.6,357.62,11.3,1549.65
+850.0,1.6,-15.4,355.0,11.84,1578.0
+849.0,1.6,-12.4,354.81,11.86,1587.48
+833.0,1.6,-10.4,351.76,12.18,1740.69
+823.89,1.3,-12.18,350.0,12.36,1829.0
+793.17,0.28,-18.32,300.0,7.21,2134.0
+785.0,0.0,-20.0,293.17,8.34,2217.13
+763.67,0.56,-4.71,275.0,11.33,2438.0
+762.0,0.6,-3.5,274.71,11.77,2455.54
+735.15,-0.18,-4.1,270.0,19.05,2743.0
+718.0,-0.7,-4.5,266.9,20.33,2932.19
+707.67,-0.84,-6.83,265.0,21.11,3048.0
+703.0,-0.9,-7.9,265.0,19.86,3100.88
+700.0,-1.3,-8.3,265.0,19.05,3135.0
+655.04,-5.14,-13.67,275.0,17.5,3658.0
+642.0,-6.3,-15.3,272.44,19.08,3816.46
+626.0,-6.3,-16.3,269.23,21.07,4013.39
+605.56,-7.95,-17.1,265.0,23.68,4267.0
+559.08,-11.93,-19.02,270.0,27.8,4877.0
+516.23,-15.91,-20.93,270.0,31.92,5486.0
+500.0,-17.5,-21.7,265.0,32.43,5730.0
+476.0,-20.7,-27.7,265.0,32.43,6096.72
+412.0,-28.9,-31.7,273.3,34.57,7148.81
+400.0,-30.3,-35.3,275.0,35.01,7360.0
+386.0,-31.7,-40.7,270.13,35.01,7613.15
+385.62,-31.76,-40.7,270.0,35.01,7620.0
+369.28,-34.33,-40.7,270.0,34.49,7925.0
+367.0,-34.7,-40.7,270.0,34.83,7968.52
+339.0,-38.5,-52.5,270.0,39.21,8519.31
+313.0,-42.5,-55.5,270.0,43.61,9063.95
+309.31,-42.95,-56.23,270.0,44.27,9144.0
+300.0,-44.1,-58.1,270.0,46.33,9350.0
+279.0,-45.9,-62.9,268.01,53.09,9835.73
+250.0,-50.5,-63.5,265.0,63.32,10560.0
+245.85,-50.94,-63.57,265.0,65.38,10668.0
+234.49,-52.17,-63.78,265.0,71.04,10973.0
+231.0,-52.56,-63.84,265.0,71.04,11069.62
+228.0,-52.9,-63.9,265.45,70.85,11153.9
+200.0,-60.3,-69.3,270.0,68.98,11980.0
+193.0,-61.1,-70.1,270.0,65.89,12202.11
+185.0,-57.7,-68.7,270.0,63.73,12467.66
+174.0,-58.3,-70.3,270.0,60.6,12854.69
+151.55,-62.77,-73.84,270.0,53.54,13716.0
+150.0,-63.1,-74.1,270.0,53.54,13780.0
+130.0,-66.3,-77.3,265.0,46.33,14651.96
+124.1,-64.8,-76.68,255.0,43.76,14935.0
+118.05,-63.19,-76.02,270.0,45.3,15240.0
+117.0,-62.9,-75.9,270.27,44.64,15294.27
+101.0,-66.7,-77.7,274.7,33.69,16189.89
+100.0,-66.3,-77.3,275.0,32.95,16250.0
+91.8,-68.57,-78.94,280.0,28.83,16764.0
+87.3,-69.9,-79.9,265.16,27.3,17065.63
+87.25,-69.85,-79.88,265.0,27.28,17069.0
+81.9,-63.7,-77.7,279.73,23.16,17451.51
+75.07,-64.81,-77.7,300.0,17.5,17983.0
+71.41,-65.45,-77.7,290.0,15.44,18288.0
+70.0,-65.7,-77.7,290.0,15.44,18410.0
+64.59,-66.09,-78.09,275.0,13.9,18898.0
+55.55,-66.81,-78.81,285.0,11.33,19812.0
+52.3,-67.1,-79.1,302.96,9.48,20177.21
+50.23,-65.66,-78.56,315.0,8.24,20422.0
+50.0,-65.5,-78.5,315.0,7.21,20450.0
+47.9,-63.1,-78.1,291.22,3.78,20712.52
+47.79,-63.11,-78.1,290.0,3.6,20726.0
+43.27,-63.78,-78.21,265.0,4.63,21336.0
+40.0,-64.3,-78.3,316.48,6.26,21818.41
+39.18,-63.86,-78.17,330.0,6.69,21946.0
+35.51,-61.74,-77.57,0.0,12.36,22555.0
+32.9,-60.1,-77.1,2.26,10.73,23025.97
+30.0,-61.5,-77.5,5.0,8.75,23600.0
+21.7,-60.3,-78.3,36.95,8.75,25615.8
+20.0,-56.1,-77.1,45.0,8.75,26130.0
+18.5,-55.1,-77.1,58.32,8.36,26626.25
+16.3,-55.77,-78.45,80.0,7.72,27432.0
+12.7,-57.1,-81.1,27.95,4.24,29015.18
+10.8,-53.9,-80.9,354.11,1.97,30047.19
+10.09,-54.6,-81.6,340.0,1.03,30480.0
+10.0,-54.7,-81.7,320.0,2.06,30540.0
+9.1,-57.3,-83.3,-9999.0,-9999.0,31139.48
+7.9,-48.9,-81.9,-9999.0,-9999.0,32050.37
+7.6,-50.1,-83.1,-9999.0,-9999.0,32303.82
+7.3,-52.5,-83.5,-9999.0,-9999.0,32565.37

tests/io/test_gempak.py

@@ -7,7 +7,7 @@
 import logging
 
 import numpy as np
-from numpy.testing import assert_allclose, assert_almost_equal
+from numpy.testing import assert_allclose, assert_almost_equal, assert_equal
 import pandas as pd
 import pytest
 
@@ -17,6 +17,18 @@
 logging.getLogger('metpy.io.gempak').setLevel(logging.ERROR)
 
 
+@pytest.mark.parametrize('order', ['little', 'big'])
+def test_byte_swap(order):
+    """"Test byte swapping."""
+    g = get_test_data(f'gem_{order}_endian.grd')
+
+    grid = GempakGrid(g).gdxarray()[0].squeeze()
+
+    reference = np.ones((113, 151), dtype='int32')
+
+    assert_equal(grid, reference)
+
+
 @pytest.mark.parametrize('grid_name', ['none', 'diff', 'dec', 'grib'])
 def test_grid_loading(grid_name):
     """Test reading grids with different packing."""
@@ -94,6 +106,35 @@ def test_merged_sounding():
     np.testing.assert_allclose(gdtar, ddtar, rtol=1e-10, atol=1e-2)
 
 
+def test_merged_sounding_no_packing():
+    """Test loading a merged sounding without data packing."""
+    gso = GempakSounding(get_test_data('gem_merged_nopack.snd')).snxarray(
+        station_id='OUN')
+
+    gpres = gso[0].pressure.values
+    gtemp = gso[0].temp.values.squeeze()
+    gdwpt = gso[0].dwpt.values.squeeze()
+    gdrct = gso[0].drct.values.squeeze()
+    gsped = gso[0].sped.values.squeeze()
+    ghght = gso[0].hght.values.squeeze()
+
+    gempak = pd.read_csv(get_test_data('gem_merged_nopack.csv', as_file_obj=False),
+                         na_values=-9999)
+    dpres = gempak.PRES.values
+    dtemp = gempak.TEMP.values
+    ddwpt = gempak.DWPT.values
+    ddrct = gempak.DRCT.values
+    dsped = gempak.SPED.values
+    dhght = gempak.HGHT.values
+
+    assert_allclose(gpres, dpres, rtol=1e-10, atol=1e-2)
+    assert_allclose(gtemp, dtemp, rtol=1e-10, atol=1e-2)
+    assert_allclose(gdwpt, ddwpt, rtol=1e-10, atol=1e-2)
+    assert_allclose(gdrct, ddrct, rtol=1e-10, atol=1e-2)
+    assert_allclose(gsped, dsped, rtol=1e-10, atol=1e-2)
+    assert_allclose(ghght, dhght, rtol=1e-10, atol=1e-1)
+
+
 @pytest.mark.parametrize('gem,gio,station', [
     ('gem_sigw_hght_unmrg.csv', 'gem_sigw_hght_unmrg.snd', 'TOP'),
     ('gem_sigw_pres_unmrg.csv', 'gem_sigw_pres_unmrg.snd', 'WAML')
@@ -160,6 +201,24 @@ def test_unmerged_sigw_pressure_sounding():
     assert_allclose(ghght, dhght, rtol=1e-10, atol=1e-1)
 
 
+def test_climate_surface():
+    """Test to read a cliamte surface file."""
+    gsf = GempakSurface(get_test_data('gem_climate.sfc'))
+    gstns = gsf.sfjson()
+
+    gempak = pd.read_csv(get_test_data('gem_climate.csv', as_file_obj=False))
+    gempak['YYMMDD/HHMM'] = pd.to_datetime(gempak['YYMMDD/HHMM'], format='%y%m%d/%H%M')
+    gempak = gempak.set_index(['STN', 'YYMMDD/HHMM'])
+
+    for stn in gstns:
+        idx_key = (stn['properties']['station_id'],
+                   stn['properties']['date_time'])
+        gemsfc = gempak.loc[idx_key, :]
+
+        for param, val in stn['values'].items():
+            assert val == pytest.approx(gemsfc[param.upper()])
+
+
 def test_standard_surface():
     """Test to read a standard surface file."""
     skip = ['text', 'spcl']
@@ -261,6 +320,20 @@ def test_date_parsing():
     assert dat == datetime(2000, 1, 2)
 
 
+@pytest.mark.parametrize('access_type', ['STID', 'STNM'])
+def test_surface_access(access_type):
+    """Test for proper surface retrieval with multi-parameter filter."""
+    g = get_test_data('gem_surface_with_text.sfc')
+    gsf = GempakSurface(g)
+
+    if access_type == 'STID':
+        gsf.sfjson(station_id='MSN', country='US', state='WI',
+                   date_time='202109070000')
+    elif access_type == 'STNM':
+        gsf.sfjson(station_number=726410, country='US', state='WI',
+                   date_time='202109070000')
+
+
 @pytest.mark.parametrize('text_type,date_time', [
     ('text', '202109070000'), ('spcl', '202109071600')
 ])
@@ -276,6 +349,20 @@ def test_surface_text(text_type, date_time):
     assert text == gem_text
 
 
+@pytest.mark.parametrize('access_type', ['STID', 'STNM'])
+def test_sounding_access(access_type):
+    """Test for proper sounding retrieval with multi-parameter filter."""
+    g = get_test_data('gem_merged_nopack.snd')
+    gso = GempakSounding(g)
+
+    if access_type == 'STID':
+        gso.snxarray(station_id='OUN', country='US', state='OK',
+                     date_time='202101200000')
+    elif access_type == 'STNM':
+        gso.snxarray(station_number=72357, country='US', state='OK',
+                     date_time='202101200000')
+
+
 @pytest.mark.parametrize('text_type', ['txta', 'txtb', 'txtc', 'txpb'])
 def test_sounding_text(text_type):
     """Test for proper decoding of coded message text."""
@@ -313,6 +400,22 @@ def test_special_surface_observation():
     assert stn['vsby'] == 2
 
 
+def test_multi_level_multi_time_access():
+    """Test accessing data with multiple levels and times."""
+    g = get_test_data('gem_multilevel_multidate.grd')
+
+    grid = GempakGrid(g)
+
+    grid.gdxarray(
+        parameter='STPC',
+        date_time='202403040000',
+        coordinate='HGHT',
+        level=0,
+        date_time2='202403050000',
+        level2=1
+    )
+
+
 def test_multi_time_grid():
     """Test files with multiple times on a single grid."""
     g = get_test_data('gem_multi_time.grd')