From 33804b9b782441ce390d864bb826e67e3f6e7c81 Mon Sep 17 00:00:00 2001 From: Metamess Date: Tue, 20 Feb 2024 11:28:27 +0100 Subject: [PATCH 1/4] Explicitly request value for 'endStep' as integer --- cfgrib/cfmessage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cfgrib/cfmessage.py b/cfgrib/cfmessage.py index 506d80dc..25624a55 100644 --- a/cfgrib/cfmessage.py +++ b/cfgrib/cfmessage.py @@ -87,7 +87,7 @@ def to_grib_date_time( message[time_key] = int(datetime_iso[11:16].replace(":", "")) -def from_grib_step(message, step_key="endStep", step_unit_key="stepUnits"): +def from_grib_step(message, step_key="endStep:int", step_unit_key="stepUnits:int"): # type: (abc.Field, str, str) -> float step_unit = message[step_unit_key] to_seconds = GRIB_STEP_UNITS_TO_SECONDS[step_unit] @@ -97,7 +97,7 @@ def from_grib_step(message, step_key="endStep", step_unit_key="stepUnits"): return int(message[step_key]) * to_seconds / 3600.0 -def to_grib_step(message, step_ns, step_unit=1, step_key="endStep", step_unit_key="stepUnits"): +def to_grib_step(message, step_ns, step_unit=1, step_key="endStep:int", step_unit_key="stepUnits:int"): # type: (abc.MutableField, int, int, str, str) -> None step_s = step_ns * 1e-9 to_seconds = GRIB_STEP_UNITS_TO_SECONDS[step_unit] From 392cad7372f9b992e9d5f2e14775407818624e0f Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Sat, 20 Apr 2024 18:29:08 +0100 Subject: [PATCH 2/4] Handle sub-hourly steps (#370) --- cfgrib/cfmessage.py | 13 +++++++++++++ tests/sample-data/step_60m.grib | Bin 0 -> 17520 bytes tests/test_25_cfmessage.py | 6 +++--- tests/test_40_xarray_store.py | 12 ++++++++++++ tests/test_50_sample_data.py | 1 + 5 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 tests/sample-data/step_60m.grib diff --git a/cfgrib/cfmessage.py b/cfgrib/cfmessage.py index 25624a55..313daf4d 100644 --- a/cfgrib/cfmessage.py +++ b/cfgrib/cfmessage.py @@ -107,6 +107,17 @@ def to_grib_step(message, step_ns, step_unit=1, step_key="endStep:int", step_uni message[step_unit_key] = step_unit +def from_grib_step_units(message): + # type: (abc.Field) -> float + # we always index steps in hours + return 1 + + +def to_grib_step_units(message, step_unit=1, step_unit_key="stepUnits:int"): + # type: (abc.MutableField, int, int, str, str) -> None + message[step_unit_key] = step_unit + + def from_grib_month(message, verifying_month_key="verifyingMonth", epoch=DEFAULT_EPOCH): # type: (abc.Field, str, datetime.datetime) -> int date = message[verifying_month_key] @@ -149,6 +160,8 @@ def build_valid_time(time, step): COMPUTED_KEYS = { "time": (from_grib_date_time, to_grib_date_time), "step": (from_grib_step, to_grib_step), + "endStep": (from_grib_step, to_grib_step), + "stepUnits": (from_grib_step_units, to_grib_step_units), "valid_time": ( functools.partial(from_grib_date_time, date_key="validityDate", time_key="validityTime"), functools.partial(to_grib_date_time, date_key="validityDate", time_key="validityTime"), diff --git a/tests/sample-data/step_60m.grib b/tests/sample-data/step_60m.grib new file mode 100644 index 0000000000000000000000000000000000000000..04bedb659bd0ee3550e14d272f1e4ba1ae825851 GIT binary patch literal 17520 zcmdVie{2)y83*uhoRB!dN!)1xceu!-AiOqZUWnT376-43L|Rc4Q8uMg{zxa(h8Are zq~NycVv;}{ni!WK!EAxKEZKs{fwF*V3XCNv3t#5 zQ=ezrpL(%QXe(2UM7;t-;(l-@(Xuz%Vyu6}xUb+4{FHc_; z8K5@x`T7dQYE^jaKNtNiwKMktwa0MguU_=5cR38BLoNjSy(oIcmV4-*T2i2|^u>UG z-F^Z*m3w{rJN2g(^Ym3A5Hmi%*1G_(sNKdw{8Vih7Tp_E+bdyw5WsmOKDFZkeNpfI z_ht3}i?@dJ8s<^n#W~hg2GJzOjrfa0n%Bd4Cr(XTiF^wj3cNpy?IFCF#>u2K?@7)Z ze~9v0Gs=5l7~7KA7{_?MG;fCU&VD{XbcgH^J*AMV^>IT#ie-%Iq#(_l=neRd57#c)`EQjL?O+a z;=CWXQ^#*@u6-VA##j;~7MzyWJ{zo@cZ$sS@vO@CH+GzgW5R`mS6IG=M$X&6m8$2* z1Il|`_qy?F44dG~SN4W$~>4Uyc1Ops-Ew1Dep_w@He5W4tolud1IWn|9&e`&(;gd>l;Cr1?@)c?Um-e$$6vS z3ld(-pz=PS#FiA?R_u-l^A>2F_p>J(3GdfT%DZh4The$M=x|H(wldyW;kSB;^Y1s4 z%6rv=PZD_7hHKLL8wFv`n?F29co&70cP5FcFeXyC?2(r59Oo_GVE{ z_X(HE_lp$%=EGYFyl<4|&2rwlb!&<7uQRQ@ffTwk=(gZT)55&QYR>Ci?kBw8QCu9l zfdkDLGUM4{Xy}-lVwD{_Ewee&{B-f;disx zRxiz)=DbT+P~N338V+H6Qk>(xXyv`U^&!u(B`^y#_7{M|Gz8mmouxkK)(#HGqBb?V6qP%xBseF4Z=(&j=2Yl7i^1Z=%tG~aN$oJ0m%G-4l zJA>GfLT5mj*W}>5_svqruf?mp9VT?f(Gfv=T$*2fuRaWH%fdU0=eneM zhdHlthWfqkp0M(!VwjC%E{^F2Y2I1JJ8|_9>VDwmDdnBA-(h_IPcRtsr#S5VN>4Ux^X;_8Igymai=DaKVDev7D~l-=u3@~<>#c6$`q84?cyK^NFo{=Murngf>*KtWPqq+VYe?mLFope%IOsx$QJVKG z=WQ8I5Z(n2nn6t3{W;|)8L!h>86&(Z zU41^g8#69k(P!0(%QwS#YZqO05Z*^)%KLT-Z-nr^8HXE$)w8*h@j5* za{*io3#d&8)@0}`MwEM6_gFmRI@c(cBzwvL92(X<1 literal 0 HcmV?d00001 diff --git a/tests/test_25_cfmessage.py b/tests/test_25_cfmessage.py index 1bd3293d..c3c83f85 100644 --- a/tests/test_25_cfmessage.py +++ b/tests/test_25_cfmessage.py @@ -28,7 +28,7 @@ def test_to_grib_date_time() -> None: def test_from_grib_step() -> None: - message = {"endStep": 1, "stepUnits": 1} + message = {"endStep:int": 1, "stepUnits:int": 1} step_seconds = cfmessage.from_grib_step(message) assert step_seconds == 1 @@ -40,8 +40,8 @@ def test_to_grib_step() -> None: cfmessage.to_grib_step(message, step_ns, step_unit=1) - assert message["endStep"] == 1 - assert message["stepUnits"] == 1 + assert message["endStep:int"] == 1 + assert message["stepUnits:int"] == 1 with pytest.raises(ValueError): cfmessage.to_grib_step(message, 0, step_unit=3) diff --git a/tests/test_40_xarray_store.py b/tests/test_40_xarray_store.py index d37fb04a..541c8562 100644 --- a/tests/test_40_xarray_store.py +++ b/tests/test_40_xarray_store.py @@ -2,6 +2,7 @@ import gribapi # type: ignore import numpy as np +import pandas as pd import pytest xr = pytest.importorskip("xarray") # noqa @@ -17,6 +18,7 @@ TEST_DATA_MULTIPLE_FIELDS = os.path.join(SAMPLE_DATA_FOLDER, "regular_gg_ml_g2.grib") TEST_DATA_DIFFERENT_STEP_TYPES = os.path.join(SAMPLE_DATA_FOLDER, "cfrzr_and_cprat.grib") TEST_DATA_DIFFERENT_STEP_TYPES_ZEROS = os.path.join(SAMPLE_DATA_FOLDER, "cfrzr_and_cprat_0s.grib") +TEST_DATA_STEPS_IN_MINUTES = os.path.join(SAMPLE_DATA_FOLDER, "step_60m.grib") TEST_DATA_ALTERNATE_ROWS_MERCATOR = os.path.join(SAMPLE_DATA_FOLDER, "ds.waveh.5.grib") @@ -157,6 +159,16 @@ def test_open_datasets_differet_step_types_zeros() -> None: assert res[1].cfrzr.attrs["GRIB_stepType"] == "avg" +def test_open_dataset_steps_in_minutes() -> None: + res = xarray_store.open_dataset(TEST_DATA_STEPS_IN_MINUTES) + + var = res["t2m"] + steps = var.step + assert steps[0] == pd.Timedelta("0 hours") + assert steps[1] == pd.Timedelta("1 hours") + assert steps[5] == pd.Timedelta("5 hours") + + def test_alternating_scanning_mercator() -> None: ds = xarray_store.open_dataset(TEST_DATA_ALTERNATE_ROWS_MERCATOR) values = ds.variables["shww"].data diff --git a/tests/test_50_sample_data.py b/tests/test_50_sample_data.py index c4384dd7..49765d28 100644 --- a/tests/test_50_sample_data.py +++ b/tests/test_50_sample_data.py @@ -29,6 +29,7 @@ "single_gridpoint", "spherical_harmonics", "t_analysis_and_fc_0", + "step_60m", ], ) def test_open_dataset(grib_name: str) -> None: From 453d53fb633318f5b96a8a88e2cbbea3520e6573 Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Tue, 23 Apr 2024 10:27:21 +0100 Subject: [PATCH 3/4] Updated changelog for #370 --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 012efc41..4e6a33fd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,13 @@ Changelog for cfgrib ==================== +x.x.x.x (xxxx-xx-xx) +-------------------- + +- fixed issue where GRIB messages with non-hourly steps could not be read + See `#370 `_. + + 0.9.11.0 (2024-04-05) --------------------- From cc5e3c475e0490fb86a0b2ca4eed4c66e6c6fdfc Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Tue, 23 Apr 2024 10:32:07 +0100 Subject: [PATCH 4/4] mypy fix for #370 --- cfgrib/cfmessage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfgrib/cfmessage.py b/cfgrib/cfmessage.py index 313daf4d..c6d3500b 100644 --- a/cfgrib/cfmessage.py +++ b/cfgrib/cfmessage.py @@ -114,7 +114,7 @@ def from_grib_step_units(message): def to_grib_step_units(message, step_unit=1, step_unit_key="stepUnits:int"): - # type: (abc.MutableField, int, int, str, str) -> None + # type: (abc.MutableField, int, str) -> None message[step_unit_key] = step_unit