From 7fa1d1eed9f33863468780852a2edfc210286882 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 27 Feb 2023 14:14:12 +0300 Subject: [PATCH] Fix incomplete json parsing of records due to nested ForwardRefs in field annotations --- funml/data/records.py | 31 ++++++-- funml/json.py | 1 + tests/conftest.py | 32 ++++++++ tests/test_enum.py | 10 +-- tests/test_list.py | 132 ++++++++++++--------------------- tests/test_records.py | 166 +++++++++++++----------------------------- 6 files changed, 159 insertions(+), 213 deletions(-) create mode 100644 tests/conftest.py diff --git a/funml/data/records.py b/funml/data/records.py index 3aa2068..397049f 100644 --- a/funml/data/records.py +++ b/funml/data/records.py @@ -42,6 +42,7 @@ class Color: Set, List, Union, + ForwardRef, ) from typing_extensions import dataclass_transform @@ -234,12 +235,14 @@ def _normalize(cls, _globals: Dict[str, Any], _locals: Dict[str, Any]): module = importlib.import_module(cls.__module_path__) _globals.update(_default_globals) _globals.update(getattr(module, "__dict__", {})) - _annotations = { - key: value - if not isinstance(value, str) - else _parse_lazy_type(_to_generic(value), _globals, _locals) - for key, value in cls.__annotations__.items() - } + _annotations = {} + + for key, value in cls.__annotations__.items(): + if isinstance(value, str): + value = _parse_lazy_type(_to_generic(value), _globals, _locals) + + value = _evaluate_forward_refs(value, _globals, _locals) + _annotations[key] = value cls.__annotations__ = _annotations cls._validate_class_defaults() @@ -322,6 +325,22 @@ def _parse_lazy_type( raise exp +def _evaluate_forward_refs( + type_: Type, + __globals: Optional[Dict[str, Any]] = ..., + __locals: Optional[Mapping[str, Any]] = ..., +) -> Type: + """Evaluates any forward refs in the given type""" + try: + type_.__args__ = tuple( + _evaluate_forward_refs(arg, __globals, __locals) for arg in type_.__args__ + ) + except AttributeError: + if isinstance(type_, ForwardRef): + return eval(type_.__forward_arg__, __globals, __locals) + return type_ + + def _get_cls_defaults(cls: type, _annotations: Dict[str, type]) -> Dict[str, Any]: """Retrieves all default values of a class attributes. diff --git a/funml/json.py b/funml/json.py index 1146690..8d1a07b 100644 --- a/funml/json.py +++ b/funml/json.py @@ -2,6 +2,7 @@ """ import inspect import json +import typing from typing import Any, TypeVar, Type, Mapping, Tuple, Union, Dict from funml import Enum, Record diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..fb73f1b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,32 @@ +from __future__ import annotations +from typing import List, Any + +from funml import record, Enum + + +@record +class Student: + name: str + favorite_color: "Color" + + +@record +class Color: + r: int + g: int + b: int + a: List["Alpha"] + + +class Alpha(Enum): + OPAQUE = None + TRANSLUCENT = float + + +@record +class Department: + seniors: list[str] + juniors: List[str] + locations: tuple[str, ...] + misc: dict[str, Any] + head: str diff --git a/tests/test_enum.py b/tests/test_enum.py index d8a71fe..72a8469 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -3,6 +3,7 @@ import pytest from funml import Option, Result, Enum, record, to_json, from_json +from tests import conftest def test_enum_creation(): @@ -180,11 +181,6 @@ class Alpha(Enum): def test_from_json_strict(): """from_json with strict transforms a JSON string representation into an Enum or errors""" - - class Alpha(Enum): - OPAQUE = None - TRANSLUCENT = float - test_data = [ '"Alph.OPAQUE: "OPAQUE""', '"OPAQUE: "OPAQUE""', @@ -193,6 +189,6 @@ class Alpha(Enum): for item in test_data: with pytest.raises(ValueError, match=r"unable to deserialize JSON.*"): - from_json(Alpha, item) + from_json(conftest.Alpha, item) - assert from_json(Alpha, item, strict=False) == item + assert from_json(conftest.Alpha, item, strict=False) == item diff --git a/tests/test_list.py b/tests/test_list.py index 79bc4a6..7c6f7e0 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -2,8 +2,9 @@ import pytest -from funml import l, imap, ifilter, ireduce, Enum, record, to_json, from_json +from funml import l, imap, ifilter, ireduce, to_json, from_json from funml.data.lists import IList +from tests import conftest def test_list_creation(): @@ -96,23 +97,6 @@ def test_tail(): def test_to_json(): """to_json method transforms list into a JSON string representation of list""" - - @record - class Student: - name: str - favorite_color: "Color" - - @record - class Color: - r: int - g: int - b: int - a: "Alpha" - - class Alpha(Enum): - OPAQUE = None - TRANSLUCENT = float - test_data = [ (l(2, 3, 5), "[2, 3, 5]"), (l("foo", 6.0), '["foo", 6.0]'), @@ -120,34 +104,38 @@ class Alpha(Enum): ( l( True, - Color(r=8, g=4, b=78, a=Alpha.OPAQUE), - Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), + conftest.Color(r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE]), + conftest.Color(r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)]), ), ( "[" "true, " - '{"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}, ' - '{"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}' + '{"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}, ' + '{"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}' "]" ), ), ( l( True, - Student( + conftest.Student( name="John Doe", - favorite_color=Color(r=8, g=4, b=78, a=Alpha.OPAQUE), + favorite_color=conftest.Color( + r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE] + ), ), - Student( + conftest.Student( name="Jane Doe", - favorite_color=Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), + favorite_color=conftest.Color( + r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)] + ), ), ), ( "[" "true, " - '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}}, ' - '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}}' + '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}}, ' + '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}}' "]" ), ), @@ -160,53 +148,41 @@ class Alpha(Enum): def test_from_json_strict(): """from_json with strict transforms a JSON string representation into an IList of given annotation""" - @record - class Student: - name: str - favorite_color: "Color" - - @record - class Color: - r: int - g: int - b: int - a: "Alpha" - - class Alpha(Enum): - OPAQUE = None - TRANSLUCENT = float - test_data = [ ("[2, 3, 5]", IList[Any], l(2, 3, 5)), ( ( "[" - '{"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}, ' - '{"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}' + '{"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}, ' + '{"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}' "]" ), - IList[Color], + IList[conftest.Color], l( - Color(r=8, g=4, b=78, a=Alpha.OPAQUE), - Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), + conftest.Color(r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE]), + conftest.Color(r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)]), ), ), ( ( "[" - '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}}, ' - '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}}' + '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}}, ' + '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}}' "]" ), - IList[Student], + IList[conftest.Student], l( - Student( + conftest.Student( name="John Doe", - favorite_color=Color(r=8, g=4, b=78, a=Alpha.OPAQUE), + favorite_color=conftest.Color( + r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE] + ), ), - Student( + conftest.Student( name="Jane Doe", - favorite_color=Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), + favorite_color=conftest.Color( + r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)] + ), ), ), ), @@ -221,22 +197,6 @@ def test_from_json_not_strict(): """from_json with not strict attempts to transform each item in a JSON string IList representation to the given annotation, defaulting to the expected JSON.loads output on error""" - @record - class Student: - name: str - favorite_color: "Color" - - @record - class Color: - r: int - g: int - b: int - a: "Alpha" - - class Alpha(Enum): - OPAQUE = None - TRANSLUCENT = float - test_data = [ ('["foo", 6.0]', IList[int], l("foo", 6)), ("[true, -6.0, 7]", IList[int], l(1, -6, 7)), @@ -244,35 +204,39 @@ class Alpha(Enum): ( "[" "true, " - '{"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}, ' - '{"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}' + '{"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}, ' + '{"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}' "]" ), - IList[Color], + IList[conftest.Color], l( True, - Color(r=8, g=4, b=78, a=Alpha.OPAQUE), - Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), + conftest.Color(r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE]), + conftest.Color(r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)]), ), ), ( ( "[" - '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}}, ' + '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}}, ' '"foo", ' - '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}}' + '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}}' "]" ), - IList[Student], + IList[conftest.Student], l( - Student( + conftest.Student( name="John Doe", - favorite_color=Color(r=8, g=4, b=78, a=Alpha.OPAQUE), + favorite_color=conftest.Color( + r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE] + ), ), "foo", - Student( + conftest.Student( name="Jane Doe", - favorite_color=Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), + favorite_color=conftest.Color( + r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)] + ), ), ), ), diff --git a/tests/test_records.py b/tests/test_records.py index 1cd3531..45317d3 100644 --- a/tests/test_records.py +++ b/tests/test_records.py @@ -7,25 +7,18 @@ import pytest from funml import record, to_dict, Enum, to_json, from_json +from tests import conftest def test_records_created(): """record creates distinct records""" + blue = conftest.Color(r=0, g=0, b=255, a=[conftest.Alpha.TRANSLUCENT(0.7)]) + red = conftest.Color(r=255, g=0, b=0, a=[conftest.Alpha.OPAQUE]) + green = conftest.Color(r=0, g=255, b=0, a=[conftest.Alpha.OPAQUE]) - @record - class Color: - r: int - g: int - b: int - a: int - - blue = Color(r=0, g=0, b=255, a=1) - red = Color(r=255, g=0, b=0, a=1) - green = Color(r=0, g=255, b=0, a=1) - - another_blue = Color(r=0, g=0, b=255, a=1) - another_red = Color(r=255, g=0, b=0, a=1) - another_green = Color(r=0, g=255, b=0, a=1) + another_blue = conftest.Color(r=0, g=0, b=255, a=[conftest.Alpha.TRANSLUCENT(0.7)]) + another_red = conftest.Color(r=255, g=0, b=0, a=[conftest.Alpha.OPAQUE]) + another_green = conftest.Color(r=0, g=255, b=0, a=[conftest.Alpha.OPAQUE]) assert blue == another_blue assert green == another_green @@ -57,16 +50,8 @@ class Color: def test_no_extra_fields(): """No extra fields are allowed""" - - @record - class Color: - r: int - g: int - b: int - a: int - with pytest.raises(TypeError): - _ = Color(r=56, g=4, b=45, a=5, y=0) + _ = conftest.Color(r=56, g=4, b=45, a=[conftest.Alpha.OPAQUE], y=0) def test_records_with_defaults(): @@ -207,17 +192,8 @@ class Branch(Enum): assert hr_dept != security_dept -def test_dict(): +def test_to_dict(): """record can be cast to dict using to_dict""" - - @record - class Department: - seniors: list[str] - juniors: List[str] - locations: tuple[str, ...] - misc: dict[str, Any] - head: str - test_data = [ dict( seniors=["Joe", "Jane"], @@ -243,51 +219,39 @@ class Department: ] for data in test_data: - dept = Department(**data) + dept = conftest.Department(**data) assert to_dict(dept) == data def test_to_json(): """to_json transforms record into a JSON string representation of record""" - @record - class Student: - name: str - favorite_color: "Color" - - @record - class Color: - r: int - g: int - b: int - a: "Alpha" - - class Alpha(Enum): - OPAQUE = None - TRANSLUCENT = float - test_data = [ ( - Color(r=8, g=4, b=78, a=Alpha.OPAQUE), - '{"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}', + conftest.Color(r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE]), + '{"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}', ), ( - Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), - '{"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}', + conftest.Color(r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)]), + '{"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}', ), ( - Student( + conftest.Student( name="John Doe", - favorite_color=Color(r=8, g=4, b=78, a=Alpha.OPAQUE), + favorite_color=conftest.Color( + r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE] + ), ), - '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}}', + '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}}', ), ( - Student( + conftest.Student( name="Jane Doe", - favorite_color=Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), + favorite_color=conftest.Color( + r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)] + ), ), - '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}}', + '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}}', ), ] @@ -297,48 +261,35 @@ class Alpha(Enum): def test_from_json_strict(): """from_json with strict transforms a JSON string representation into an Record, raising ValueError if it can't""" - - @record - class Student: - name: str - favorite_color: "Color" - - @record - class Color: - r: int - g: int - b: int - a: "Alpha" - - class Alpha(Enum): - OPAQUE = None - TRANSLUCENT = float - test_data = [ ( - Color, - '{"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}', - Color(r=8, g=4, b=78, a=Alpha.OPAQUE), + conftest.Color, + '{"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}', + conftest.Color(r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE]), ), ( - Color, - '{"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}', - Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), + conftest.Color, + '{"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}', + conftest.Color(r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)]), ), ( - Student, - '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": "Alpha.OPAQUE: \\"OPAQUE\\""}}', - Student( + conftest.Student, + '{"name": "John Doe", "favorite_color": {"r": 8, "g": 4, "b": 78, "a": ["Alpha.OPAQUE: \\"OPAQUE\\""]}}', + conftest.Student( name="John Doe", - favorite_color=Color(r=8, g=4, b=78, a=Alpha.OPAQUE), + favorite_color=conftest.Color( + r=8, g=4, b=78, a=[conftest.Alpha.OPAQUE] + ), ), ), ( - Student, - '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}}', - Student( + conftest.Student, + '{"name": "Jane Doe", "favorite_color": {"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}}', + conftest.Student( name="Jane Doe", - favorite_color=Color(r=55, g=40, b=9, a=Alpha.TRANSLUCENT(0.4)), + favorite_color=conftest.Color( + r=55, g=40, b=9, a=[conftest.Alpha.TRANSLUCENT(0.4)] + ), ), ), ] @@ -350,45 +301,28 @@ class Alpha(Enum): def test_from_json_not_strict(): """non-strict from_json transforms a JSON string representation into an Record, returning the default json-parsed value if an error occurs""" - - @record - class Student: - name: str - favorite_color: "Color" - - @record - class Color: - r: int - g: int - b: int - a: "Alpha" - - class Alpha(Enum): - OPAQUE = None - TRANSLUCENT = float - test_data = [ ( - Color, - '["r", 8, "g", 4, "b", 78, "a", "Alpha.OPAQUE: \\"OPAQUE\\""]', - ["r", 8, "g", 4, "b", 78, "a", 'Alpha.OPAQUE: "OPAQUE"'], + conftest.Color, + '["r", 8, "g", 4, "b", 78, "a", ["Alpha.OPAQUE: \\"OPAQUE\\""]]', + ["r", 8, "g", 4, "b", 78, "a", ['Alpha.OPAQUE: "OPAQUE"']], ), - (Color, '"foo"', "foo"), + (conftest.Color, '"foo"', "foo"), ( - Student, + conftest.Student, '{"name": "John Doe", "favorite_color": 9}', {"name": "John Doe", "favorite_color": 9}, ), ( - Student, - '{"name": 90, "favorite_color": {"r": 55, "g": 40, "b": 9, "a": "Alpha.TRANSLUCENT: 0.4"}}', + conftest.Student, + '{"name": 90, "favorite_color": {"r": 55, "g": 40, "b": 9, "a": ["Alpha.TRANSLUCENT: 0.4"]}}', { "name": 90, "favorite_color": { "r": 55, "g": 40, "b": 9, - "a": "Alpha.TRANSLUCENT: 0.4", + "a": ["Alpha.TRANSLUCENT: 0.4"], }, }, ),