From ee96edf59e218ab2f387d58348ad3c698c08d981 Mon Sep 17 00:00:00 2001 From: Maxim V4S Date: Tue, 3 Sep 2024 16:54:49 +0300 Subject: [PATCH 1/3] feat: type converting --- qualibrate_runner/core/run_job.py | 8 +- qualibrate_runner/core/types_parsing.py | 210 ++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 qualibrate_runner/core/types_parsing.py diff --git a/qualibrate_runner/core/run_job.py b/qualibrate_runner/core/run_job.py index 35e003f..a14ab06 100644 --- a/qualibrate_runner/core/run_job.py +++ b/qualibrate_runner/core/run_job.py @@ -10,6 +10,7 @@ from qualibrate_runner.config import State from qualibrate_runner.core.models.last_run import LastRun, RunError, RunStatus +from qualibrate_runner.core.types_parsing import types_conversion def validate_input_parameters( @@ -46,8 +47,13 @@ def run_node( try: library = get_active_library_or_error() node = library.nodes[node.name] + converted_parameters = types_conversion( + passed_input_parameters, node.parameters_class.serialize() + ) + print("before", passed_input_parameters) + print("after", converted_parameters) result = library.run_node( - node.name, node.parameters_class(**passed_input_parameters) + node.name, node.parameters_class(**converted_parameters) ) except Exception as ex: state.last_run = LastRun( diff --git a/qualibrate_runner/core/types_parsing.py b/qualibrate_runner/core/types_parsing.py new file mode 100644 index 0000000..f75660b --- /dev/null +++ b/qualibrate_runner/core/types_parsing.py @@ -0,0 +1,210 @@ +import sys + +if sys.version_info >= (3, 10): + from types import NoneType +else: + NoneType = type(None) +from typing import Any, Dict, List, Mapping, Optional, Type, Union + +NOT_NONE_BASIC_TYPES = Union[bool, int, float, str] +BASIC_TYPES = Union[NOT_NONE_BASIC_TYPES, NoneType] +LIST_TYPES = Union[List[bool], List[int], List[float], List[str]] +VALUE_TYPES_WITHOUT_REC = Union[BASIC_TYPES, LIST_TYPES] +INPUT_CONVERSION_TYPE = Union[ + VALUE_TYPES_WITHOUT_REC, Dict[str, "INPUT_CONVERSION_TYPE"] +] + + +def parse_bool(value: VALUE_TYPES_WITHOUT_REC) -> VALUE_TYPES_WITHOUT_REC: + if isinstance(value, bool): + return value + if isinstance(value, str): + if value.lower() == "true": + return True + if value.lower() == "false": + return False + return value + if isinstance(value, int): + return value != 0 + if isinstance(value, float): + return abs(value) > 1e-9 + return value + + +def parse_int(value: VALUE_TYPES_WITHOUT_REC) -> VALUE_TYPES_WITHOUT_REC: + if isinstance(value, bool): + return int(value) + if isinstance(value, int): + return value + if isinstance(value, str): + if value.isdigit() or ( + len(value) > 1 and value[0] == "-" and value[1:].isdigit() + ): + return int(value) + return value + if isinstance(value, float): + if value.is_integer(): + return int(value) + return value + return value + + +def parse_float(value: VALUE_TYPES_WITHOUT_REC) -> VALUE_TYPES_WITHOUT_REC: + if isinstance(value, float): + return value + if isinstance(value, int): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return value + return value + + +def parse_str(value: VALUE_TYPES_WITHOUT_REC) -> VALUE_TYPES_WITHOUT_REC: + return value + + +def parse_none(value: VALUE_TYPES_WITHOUT_REC) -> VALUE_TYPES_WITHOUT_REC: + if value is None or (isinstance(value, str) and not value): + return None + return value + + +BASIC_PARSERS = { + int: parse_int, + float: parse_float, + bool: parse_bool, + str: parse_str, + NoneType: parse_none, +} + +STR_TO_TYPE: Mapping[str, Type[BASIC_TYPES]] = { + "integer": int, + "number": float, + "boolean": bool, + "string": str, + "null": NoneType, +} + + +def parse_typed_list( + value: List[Any], item_type: Type[BASIC_TYPES] +) -> LIST_TYPES: + if len(value) == 0: + return value + parser = BASIC_PARSERS[item_type] + try: + return list(map(parser, value)) # type: ignore + except ValueError: + return value + + +def parse_list( + value: VALUE_TYPES_WITHOUT_REC, + item_type: Optional[Type[BASIC_TYPES]], +) -> VALUE_TYPES_WITHOUT_REC: + if isinstance(value, List): + if item_type is None: + return value + return parse_typed_list(value, item_type) + if isinstance(value, str): + stripped = value.strip() + if stripped.startswith("[") and stripped.endswith("]"): + stripped = stripped[1:-1] + splitted = list(map(str.strip, stripped.split(","))) + if item_type is None: + return splitted + return parse_typed_list(splitted, item_type) + return value + + +def types_conversion(value: Any, expected_type: Mapping[str, Any]) -> Any: + if isinstance(value, Mapping): + # has sub items + new = {} + for k, v in value.items(): + if k in expected_type: + new[k] = types_conversion(v, expected_type[k]) + else: + new[k] = v + return new + if "anyOf" in expected_type: + # suppose that only `Type | None` is possible + none = parse_none(value) + if none is None: + return None + expected_type_ = dict(expected_type) + expected_type_["type"] = expected_type_.pop("anyOf")[0]["type"] + return types_conversion(value, expected_type_) + if "type" in expected_type: + if expected_type.get("type") == "array" and "items" in expected_type: + # array + item_type: Optional[Type[BASIC_TYPES]] = STR_TO_TYPE.get( + expected_type["items"]["type"] + ) + return parse_list(value, item_type) + if expected_type.get("type") in STR_TO_TYPE.keys(): + expected = STR_TO_TYPE[expected_type["type"]] + parser = BASIC_PARSERS[expected] + return parser(value) + + return value + + +if __name__ == "__main__": + types = { + "targets_name": { + "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, + "title": "Targets Name", + }, + "str_val": { + "anyOf": [{"type": "string"}, {"type": "null"}], + "default": "a", + "title": "Str Val", + }, + "list_str": { + "default": ["a", "b"], + "items": {"type": "string"}, + "title": "List Str", + "type": "array", + }, + "float_val": {"default": 1.1, "title": "Float Val", "type": "number"}, + "list_float": { + "default": ["a", "b"], + "items": {"type": "number"}, + "title": "List Float", + "type": "array", + }, + "int_val": {"default": 100, "title": "Int Val", "type": "integer"}, + "list_int": { + "default": ["a", "b"], + "items": {"type": "integer"}, + "title": "List Int", + "type": "array", + }, + "bool_val": {"default": False, "title": "Bool Val", "type": "boolean"}, + "list_bool": { + "default": ["a", "b"], + "items": {"type": "boolean"}, + "title": "List Bool", + "type": "array", + }, + "none_val": {"default": None, "title": "None Val", "type": "null"}, + } + types_conversion( + { + # "str_val": "aaa", + # "list_str": "[aa, bb]", + # "float_val": "1.2", + # "list_float": ["1.2","1.3"], + # "int_val": "-1111111", + # "list_int": [1,2,3], + # "bool_val": False, + "list_bool": "[1, 0, True, False, true, false, 1.1, 0.0]", + # "none_val": None + }, + types, + ) From de875b85920c970f6c1327920ae4cf824eabc48f Mon Sep 17 00:00:00 2001 From: Maxim V4S Date: Tue, 3 Sep 2024 17:25:39 +0300 Subject: [PATCH 2/3] fix: remove useless print --- qualibrate_runner/core/run_job.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qualibrate_runner/core/run_job.py b/qualibrate_runner/core/run_job.py index a14ab06..cac5f93 100644 --- a/qualibrate_runner/core/run_job.py +++ b/qualibrate_runner/core/run_job.py @@ -50,8 +50,6 @@ def run_node( converted_parameters = types_conversion( passed_input_parameters, node.parameters_class.serialize() ) - print("before", passed_input_parameters) - print("after", converted_parameters) result = library.run_node( node.name, node.parameters_class(**converted_parameters) ) From 600d40787a13d2976559ae603bb52f2185ba9404 Mon Sep 17 00:00:00 2001 From: Maxim V4S Date: Thu, 5 Sep 2024 12:06:55 +0300 Subject: [PATCH 3/3] fix: list type conversion --- qualibrate_runner/core/types_parsing.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/qualibrate_runner/core/types_parsing.py b/qualibrate_runner/core/types_parsing.py index f75660b..257d777 100644 --- a/qualibrate_runner/core/types_parsing.py +++ b/qualibrate_runner/core/types_parsing.py @@ -139,17 +139,18 @@ def types_conversion(value: Any, expected_type: Mapping[str, Any]) -> Any: expected_type_["type"] = expected_type_.pop("anyOf")[0]["type"] return types_conversion(value, expected_type_) if "type" in expected_type: - if expected_type.get("type") == "array" and "items" in expected_type: + if expected_type.get("type") == "array": # array - item_type: Optional[Type[BASIC_TYPES]] = STR_TO_TYPE.get( - expected_type["items"]["type"] + item_type: Optional[Type[BASIC_TYPES]] = ( + STR_TO_TYPE.get(expected_type["items"]["type"]) + if "items" in expected_type + else None ) return parse_list(value, item_type) if expected_type.get("type") in STR_TO_TYPE.keys(): expected = STR_TO_TYPE[expected_type["type"]] parser = BASIC_PARSERS[expected] return parser(value) - return value