From bcd12e41d5788aac2c8bebe4a11b345e47a0bc7c Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Tue, 10 Oct 2023 20:21:39 -0300 Subject: [PATCH 01/13] Add missing optional --- retrack/nodes/outputs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retrack/nodes/outputs.py b/retrack/nodes/outputs.py index 53c991f..9aee712 100644 --- a/retrack/nodes/outputs.py +++ b/retrack/nodes/outputs.py @@ -12,7 +12,7 @@ class OutputMetadataModel(pydantic.BaseModel): - message: str = None + message: typing.Optional[str] = None ################################################ From 6cb6884d5b5ebe42f5a657725cc23e810f781b6a Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Tue, 10 Oct 2023 20:21:56 -0300 Subject: [PATCH 02/13] Update dependencies --- pyproject.toml | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 45771bf..835e9e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "retrack" -version = "0.11.1" +version = "1.0.0-alpha.1" description = "A business rules engine" authors = ["Gabriel Guarisa ", "Nathalia Trotte "] license = "MIT" @@ -10,39 +10,18 @@ homepage = "https://github.com/gabrielguarisa/retrack" keywords = ["rules", "models", "business", "node", "graph"] [tool.poetry.dependencies] -python = "^3.7.16" -pandas = [ - { version = "1.2.0", python = "<3.8" }, - { version = "^1.2.0", python = ">=3.8" } -] -numpy = [ - { version = "1.19.5", python = "<3.8" }, - { version = "^1.19.5", python = ">=3.8" } -] -pydantic = "^1.10.4" -networkx = [ - { version = "2.6.3", python = "<3.8" }, - { version = "^2.6.3", python = ">=3.8" } -] +python = ">=3.8,<4.0.0" +pandas = "^1.2.0" +numpy = "^1.19.5" +pydantic = "2.4.2" +networkx = "^2.6.3" [tool.poetry.dev-dependencies] -pytest = [ - { version = "6.2.2", python = "<3.8" }, - { version = "^6.2.4", python = ">=3.8" } -] -pytest-cov = [ - { version = "2.11.1", python = "<3.8" }, - { version = "^3.0.0", python = ">=3.8" } -] -black = [ - { version = "20.8b1", python = "<3.8" }, - { version = "^22.6.0", python = ">=3.8" } -] +pytest = "^6.2.4" +pytest-cov = "^3.0.0" +black = "^22.6.0" isort = {extras = ["colors"], version = "*"} -pytest-mock = [ - { version = "3.5.1", python = "<3.8" }, - { version = "^3.10.0", python = ">=3.8" } -] +pytest-mock = "^3.10.0" [tool.black] # https://github.com/psf/black From 22883948bf5a8eede6aab778ba07ab8ef6abe2b5 Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Tue, 10 Oct 2023 20:50:18 -0300 Subject: [PATCH 03/13] From dict to model_dump --- retrack/engine/parser.py | 2 +- retrack/engine/runner.py | 4 ++-- retrack/nodes/base.py | 17 +++++++++++------ tests/test_nodes/test_check.py | 2 +- tests/test_nodes/test_contains.py | 2 +- tests/test_nodes/test_datetime.py | 2 +- tests/test_nodes/test_endswith.py | 2 +- tests/test_nodes/test_endswithany.py | 2 +- tests/test_nodes/test_input.py | 2 +- tests/test_nodes/test_logic.py | 2 +- tests/test_nodes/test_math.py | 2 +- tests/test_nodes/test_start.py | 2 +- tests/test_nodes/test_startswith.py | 2 +- tests/test_nodes/test_startswithany.py | 2 +- 14 files changed, 25 insertions(+), 20 deletions(-) diff --git a/retrack/engine/parser.py b/retrack/engine/parser.py index a0fa573..72aede3 100644 --- a/retrack/engine/parser.py +++ b/retrack/engine/parser.py @@ -166,7 +166,7 @@ def _set_execution_order(self): def get_node_connections( self, node_id: str, is_input: bool = True, filter_by_connector=None ): - node_dict = self.get_by_id(node_id).dict(by_alias=True) + node_dict = self.get_by_id(node_id).model_dump(by_alias=True) connectors = node_dict.get("inputs" if is_input else "outputs", {}) result = [] diff --git a/retrack/engine/runner.py b/retrack/engine/runner.py index 74d6883..66d1df8 100644 --- a/retrack/engine/runner.py +++ b/retrack/engine/runner.py @@ -111,7 +111,7 @@ def _create_initial_state_from_payload( ) -> pd.DataFrame: """Create initial state from payload. This is the first step of the runner.""" validated_payload = self.request_manager.validate(payload) - validated_payload = pd.DataFrame([p.dict() for p in validated_payload]) + validated_payload = pd.DataFrame([p.model_dump() for p in validated_payload]) state_df = pd.DataFrame([]) for node_id, input_name in self.input_columns.items(): @@ -169,7 +169,7 @@ def __run_node(self, node_id: str): return input_params = self.__get_input_params( - node_id, node.dict(by_alias=True), current_node_filter + node_id, node.model_dump(by_alias=True), current_node_filter ) output = node.run(**input_params) diff --git a/retrack/nodes/base.py b/retrack/nodes/base.py index 65f36b8..d947084 100644 --- a/retrack/nodes/base.py +++ b/retrack/nodes/base.py @@ -34,14 +34,18 @@ class NodeMemoryType(str, enum.Enum): # Connection Models ############################################################### +def cast_int_to_str(v: typing.Any, info: pydantic.ValidationInfo) -> str: + return str(v) + +CastedToStringType = typing.Annotated[typing.Any, pydantic.BeforeValidator(cast_int_to_str)] class OutputConnectionItemModel(pydantic.BaseModel): - node: str + node: CastedToStringType input_: str = pydantic.Field(alias="input") class InputConnectionItemModel(pydantic.BaseModel): - node: str + node: CastedToStringType output: str = pydantic.Field(alias="output") @@ -57,11 +61,12 @@ class InputConnectionModel(pydantic.BaseModel): # Base Node ############################################################### - class BaseNode(pydantic.BaseModel): - id: str - inputs: typing.Optional[typing.Dict[str, InputConnectionModel]] = None - outputs: typing.Optional[typing.Dict[str, OutputConnectionModel]] = None + id: CastedToStringType + inputs: typing.Optional[typing.Dict[CastedToStringType, InputConnectionModel]] = None + outputs: typing.Optional[typing.Dict[CastedToStringType, OutputConnectionModel]] = None + + def run(self, **kwargs) -> typing.Dict[str, typing.Any]: return {} diff --git a/tests/test_nodes/test_check.py b/tests/test_nodes/test_check.py index e7bffdc..a8f675b 100644 --- a/tests/test_nodes/test_check.py +++ b/tests/test_nodes/test_check.py @@ -25,7 +25,7 @@ def test_check_node(node_schema): assert isinstance(check_node, Check) assert isinstance(check_node.data.operator, CheckOperator) - assert check_node.dict(by_alias=True) == { + assert check_node.model_dump(by_alias=True) == { "id": "8", "data": {"operator": CheckOperator.EQUAL}, "inputs": { diff --git a/tests/test_nodes/test_contains.py b/tests/test_nodes/test_contains.py index 93e879d..69f85bf 100644 --- a/tests/test_nodes/test_contains.py +++ b/tests/test_nodes/test_contains.py @@ -17,7 +17,7 @@ def test_Contains_node(): assert isinstance(Contains_node, Contains) - assert Contains_node.dict(by_alias=True) == { + assert Contains_node.model_dump(by_alias=True) == { "id": "9", "inputs": { "input_list": {"connections": []}, diff --git a/tests/test_nodes/test_datetime.py b/tests/test_nodes/test_datetime.py index 550df73..0457c1c 100644 --- a/tests/test_nodes/test_datetime.py +++ b/tests/test_nodes/test_datetime.py @@ -22,7 +22,7 @@ def test_current_year_node(current_year_input_data): assert isinstance(current_year_node, CurrentYear) - assert current_year_node.dict(by_alias=True) == { + assert current_year_node.model_dump(by_alias=True) == { "id": "18", "inputs": { "input_void": {"connections": []}, diff --git a/tests/test_nodes/test_endswith.py b/tests/test_nodes/test_endswith.py index fd3c73e..ea4582f 100644 --- a/tests/test_nodes/test_endswith.py +++ b/tests/test_nodes/test_endswith.py @@ -20,7 +20,7 @@ def test_EndsWith_node(): assert isinstance(EndsWith_node, EndsWith) - assert EndsWith_node.dict(by_alias=True) == { + assert EndsWith_node.model_dump(by_alias=True) == { "id": "11", "inputs": { "input_value_0": {"connections": []}, diff --git a/tests/test_nodes/test_endswithany.py b/tests/test_nodes/test_endswithany.py index 1cb4dc9..28bf6a0 100644 --- a/tests/test_nodes/test_endswithany.py +++ b/tests/test_nodes/test_endswithany.py @@ -17,7 +17,7 @@ def test_EndsWithAny_node(): assert isinstance(EndsWithAny_node, EndsWithAny) - assert EndsWithAny_node.dict(by_alias=True) == { + assert EndsWithAny_node.model_dump(by_alias=True) == { "id": "13", "inputs": { "input_value": {"connections": []}, diff --git a/tests/test_nodes/test_input.py b/tests/test_nodes/test_input.py index 8b69aae..7a44c38 100644 --- a/tests/test_nodes/test_input.py +++ b/tests/test_nodes/test_input.py @@ -6,4 +6,4 @@ def test_input_node( ): input_node = Input(**valid_input_dict_before_validation) - assert input_node.dict(by_alias=True) == valid_input_dict_after_validation + assert input_node.model_dump(by_alias=True) == valid_input_dict_after_validation diff --git a/tests/test_nodes/test_logic.py b/tests/test_nodes/test_logic.py index c9b6d31..95cde6f 100644 --- a/tests/test_nodes/test_logic.py +++ b/tests/test_nodes/test_logic.py @@ -45,7 +45,7 @@ def test_Logic_node(): assert isinstance(Or_node, Or) assert isinstance(Not_node, Not) - assert And_node.dict(by_alias=True) == { + assert And_node.model_dump(by_alias=True) == { "id": "15", "inputs": { "input_bool_0": {"connections": []}, diff --git a/tests/test_nodes/test_math.py b/tests/test_nodes/test_math.py index 7ae9fd6..f2a3b0c 100644 --- a/tests/test_nodes/test_math.py +++ b/tests/test_nodes/test_math.py @@ -39,7 +39,7 @@ def test_math_node(math_operator_input_data): assert isinstance(math_node, Math) assert isinstance(math_node.data.operator, MathOperator) - assert math_node.dict(by_alias=True) == { + assert math_node.model_dump(by_alias=True) == { "id": "18", "data": {"operator": MathOperator.SUM}, "inputs": { diff --git a/tests/test_nodes/test_start.py b/tests/test_nodes/test_start.py index 6f42cdc..e287f64 100644 --- a/tests/test_nodes/test_start.py +++ b/tests/test_nodes/test_start.py @@ -24,7 +24,7 @@ def test_start_node(): start_node = Start(**input_data) - assert start_node.dict(by_alias=True) == { + assert start_node.model_dump(by_alias=True) == { "id": "0", "outputs": { "output_up_void": { diff --git a/tests/test_nodes/test_startswith.py b/tests/test_nodes/test_startswith.py index 79d471a..fa64247 100644 --- a/tests/test_nodes/test_startswith.py +++ b/tests/test_nodes/test_startswith.py @@ -20,7 +20,7 @@ def test_StartsWith_node(): assert isinstance(StartsWith_node, StartsWith) - assert StartsWith_node.dict(by_alias=True) == { + assert StartsWith_node.model_dump(by_alias=True) == { "id": "10", "inputs": { "input_value_0": {"connections": []}, diff --git a/tests/test_nodes/test_startswithany.py b/tests/test_nodes/test_startswithany.py index f6905cf..089b5b2 100644 --- a/tests/test_nodes/test_startswithany.py +++ b/tests/test_nodes/test_startswithany.py @@ -17,7 +17,7 @@ def test_StartsWithAny_node(): assert isinstance(StartsWithAny_node, StartsWithAny) - assert StartsWithAny_node.dict(by_alias=True) == { + assert StartsWithAny_node.model_dump(by_alias=True) == { "id": "13", "inputs": { "input_value": {"connections": []}, From ab6280271ec6885078a8bccc401e63a5632c2548 Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Tue, 10 Oct 2023 21:44:33 -0300 Subject: [PATCH 04/13] Retrack only supports dataframes from now on --- pyproject.toml | 9 ++-- retrack/engine/request_manager.py | 51 ++++++++++++++++++++--- retrack/engine/runner.py | 49 +++++++++------------- retrack/nodes/base.py | 18 +++++--- retrack/utils/constants.py | 4 +- tests/__init__.py | 0 tests/test_engine/__init__.py | 0 tests/test_engine/test_request_manager.py | 4 +- tests/test_nodes/__init__.py | 0 9 files changed, 86 insertions(+), 49 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/test_engine/__init__.py delete mode 100644 tests/test_nodes/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 835e9e4..8c2c314 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,13 +15,14 @@ pandas = "^1.2.0" numpy = "^1.19.5" pydantic = "2.4.2" networkx = "^2.6.3" +pandera = "^0.17.2" [tool.poetry.dev-dependencies] -pytest = "^6.2.4" -pytest-cov = "^3.0.0" +pytest = "*" +pytest-cov = "*" black = "^22.6.0" isort = {extras = ["colors"], version = "*"} -pytest-mock = "^3.10.0" +pytest-mock = "*" [tool.black] # https://github.com/psf/black @@ -72,7 +73,7 @@ indent = 4 color_output = true [tool.pytest.ini_options] -addopts = "--junitxml=pytest.xml -p no:warnings --cov-report term-missing:skip-covered --cov=retrack" +addopts = "-vv --junitxml=pytest.xml -p no:warnings --cov-report term-missing:skip-covered --cov=retrack" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/retrack/engine/request_manager.py b/retrack/engine/request_manager.py index 295c4eb..21a6492 100644 --- a/retrack/engine/request_manager.py +++ b/retrack/engine/request_manager.py @@ -1,5 +1,7 @@ import typing +import pandas as pd +import pandera import pydantic from retrack.nodes.base import BaseNode, NodeKind @@ -8,6 +10,7 @@ class RequestManager: def __init__(self, inputs: typing.List[BaseNode]): self._model = None + self._dataframe_model = None self.inputs = inputs @property @@ -41,13 +44,19 @@ def inputs(self, inputs: typing.List[BaseNode]): if len(self.inputs) > 0: self._model = self.__create_model() + self._dataframe_model = self.__create_dataframe_model() else: self._model = None + self._dataframe_model = None @property def model(self) -> typing.Type[pydantic.BaseModel]: return self._model + @property + def dataframe_model(self) -> pandera.DataFrameSchema: + return self._dataframe_model + def __create_model( self, model_name: str = "RequestModel" ) -> typing.Type[pydantic.BaseModel]: @@ -62,9 +71,12 @@ def __create_model( fields = {} for input_field in self.inputs: fields[input_field.data.name] = ( - (str, ...) - if input_field.data.default is None - else (str, input_field.data.default) + str, + pydantic.Field( + default=Ellipsis + if input_field.data.default is None + else input_field.data.default, + ), ) return pydantic.create_model( @@ -72,6 +84,33 @@ def __create_model( **fields, ) + def __create_dataframe_model( + self, model_name: str = "RequestModel" + ) -> pandera.DataFrameSchema: + """Create a pydantic model from the RequestManager's inputs + + Args: + model_name (str, optional): The name of the model. Defaults to "RequestModel". + + Returns: + typing.Type[pydantic.BaseModel]: The pydantic model + """ + fields = {} + for input_field in self.inputs: + fields[input_field.data.name] = pandera.Column( + str, + nullable=input_field.data.default is not None, + coerce=True, + default=input_field.data.default, + ) + + return pandera.DataFrameSchema( + fields, + index=pandera.Index(int), + strict=True, + coerce=True, + ) + def validate( self, payload: typing.Union[ @@ -92,7 +131,7 @@ def validate( if self.model is None: raise ValueError("No inputs found") - if not isinstance(payload, list): - payload = [payload] + if not isinstance(payload, pd.DataFrame): + raise TypeError(f"payload must be a pandas.DataFrame, not {type(payload)}") - return pydantic.parse_obj_as(typing.List[self.model], payload) + return self.dataframe_model.validate(payload) diff --git a/retrack/engine/runner.py b/retrack/engine/runner.py index 66d1df8..bb4c460 100644 --- a/retrack/engine/runner.py +++ b/retrack/engine/runner.py @@ -107,11 +107,12 @@ def __set_output_connection_filters( ) def _create_initial_state_from_payload( - self, payload: typing.Union[dict, list] + self, payload_df: pd.DataFrame ) -> pd.DataFrame: """Create initial state from payload. This is the first step of the runner.""" - validated_payload = self.request_manager.validate(payload) - validated_payload = pd.DataFrame([p.model_dump() for p in validated_payload]) + validated_payload = self.request_manager.validate( + payload_df.reset_index(drop=True) + ) state_df = pd.DataFrame([]) for node_id, input_name in self.input_columns.items(): @@ -186,34 +187,14 @@ def __run_node(self, node_id: str): f"{node_id}@{output_name}", output_value, current_node_filter ) - def __get_output_states(self) -> pd.DataFrame: - """Returns a dataframe with the final states of the flow""" - return pd.DataFrame( - { - "output": self.states[constants.OUTPUT_REFERENCE_COLUMN], - "message": self.states[constants.OUTPUT_MESSAGE_REFERENCE_COLUMN], - } - ) - - def __parse_payload( - self, payload: typing.Union[dict, list, pd.DataFrame] - ) -> typing.List[dict]: - if isinstance(payload, dict): - payload = [payload] - - if not isinstance(payload, pd.DataFrame): - payload = pd.DataFrame(payload, index=list(range(len(payload)))) - - for column in payload.columns: - payload[column] = payload[column].astype(str) - - return payload.to_dict("records") - - def execute(self, payload: typing.Union[dict, list, pd.DataFrame]) -> pd.DataFrame: + def execute( + self, + payload_df: typing.Union[dict, pd.DataFrame], + return_all_states: bool = False, + ) -> pd.DataFrame: """Executes the flow with the given payload""" self.reset() - payload = self.__parse_payload(payload) - self._states = self._create_initial_state_from_payload(payload) + self._states = self._create_initial_state_from_payload(payload_df) for node_id in self.parser.execution_order: try: @@ -224,4 +205,12 @@ def execute(self, payload: typing.Union[dict, list, pd.DataFrame]) -> pd.DataFra if self.states[constants.OUTPUT_REFERENCE_COLUMN].isna().sum() == 0: break - return self.__get_output_states() + if return_all_states: + return self.states + + return self.states[ + [ + constants.OUTPUT_REFERENCE_COLUMN, + constants.OUTPUT_MESSAGE_REFERENCE_COLUMN, + ] + ] diff --git a/retrack/nodes/base.py b/retrack/nodes/base.py index d947084..63560e4 100644 --- a/retrack/nodes/base.py +++ b/retrack/nodes/base.py @@ -34,10 +34,15 @@ class NodeMemoryType(str, enum.Enum): # Connection Models ############################################################### + def cast_int_to_str(v: typing.Any, info: pydantic.ValidationInfo) -> str: return str(v) -CastedToStringType = typing.Annotated[typing.Any, pydantic.BeforeValidator(cast_int_to_str)] + +CastedToStringType = typing.Annotated[ + typing.Any, pydantic.BeforeValidator(cast_int_to_str) +] + class OutputConnectionItemModel(pydantic.BaseModel): node: CastedToStringType @@ -61,12 +66,15 @@ class InputConnectionModel(pydantic.BaseModel): # Base Node ############################################################### + class BaseNode(pydantic.BaseModel): id: CastedToStringType - inputs: typing.Optional[typing.Dict[CastedToStringType, InputConnectionModel]] = None - outputs: typing.Optional[typing.Dict[CastedToStringType, OutputConnectionModel]] = None - - + inputs: typing.Optional[ + typing.Dict[CastedToStringType, InputConnectionModel] + ] = None + outputs: typing.Optional[ + typing.Dict[CastedToStringType, OutputConnectionModel] + ] = None def run(self, **kwargs) -> typing.Dict[str, typing.Any]: return {} diff --git a/retrack/utils/constants.py b/retrack/utils/constants.py index 65d5663..d2c9ee7 100644 --- a/retrack/utils/constants.py +++ b/retrack/utils/constants.py @@ -1,5 +1,5 @@ -OUTPUT_REFERENCE_COLUMN = "graph_output" -OUTPUT_MESSAGE_REFERENCE_COLUMN = "graph_output_message" +OUTPUT_REFERENCE_COLUMN = "output" +OUTPUT_MESSAGE_REFERENCE_COLUMN = "message" NULL_SUFFIX = "_void" FILTER_SUFFIX = "_filter" INPUT_OUTPUT_VALUE_CONNECTOR_NAME = "output_value" diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_engine/__init__.py b/tests/test_engine/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_engine/test_request_manager.py b/tests/test_engine/test_request_manager.py index 2a04fc7..c2a5239 100644 --- a/tests/test_engine/test_request_manager.py +++ b/tests/test_engine/test_request_manager.py @@ -2,7 +2,7 @@ from retrack.engine.request_manager import RequestManager from retrack.nodes.inputs import Input - +import pandas as pd def test_create_request_manager(valid_input_dict_before_validation): with pytest.raises(TypeError): @@ -36,4 +36,4 @@ def test_create_request_manager_with_invalid_input(valid_input_dict_before_valid def test_validate_payload_with_valid_payload(valid_input_dict_before_validation): pm = RequestManager([Input(**valid_input_dict_before_validation)]) payload = pm.model(example="test") - assert pm.validate({"example": "test"})[0] == payload + assert pm.validate(pd.DataFrame([{{"example": "test"}}]))[0] == payload diff --git a/tests/test_nodes/__init__.py b/tests/test_nodes/__init__.py deleted file mode 100644 index e69de29..0000000 From 0ec0ab7251fc7960d99cb7a78abb7d0f1bbc2da0 Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Wed, 25 Oct 2023 10:44:33 -0300 Subject: [PATCH 05/13] Fix broken tests --- retrack/engine/parser.py | 27 +++++++++++++++++++++++ retrack/engine/request_manager.py | 19 ++++------------ retrack/engine/runner.py | 15 +++++++++++-- tests/test_engine/test_request_manager.py | 19 ++++++++++++---- tests/test_engine/test_runner.py | 8 +++---- 5 files changed, 63 insertions(+), 25 deletions(-) diff --git a/retrack/engine/parser.py b/retrack/engine/parser.py index 72aede3..8bcbd86 100644 --- a/retrack/engine/parser.py +++ b/retrack/engine/parser.py @@ -1,5 +1,7 @@ import typing +import hashlib + from retrack import nodes, validators from retrack.utils.registry import Registry @@ -29,10 +31,35 @@ def __init__( self._set_execution_order() self._set_indexes_by_memory_type_map() + self._version = self.graph_data.get("version", None) + + if self._version is None: + self._version = "{}.dynamic".format( + hashlib.sha256(str(self.graph_data).encode("utf-8")).hexdigest()[:10], + ) + else: + graph_data_without_version = self.graph_data.copy() + file_version_hash = graph_data_without_version["version"].split(".")[0] + del graph_data_without_version["version"] + + if ( + file_version_hash + != hashlib.sha256( + str(graph_data_without_version).encode("utf-8") + ).hexdigest()[:10] + ): + raise ValueError( + "Invalid version. Graph data has changed and the hash is different" + ) + @property def graph_data(self) -> dict: return self.__graph_data + @property + def version(self) -> str: + return self._version + @staticmethod def _check_input_data(data: dict): if not isinstance(data, dict): diff --git a/retrack/engine/request_manager.py b/retrack/engine/request_manager.py index 21a6492..a4b484f 100644 --- a/retrack/engine/request_manager.py +++ b/retrack/engine/request_manager.py @@ -84,17 +84,8 @@ def __create_model( **fields, ) - def __create_dataframe_model( - self, model_name: str = "RequestModel" - ) -> pandera.DataFrameSchema: - """Create a pydantic model from the RequestManager's inputs - - Args: - model_name (str, optional): The name of the model. Defaults to "RequestModel". - - Returns: - typing.Type[pydantic.BaseModel]: The pydantic model - """ + def __create_dataframe_model(self) -> pandera.DataFrameSchema: + """Create a pydantic model from the RequestManager's inputs""" fields = {} for input_field in self.inputs: fields[input_field.data.name] = pandera.Column( @@ -113,14 +104,12 @@ def __create_dataframe_model( def validate( self, - payload: typing.Union[ - typing.Dict[str, str], typing.List[typing.Dict[str, str]] - ], + payload: pd.DataFrame, ) -> typing.List[pydantic.BaseModel]: """Validate the payload against the RequestManager's model Args: - payload (typing.Union[typing.Dict[str, str], typing.List[typing.Dict[str, str]]]): The payload to validate + payload (pandas.DataFrame): The payload to validate Raises: ValueError: If the RequestManager has no model diff --git a/retrack/engine/runner.py b/retrack/engine/runner.py index bb4c460..1f30eae 100644 --- a/retrack/engine/runner.py +++ b/retrack/engine/runner.py @@ -189,10 +189,21 @@ def __run_node(self, node_id: str): def execute( self, - payload_df: typing.Union[dict, pd.DataFrame], + payload_df: pd.DataFrame, return_all_states: bool = False, ) -> pd.DataFrame: - """Executes the flow with the given payload""" + """Executes the flow with the given payload. + + Args: + payload_df (pd.DataFrame): The payload to be used as input. + return_all_states (bool, optional): If True, returns all states. Defaults to False. + + Returns: + pd.DataFrame: The output of the flow. + """ + if not isinstance(payload_df, pd.DataFrame): + raise ValueError("payload_df must be a pandas.DataFrame") + self.reset() self._states = self._create_initial_state_from_payload(payload_df) diff --git a/tests/test_engine/test_request_manager.py b/tests/test_engine/test_request_manager.py index c2a5239..34c4413 100644 --- a/tests/test_engine/test_request_manager.py +++ b/tests/test_engine/test_request_manager.py @@ -1,8 +1,11 @@ +import pandas as pd +import pandera +import pydantic import pytest from retrack.engine.request_manager import RequestManager from retrack.nodes.inputs import Input -import pandas as pd + def test_create_request_manager(valid_input_dict_before_validation): with pytest.raises(TypeError): @@ -34,6 +37,14 @@ def test_create_request_manager_with_invalid_input(valid_input_dict_before_valid def test_validate_payload_with_valid_payload(valid_input_dict_before_validation): - pm = RequestManager([Input(**valid_input_dict_before_validation)]) - payload = pm.model(example="test") - assert pm.validate(pd.DataFrame([{{"example": "test"}}]))[0] == payload + rm = RequestManager([Input(**valid_input_dict_before_validation)]) + + assert issubclass(rm.model, pydantic.BaseModel) + + assert isinstance(rm.dataframe_model, pandera.api.pandas.container.DataFrameSchema) + + payload = rm.model(example="test") + + assert isinstance(payload, pydantic.BaseModel) + result = rm.validate(pd.DataFrame([{"example": "test"}])) + assert isinstance(result, pd.DataFrame) diff --git a/tests/test_engine/test_runner.py b/tests/test_engine/test_runner.py index 672ebad..ceb9a24 100644 --- a/tests/test_engine/test_runner.py +++ b/tests/test_engine/test_runner.py @@ -30,7 +30,7 @@ def test_flows_with_single_element(filename, in_values, expected_out_values): rule = json.load(f) runner = Runner(Parser(rule)) - out_values = runner.execute(in_values) + out_values = runner.execute(pd.DataFrame([in_values])) assert isinstance(out_values, pd.DataFrame) assert out_values.to_dict(orient="records") == expected_out_values @@ -90,7 +90,7 @@ def test_flows(filename, in_values, expected_out_values): rule = json.load(f) runner = Runner(Parser(rule)) - out_values = runner.execute(in_values) + out_values = runner.execute(pd.DataFrame(in_values)) assert isinstance(out_values, pd.DataFrame) assert out_values.to_dict(orient="records") == expected_out_values @@ -147,7 +147,7 @@ def test_flows(filename, in_values, expected_out_values): ) def test_create_from_json(filename, in_values, expected_out_values): runner = Runner.from_json(f"tests/resources/{filename}.json") - out_values = runner.execute(in_values) + out_values = runner.execute(pd.DataFrame(in_values)) assert isinstance(out_values, pd.DataFrame) assert out_values.to_dict(orient="records") == expected_out_values @@ -171,7 +171,7 @@ def test_csv_table_with_if(): {"in_a": 1, "in_b": 1, "in_d": -1, "in_e": 0}, ] - out_values = runner.execute(in_values) + out_values = runner.execute(pd.DataFrame(in_values)) assert isinstance(out_values, pd.DataFrame) assert len(out_values) == len(in_values) From 369bff32e3d3e32ee7f462f671821c35446b6bca Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Wed, 25 Oct 2023 10:45:14 -0300 Subject: [PATCH 06/13] Move from black and isort to ruff --- .gitignore | 1 + Makefile | 6 ++---- pyproject.toml | 3 +-- tests/test_nodes/test_contains.py | 1 - tests/test_nodes/test_endswith.py | 1 - tests/test_nodes/test_endswithany.py | 1 - 6 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index db6e11f..559475b 100644 --- a/.gitignore +++ b/.gitignore @@ -282,3 +282,4 @@ $RECYCLE.BIN/ # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) +.vscode/settings.json diff --git a/Makefile b/Makefile index b5478bf..a894f70 100644 --- a/Makefile +++ b/Makefile @@ -4,13 +4,11 @@ init: .PHONY: formatting formatting: - poetry run isort --settings-path pyproject.toml ./ - poetry run black --config pyproject.toml ./ + poetry run ruff format . .PHONY: check-formatting check-formatting: - poetry run isort --settings-path pyproject.toml --check-only ./ - poetry run black --config pyproject.toml --check ./ + poetry run ruff check . .PHONY: tests tests: diff --git a/pyproject.toml b/pyproject.toml index 8c2c314..1e2b7c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,12 +16,11 @@ numpy = "^1.19.5" pydantic = "2.4.2" networkx = "^2.6.3" pandera = "^0.17.2" +ruff = "^0.1.2" [tool.poetry.dev-dependencies] pytest = "*" pytest-cov = "*" -black = "^22.6.0" -isort = {extras = ["colors"], version = "*"} pytest-mock = "*" [tool.black] diff --git a/tests/test_nodes/test_contains.py b/tests/test_nodes/test_contains.py index 69f85bf..0d70d56 100644 --- a/tests/test_nodes/test_contains.py +++ b/tests/test_nodes/test_contains.py @@ -28,7 +28,6 @@ def test_Contains_node(): def test_Contains_node_run(): - Contains_node = Contains(**input_data) output = Contains_node.run(pd.Series(["1", "2"]), pd.Series(["2"])) diff --git a/tests/test_nodes/test_endswith.py b/tests/test_nodes/test_endswith.py index ea4582f..29ff285 100644 --- a/tests/test_nodes/test_endswith.py +++ b/tests/test_nodes/test_endswith.py @@ -31,7 +31,6 @@ def test_EndsWith_node(): def test_EndsWith_node_run(): - EndsWith_node = EndsWith(**input_data) output = EndsWith_node.run(pd.Series(["100"]), pd.Series(["2"])) diff --git a/tests/test_nodes/test_endswithany.py b/tests/test_nodes/test_endswithany.py index 28bf6a0..0de017b 100644 --- a/tests/test_nodes/test_endswithany.py +++ b/tests/test_nodes/test_endswithany.py @@ -28,7 +28,6 @@ def test_EndsWithAny_node(): def test_EndsWithAny_node_run(): - EndsWithAny_node = EndsWithAny(**input_data) output = EndsWithAny_node.run(pd.Series(["100"]), pd.Series(["2", "1"])) From fdbf3a99724724bf0a6437c07e972292d57b35ac Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Wed, 25 Oct 2023 11:53:51 -0300 Subject: [PATCH 07/13] Fix pyproject.toml --- pyproject.toml | 50 +------------------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1e2b7c9..252c4f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,61 +16,13 @@ numpy = "^1.19.5" pydantic = "2.4.2" networkx = "^2.6.3" pandera = "^0.17.2" -ruff = "^0.1.2" [tool.poetry.dev-dependencies] pytest = "*" pytest-cov = "*" +ruff = "^0.1.2" pytest-mock = "*" -[tool.black] -# https://github.com/psf/black -target-version = ["py37"] -line-length = 88 -color = true - -exclude = ''' -/( - \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - | env - | venv -)/ -''' - -[tool.isort] -# https://github.com/timothycrosley/isort/ -py_version = 37 -line_length = 88 - -known_typing = [ - "typing", - "types", - "typing_extensions", - "mypy", - "mypy_extensions", -] -sections = [ - "FUTURE", - "TYPING", - "STDLIB", - "THIRDPARTY", - "FIRSTPARTY", - "LOCALFOLDER", -] -include_trailing_comma = true -profile = "black" -multi_line_output = 3 -indent = 4 -color_output = true - [tool.pytest.ini_options] addopts = "-vv --junitxml=pytest.xml -p no:warnings --cov-report term-missing:skip-covered --cov=retrack" From 64ac1f70e5e5d97abe0013c8d2611d223a71389e Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Wed, 25 Oct 2023 12:52:03 -0300 Subject: [PATCH 08/13] Add logic to check versions --- retrack/engine/parser.py | 39 +++--- tests/resources/rule-with-version.json | 168 +++++++++++++++++++++++++ tests/resources/to-lowercase.json | 118 +++++++++++++++++ tests/test_engine/test_runner.py | 9 ++ 4 files changed, 315 insertions(+), 19 deletions(-) create mode 100644 tests/resources/rule-with-version.json create mode 100644 tests/resources/to-lowercase.json diff --git a/retrack/engine/parser.py b/retrack/engine/parser.py index 8bcbd86..32a913d 100644 --- a/retrack/engine/parser.py +++ b/retrack/engine/parser.py @@ -4,6 +4,7 @@ from retrack import nodes, validators from retrack.utils.registry import Registry +import json class Parser: @@ -30,27 +31,9 @@ def __init__( self._set_indexes_by_kind_map() self._set_execution_order() self._set_indexes_by_memory_type_map() + self._set_version() - self._version = self.graph_data.get("version", None) - - if self._version is None: - self._version = "{}.dynamic".format( - hashlib.sha256(str(self.graph_data).encode("utf-8")).hexdigest()[:10], - ) - else: - graph_data_without_version = self.graph_data.copy() - file_version_hash = graph_data_without_version["version"].split(".")[0] - del graph_data_without_version["version"] - if ( - file_version_hash - != hashlib.sha256( - str(graph_data_without_version).encode("utf-8") - ).hexdigest()[:10] - ): - raise ValueError( - "Invalid version. Graph data has changed and the hash is different" - ) @property def graph_data(self) -> dict: @@ -227,3 +210,21 @@ def _walk(self, actual_id: str, skiped_ids: list): self._walk(next_id, skiped_ids) return skiped_ids + + def _set_version(self): + self._version = self.graph_data.get("version", None) + + graph_json_content = json.dumps(self.graph_data["nodes"]).replace(": ", ":").replace(", ", ",").encode( + "utf-8" + ) + calculated_hash = hashlib.sha256(graph_json_content).hexdigest()[:10] + + if self.version is None: + self._version = f"{calculated_hash}.dynamic" + else: + file_version_hash = self.version.split(".")[0] + + if file_version_hash != calculated_hash: + raise ValueError( + f"Invalid version. Graph data has changed and the hash is different: {calculated_hash} != {file_version_hash}" + ) \ No newline at end of file diff --git a/tests/resources/rule-with-version.json b/tests/resources/rule-with-version.json new file mode 100644 index 0000000..581fb0d --- /dev/null +++ b/tests/resources/rule-with-version.json @@ -0,0 +1,168 @@ +{ + "id": "demo@0.1.0", + "nodes": { + "0": { + "id": 0, + "data": {}, + "inputs": {}, + "outputs": { + "output_up_void": { + "connections": [ + { + "node": 2, + "input": "input_void", + "data": {} + } + ] + }, + "output_down_void": { + "connections": [ + { + "node": 3, + "input": "input_void", + "data": {} + } + ] + } + }, + "position": [ + 0, + 0 + ], + "name": "Start" + }, + "2": { + "id": 2, + "data": { + "name": "variable", + "default": null + }, + "inputs": { + "input_void": { + "connections": [ + { + "node": 0, + "output": "output_up_void", + "data": {} + } + ] + } + }, + "outputs": { + "output_value": { + "connections": [ + { + "node": 4, + "input": "input_value_0", + "data": {} + } + ] + } + }, + "position": [ + 338.42578125, + -177.6171875 + ], + "name": "Input" + }, + "3": { + "id": 3, + "data": { + "value": "100" + }, + "inputs": { + "input_void": { + "connections": [ + { + "node": 0, + "output": "output_down_void", + "data": {} + } + ] + } + }, + "outputs": { + "output_value": { + "connections": [ + { + "node": 4, + "input": "input_value_1", + "data": {} + } + ] + } + }, + "position": [ + 337.4140625, + 77.703125 + ], + "name": "Constant" + }, + "4": { + "id": 4, + "data": { + "operator": "==" + }, + "inputs": { + "input_value_0": { + "connections": [ + { + "node": 2, + "output": "output_value", + "data": {} + } + ] + }, + "input_value_1": { + "connections": [ + { + "node": 3, + "output": "output_value", + "data": {} + } + ] + } + }, + "outputs": { + "output_bool": { + "connections": [ + { + "node": 5, + "input": "input_value", + "data": {} + } + ] + } + }, + "position": [ + 670.08984375, + -83.58203125 + ], + "name": "Check" + }, + "5": { + "id": 5, + "data": { + "message": null + }, + "inputs": { + "input_value": { + "connections": [ + { + "node": 4, + "output": "output_bool", + "data": {} + } + ] + } + }, + "outputs": {}, + "position": [ + 965.2007983378469, + -54.070333506085134 + ], + "name": "Output" + } + }, + "version": "93df1616e7.2023-10-25" +} \ No newline at end of file diff --git a/tests/resources/to-lowercase.json b/tests/resources/to-lowercase.json new file mode 100644 index 0000000..225d5e6 --- /dev/null +++ b/tests/resources/to-lowercase.json @@ -0,0 +1,118 @@ +{ + "id": "demo@0.1.0", + "nodes": { + "0": { + "id": 0, + "data": {}, + "inputs": {}, + "outputs": { + "output_up_void": { + "connections": [ + { + "node": 2, + "input": "input_void", + "data": {} + } + ] + }, + "output_down_void": { + "connections": [] + } + }, + "position": [ + 0, + 0 + ], + "name": "Start" + }, + "2": { + "id": 2, + "data": { + "name": "var", + "default": null + }, + "inputs": { + "input_void": { + "connections": [ + { + "node": 0, + "output": "output_up_void", + "data": {} + } + ] + } + }, + "outputs": { + "output_value": { + "connections": [ + { + "node": 3, + "input": "input_value", + "data": {} + } + ] + } + }, + "position": [ + 338.91796875, + -68.23046875 + ], + "name": "Input" + }, + "3": { + "id": 3, + "data": {}, + "inputs": { + "input_value": { + "connections": [ + { + "node": 2, + "output": "output_value", + "data": {} + } + ] + } + }, + "outputs": { + "output_value": { + "connections": [ + { + "node": 4, + "input": "input_value", + "data": {} + } + ] + } + }, + "position": [ + 654.453125, + -7.6953125 + ], + "name": "LowerCase" + }, + "4": { + "id": 4, + "data": { + "message": null + }, + "inputs": { + "input_value": { + "connections": [ + { + "node": 3, + "output": "output_value", + "data": {} + } + ] + } + }, + "outputs": {}, + "position": [ + 922.9765625, + -18.54296875 + ], + "name": "Output" + } + }, + "version": "e7dd59c058.2023-10-25" +} \ No newline at end of file diff --git a/tests/test_engine/test_runner.py b/tests/test_engine/test_runner.py index ceb9a24..ad88161 100644 --- a/tests/test_engine/test_runner.py +++ b/tests/test_engine/test_runner.py @@ -83,6 +83,15 @@ def test_flows_with_single_element(filename, in_values, expected_out_values): {"message": None, "output": "group 3"}, ], ), + ( + "rule-with-version", + [{"variable": 0}, {"variable": 100}, {"variable": 200}], + [ + {"message": None, "output": False}, + {"message": None, "output": True}, + {"message": None, "output": False}, + ], + ), ], ) def test_flows(filename, in_values, expected_out_values): From 39b1fddc9b561bf5caf85a31faa2e86eeb658b80 Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Wed, 25 Oct 2023 12:59:00 -0300 Subject: [PATCH 09/13] Add lowercase node --- retrack/engine/parser.py | 11 ++++++----- retrack/nodes/__init__.py | 2 ++ retrack/nodes/lowercase.py | 27 +++++++++++++++++++++++++++ tests/test_engine/test_runner.py | 9 +++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 retrack/nodes/lowercase.py diff --git a/retrack/engine/parser.py b/retrack/engine/parser.py index 32a913d..064de50 100644 --- a/retrack/engine/parser.py +++ b/retrack/engine/parser.py @@ -33,8 +33,6 @@ def __init__( self._set_indexes_by_memory_type_map() self._set_version() - - @property def graph_data(self) -> dict: return self.__graph_data @@ -214,8 +212,11 @@ def _walk(self, actual_id: str, skiped_ids: list): def _set_version(self): self._version = self.graph_data.get("version", None) - graph_json_content = json.dumps(self.graph_data["nodes"]).replace(": ", ":").replace(", ", ",").encode( - "utf-8" + graph_json_content = ( + json.dumps(self.graph_data["nodes"]) + .replace(": ", ":") + .replace(", ", ",") + .encode("utf-8") ) calculated_hash = hashlib.sha256(graph_json_content).hexdigest()[:10] @@ -227,4 +228,4 @@ def _set_version(self): if file_version_hash != calculated_hash: raise ValueError( f"Invalid version. Graph data has changed and the hash is different: {calculated_hash} != {file_version_hash}" - ) \ No newline at end of file + ) diff --git a/retrack/nodes/__init__.py b/retrack/nodes/__init__.py index ca16e13..4d01912 100644 --- a/retrack/nodes/__init__.py +++ b/retrack/nodes/__init__.py @@ -15,6 +15,7 @@ from retrack.nodes.start import Start from retrack.nodes.startswith import StartsWith from retrack.nodes.startswithany import StartsWithAny +from retrack.nodes.lowercase import LowerCase from retrack.utils.registry import Registry _registry = Registry() @@ -49,5 +50,6 @@ def register(name: str, node: BaseNode) -> None: register("Contains", Contains) register("CurrentYear", CurrentYear) register("IntervalCatV0", IntervalCatV0) +register("LowerCase", LowerCase) __all__ = ["registry", "register", "BaseNode", "dynamic_registry", "BaseDynamicNode"] diff --git a/retrack/nodes/lowercase.py b/retrack/nodes/lowercase.py new file mode 100644 index 0000000..bc57b59 --- /dev/null +++ b/retrack/nodes/lowercase.py @@ -0,0 +1,27 @@ +import typing + +import enum + +import pandas as pd +import pydantic + +from retrack.nodes.base import BaseNode, InputConnectionModel, OutputConnectionModel + + +class LowerCaseOutputsModel(pydantic.BaseModel): + output_value: OutputConnectionModel + + +class LowerCaseInputsModel(pydantic.BaseModel): + input_value: InputConnectionModel + + +class LowerCase(BaseNode): + inputs: LowerCaseInputsModel + outputs: LowerCaseOutputsModel + + def run( + self, + input_value: pd.Series, + ) -> typing.Dict[str, pd.Series]: + return {"output_value": input_value.astype(str).str.lower()} diff --git a/tests/test_engine/test_runner.py b/tests/test_engine/test_runner.py index ad88161..4701d41 100644 --- a/tests/test_engine/test_runner.py +++ b/tests/test_engine/test_runner.py @@ -92,6 +92,15 @@ def test_flows_with_single_element(filename, in_values, expected_out_values): {"message": None, "output": False}, ], ), + ( + "to-lowercase", + [{"var": "EXAMPLE"}, {"var": "test with numbers 120"}, {"var": 200}], + [ + {"message": None, "output": "example"}, + {"message": None, "output": "test with numbers 120"}, + {"message": None, "output": "200"}, + ], + ), ], ) def test_flows(filename, in_values, expected_out_values): From 9eee09d2c3d58132b2c59b60cd44583e5ca5d3b3 Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Wed, 25 Oct 2023 14:35:23 -0300 Subject: [PATCH 10/13] Fix typos --- README.md | 4 +++- retrack/engine/runner.py | 20 ++++++++++++++------ retrack/nodes/dynamic/flow.py | 1 + tests/resources/rule-of-rules.json | 10 ++++++---- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9fffb6a..24e5172 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ import retrack parser = retrack.Parser(rule) # Create a runner -runner = retrack.Runner(parser) +runner = retrack.Runner(parser, name="your-rule") # Run the rule/model passing the data runner.execute(data) @@ -48,6 +48,8 @@ runner.execute(data) The `Parser` class parses the rule/model and creates a graph of nodes. The `Runner` class runs the rule/model using the data passed to the runner. The `data` is a dictionary or a list of dictionaries containing the data that will be used to evaluate the conditions and execute the actions. To see wich data is required for the given rule/model, check the `runner.request_model` property that is a pydantic model used to validate the data. +Optionally you can name the rule by passing the `name` field to the `retrack.Runner` constructor. This is useful to identify the rule when exceptions are raised. + ### Creating a rule/model A rule is a set of conditions and actions that are executed when the conditions are met. The conditions are evaluated using the data passed to the runner. The actions are executed when the conditions are met. diff --git a/retrack/engine/runner.py b/retrack/engine/runner.py index 1f30eae..76da78e 100644 --- a/retrack/engine/runner.py +++ b/retrack/engine/runner.py @@ -13,8 +13,9 @@ class Runner: - def __init__(self, parser: Parser): + def __init__(self, parser: Parser, name: str = None): self._parser = parser + self._name = name self._internal_runners = {} self.reset() self._set_constants() @@ -23,19 +24,25 @@ def __init__(self, parser: Parser): self._set_internal_runners() @classmethod - def from_json(cls, data: typing.Union[str, dict], **kwargs): + def from_json(cls, data: typing.Union[str, dict], name: str = None, **kwargs): if isinstance(data, str) and data.endswith(".json"): + if name is None: + name = data data = json.loads(open(data).read()) elif not isinstance(data, dict): raise ValueError("data must be a dict or a json file path") parser = Parser(data, **kwargs) - return cls(parser) + return cls(parser, name=name) @property def parser(self) -> Parser: return self._parser + @property + def name(self) -> str: + return self._name + @property def request_manager(self) -> RequestManager: return self._request_manager @@ -68,8 +75,9 @@ def _set_internal_runners(self): constants.FLOW_NODE_NAME, [] ): try: + node_data = self.parser.get_by_id(node_id).data self._internal_runners[node_id] = Runner.from_json( - self.parser.get_by_id(node_id).data.parsed_value() + node_data.parsed_value(), name=node_data.name ) except Exception as e: raise Exception( @@ -211,8 +219,8 @@ def execute( try: self.__run_node(node_id) except Exception as e: - print(f"Error running node {node_id}") - raise e + raise Exception(f"Error running node {node_id} in {self.name}") from e + if self.states[constants.OUTPUT_REFERENCE_COLUMN].isna().sum() == 0: break diff --git a/retrack/nodes/dynamic/flow.py b/retrack/nodes/dynamic/flow.py index a8b8b00..36e70c6 100644 --- a/retrack/nodes/dynamic/flow.py +++ b/retrack/nodes/dynamic/flow.py @@ -11,6 +11,7 @@ class FlowV0MetadataModel(pydantic.BaseModel): value: str + name: typing.Optional[str] = None default: typing.Optional[str] = None def parsed_value(self) -> typing.Dict[str, typing.Any]: diff --git a/tests/resources/rule-of-rules.json b/tests/resources/rule-of-rules.json index 7acada2..3c5aabb 100644 --- a/tests/resources/rule-of-rules.json +++ b/tests/resources/rule-of-rules.json @@ -34,8 +34,9 @@ "2": { "id": 2, "data": { - "value": "{\n\t\"id\": \"demo@0.1.0\",\n\t\"nodes\": {\n\t\t\"0\": {\n\t\t\t\"id\": 0,\n\t\t\t\"data\": {},\n\t\t\t\"inputs\": {},\n\t\t\t\"outputs\": {\n\t\t\t\t\"output_up_void\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 2,\n\t\t\t\t\t\t\t\"input\": \"input_void\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"output_down_void\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 3,\n\t\t\t\t\t\t\t\"input\": \"input_void\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"position\": [\n\t\t\t\t0,\n\t\t\t\t0\n\t\t\t],\n\t\t\t\"name\": \"Start\"\n\t\t},\n\t\t\"2\": {\n\t\t\t\"id\": 2,\n\t\t\t\"data\": {\n\t\t\t\t\"name\": \"var_a\",\n\t\t\t\t\"default\": null\n\t\t\t},\n\t\t\t\"inputs\": {\n\t\t\t\t\"input_void\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 0,\n\t\t\t\t\t\t\t\"output\": \"output_up_void\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"outputs\": {\n\t\t\t\t\"output_value\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 4,\n\t\t\t\t\t\t\t\"input\": \"input_value_0\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"position\": [\n\t\t\t\t364.0234375,\n\t\t\t\t-195.8359375\n\t\t\t],\n\t\t\t\"name\": \"Input\"\n\t\t},\n\t\t\"3\": {\n\t\t\t\"id\": 3,\n\t\t\t\"data\": {\n\t\t\t\t\"name\": \"var_b\",\n\t\t\t\t\"default\": null\n\t\t\t},\n\t\t\t\"inputs\": {\n\t\t\t\t\"input_void\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 0,\n\t\t\t\t\t\t\t\"output\": \"output_down_void\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"outputs\": {\n\t\t\t\t\"output_value\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 4,\n\t\t\t\t\t\t\t\"input\": \"input_value_1\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"position\": [\n\t\t\t\t363.8376785973601,\n\t\t\t\t108.76168656142642\n\t\t\t],\n\t\t\t\"name\": \"Input\"\n\t\t},\n\t\t\"4\": {\n\t\t\t\"id\": 4,\n\t\t\t\"data\": {\n\t\t\t\t\"operator\": \"*\"\n\t\t\t},\n\t\t\t\"inputs\": {\n\t\t\t\t\"input_value_0\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 2,\n\t\t\t\t\t\t\t\"output\": \"output_value\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"input_value_1\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 3,\n\t\t\t\t\t\t\t\"output\": \"output_value\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"outputs\": {\n\t\t\t\t\"output_value\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 6,\n\t\t\t\t\t\t\t\"input\": \"input_value\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"position\": [\n\t\t\t\t717.8109142505134,\n\t\t\t\t-74.6040121470534\n\t\t\t],\n\t\t\t\"name\": \"Math\"\n\t\t},\n\t\t\"6\": {\n\t\t\t\"id\": 6,\n\t\t\t\"data\": {},\n\t\t\t\"inputs\": {\n\t\t\t\t\"input_value\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 4,\n\t\t\t\t\t\t\t\"output\": \"output_value\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"outputs\": {\n\t\t\t\t\"output_value\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 7,\n\t\t\t\t\t\t\t\"input\": \"input_value\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"position\": [\n\t\t\t\t1005.7455832447746,\n\t\t\t\t-52.25458299824986\n\t\t\t],\n\t\t\t\"name\": \"Round\"\n\t\t},\n\t\t\"7\": {\n\t\t\t\"id\": 7,\n\t\t\t\"data\": {\n\t\t\t\t\"message\": null\n\t\t\t},\n\t\t\t\"inputs\": {\n\t\t\t\t\"input_value\": {\n\t\t\t\t\t\"connections\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"node\": 6,\n\t\t\t\t\t\t\t\"output\": \"output_value\",\n\t\t\t\t\t\t\t\"data\": {}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"outputs\": {},\n\t\t\t\"position\": [\n\t\t\t\t1279.293661227661,\n\t\t\t\t-54.172931549720175\n\t\t\t],\n\t\t\t\"name\": \"Output\"\n\t\t}\n\t}\n}", - "default": null + "value": "{\"id\":\"demo@0.1.0\",\"nodes\":{\"0\":{\"id\":0,\"data\":{},\"inputs\":{},\"outputs\":{\"output_up_void\":{\"connections\":[{\"node\":2,\"input\":\"input_void\",\"data\":{}}]},\"output_down_void\":{\"connections\":[{\"node\":3,\"input\":\"input_void\",\"data\":{}}]}},\"position\":[0,0],\"name\":\"Start\"},\"2\":{\"id\":2,\"data\":{\"name\":\"var_a\",\"default\":null},\"inputs\":{\"input_void\":{\"connections\":[{\"node\":0,\"output\":\"output_up_void\",\"data\":{}}]}},\"outputs\":{\"output_value\":{\"connections\":[{\"node\":4,\"input\":\"input_value_0\",\"data\":{}}]}},\"position\":[364.0234375,-195.8359375],\"name\":\"Input\"},\"3\":{\"id\":3,\"data\":{\"name\":\"var_b\",\"default\":null},\"inputs\":{\"input_void\":{\"connections\":[{\"node\":0,\"output\":\"output_down_void\",\"data\":{}}]}},\"outputs\":{\"output_value\":{\"connections\":[{\"node\":4,\"input\":\"input_value_1\",\"data\":{}}]}},\"position\":[363.8376785973601,108.76168656142642],\"name\":\"Input\"},\"4\":{\"id\":4,\"data\":{\"operator\":\"*\"},\"inputs\":{\"input_value_0\":{\"connections\":[{\"node\":2,\"output\":\"output_value\",\"data\":{}}]},\"input_value_1\":{\"connections\":[{\"node\":3,\"output\":\"output_value\",\"data\":{}}]}},\"outputs\":{\"output_value\":{\"connections\":[{\"node\":6,\"input\":\"input_value\",\"data\":{}}]}},\"position\":[717.8109142505134,-74.6040121470534],\"name\":\"Math\"},\"6\":{\"id\":6,\"data\":{},\"inputs\":{\"input_value\":{\"connections\":[{\"node\":4,\"output\":\"output_value\",\"data\":{}}]}},\"outputs\":{\"output_value\":{\"connections\":[{\"node\":7,\"input\":\"input_value\",\"data\":{}}]}},\"position\":[1005.7455832447746,-52.25458299824986],\"name\":\"Round\"},\"7\":{\"id\":7,\"data\":{\"message\":null},\"inputs\":{\"input_value\":{\"connections\":[{\"node\":6,\"output\":\"output_value\",\"data\":{}}]}},\"outputs\":{},\"position\":[1279.293661227661,-54.172931549720175],\"name\":\"Output\"}}}", + "default": null, + "name": "example" }, "inputs": { "input_var_a": { @@ -70,7 +71,7 @@ }, "position": [ 588.546875, - -135.41796875 + -135.7265625 ], "name": "FlowV0" }, @@ -165,5 +166,6 @@ ], "name": "Output" } - } + }, + "version": "8a3b4cc507.2023-10-25" } \ No newline at end of file From d093cb72b4caf455bca58cd027d13551f1bcf25a Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Wed, 25 Oct 2023 14:43:58 -0300 Subject: [PATCH 11/13] Fix linter and gh actions --- .github/workflows/checks.yml | 2 +- retrack/engine/runner.py | 2 +- retrack/nodes/lowercase.py | 2 -- retrack/nodes/match.py | 4 ++-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ec5708a..7d6a410 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7.16, 3.11] + python-version: [3.8, 3.11] poetry-version: [1.4.2] os: [ubuntu-latest] runs-on: ${{ matrix.os }} diff --git a/retrack/engine/runner.py b/retrack/engine/runner.py index 76da78e..8a99544 100644 --- a/retrack/engine/runner.py +++ b/retrack/engine/runner.py @@ -220,7 +220,7 @@ def execute( self.__run_node(node_id) except Exception as e: raise Exception(f"Error running node {node_id} in {self.name}") from e - + if self.states[constants.OUTPUT_REFERENCE_COLUMN].isna().sum() == 0: break diff --git a/retrack/nodes/lowercase.py b/retrack/nodes/lowercase.py index bc57b59..88a6413 100644 --- a/retrack/nodes/lowercase.py +++ b/retrack/nodes/lowercase.py @@ -1,7 +1,5 @@ import typing -import enum - import pandas as pd import pydantic diff --git a/retrack/nodes/match.py b/retrack/nodes/match.py index df76e2f..3c77c49 100644 --- a/retrack/nodes/match.py +++ b/retrack/nodes/match.py @@ -39,8 +39,8 @@ def kind(self) -> NodeKind: def run(self, input_bool: pd.Series) -> typing.Dict[str, pd.Series]: return { - f"output_then_filter": input_bool, - f"output_else_filter": ~input_bool, + "output_then_filter": input_bool, + "output_else_filter": ~input_bool, } def memory_type(self) -> NodeMemoryType: From 869787b0942788ffa1476c0ee6298af36817b279 Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Wed, 25 Oct 2023 14:57:49 -0300 Subject: [PATCH 12/13] Change python min version to 3.9 --- .github/workflows/checks.yml | 2 +- .github/workflows/release.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7d6a410..d8212fe 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.11] + python-version: [3.9, 3.10, 3.11] poetry-version: [1.4.2] os: [ubuntu-latest] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5faafe7..f54d81b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7.16] + python-version: [3.11] poetry-version: [1.4.2] os: [ubuntu-latest] runs-on: ${{ matrix.os }} diff --git a/pyproject.toml b/pyproject.toml index 252c4f8..627636c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ homepage = "https://github.com/gabrielguarisa/retrack" keywords = ["rules", "models", "business", "node", "graph"] [tool.poetry.dependencies] -python = ">=3.8,<4.0.0" +python = ">=3.9,<4.0.0" pandas = "^1.2.0" numpy = "^1.19.5" pydantic = "2.4.2" From 57148cd14ebbc966ee879b46da7d32c28cdc8d53 Mon Sep 17 00:00:00 2001 From: Gabriel Guarisa Date: Wed, 25 Oct 2023 14:59:43 -0300 Subject: [PATCH 13/13] Change python version --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d8212fe..9d67d41 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.9, 3.10, 3.11] + python-version: [3.9, 3.11] poetry-version: [1.4.2] os: [ubuntu-latest] runs-on: ${{ matrix.os }} @@ -47,7 +47,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.9] + python-version: [3.11] poetry-version: [1.4.2] os: [ubuntu-latest] runs-on: ${{ matrix.os }}