From 17c21ec12632b9f4ce1a0b2038cfa0f590742eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E6=95=B0?= <33194175+hanshuaikang@users.noreply.github.com> Date: Wed, 18 Oct 2023 15:28:41 +0800 Subject: [PATCH 01/24] =?UTF-8?q?feature:=20=E5=9B=9E=E6=BB=9A=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E6=94=AF=E6=8C=81=20(#172)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feature: 回滚功能支持 --- bamboo_engine/builder/builder.py | 173 +++++- bamboo_engine/config.py | 2 + bamboo_engine/engine.py | 41 +- bamboo_engine/eri/interfaces.py | 30 + bamboo_engine/eri/models/node.py | 6 +- bamboo_engine/states.py | 18 +- bamboo_engine/utils/constants.py | 2 + bamboo_engine/utils/graph.py | 49 ++ docs/assets/img/rollback/rollback.png | Bin 53832 -> 51816 bytes docs/user_guide/rollback.md | 125 +++- .../pipeline/conf/default_settings.py | 2 + .../pipeline/contrib/rollback/api.py | 35 +- .../pipeline/contrib/rollback/apps.py | 21 + .../pipeline/contrib/rollback/constants.py | 9 + .../pipeline/contrib/rollback/graph.py | 132 +++++ .../pipeline/contrib/rollback/handler.py | 536 +++++++++++++----- .../rollback/migrations/0001_initial.py | 65 +++ .../contrib/rollback/migrations/__init__.py | 0 .../pipeline/contrib/rollback/models.py | 52 ++ .../pipeline/contrib/rollback/tasks.py | 301 ++++++++++ .../core/flow/activity/service_activity.py | 3 + .../bamboo-pipeline/pipeline/engine/tasks.py | 31 +- .../bamboo-pipeline/pipeline/eri/imp/hooks.py | 5 + .../bamboo-pipeline/pipeline/eri/imp/node.py | 22 +- .../pipeline/eri/imp/process.py | 31 +- .../pipeline/eri/imp/rollback.py | 83 +++ .../bamboo-pipeline/pipeline/eri/runtime.py | 5 +- .../pipeline/tests/contrib/test_graph.py | 56 ++ .../pipeline/tests/contrib/test_rollback.py | 415 ++++++++------ .../tests/control/test_rollback_node.py | 149 ----- .../test/pipeline_sdk_use/settings.py | 9 +- tests/builder/__init__.py | 12 + tests/builder/test_token.py | 205 +++++++ tests/engine/test_engine_execute.py | 78 ++- tests/engine/test_engine_schedule.py | 70 ++- tests/utils/test_graph.py | 206 +++++++ 36 files changed, 2435 insertions(+), 544 deletions(-) create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/rollback/apps.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/rollback/constants.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/rollback/graph.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0001_initial.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/__init__.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/rollback/tasks.py create mode 100644 runtime/bamboo-pipeline/pipeline/eri/imp/rollback.py create mode 100644 runtime/bamboo-pipeline/pipeline/tests/contrib/test_graph.py delete mode 100644 runtime/bamboo-pipeline/test/eri_imp_test_use/tests/control/test_rollback_node.py create mode 100644 tests/builder/__init__.py create mode 100644 tests/builder/test_token.py create mode 100644 tests/utils/test_graph.py diff --git a/bamboo_engine/builder/builder.py b/bamboo_engine/builder/builder.py index 92a5eb53..26a7e84c 100644 --- a/bamboo_engine/builder/builder.py +++ b/bamboo_engine/builder/builder.py @@ -16,10 +16,10 @@ from bamboo_engine.utils.string import unique_id +from ..validator.connection import validate_graph_without_circle from .flow.data import Data, Params from .flow.event import ExecutableEndEvent - __all__ = ["build_tree"] __skeleton = { @@ -94,6 +94,177 @@ def build_tree(start_elem, id=None, data=None): return tree +def _get_next_node(node, pipeline_tree): + """ + 获取当前节点的下一个节点 + """ + + out_goings = node["outgoing"] + + # 当只有一个输出时, + if not isinstance(out_goings, list): + out_goings = [out_goings] + + next_nodes = [] + for out_going in out_goings: + target_id = pipeline_tree["flows"][out_going]["target"] + if target_id in pipeline_tree["activities"]: + next_nodes.append(pipeline_tree["activities"][target_id]) + elif target_id in pipeline_tree["gateways"]: + next_nodes.append(pipeline_tree["gateways"][target_id]) + elif target_id == pipeline_tree["end_event"]["id"]: + next_nodes.append(pipeline_tree["end_event"]) + + return next_nodes + + +def _get_all_nodes(pipeline_tree: dict, with_subprocess: bool = False) -> dict: + """ + 获取 pipeline_tree 中所有 activity 的信息 + + :param pipeline_tree: pipeline web tree + :param with_subprocess: 是否是子流程的 tree + :return: 包含 pipeline_tree 中所有 activity 的字典（包括子流程的 acitivity） + """ + all_nodes = {} + all_nodes.update(pipeline_tree["activities"]) + all_nodes.update(pipeline_tree["gateways"]) + all_nodes.update( + { + pipeline_tree["start_event"]["id"]: pipeline_tree["start_event"], + pipeline_tree["end_event"]["id"]: pipeline_tree["end_event"], + } + ) + if with_subprocess: + for act in pipeline_tree["activities"].values(): + if act["type"] == "SubProcess": + all_nodes.update(_get_all_nodes(act["pipeline"], with_subprocess=True)) + return all_nodes + + +def _delete_flow_id_from_node_io(node, flow_id, io_type): + """ + 删除节点的某条连线，io_type(incoming or outgoing) + """ + if node[io_type] == flow_id: + node[io_type] = "" + elif isinstance(node[io_type], list): + if len(node[io_type]) == 1 and node[io_type][0] == flow_id: + node[io_type] = ( + "" if node["type"] not in ["ExclusiveGateway", "ParallelGateway", "ConditionalParallelGateway"] else [] + ) + else: + node[io_type].pop(node[io_type].index(flow_id)) + + # recover to original format + if ( + len(node[io_type]) == 1 + and io_type == "outgoing" + and node["type"] in ["EmptyStartEvent", "ServiceActivity", "ConvergeGateway"] + ): + node[io_type] = node[io_type][0] + + +def _acyclic(pipeline): + """ + @summary: 逆转反向边 + @return: + """ + + pipeline["all_nodes"] = _get_all_nodes(pipeline, with_subprocess=True) + + deformed_flows = { + "{}.{}".format(flow["source"], flow["target"]): flow_id for flow_id, flow in pipeline["flows"].items() + } + while True: + no_circle = validate_graph_without_circle(pipeline) + if no_circle["result"]: + break + source = no_circle["error_data"][-2] + target = no_circle["error_data"][-1] + circle_flow_key = "{}.{}".format(source, target) + flow_id = deformed_flows[circle_flow_key] + pipeline["flows"][flow_id].update({"source": target, "target": source}) + + source_node = pipeline["all_nodes"][source] + _delete_flow_id_from_node_io(source_node, flow_id, "outgoing") + + target_node = pipeline["all_nodes"][target] + _delete_flow_id_from_node_io(target_node, flow_id, "incoming") + + +def generate_pipeline_token(pipeline_tree): + tree = copy.deepcopy(pipeline_tree) + # 去环 + _acyclic(tree) + + start_node = tree["start_event"] + token = unique_id("t") + node_token_map = {start_node["id"]: token} + inject_pipeline_token(start_node, tree, node_token_map, token) + return node_token_map + + +# 需要处理子流程的问题 +def inject_pipeline_token(node, pipeline_tree, node_token_map, token): + # 如果是网关 + if node["type"] in ["ParallelGateway", "ExclusiveGateway", "ConditionalParallelGateway"]: + next_nodes = _get_next_node(node, pipeline_tree) + node_token = unique_id("t") + target_node = None + for next_node in next_nodes: + # 分支网关各个分支token相同 + node_token_map[next_node["id"]] = node_token + # 并行网关token不同 + if node["type"] in ["ParallelGateway", "ConditionalParallelGateway"]: + node_token = unique_id("t") + node_token_map[next_node["id"]] = node_token + + # 如果是网关，沿着路径向内搜索，最终遇到对应的分支网关会返回 + target_node = inject_pipeline_token(next_node, pipeline_tree, node_token_map, node_token) + + if target_node is None: + return + + # 汇聚网关可以直连结束节点，所以可能会存在找不到对应的汇聚网关的情况 + if target_node["type"] == "EmptyEndEvent": + node_token_map[target_node["id"]] = token + return + # 汇聚网关的token等于对应的网关的token + node_token_map[target_node["id"]] = token + # 到汇聚网关之后，此时继续向下遍历 + next_node = _get_next_node(target_node, pipeline_tree)[0] + # 汇聚网关只会有一个出度 + node_token_map[next_node["id"]] = token + return inject_pipeline_token(next_node, pipeline_tree, node_token_map, token) + + # 如果是汇聚网关，并且id等于converge_id，说明此时遍历在某个单元 + if node["type"] == "ConvergeGateway": + return node + + # 如果是普通的节点，说明只有一个出度，此时直接向下遍历就好 + if node["type"] in ["ServiceActivity", "EmptyStartEvent"]: + next_node = _get_next_node(node, pipeline_tree)[0] + node_token_map[next_node["id"]] = token + return inject_pipeline_token(next_node, pipeline_tree, node_token_map, token) + + # 如果遇到结束节点，直接返回 + if node["type"] == "EmptyEndEvent": + return node + + if node["type"] == "SubProcess": + subprocess_pipeline_tree = node["pipeline"] + subprocess_start_node = subprocess_pipeline_tree["start_event"] + subprocess_start_node_token = unique_id("t") + node_token_map[subprocess_start_node["id"]] = subprocess_start_node_token + inject_pipeline_token( + subprocess_start_node, subprocess_pipeline_tree, node_token_map, subprocess_start_node_token + ) + next_node = _get_next_node(node, pipeline_tree)[0] + node_token_map[next_node["id"]] = token + return inject_pipeline_token(next_node, pipeline_tree, node_token_map, token) + + def __update(tree, elem): node_type = __node_type[elem.type()] node = tree[node_type] if node_type == "end_event" else tree[node_type][elem.id] diff --git a/bamboo_engine/config.py b/bamboo_engine/config.py index bebf2d21..f4a02cc5 100644 --- a/bamboo_engine/config.py +++ b/bamboo_engine/config.py @@ -70,3 +70,5 @@ class Settings: PIPELINE_EXCLUSIVE_GATEWAY_EXPR_FUNC = default_expr_func PIPELINE_EXCLUSIVE_GATEWAY_STRATEGY = ExclusiveGatewayStrategy.ONLY.value + + PIPELINE_ENABLE_ROLLBACK = False diff --git a/bamboo_engine/engine.py b/bamboo_engine/engine.py index 6f929705..c7d5ed7b 100644 --- a/bamboo_engine/engine.py +++ b/bamboo_engine/engine.py @@ -57,6 +57,7 @@ setup_gauge, setup_histogram, ) +from .utils.constants import RuntimeSettings from .utils.host import get_hostname from .utils.string import get_lower_case_name @@ -122,6 +123,7 @@ def run_pipeline( process_id = self.runtime.prepare_run_pipeline( pipeline, root_pipeline_data, root_pipeline_context, subprocess_context, **options ) + # execute from start event self.runtime.execute( process_id=process_id, @@ -912,7 +914,8 @@ def execute( # 设置状态前检测 if node_state.name not in states.INVERTED_TRANSITION[states.RUNNING]: logger.info( - "[pipeline-trace](root_pipeline: %s) can not transit state from %s to RUNNING for exist state", # noqa + "[pipeline-trace](root_pipeline: %s) can not transit state from %s to RUNNING " + "for exist state", process_info.root_pipeline_id, node_state.name, ) @@ -1010,6 +1013,15 @@ def execute( hook=HookType.NODE_FINISH, node=node, ) + if node.type == NodeType.ServiceActivity and self.runtime.get_config( + RuntimeSettings.PIPELINE_ENABLE_ROLLBACK.value + ): + self._set_snapshot(root_pipeline_id, node) + # 判断是否已经预约了回滚，如果已经预约，则kill掉当前的process，直接return + if node.reserve_rollback: + self.runtime.die(process_id) + self.runtime.start_rollback(root_pipeline_id, node_id) + return # 进程是否要进入睡眠 if execute_result.should_sleep: @@ -1177,7 +1189,9 @@ def schedule( # only retry at multiple calback type if schedule.type is not ScheduleType.MULTIPLE_CALLBACK: logger.info( - "root pipeline[%s] schedule(%s) %s with version %s is not multiple callback type, will not retry to get lock", # noqa + "root pipeline[%s] schedule(%s) %s with version %s is not multiple callback type, " + "will not retry to get lock", + # noqa root_pipeline_id, schedule_id, node_id, @@ -1290,6 +1304,15 @@ def schedule( node=node, callback_data=callback_data, ) + if node.type == NodeType.ServiceActivity and self.runtime.get_config( + RuntimeSettings.PIPELINE_ENABLE_ROLLBACK.value + ): + self._set_snapshot(root_pipeline_id, node) + # 判断是否已经预约了回滚，如果已经预约，启动回滚流程 + if node.reserve_rollback: + self.runtime.start_rollback(root_pipeline_id, node_id) + return + self.runtime.execute( process_id=process_id, node_id=schedule_result.next_node_id, @@ -1302,6 +1325,20 @@ def schedule( time.time() - engine_post_schedule_start_at ) + def _set_snapshot(self, root_pipeline_id, node): + inputs = self.runtime.get_execution_data_inputs(node.id) + outputs = self.runtime.get_execution_data_outputs(node.id) + root_pipeline_input = {key: di.value for key, di in self.runtime.get_data_inputs(root_pipeline_id).items()} + self.runtime.set_node_snapshot( + root_pipeline_id=root_pipeline_id, + node_id=node.id, + code=node.code, + version=node.version, + context_values=root_pipeline_input, + inputs=inputs, + outputs=outputs, + ) + def _add_history( self, node_id: str, diff --git a/bamboo_engine/eri/interfaces.py b/bamboo_engine/eri/interfaces.py index 99c722db..8ac6fb23 100644 --- a/bamboo_engine/eri/interfaces.py +++ b/bamboo_engine/eri/interfaces.py @@ -1577,6 +1577,35 @@ def get_config(self, name): """ +class RollbackMixin: + @abstractmethod + def set_pipeline_token(self, pipeline_tree: dict): + """ + 设置pipeline token + """ + + @abstractmethod + def set_node_snapshot( + self, + root_pipeline_id: str, + node_id: str, + code: str, + version: str, + context_values: dict, + inputs: dict, + outputs: dict, + ): + """ + 创建一份节点快照 + """ + + @abstractmethod + def start_rollback(self, root_pipeline_id: str, node_id: str): + """ + 开始回滚 + """ + + class EngineRuntimeInterface( PluginManagerMixin, EngineAPIHooksMixin, @@ -1591,6 +1620,7 @@ class EngineRuntimeInterface( ExecutionHistoryMixin, InterruptMixin, ConfigMixin, + RollbackMixin, metaclass=ABCMeta, ): @abstractmethod diff --git a/bamboo_engine/eri/models/node.py b/bamboo_engine/eri/models/node.py index 05c7ce21..ba34862a 100644 --- a/bamboo_engine/eri/models/node.py +++ b/bamboo_engine/eri/models/node.py @@ -12,7 +12,7 @@ """ from enum import Enum -from typing import List, Dict +from typing import Dict, List from bamboo_engine.utils.object import Representable @@ -49,7 +49,8 @@ def __init__( parent_pipeline_id: str, can_skip: bool = True, can_retry: bool = True, - name: str = None + name: str = None, + reserve_rollback: bool = False, ): """ @@ -82,6 +83,7 @@ def __init__( self.can_skip = can_skip self.can_retry = can_retry self.name = name + self.reserve_rollback = reserve_rollback class EmptyStartEvent(Node): diff --git a/bamboo_engine/states.py b/bamboo_engine/states.py index 6a4a460d..319960b1 100644 --- a/bamboo_engine/states.py +++ b/bamboo_engine/states.py @@ -29,6 +29,9 @@ class StateType(Enum): FINISHED = "FINISHED" FAILED = "FAILED" REVOKED = "REVOKED" + ROLLING_BACK = "ROLLING_BACK" + ROLL_BACK_SUCCESS = "ROLL_BACK_SUCCESS" + ROLL_BACK_FAILED = "ROLL_BACK_FAILED" CREATED = StateType.CREATED.value @@ -39,8 +42,11 @@ class StateType(Enum): FINISHED = StateType.FINISHED.value FAILED = StateType.FAILED.value REVOKED = StateType.REVOKED.value +ROLLING_BACK = StateType.ROLLING_BACK.value +ROLL_BACK_SUCCESS = StateType.ROLL_BACK_SUCCESS.value +ROLL_BACK_FAILED = StateType.ROLL_BACK_FAILED.value -ALL_STATES = frozenset([READY, RUNNING, SUSPENDED, BLOCKED, FINISHED, FAILED, REVOKED]) +ALL_STATES = frozenset([READY, RUNNING, SUSPENDED, BLOCKED, FINISHED, FAILED, REVOKED, ROLLING_BACK]) ARCHIVED_STATES = frozenset([FINISHED, FAILED, REVOKED]) SLEEP_STATES = frozenset([SUSPENDED, REVOKED]) @@ -51,18 +57,20 @@ class StateType(Enum): TRANSITION = ConstantDict( { READY: frozenset([RUNNING, SUSPENDED]), - RUNNING: frozenset([FINISHED, FAILED, REVOKED, SUSPENDED]), - SUSPENDED: frozenset([READY, REVOKED, RUNNING]), + RUNNING: frozenset([FINISHED, FAILED, REVOKED, SUSPENDED, ROLLING_BACK]), + SUSPENDED: frozenset([READY, REVOKED, RUNNING, ROLLING_BACK]), BLOCKED: frozenset([]), - FINISHED: frozenset([RUNNING, FAILED]), + FINISHED: frozenset([RUNNING, FAILED, ROLLING_BACK]), FAILED: frozenset([READY, FINISHED]), REVOKED: frozenset([]), + ROLLING_BACK: frozenset([ROLL_BACK_SUCCESS, ROLL_BACK_FAILED]), + ROLL_BACK_SUCCESS: frozenset([READY, FINISHED]), + ROLL_BACK_FAILED: frozenset([READY, FINISHED, ROLLING_BACK]), } ) def can_transit(from_state, to_state): - if from_state in TRANSITION: if to_state in TRANSITION[from_state]: return True diff --git a/bamboo_engine/utils/constants.py b/bamboo_engine/utils/constants.py index 1eb0f7df..5f458433 100644 --- a/bamboo_engine/utils/constants.py +++ b/bamboo_engine/utils/constants.py @@ -35,9 +35,11 @@ class ExclusiveGatewayStrategy(Enum): class RuntimeSettings(Enum): PIPELINE_EXCLUSIVE_GATEWAY_EXPR_FUNC = "PIPELINE_EXCLUSIVE_GATEWAY_EXPR_FUNC" PIPELINE_EXCLUSIVE_GATEWAY_STRATEGY = "PIPELINE_EXCLUSIVE_GATEWAY_STRATEGY" + PIPELINE_ENABLE_ROLLBACK = "PIPELINE_ENABLE_ROLLBACK" RUNTIME_ALLOWED_CONFIG = [ RuntimeSettings.PIPELINE_EXCLUSIVE_GATEWAY_EXPR_FUNC.value, RuntimeSettings.PIPELINE_EXCLUSIVE_GATEWAY_STRATEGY.value, + RuntimeSettings.PIPELINE_ENABLE_ROLLBACK.value, ] diff --git a/bamboo_engine/utils/graph.py b/bamboo_engine/utils/graph.py index c3232c25..df162a6f 100644 --- a/bamboo_engine/utils/graph.py +++ b/bamboo_engine/utils/graph.py @@ -60,6 +60,55 @@ def get_cycle(self): return [] +class RollbackGraph(Graph): + def __init__(self, nodes=None, flows=None): + self.nodes = nodes or [] + self.flows = flows or [] + super().__init__(self.nodes, self.flows) + self.edges = self.build_edges() + self.path = [] + self.last_visited_node = "" + self.graph = {node: [] for node in self.nodes} + for flow in self.flows: + self.graph[flow[0]].append(flow[1]) + + def build_edges(self): + edges = {} + for flow in self.flows: + edges.setdefault(flow[0], set()).add(flow[1]) + return edges + + def add_node(self, node): + if node not in self.nodes: + self.nodes.append(node) + + def add_edge(self, source, target): + self.flows.append([source, target]) + self.edges.setdefault(source, set()).add(target) + + def next(self, node): + return self.edges.get(node, {}) + + def reverse(self): + graph = RollbackGraph() + graph.nodes = self.nodes + for flow in self.flows: + graph.add_edge(flow[1], flow[0]) + + return graph + + def in_degrees(self): + ingress = {node: 0 for node in self.nodes} + for node, targets in self.edges.items(): + for target in targets: + ingress[target] += 1 + + return ingress + + def as_dict(self): + return {"nodes": self.nodes, "flows": self.flows} + + if __name__ == "__main__": graph1 = Graph([1, 2, 3, 4], [[1, 2], [2, 3], [3, 4]]) assert not graph1.has_cycle() diff --git a/docs/assets/img/rollback/rollback.png b/docs/assets/img/rollback/rollback.png index 2029187e066fbd45294e6c94eeb5832e1e3ce071..537ef21b3d7db5dd8686ab867a7d043c354c8ea3 100644 GIT binary patch literal 51816 zcmc$`by$?|+64;3AVUufBHf`hBApUqkOETDNH+pX$>2~@DoQts(hZ8_P|_+5(%s#4 zo?-v?KHs;`xz1nbT;~t23mtf8-sinz-D|CzFby?jVgechEG#VI2lo{pV`1SSu&}W0 zVb{R_ zq`OsRWvcYc^m4peOpPnzfJT`GW>956&-t}35SRA^OH)w;&Woe#NRX}6{ygnTmJ%t+ zVcT>~=R6ieeMkH>?puC6D;62{!U9eulk3<=59p~hDpDGFD)Hv%t$TK(a@dc#&)P9*wlg)Ff9jm3hP+mT=9TI~?u1g`gg;vM0-CQj%l@2_rK^g9#D$TYrOu!CFe8ZYOISr3DfxDm3;VSwqGrv5^oB90qq^N zxb)IzkOhY=_Dt`Am8vQh7x*5Ag&ksx1qI(>gD)EJg@px4hy440a1iM@|Nb7^9`oYr z1L%|n+rr@xIyD#^!u;*gTROGCA9(m1P_aPz4XBrw7X;Nb zKECe)Yov!XqDxFJjq9qGQ@529ibHi#oEe0KZz24n0_t8xHGiv^h<4TcRAufO#;ZK;eenktJx=N067RxB88__K zpR-6xk@%Q*hO=9-Pq07rzdzeM*mK^l7e?Pqa#N)`BSj}(JQH&Ek=&`__h{|^av*-v zVn3}Lfv;@t62|*wk)PJ$pM%H2_A@a!q=U&$FT&=If2gMj?(1~!a(1I+aSaHsj`jaM zGz~IvE?j|+g8us;U7&0#V&wmS{Cr0MJ$BsfJCv`r!>MV)eFP6eTdSz3Bb@~~=)zjq zE9pE|j&Xvr+0GJYrhTqr$XNPS)W)}pE!q$zZ|P`o9Cs z<{CCWqB|f<>pwyN&$T|>0}*KJ@cjRkw|Lgzbd65-Gf^anNe^Bt8@i#BY(tvK>=(sf zws~7Wk^hshey;zI32XT=#)^eji~hfpBAp1*5l}!hoDdy)JKhRS{!uWVJ1A`{+>a!e z{k82usE&HDPK7fa_c}a6p;X)Uho84- zKita^^1rP1+Dr{j?{ns?Zwf~GPrhLc>N)cNl=d3!jCVR?@+J)twR9-T5SRbHpwYQz z&OC(n>Kb|Ih%JALLCc)3Aw(v9cU`s`@opH?+>4+hQ6JNki1*!T zZz2}`3y;;eO4#b;um7ZOZ-~iCjWXCz>>pMkYQ#4%@zl=;S_*E@UPHcpUv9VbPFz@#*P9c=+g;o*peG*^xBzAj64?V-vYB@$&TsG(IQo)x~ks<43&L2R?sE z_2!Z~-|LHfrfcH2zjZVl9U?ra$GW#UISGAJdpI%H(^Sq1mD{X6G#++sqo+L-uiS2> z6(vBFPZ$2!Ac#?uu#LBT|G>eF7Kt;Bt12%(>NV-7sv)-WHLL5>3b=8;YN$K6O>Ry$ zK~pd>VP1ae!}Px@LfzWhPw=jaR0S#k6=p!gTcLAxoXW8D6sp^n_qC%1Y`teAMH4@? zJcYS`y8HTC!uP(fnpx;qm#34(E4g`zgOI*YvX^H_Ag7O`Jd!CvNT)%{=S$>J80~)-m0a{@bc4u-{tvX2qAxsfYi~{>&V67qk}PL z$5%r}UcyLR?4<-#TFdBE$xb`>1O?;(JrTFp$#t#k9-F zv$zq@>9p4*5;x&**&cPujew_FwDsPl0UHl>elW)W*LSzuY}ED(uS75Q9S$3*^jBxI zjFp3oR|SBVmB}nKR$1NgUlE55sT%h*q!M#6wNiHpm@p6LebZy)*3Ld1${Ga&kJoP4 zzNoe<&K%@bRjCb=fOPfD8*UYDq{2Qp0Mo!0mT(?&hL>~i1jZc7f>Nb@MoWLm69$3A zt!A2~d7cZ57W<_v&nWFb;iT>g0iWSkAbP z_xsF3=wl~%e?+!_2qArD^i2c1t=epWEwG-%L_k1&`w|6q({RAzpKNaf%dfGwo6r zn?|`Dj97<+FnHhWlG^Pudrd4-e4A8iD?rJDtLx$UsC`kw6KvUQUjXWyq_U#>ma_3} zmIPt!n^qB$(2R%DD2E~RQL@)QTyMe0#tppd9Cc<4!`*=OjNaHGEmjqmZB3l@@2S$) ziYj?MZ9NuUl4881ds4Gm##gnaq7hu-+{dwzVC)iVUGEYZ5O^(QwvH+PQ`PqakW z-y}biZ^4c$7ODy8plU^M+uFf$x-xJ$zYo#8QIyX~ThaH^MZd}6lm5CjO)~$2CZ!{M zS>^Qio0Chwvv3H_G|MQdbfTJy%Yu)siJRh$&*BO3^2Ik#9VBCLaqfNkbjN*Rtj@DF z$}K2@wrqkfi%6(H|IzQ~eX6=E+6%5{^a9lAWLprm!m{{L85IpuVg55e0-RewLfAgr z^7zS;DWN&B9L`@?Tx;uiW#rji<3pte#p0#KK-W^$p^8wGU6aD>^S*yz?{hdrU<2Mo7@m}H#+KY~CdgLu~rPCue#a~lv^q+;eVK9JpP_a5=Kt^yd|>PVr* zO3u)2sd}lrc@*MglCsz2qz-lic85=eHGj7L^{)_@uqRQDP!AI^HSy#XalG9{bSNP= zjEnE4D}y8X;!`1L45=oGl%jlkEZLI5*C)Jx`3JsBb#A|(*JpI>DU+!v>eqsYlwk$} zxztT|0Ni!?dl`xFGBgs9Uc^E6w|p6`LL8}RtzYPsy~RPET@z6CPl=nQdS>bmXZu5_ zA`IS+dRcgk$CkPEkZ|aOtNVwR8zz4l!#M0!g1A0&roLd|`LR~IfGAncP7+DH<4WYt zNp&K=5Z|;T%TwF{u{tTA3M8a)kE_$&oT%m1E?~^+g*`(>d9Y*{5xLSwO&VE4K#c&VmBrG*8U>r`7qLr(y=_U( zb=0wjPT~l~KWLwVDCcP*`myK(?~DFA-+J1hS?+GdK}5F2RY~DezzRlKRerA_a;h|l z8sX82qSmI8HZlFPWf4Gao-~_ZR%zA+*ArXpGW1TesEZbQo>)_ha^_`=FI#}&t%)xt z3r+PCSp;%m-3^qYc#TI|DUt$!f?2ol&>cry{jcqoDN&WEN=OE(1z(H(j<7-q$-Y8m z_J&*Yn){%G5Sj==4;>W96o^xNWu9cO7iVxrQ+PHNqm|*6b7{uSh&NjP|s@Ez!EA z*ScVtv(0Sbyhhf?bfx(`Bj4S)bqFRIBqQeEGt{p#Y z3I!1#MsI7=$4n@z<6sc}oDS)!o&(8MO6o{*G!Cfq=TC>8WrR zmvvC^ywrE@pFJr$oHPlbu21CNb!pT;d&gEv@nDVVhMZ|S;SJfg>hD2-@8kMBt@g{!}=w_AZq6a9q$mF*9}S4w74NlNn#Yy!n$ z3AM6_7h}$!qKFE(J0o3cnU%tnBzl1mnBp^?Ya!A_-x=aAGrl9H;~C)60<*U91qnS! z1}KJA)~^R(>iBgFE@ue0FEil|Xi$u%6ThAN-LhH@q3N$Cxqb&hx243yvj#vSC5$s% z0K4oSx#VrVNq`kS8KHP9-#4mPySnflLxpyht4EBTYxf-}>-%38dLV;#6kiAa5}Dds zf9EamMgD8UoL$Rx-SX2Jx~Nw_t&$wAhL+S96{bL!1fb_QxYo&s$1Kw+y^p-!Z)(!mmi?`v-ghC%lVI$o%yomO(c@TGD{~mK<+Sf;SUOZsE*Qs8JdYF^z3KOI&(m!DhVhGxkp?a7 zg@?3G-^{FxV6uvH=rgILd>` z28Dp1!oyWUG5S4~b(9Sw0A+F6S!GNiU6gJP%O^TkjzAvWO7xm|i9@I;1B;b{S&!=N zXq1%9>lD_(8_J`*)}Y$pL7P~N>HB064U@6HLwPhE5Sc(ywj#iyPvJ8;?DjW{w~l|GiRjt%_&wD`_U(#F`JXjK@N3 z89CHo2!_hIx>hh&6&i16ndnnsh_H>{l#})Om0zL<%ulO7Nv1nu;3L3QYEYLYc9eDP zv8p!bsNF}njU@Ev_@6Rsbp+cVwKGRA54v7l z(LK#l{2#i@p04dG-{FK3P`z_I;b!XR2Hz?WDfb(Yf5#4y4+_55qw z0U3Eb%+iMaXOm{A?sT0r9rY{ojsaGGYK=SM6t4+yG=D(&tEM}a#KBfQx zq=7zK@e8$^N4Xjqm9%$x0?&KrKzztM_*V}M7(IdIo`WkrCxCLKJb&EzEut3`6&P|< zY3gwDRn^)G41rJ+Gh6pmkt;+=#W$RE}nNI)O6Zq&1A{uoxTq#LVuA`G?wi z=QV?)3X-_WR_Dwxj?%Y{M|VFfFttsHRwhqG4@%>q`56GOQd;sy6BA`}I?Mr9kUkhH z)Vcn=GIun)70gobXsl!NEk72r&;NWh44KpGDp6);g}jud(o&;ldjv>`(vq;oAa09} zfzI8K2T-(PZ>lr^@@KysCM{f+9gbQKL^s{`ch2S`A36B#hce1T^^3nGyLEIO>TU9? zD+|tt5mvPI>qhsQ2$rfwH%~ndO}m3o{fGEW=OT0IZobo}Vi9p)N91uNIfxV>+TOiWzKcj#jMah66_jeQMF)|09$ z+&mVXmzTZZ;adsD^|cXH)%sTy!4u%9v{czzM%6Urp5bHlO+0@DU?RRGh#yx;jt}K8 zrf-eqZMxHxWRIbXZZXuw1%;Jjd$-GP!*;}XezIDm`hsyf&Ofo$(R%v! z#@e@nd^hilH)-{*p~B;K#QsK_Rd)eLaOq&gw`(*p|LvCSn+Xzo#n!YM5XNbvae`Fs z%>NM&xQ;KT`H{OED)$FPg$};I;Z>&PwoUVw_0?`G2$~e*@qN^50s|yRj%s!G&0P)-aa}2@o=$F8~z_HTo^x+I=84ZoUr4Ed7Xm z|BF2VCbWc~+U0}LN}oeay+thiT-vHhu`-P4ARjsUm!MpMC90tW)-e4-cD%5*#iOm0G z`OBz7jR7z8e+d}_Z?F>aRZ@NmGn-`^C_h*yhs=}3dyd+&t9%G~z-|Y{e8}4xcG*|) zAK45jCFX7v11_`1EUXJ4c!EzFvld?$qJaj|EXGrQML~bHgFY3SmRHgdeG-X*4Q!CI zuG`^3FZlH3I8k!M%)bU^TP^^vr`g@}gHDuPGq@^Cn{}to*}>1wA{EU@@j4Z8QQ8r1Au^VYIf?Sj9gncNI;Q%)*d zkHQ7b&H$0tC72{EbG>G)lg6qaU8cbIA#A9k+NliN15eXuG{K-tJvcvXB#X z&SZ@!qf7+(^f#4|Gnt@43(U0W-X*9UZx&1~U+;gevbEHYlhspo3t~hl6ic}gSM8HJ z|KjdWezeGLS z{Dk)OX-h1^|DybQkwt4(#nE2)ihpL~AmXSnXpmRPvk9F*9%|3K(Bh5b4xIZV)8N^BI2t~lZM2dhlG zkO#!8a!}(}=lfgzTIxa8X?`L#M>FBm6+^~^yFHRc-G7>X9 z9P#f8g9`}oo`71_1Q4pBzt(QO;yr^l5U8|9xgMi29EmWmQHpZAISqi zg=I$OKhvlpRx1_<*0r0RP89@6`5gF25hbkPOK=GiQkf&u7oe!%Slz`8RmW^=m19n= z_fLtqh(wwl#L5nN82yhj3MK)+955JoqUYWX7%=CqJ7{d7le$ZAqam1iekCv=Butq~ z4vU$L-hdPu63$os_4i^Uwf{y+_MQxy`0!bGpl36nuK2rTCXryg7PZqDw$30f>ygMC zxGU|;srTgu6teNaClyrfEx@}DpM-Iy!U*}kna0}}ey0f{hL!@dyz34xqwf|H{~^pO z-io(+DHD7Bnw1WyUuN>XSE^I`J-DeSLedsfXH4y^l9;e&U#%8Y1%pXIy5?jh&ul

*2>pFA8)*iR0V#Lkw@iFW8}{sY-#5kG zB4a4ts=t3!y0)krwFxpT+vvTBfPA7hCZb*8MFsh@>p`_wU4rRD>9?WA-pil2fPfQ^ z^BEP^QLMGRqKu`*j)JljyGX_L=!Qi5 zLR%SxSQ4u#(;OML-N3*TVnCWFCA>8=_nZmiyp}v57$b3UG^^_Kd?CTw0GfN?T(L16 z3%iNq39UY}i7Qi&ip+oie>bW>u0K41l%chOggbg6DG$m)2ZDwcDM`FFp0J4+2;g;E zP{bN}>I-?H=a%0oGvH&S4KW~9bXyW5xVnB5)LR6dbNLz*^q%@p%zxFLlvVcnD?w;& z%h5WW#S6+Nz~8I^FA-LZS3oTp%%!n8)Qm9s9JI}Q6J&yuRSo`Yu3Td1FOF$-q&Q;a zhoCEl{NT}T;rp>>PL_UhxaoXBG1n>Agkf(&a7d+oWhxKc;~(gFj5!nX?F_nYoTw%X zA`eCJilV)r%@HA2PXb@3 zJcxSek&_uA3^aT-Tk?eNHKTTUL3j)=8d&(RAId4)5o5Q;2F)c#tm^0*J?q)cCllNH z(x4LQZX*;WzCeYLf$@i9$p_XvruY|&C^2)sbJI@uy4r4RNqPvirBx*t7(_rzKHTTU9=<$Oj1 zTF>I8o4|-KfKR+#JPGyIpgqe)k)jCP^g`_j0X(V@HnuCkU^psltBm65j{?8L7z1h6 zBrf1Fsv`Q|02hiTkz6y?&lsWfV1R}$p{Oa^xAk`ES#^KXxp$eFS^j>Q3;>te+|PGr zAZdutPxc9ijC~vzvA?$ZNT^vG&(dq|Ug=HyQ2>j$=tXEEha?$ROgvcS6th{Q#8}RD zI{D4Mqe|TNrLDK+>%T?Dv!223 z3;n+R_xZ=Gj2Z{1KhTE8_A<=VIn?PUH1X~fFxlL0TLGA~ZwD?w8)?c+4he!yIYe(l z!?pkzLV$9cx&>fd5|b-HE_m|o@AJ1nE#a!sKGng&ZTZzqQIFFW0V=Ci}Kl+KGP4tk88vf@k1H1SY7@ zSCg07aIu-^2q^z%G`!`-@k;NYn*+Kg#9&H%_<11LmzEWb+O#wo|usOiA zRgG+iTVSQ&3 z1!IgDvZUcDH9Y)o?wk^(KbrsGW&01hpF?B2A-nrPZnYI^2uiRBGTHvo`mV+`tRUJ4 zyri6#oceMhp?Jc3Ri6=~us%PLGZB$^IfJG*U&-4$&YA!?O!Du`#gy3P zqo1tNb8qj%$_XJtJz^FwK;WRk3pciMmPX2~Ux0h%SxymUTKN9OLFIdF4I?l&t;U}Y z8k~MgxG6j;0VohK#M}lm1ybPdQ_pvB8)qm+INbu5jbE>Zh8^59Idq3!BW0iiBEDoe z#o!OG&3j)QWs5vcSwJKA+a4Uh8lHIX1?b;Kvt0lPn_*bx#9OF*g8=!EtfAZq==ddc zSQs3PK?9$4zSh8hUPprV@)!CBpgb9en(fxuyOKk+UrTKONK;h!9C%e%qLi@q#kl-Y z__iDpo;(tLgk*x*CJ_zxiElSWc9R_*16y4vOnYG zt|?ISh%qh%rD{naWMYb&mTXEMG8L%W$h(gbS1AyM^_~6GN0&S0PLH6JdZx$#BXGu* ziuxXlL0AtE;I~)hz71z6gaiY4B3P8GqfQNuW(fmg*R=AT)s%^~dT^vt<@e#UIu;*1+fa z4K8GYvAa~w?yveDMEDgI6-UH9I|q33Qmc2Mh?;+_*}pnmms?S1pR_LG z_jnnjTCZG9H|C#=UK;|0tQugI2KComGTai5clg?*@g*16O=ZKcrs8s?b#4Z-fBEh( zroI&K3%QJ29#77SO%`dTfy`p%FJu8;z+3UYNR;te8{qZ_koztZ8GS+!`+bV&A5r}z z5Jit_&?a~l4sdTgg%6s%I{#H{4`=|sBL>fL4_zVW6CegR0b_|F>3D93 z%>N957MSUyaDfb>=9b~5`<-MBmkR)&lUS#|w89W!2FC9SDIhw*4WQohEJEd5^sQ5# z6|f$ExoHAL+0?zr6}if+B<28tMi#ToZl;Y7*GXya+~fi94AW%SKb-4+S-pV-{+*Vy zSAZ@p+5%sOQv}J$eQLk<$4(Dc!Mz+b8F&FsP!3j3fNjfRfSbnGBRBQu5kIk*aVL}w zhmJm%5c*Tp6$pGNa7z=(rOO{sVZI?x(2D=#6M%k_DISKBV}Jo#@)pKm0K?z{hJhRs zW(^P6Tk2Dws1RZXIsEIs%1XzeWU-h?=_Fk}h;BI3dxR{CoMV(|MoniM<2kAWGm+e& zDIlS(p@RUYZ2^yh{)=Hlr|(LMVaa$u^4dPOr1At~f=K$WnfqbyKQ_^<|JX$3*eB}+ zEn4FR?%v_N^z@O=zv5Q<^&gzxoTO_394_7ArMP__}7XfhqIc%9+j2>FbR2NbwKOl8XQ)-sU)bGky$;d-zJG+WSK@ z=QIw1-O~~ofaLqW&{pNR0{}$QrzTm@9oh@bA;`Lem4VKVbh3v-3Jb1KDOnLPtcVWk zI9UK)AadwLw}@V;V{t}z;fjLh#Z#!pPTl*!wNB`fBpU*OLsrMHyEh&naqNX_F{8y6=*Pqo?y0J zm!{jDJQi($f$46w8+z)lD(SgiF(ksaeM#$7yFVBVI$a&cJ5_8gMix(tfp@_`z-a7X zXSSyW-6AR&Lo1Y!h+w3*U{Ll%*+xGGW8+-Av*nMm(5ZH4mo`b;UPei7*mxMdm=3~B z?eFPlFVukk7@P8ihtN?y-b~hU73Esx>B>hINFxJkt9_nHj1BcV)#mNCU=sRK0k^rI z!HcOrJqL~J7^BH!>+dPeVRdF0k6c$HIu5X}?JFuMu{@USbBa$gD;Rs+x_2dYUbETc zN?6M!5|6<~&u@Q#7K0F4MKpC;#Pn;!7Ft(+07=y061W{5zJYWnMr`4vv607owCzq) zbg?~}i&3W`XD-0dcTu?y&+ZDUEN_OVDJeO8@c7gFc?g&eors^%OqYz6h=Go=1|T=a za-FAk<^+&Iu;r7o>4hZc9tYF4Z!NL2949h26pXl!*vkfp+s`uai2(tqu5f#Id^mw^ ztU|DHk6k@*`}M8o@}X11ps6{AkP`>5n*elX+&n0-`eByZOI~0~FIHggnI0#}95H5) zKG6uTGBd6%(B1)n+mMz-YO8_6gX6pSnwW~9BPI}%lIwwzx+Zq68Y6T7qlKgChGh4C zDRA&GYcSLK0U}_zGqWmY?lp0eLuuGY3@uM91Il@@uJOy3jcGKLv$I0hPr-VpukFv6 z;L#??Fk;}&dt($l6*M*H=wb%B$H)VJn9?ANzd8Cgp7S_^vM3JBqzE^JqbANQLgD1% zde(>ibHIKhv+rTl2N=>$2`dfXm)+9YsDM@5XdRvW3FxvI;Klq~K3-fcnn4;3XgEgl zd(vINZ|PaXYnUR?^5h(ld5-p78R@`$K0J6~T`}-PXfhIy(f1drPRmLC<#|yBE7|SD za!xW9bY?go4W^X|fao*)Q~w-$ZAVaRWwQR&iOc=?6&FB}-C)OSRtfuR5QYUOZ{&Io z{3z`nA2--|Ouwxb{(w9f1B!R0%Th*&NV~dL;rYi2{z|EJ^9ZkDvk-wcI`{aTeno_; z_KsWX5a>~oY@?9@(W>^a-BLvb3Q7YTW}$6+Rio-kz|SUa*j-WYfTO&QjS91Tav#7S z5{-L5sD!Az;vYNIBl_TkTE~E&fjX>|s9LL7C)#pYJl`@gT^_B-6FfnvOS#4a%Uk5J zN|!H0#qqYGo#!D8#sl}5nhJN9k%y{Kbs-|CB4{Btw^to2mw}TnSG-WPt@+t=_7e?r z0t=G1y<>2iRh4X3e4f(cyf!!1RS@R^w_Y8Nd-a7#TAl(pD!Qhfk`~M8rtKly8yMg@|0YgKc0gMTcY+y3 zg#x|>ppX3|eKf#^f%7(6P*WBCPYE_!@$Pwa%I5Kx_#E*KRA<0I_J_*9H5ztLEdI7u z`0g@){h4_nb{7d6(zOqSI)tf=*)=jQQ;1n8&@(vm5WLsRD#Qlecm!=rs=r1D%>pr6 zLGs;jU6O)e7Y_t?-JbBFN|L;JB86!y8J&x-(k~>8>6ZaZqp^AAb??Isqf)Mu*m#lz z3$J#0b~O!Al_Nl2S^It4>EMo}R`x48kkjQVzI(W+OkNj&Q18 z&R(L~g~1!T;TNz)mR$Vg)lE-Bj*!mP^v>k-jyCBZrrh}hIm!adakO~Hp*bL4!+xPT*)+;(iW-%_& zF{dAn7|ZO#SfzjrB4#=6&&^@tV7BIqjKkb9#ULY#5HYPtSC0|kP$QoW-%rk=4GOFC zI}MIA_HGy86E%LCB-V_lQVTyK5t|BlE0z-i3?44I=9#q^UfvcZ;LiOi{@{kO*0Mk4 zMTF?Oe1lcwRI-?%AvAAcMfqkByv|TTYLoBNE=lN)f}9NE*eoaUYAkRBdM5@`iUf&Q zbtI$&zeI*HTP|aWfkf;xT+>Q~vVK`i8#Gkkg-V0!C(qr?&ynWW;@p&GVA4i5dHk$Jr%)HzI^*zPH(_XCWf6A zYpY(BO+fSC!$QEUoksVpa%%=)4Qv@6_t9yU)rGOzSU99j;R&$zYEz98S!+r1_tS0vgN8D4b@*fGoZHWvF)5{e66~* zWofQ1bNs1$O=}VmHg*QrE1A=DTD61hU2}sp6>;WJX%E?2Eo0%@-?M zG;N8HxS)W`o0EsNM>TF$-y)H}CdN7a497|{UzrVmbxW7I3U852LhnWN_;hlDFWrn_ z<6T%2BWu{VXK0SmJhx`A>5?`EpAKJTu`-_jb|crM|3|mxzgNVr|4)^defzWtFc3pxovR*45?4xZ z?Z2-2# zf47;vA=671$dk80nx_Abm}>R~_dpdBHR~urO(%3LP~ByKz$UP&5PvA&nHVGF$E{@Z z(ghX83`O{|6X5Tbv*{{}0g5}~d&}9+FgADCeKDylTem>$)?2h0cp104VObgSr*h=N z?+48W=TKNk{2TH&=i%}%f(3}Vl?5g-Y)Xu|4aZM`4OB7Nn18voO?fG-krX@Jx|;6d z!9>KpFMIjb4Wjt?_808&Z$IO|qD83+*cOB#_F3*D=&+_E7Kyv*iJfOIJ1mC@`9I;9g5%uC4aJK3Y^k}3`(cklCKMC_B zFo~BB{%(BwH__jWGF0}Ry^n!boJqJo0wQdcX;Zz$J|q2BU>a7Tmh-DZV?gVh?1qSH z)ecXaLX5Rv`JGzPg{b*?Cb<9FXJgKs-lHNND3Joa}q}rlnsmPIV0@F4J_X&8Kg{R{FOFvs~a4f zd@obgB8>XUzQ>1JrZTfr`tOF&Kz+OSzO3V@o{m%Hroeit()UEB#=SREP=y03^EzUb zaLZiI1ngB1m9@j#rE5=tNbdfDE@5i?9WxGsX~{r~nwzqKp_vQ$Z1=BW_7?DQ{8V59 zjDK<8y%5EFCKrHj^(nZ0b$_0gG+v`c)zccJg-OS?4m@B7 zP8mz40b(%9uP|a;^}|wbfzSL04HdtZ8;Ri$I)mhXsL349VMgE}%mh7}m~#L_K4F%d zX$3fWa)rtM7$CgC+>__7g1{N`M)3F5UY?u|OMrl;YW8g%{KwaYJA&|M_TvZ^$KG|b z<3ZGhfqAQ5_%$6IMn#9f_^SdGUs}>Jf{h9$QT%p1*UB67;eg%t-PU&eSG`u4ik7|5 zNInr!5WRdu#=((j!}91tClU>no_KY6&YM9uW@&p*p#=CNi@C`m!qHerTre#X?i$UE zsBz-wANS~J;#I$=gYH<0&5y={6<6gyaA)P5|5*FU{-?E{@g-rXnAVC9HAc|Siu_pK zd#Y_mk8uSKOvum`ZznM64+1HC&}sGwzCW#Fx#kOiZqaw~=qWI23E}l{D|2w(6$yVI zTjb2jr&Ge2I>iVBq144-5O~&ENk8W4clqXB$4T9NAmIxw_2!>hB>uUt6hwxNHh;g3F&`+t7B>`R?&k$U4-M;Rt_ZP=`U*RAvjR=?)O zr~cGDRjcbyC=-s5g3U0-SUBPH(OP_h6sk>E6t)HMpit1~IAaOm(iO%`h*g!F<$Y@Y z_60$5-AtH$8{D(N(8nUrmwpuk6-LC?r^`oMy`E9<95+4oLyT2sd)8Bm0<;kQ&-{+>L&ESTVTgzUvX{MLOK9~%4+_lmlu55XcS=^_cY!vnUoKygN}JIACB zlc5+;OLD5tQ6e4#tWML9%{p2V@wvAyiz!}-_!Lp2iUH*KNIS(Ta2NQeYh~&fqonP? zufniV2JB;%a_QFU*H^%H00zwPoWhNvmDO1u+Q=mF-=A}`6S>X+11Q}01JDR4z*(SU z0mUW21M*`!)n@}=%);dPZzbVAQsHvo2wrdi%#7J^WE1)tt;4VH5|hYUAh!KS3etoA z)RL=%Xs$B>%FRG({)1NHveJ6F3~Hj*CkaciS85l9OR3k8{FCv;6n6MXQE^Gdo%bnO z1y>7O0M#}9YhTNGyyy2a*D{cmxu{{n3jYXLmKv$vD>+HrneIBl0jDaz@_{%|vw*@6 z*##;@`F(Y^3dz|BA)abP<y?7g`OR4@rl zvlxrbJ8^&g!+nv@L;CXueKIRieit`8YcIfBKc3rTI-j+Y%FEtY7ky{9ekRrew}GY# z;QVosK>Qv$^8&8gKU-6;NrS;2TB6D06R+&r$>;?MW8zI`!-}H3yzcgVf{&fPU?p~= zJ!EjK(Vuc2vkhevKB_Cl@{isQHcKV;&yez3D?~?rz$Suxc5%E|wfIJV{rf^pd_oYw^*(_uCk|I}Tyguri&i zqMW_!U)Q#TgmysU7{x|U)d3mvF%+``Pz>hdkxv2$9#yKb|KoZ3VTo~xYsm71KAAOcbMr$1XWVOCJ>frP=?byq{q~Cg*c+1D494 zDZIZj!2K7@n+JbA96FzSDa)ZztQi$oMW6#d45C(*6**N51eOpS60ruVe$c8-ve|S* zo&RIT; z!iX!frZ~v_n||5?aE==;fEOF`85*chhBrvL0Me|=rQ>K0esEn&okhg;skR?x`?F3^ zGyDqeM8c9mjAOS_;5o><@YNm7L@t(-g(m<$TDPw;lXT(% z{sClh3eZSX@tcoMrqo?Q-L16Xex7F_7vM}gGC(S^#6+kau>2QnR9puf;*H%FX8iGc zw2s<@M$6@nYooS4g;7_1!ww6HLFqubyG9;NW1VmN%wc_a_6zmp$Rxb@A9C&be zo`Q>*0a^?n1lY?LpsUj6vGUA=YGf~zbM#$|m`Z@jbYf1CT+(V8zv!XfqgI6VnQBeT)641m535&$Lj8bpN$_&Tv26(zQlmG5@cv*!x(LtFj! zcXBoNuiyzvt`G*lM&?e2X0i(|bh9Dj2#u-$f}!E_d^h1K6<@I5GVCCWhH6>>no)=r zhlSFts5@Q`klAJdW>zDB>FD~QooS2z~Yr87{yknuyt3fDRD_^%J3Ek{qXhiY6uj=mj_(i z0KUKyTxPoNreyaXTf$8OG{j%z$9iRwZ4uX3W2HPOvXF0^nYmR1C<2^}-T?K9%>A>_ z`48&zU^o+yMlORK-&6_O6q*yr`85}#no8Vy$;jTw(cTb$j$z|z5m-Cp&v({`7r83w z_J^7dM@lfOzdW3Qvrp{K8*oVD1>VW1-$}qM>CwFz0;*9l25Mhz#F9{1W(qgbS)|_m z_AvQgA8J>ateUAXNQza2sKUCV64B1M6MD94a=Bw5JK%D5{ z07kAR&>a7#*eZh|RCwg0nN>nqs{{0lV-!DdCVNf-fPhE&TMrt4%j2F*BtDT}>0nUS_E4l5Yr60iqbwbf&G@PS_} zF#xCOV>WPdaMq4}I|n=Oi3+)l>Adk~86Fulw~!83Ud=)tI(J?(>4x=Hfwzp&yhIe%CkRR`gTnkIcHtWJZ&%fxSBC3loVo z(3$9vQj?EXB65nPQ)O>3xbU%)<5MFA9|evmrd$FOPVnW~qU<%CX(Ka5v!@Bjl|P@u zM>=~2nuuCa=8|23M&cY;z>4twoQaig>-D2^;2IH8#S_12djHt$(D4paJUP=_%+fl? z2N%+KwUwE=;Q`ja8nR{kU73=vS1smO4;p$mb-wl3`w~)L-)f7PSeEPcUcZE5>iHqGfVYqZoK7mgp2DmT%Bnq3Vv zX31Wd?YBGFBNCrw*WX}GeguCHfQ_kZ=jp4x-^ymopW`a$KgFn(3Qs27nfC1!wE<2d zdd-s;ni0Y-)lIcM)DbVcZRev)5@EBBikzlC%Jdx3U?c5fxECkrf zYw=la9{B$sM7?())$bes&v9^!ILEO^_R2a&_I9isN+P73lR8eJjD(DgWAE89%JwEI zBB^B099w0S5mBP-9pC%(`ToAYKmAjO^LpL)bzRTr^Kk)JxhJSl9CEWO{*$A4jInlk z+dzc<7IPT^sF5#u?1HFJ z!*4)+(VsxyA9`qo(fFQ$_s-xHYhVo*8XIv9U5Zs9AQH2M_J3gI`-M}!D_vjWN6JP;V92`eAlV$$>J=A)~swT4d&RPGU}@dl!CgG@OR zO@T!NVW4{(P#jn;1trikbZE8)ISb3iu(S`(^c(>l<-eqgf|2MqVtG>Jo6q19)OAy< zq+4pdOx#xuA1JlMHly^YnAMKyDxbf7%3wtvYj3>I*@o^%4$i?&jj{-r7?6?#ynBSp z)?4@)=hzJQ#GK8f+ZxZr`wyUAoI{dxbG?K@g<{}IoUU$9j6a*f!|OjW*vQod!tN4V zI-TfD{h^FcbVP2iW733Jg-ORqZ>}iLcfpu)_*!|i@udW+Nk#r|4r^22vJJU~W{oEK zrGv!>`4_E7I&+xBH^^{M*IeV^yy`xT7dRf>CcF!wB#ZQ1DxOe`e{`_5h}fUBybOk1 zN{4+IellO}*f_Q5jf|W(ZV?i%0y0V8TP{>%|0LTeVt8uXQ3%*Ye$fx?RvB2TO(zR9 zCa})Tiu5c~6!Up4x`l_&5*i{4<3?Q~u5>=6E@Yy2hgO50p`ZIAoQPW@d5`d{1c5t##8ZO#%NyvDAh6SFC@NeasQf0PGD6sr_g+AvC^% zhv<+WEn*B4Hf370-mBjWpEbQ9t#bcy;!sK$Jp^jVo%CCll(e4o9|O|rgHScDDOQO3 zenzJ|EAqfFwnzXET5-KTuKSp7_(esT z_VZPxy1HjsOEfRA0Yj{K;V=bGq{#;%J;N6FLzG*qlyzyzW=M0oW@;6V(y^26s7KK< zogIA)Ck#qlwf}-M;D}f2N{G3D8W}&w%A+K<@@nZ?VRv0=t!7}bK#fEia`icJNVgz0 zEZb1a7%j!^hspc$BCNxWoKkyU3X?Zp8~hI<0_nY zcB*7!84{zqTgtqT(qszTJtOnt`oMdb{lfIDfN`}ERfjXJr8Vf+d9%N%sP2JyQ+3Nd z<;|pYohj4^b};zVv_?tp|hb>VXRyRg7$l5!h0jmL}@jb1p&zszD2C| z741GON%hhE6)84|11#ZDqkGnVk09IYPVyOYO=F+6mh}!ceU! zzDt;-4lba6gxyQSYuDuszdEtb#}{is=S|qO>v&-QM(t#58uUXnoe- zN_OLaXvk#am%wnIIpkkEeDEn}=Zz=TB1ueSw$Py?AJWRy#bfi|MPAr%OLmnkg!v5- z=i2^o2giKfVDf}OYbxyuk4+aB{mDJLdk5)~h?@*PMnoqvN^#04x1}}$9vXU{U8yb> zc3hY3%3T=a8o_{+FtJCr_Diy!dsQ>p^}71;ZD(N-m)*@S@k-sBEdyOmz1F(lSgxfb zD+F}c6f?jaz@aI4ej?fO(fV6^hv#0eB&KeCi%xY=TZxTz51-^#X4N&JL=5LzUX3#< z$G3K8NS!VJx*xisxXrqG(pQ04jUU__SKO-z{^?rc`R-`*^83BQ5m$-Fu6YjE@{$lz z*qxrP_$UaH-8ujYuebLM?;H%z#*)g; z-96-pa9;6WxSG`Tx-7WNlH{w}9`}|YI3LQunxGaSOG=I2zhB0SOZjCV6JwkqnsMdAG)8tSt14oXw>Hx1DEn} z&bpkt(MqsOUqx^v;1LeCoyS{E*nixzM)DdOt(_J8Z@v4^$hf4gVMq9)Xx!iOO27Tq zg@|r_Z0f%D{ zBB(^e_LQ5np>zr5j)Z#+S;0`~5j9-bGqm#ty-0>a8(*1@_B%q96g}=+xV}LyspfZ8 z$K6vWd#tUVyH^`t;Iwyo#+y;cs9T%Q7gMk_jtY2Aft|W;HYkQub!RNcOoTAxX)%Eoom0der zPWL;Se!2F}@^GZ%Z-W(>sIgIJYltgTe~)1`IGc-0Jh*kqJla%OEQs5BePg~ofmtsI zwNGxrZuXj{uyQD?lJ9vvc}X}uz7v^^gfig}3ycmqR@wcU{Tk?bEt36H9VfnDt`!4j>Xd| z-U_!lqqsXt%?)K=yEc_ucJFYcySDyp zQ)fAh;vi;=x3JRy2{w7mn!&ticN>NtPkU#+S9b|_w9g%|z=1cq5L|1L$V4+pAD_%p z8&!T|*bWugRj=T%(>#%UW8GT!<Z@|Q?HC>({}TRqMm5)qH4&KY?iLMV&wnIR}n1%+1mUVUTl49U{Vz1h-p)y zF4*lJB?S-Kf*vF&8v=^5ZLY$YP?O}@d2%Fs%q=(sl5~mud+tZ%a-Nh33<cy6djf_RfO1pQKY zfTPWs%G3p_4b_ogi%B55)a7c|->tl-_P-B6kb{&a+FiL)`}_rr+Fh~cy99*Z`hsVj zi-h16dva!VqWA;)f&0HRx-q(CMthp5cb8?t(VQrC0@9p*k(HWe5fbge4>M2dbT2c5 zWf)Pd@-?xd8w`Xx6*EGP+BB(Ae#n}&hM?m2$o{SwDIcpEqXDD9;US$ea-L zyH*E|Zc~0cucs1jGO;Zv4zY9T<(hDzeZde}^;5{r(_yU5nZR36tjS9Ln_6^%M8k@@0hjw-`~3RCfu(B73?%-c^wY4dbvC zgaB=^KKs^&fS8KJ9A=*`ZQ=Q87*518Xp|ueq>w;~Jx1ZdOa`F$hdc0mJ`&e2&YPX8 zogX%fF-p82zF^$(R#4TQ^mq1~|oOi z3ay-ixT0FdX-mY@$2w5PBk`C$v_1p*#?w!`5oUq1)4HsqkGHR|9qRPF z;~#%;n9~kOZ(X1M+qsz>x8^1w-=^^Js3g0mOJ%pX!(Nh4cG$v`(UOU6#btnaRjv0M zHVq6^2EI&P_{9M}b?4#@maOrvdDrlINIxxvRK6*se;vKDYL4;)Q}HW5K7$OQX*fDj z7qS{<&x01oDvK&fK&z&frX5K zQbw6aJ-a2y%!05mV~eg}+2_oQ>F7w%MDu&cXp}391}|6q zm-7=jFm)>~fW=X{>1SZ+YnR^^`y}t%&b$*fQ|SFRG59C~|MHRJ?TT-Pnip+g?!auP zTDFH?_hKPKU1#Z_(eBO0 zyY6~pO%zaU5)bR#XZG5yCu7;k1tqB)mR z-isrRXs*J{PO=~0=5jGiQJ`N zMv4(P?L3~n`t*`KFutXQU`aJo1`g7iicQe%Z@SgVn}mu5FU;OCwymU_(?D$VD4a6j6n*g~sVtH~g+SrfP+}Ihp9|iu z{&xUTpx7UT$HF(sWvQanwmP-taO=Dt$*}aZ8e#xQRcB&p%yg&Ff{l_GqP8HIE}Y)* zz#gD4Dy1FAF0)dTJ^*{h*hW%{i}9sq81{piy>pt^#x^Vye76>j1phI+RJ+l zR2zSnu`Q=O1TKW4c1X`IeEz7qa9df^V-zWIwdX9w{O_2d6;uKfydpzH8D!|faF8&w ze`eHjQ;I!C{i!;=%Lw|*79|{iTNgB3az1DEi0<(lrlHjZFm{5}TCvc`DDbGc6dGOip>)%v#c)|FmJ-;AJQO{$+GV^kV}%yM7f!tuUExJLsj?8Ir(q2}h< zbKLPlJ&mIV^r3!H| zqTel%T#}yNt>yiP2PQB;Mlf~ISK)4E1ty%+z#%ZaA=E-nyBGUI;D%48kTsP(-|9w$ zWfQ}oh}FhRVDY~)KCb$<^r?pnlr(&^HR{@!215NuL4<7M8C_+gC}xJpx%^6w%vgyW zsUzS%{-mu^rt@H{{8QP}tN-R8cEH+NWV=S2s~8!k0}{(}e1CdUMcWx=s;~j#&)+Qa z9wOYo%iRm0g;8{_#jR=JWLN`E`Y2#H=q+lc$a@@PMMWS}#w*b#@d9MeK8}dJDyD(! zhYE!WEEWTHhA&_~C}3E!po>-J=CfID=JA zah12zeD`~n!K0ts99iZw5v=lp(~CX7b`Q>F_ESP(JDax!h0jO=o>T3xsS))kka(s@ zwS)>)0K^1R{;6?xHPif{jo3&Fxo8g`-;O+&ol{fzA{jYAL{xlK7r@iy_dWA+y`T@C z7TojCGfTyveqS04BR}~4*XaRJ+2v+foBcioQ+|;58#d<3Hqt@7jrkjOC6q&TY2oPV0cGPz}uu65RLbDcbVM_nl6@nOXU1_w!WMPHgj;AiARo z?A2#~Uc9zmji3L6fFw}TOmtb^ly*2;-zMo_gslISg4c$J8Q${Q&t6&ocEC0p@$BMp z&G>b$a!|6q-7MskWS*yC?^|6UNXn$E6CcUi^v0sG1hWtrUs}&4<458MyOnEzNfnPUMc0;R1P}7RjwKInf^{Nx-tjY#G_}$-Z+kY9lI}S zDwJJy%Z&f!zf^{a*yQrQTcbgj2{~{J&?wZ_m`9sB9#sktr{Tv`b}nFoGsy4weeGmI z>P}re?E02+%y-BSYgV~EUr&MX{9nutO?`Ym$8*I(;bRS!Pn5W4H=MPEV zC&zye=?}WLi%k+=x&dkGj55!aTcw$@=WN6g*09bV%wi}tjXP($m@765h7&(5Kz4j2WVyQ9;h7bp88_JTFX>WAAMK=i-fDx?mQd9uWN7+@-TKvB1u zOw*j$P-O5lzRwDKMrrjNlL!~$#r_Ciy86gup{hJ$*?rwVV6WGaXnB9NW>ztx)OrCQkEU56MJ z#x{e?wW!wJf#z5MX8b8_a|QU4%Gp7|nJJQ$(S4tb&L8|!DA!*g96m{YfUzal8NHqf zc_-(dE=gTharwoyt{)ep1Vyqa$z2j5O>KQX7M1$sarF8Ouy{Z;@>NIGZnFD#k#OP@Of z-==J_?{)c4IOa#${e>E9YaB~SjUX2I|GUeV{a z-#C{dv|8QH68~E|DZ+~MwWWcgz=f1AOlI%j3$u`a)3G6qWeZ;ve#Ya~bs8x;v*R|= z-9gc)7E=Oi|g2V6gR<%+=8!%KG0l2paZK(wl}1wP_@W)*E&r zmMbvF)#xGgK`#%DNjHxbt-j5^C+wN1uBjK8T z)}Y=0fMVM=I{82LG%aL)<>qpcmd6YOSpiOER?xwGyHs&-or@A$uPOwIW0wpe1Kg2R z9vm)&@dwFWr76Ia{_3In`F@(-@8%S7OhKuBiqOg;==69&&EkxdFgvmXM7mDe^jcc6xL?z8usEn3iE^P`5RcKJi!}p>$%-fG$1v1 z!zszh)69+Q_-OO|J7&DVDVh)@90$7^G51{&>76bqlW`<0sz$8fsNE!QDF=SypBtr~ zyAmdsL<_bgq`C}%NjGAdQF1ERTv$QZcG5&u|7KqJn!@nxI-sHEB6_^vGMP-FD>a2D#(gSkj9fc9dM`g2n% zzbPx2tpPWQo~8|wuoq%89M;=VrkL&$B`nnmg20t!M~7sW#3c4?EVm`jjGjfJuN2+| zQf2AB$LwL~{MnuMz2Jrm^V^wM(nDEr3^UPLY`Vz3By4!3Xb_v`>r&gFZ&mfFEO`b; z!Cf>%_p0*414AtEMJYc#lUGx@ts_yy--b?*cGF2rJOD-={UyT!nKI3k!O%y)0K%j} zko@24;<3M*6eQTRF86n_7sTud6rV{+cGg9rEy=dnZ_d*P*F z$7)v59RA7+l zG+82>lLMk$Ux+FPlmumtpoy@FrTn@3|I(YoNsj({8*Qu}JcF6^XQcS)SDC0yL;r|5 zUT?Y?Im4?k|Em#@8xG(Suuy}s=EdpM@q-O{dfmrA4@HiS5(#ey_#TZPY#p5ypr5HU z1})p#up9oAj}cU|NG|m=b>P=5w))!NQm1^9B&e2L^WwcmL_^*x|NI$PA}GI`Z)SSe zutE46vI%*`<5lLzzHN1k+OB96Hi8w#)KP_+MARvOn(39|AN0+l$Eazv|MKS-`I*~# zM_5C7_5c9zOs0TY4~Uz%x9>wd$h-!S*7)+vdlqov(z6}hLv{yoYne%oA9Eh~di+A@ zqr_jFv&hf<^tvGU_N>6%;~j>ut*aU!jJPde_NcPcqbY!(oV)oxZk20>9o41~)^l&q z*p?4*2~zw1T${t?D}rOE|JB19(6?pn2^SkYn=jD@WJP%O)HXF9wz6_2XqwkKy16Fj zqtjwFzJQZr4p>>-RRqsO)(W0yLIMzCv4DIBbC-ZKi?=eWovCnNie17a`SjHhG}B@f z+7G;C|6OgKi{hKii8(h!Qgm^Dd_3O+`TSL5muRX< zC+N#9633m*)9|?LrQ`=WqD@SwUQiEO_;m6P3IL4%PEzU=EneM{AQr$H`^nCXvIoN- zG}4wnvGvs#+cMai{}a=yeP5}r2>Q_}GH44*72klw}jT zpo=-GCCNSl^PimFXGn{MI}|9IxWtZzB;0ZWsP9$j%Yx}=Nxs(?FFkk11Sc6_uGVaa z1wPrPgu3%4mx`}7Ch@3&Oh>LDA|wet>dP$elqwH40ogDs?~>%ccOJ`KbbhY2|d zw4uez_-{p(U%DZ#%S*!QfG#J|5i*%Ou)ap|w6v=pGaYej34407&`r_C0*CZ-MX1ao z6u#3O2^IZ~zkt$2(OoEf5#`*yB8EXEP%+!jt;qg0^3>M~rwgloj`rwvQ3be$oC^&o zn}o=v8MFM)_YXx|0!_R-&&3Ks_BLA@gj4k8sanR;G_v`{Z-rMF?{zuHzC<;U@4N%x z%`Es(t#z>ze@d(@C6g_Tz2{wc zGhF!)b|Pw;#0)S3W$&3_ypeNxreeb}PSwKfs*A~y9Z>pU+{+svqL0$6pfISRTnZKs z3%ArVb_0J56)r9r8xEe6WZGL)oHO_xdyZj_t~LG7DVJ0ALFpq%ksz&o-Sx+9rb+e6 zszIAa`?r5Lk<`KxgCuf{%$LB}=pQr|oghSp>WP85r4Rb&p2$zZkKUKP9)=tVam!1| ziH(OXurfm+k(EdLtrkqDP*z8w^OwVJA#00J+$a_PXit0xd!In3#-iSaLmPB8w6#T+ z(MC!wTV#?$`2@vHFe(?6)6a>a`<_5$$Y+%aatRF_!bCZbR@Jw11mI1oKSq8BnyYoU zNa*0H90l&`a|}_*4k3o($dAmz2oZm(5KR&*TW24`9Hd`JSDiGpeoe@%xVQN9li-KV zBee=4!E-q;zkgU~5vmIT9*;y{kG0FeqJTq4QIIAwijrsk%h zs3plnWFbX*`c-PkX@kW3qY}gnu%uoi*FhoqU%1txH%HqpM;!1d; zyg#Q@tpYej&GjSFSv8rchct*}>~CDjUTU#+I87PVQMKY|%sw5nE?KkoG1@CUwMQvG zbiIs)m?Ap~lA^1o!C-rbN81WYrrVcX)Ybzq-)A)X4N$FB}dRTBy=j56uS_8`ifWEn38-^l2J>3r#B|RjtNJV9gF7_ zun{5NBpKAj?Xi`<@&UA`IK>)NtGw9-N%FVv=Jq#9mUX`J=#D!2xdNv*brXVv0=1Jc z1+-eb8ci8*_|F|7I@nv7QJ$*=zp^3eCh0R6KJ<|s42{||x>KUPOk`TLy4`gbjmPwE z^uS%7jBcai2X_}i>vSgpYq}-&SwduQ&EuQaPyf;v=kJcX#?*`0TTAj@sH2t>ZYtdK z(jU~_bl+S4^K=K^2I*CNwk(wT z_1XJN^E2&sS%ML_%w)@Za}=YU?AYMzs64g*c_aVvg2_@!fl+d=PV6(JPy6NwDx4q^ z-p-yd1@JMt9-aG`iobI-+4izDi7RWKzBWV1+P>ayrT5IjIbKuoBwALwpe`RFS zPHbna1D0Ek`_;MLd|yo7kV!#j`(=^%Aeugk?>|omX2)_)o+j%n))+=u@Q?!3UfoDEp^@We*5j4slt|##BU(IRYKpJg z|6Ox|ZWYSQqGC_7v!-hm_F(KJS*r!p8Z~{CC7jt?EC%cdS)KKZPapnY-V_Iz31;v} zv-Py@#ee~(s}PFzaJHRw`9*~7q^RLdPzW@2m9#eA0ew-|m$shvmA4+j;{ChXLC;%D zO4uV#ZFZBJl=~cZ3z)-|2kF^GF&FBcG`KHHLYY3t&C-X>ul{60}x9{3kmmBV{Q ztj5(fMC}zv2X%UdSZ`()yL~rg)@K?SednQ z(vM?@UW17|2^Dcad`aLL+n-a0u!hXrePB6}`yZx1r-=2Dw1RLMQMM;6^st&Tqg27FFu7#PN)4k_wm2^e{;+v6MdfQTpnr+16EhN z+fdoq_bDABl=^2nSY?7niJHgQcYP^--|z;CKSj2C`(6cuWMZanqk$ODwRy$xoO^ER zOykq{stG^EMPesAV==*>(%rK6JIcT!mWf-F%rvmg{wi5FJRVm?`0q7G8M2E5|M@j8 z4Yv~1U;ih9f$6i4fZM@06(#Dky(c zr*^Rdr5RNoeY$;-0Nel?^8kHGQ6v&)WmG_YB$NlcK&5QqZ_64$)>sz8NdN7wLp+2? z$t8@3CFmu1z1exd_l7^LHIiXRu|lM76|BhWe6F}XP<`wi=n8q!xplH0v2xVQIq^tJ zSnNuch**YrAdz_Ce(8lv^=ZM*A1>CPg9%GtIkyuhyKd8S?Nim@!|9p4rGenAL-KpS z-|q)E_M5KxxIW2NcHJjuznS^7e}^=WJv|>QmbAyWu#vm*y**F?GfM;Kl<@CbaBk({ z4-7%Q1w0{HZ{%M_q=wR(@O-0RUKjtFkN4sLM6u96D4b-AHFNa~DrfAYmYE(0co4SmBB<57p5KIe6~{N7pr>Kkjus2e|1`0c&y zT=w5wn&T+>a$mmXW4G@wA*6n+m z$q4;l&~&ESf#oy2QcnW?iw}cVj_8x$%x2?B5LME*wZ&`#Nt8&%t^Q#@utJayw#X(C z;X(=^$q{)!w8aEaQ$;;-`EW7nWstje#9+XT>Evt(x25Vqo5@L#AR)iLf*#1q*%uyx z)j-mylNHF@N`wmcR2fg^ z-ef4aTlK{*>&GL>X$KW1RIb<+BdkHSP$;z_7ldZ#WbY1up?tM_-728R6hz&XB;aNr z#sHmX&Z?WE)o!MjJQ&>Tp{3?L{R?H6VftgUJ4K4MZ`7@T;c?a-s2tG(?l=>KYPyt}YQ!N#Pz|1E$68L2(Oy#I)_f$et^+(bb4Z zvk$QknL7eQks9GQgyDaHKlGn=BgF~Lpw_}_zeJog;$VB-WIZEmFJb)>F1tQl8Sc70 zBDyzHakoP+nET#9Yl(>eC3Fa_(&sb2W8QU{w|yR01TRPM5O3Y!C)kr-SCQIhMS@VA zP(~F&7&XBXyApGTNQVQ;k(J$5Q`0b3(X<@oidC#(I2dSF6i#(~>D3GadHxV(1GVmvw(@~H#*+cQ6NKOg$ z^F3D%l8P0|thHUB20?rP^};;Jp)5}FhovhJyz>Wwxi?-CyFRy< z^dYFwJ@W%CCe3>7yi#jc{5N^iz*fCzr+&wo+wUH>p23Rj{g(%`>-XL{Ps_*}7W(rX zRPh_KmC;?q^W_UqDzCxA^6lY@pkO_1Z04$Si@!v8<}q}SKEYQ6>xiW2^0xHy)bqKp zAi<#RniU!oRYcWL_r=TnnNY~}Pq!h$VoR^pd$dwv#Of;Fg|}3v!z83H*HnF`k+Ttj zUmSftx|>th(s%{|%Ztxp$eBOnN2fono+$rw<;(XUmzEC^Fw0ndg3RZchN(nH=d$C> zU*iw;iav4qX>2Quj+&ttqrsYCny`kMKFoGzfW(7g<3lkBK^(Qut#-$0BabDHh8p8- zFYLOVSz^NnZ9*31m{O(7@yAE}$$i%7;+LGRuxR$A=XwHzWgan`Q)X`8kZ|gybw(yy zMg3#14b|yuOsUTVIUbv8h5fvIW_ULTI0s3NJhc4?%}3~nNJ9eSuwm>!3qogjXcK`$ z+Z=;LnG03^%+v5WBUND$8AVk_NAlMvMbCQ5JPm+i4U?JNR~VGzM$ek}t+B$8JVWL^ zs(u4s)@}(fQE$59Qe7z%brm=oVZ|gJq!waq-4EO9`4s68p@GHFaU5TTRnKaP^rMBfrMIntrcn;Nv|`EoDoaEz8XulHX7yO} z4JH1Wr|8A`Oxh486wWx1Qc?M;$+D;xu0gltY4_UGu`^W_1EU36X`Y9A{o?fugaj-< zwpP5d%Q^<8NHLhKm8j>?soq$dat#zehfs`rGoGw6VLfT$0a@aA(Y(AE`=4jMXoGlQM9V4YUJ@7_JbquCl#a6~9 zGjLQ~B5(Nc;Nj6Y!{`go8T+su{Eep5BK`2MrDgp|s2UAGUy>8CEF@-Zac8zYL6s3` z-hl;+_^478KC$LHw8tSVAfwM&+nu z_10QwvfLS-8GuYjaIF3Af_CV^_Al4K;}$2EJDt<61;*q0E^S#)eTC2>J3k1#*bfYa zq@1m|2j=4(XnjRoTjdv@NS*Jx=Lw=hYeIH`+d&bI zY+u6lrDlv+G&&3=yFXfaj420f++Y%Q;`6AW03wL`9zq4o;~Ezvkg}?8NRS#OI+{t4 z%POJ4CQZrs@}!HyYFRLuLve7+r%DUOvqDfDp7;hi5!$Q7Cjn(Se+{-qey`^|Jk5k8 z$(80f*xnPo***~@ZE<*m$5cw#naYqwapir)J>zZL?_eBu(KsRSrP<-$A;-r^p3CfU z@Ay5gt7f^0EC0S<{ONM~U(p%pq{4=b5V#q*s8!(Vi<|N(&EYExsPBzjkvk!4k4;DU zL;fpny6g3*vUm4&(FCtA|FSclJ<@sG6A`jLFG;^bCP@r_Qwx)ZlXc9ipM#uOyne)8 z{V!mXu1}>m;Q68i6F)Gr5xuv!2Ierkm2ul>g%aLzF1Fe4wh0bvQIfJ)S zb|v2ndN|y~rrl~&_+IcDsb}jsUUR9NXS&mvOcPyw%8bt<+}jd?hP+Xy$X zRY7+6n^NghiJU{;?^nMzN~ZuBPCH`+MvwQ@vylds5-{qNxBc0hRZ6Y{IRp#et_c0t zHg6~5K9qI(6FA-tP4SxNzSQbII%(?lNCaB>ia5q(u(_{v{XjGxY zt+Db{4bumLDQnwtNdQ=J)^8!z)Un|Tu|7xW319tZ?DxpZ9D?b~k5i2Q9}B>@D@Gp< z3zO|`bR&BZZP^~h#_>@@&H`>CU%gPPS~8YFOhfjbA*r0)W?7hg`7h8C3-*9EhgjzU z^yA~j<0$6C29;q6L@$7TzemaCDU zpp?L(;s&rK*^CTl6O7S!i$cigq5H(%+%u7uWo@681p+QC`Ar>E-?i{aW)IYSXq&4zSb; z12IP5t%XmiDC_d`=l_DW;NS)EZW$n+%ko)p$J*wW7+vU25=iuNJ*&fF=rVhG(;-&8 zQ3j78Im~6`S*GY)P(T`*A~H3me263xT42uw`^|`NPD*5Jiis<8pML2qvd*aCa*>Ly zaf&`6{FeUStuGBVM&koAVoOXT$blPXtnlGm97^y{m z;HJ?k3QhDRaZV<4Woy~<7w5q%+yd^ZHcV@La&a_o1W4&fq?59cFeFQrR6 zN7bX=x6RP##a$8+RUOWNKY{1KN3I2Y=jhW*byQ878Fv)TC@dAGE-XhOW_gMt+mC3| zb*$HgXNw2>^E5`GCx(5VkAKC&L8`Foe_l6PO5T1Uo%l1-x@j@TEzPg!lu(7hB(VWz zX?SVljW`8^S@nAF#(A;q&G1kKZPx*Qts9tAyegjCzdrj>JY$gx=&6>d3B>0f`r#^v z+7i`>dPQoNQf@&u^5dWYJ9b5SJ{3aB-47zt)|{(UmB|Eaf}l^$2ZBi&hl#rW_*-DQ z!Vum9_?_&D332L@*m;CLGA4^bj#D_qxHE#P2@}MWWgo=J6Yhn)Ol8ZVV0@b&_VxB~1CatQM;E7$Nn?W>tC26yh46W&Wq`D$e zVHcuK3~#ESCc3hIX}h%J1cbr5qk%qKr_W4O)chyIB{gx0FHRL2)n-$3ws?+pQc)j? z!ElyqvD=@V2UQ+*!kBfD3;pna?%p_BFrl^ePphi09ZB3&tSm+?>$(f~zM^fL@I5KQXUEQc zy=<}7XlU7RdG|*JJ;fj)3W=v|>ds=BP>u>B%M@Y0H+1JOT?;w1b$52Yo$bkoMuyK- z#svLsYI47+oN3{aJL9$W0^o3d1IV4XOLvmxeflyy0NvK+vG%5GmLs8Sy`j>rei4~& zY1=^FI)97hN2KOSTLC_2ckb7*y3L!M2YNxdE6T63Lf&5i_Ycn`y_es)9?T+|a_d^;i^F-$ccuR3<*mCEUc=f1$ap8;Ro&Sw z*8FR6|IEDh!VtvVWzG70dYP)ib{pp9wAM;9PVNia&u&C%06x`r?-JCJftVJbvv+ z+5UT#SxWi-ba`ZsVatmUYrS4FfF`iS^P8NGqlBN!a%|r#iiEC(vO3zQ#MB%#R-MCJ z1?=08NrP+eDJ4^TJGA>*6$a6H3vABlwf}*txpB8sV@xu=60iJLM+AbtBhtnyfl?1h zuR4E;7givir|wmBiqj^I0#8+;yH-%V@Kx*~ba=Yaq)0#dV#OT_6?Y@Moc;P4%976} zN<|+fiQX!*E;e~X*+x{6NC4L#13Q4=RRsdHfD6IccEK=|xtBt@aQV*qpA!Sy8@No< zFEAT^7YN1%vSNybX$%=c9~7WNQiU4mbRx?7TsZW6c!@9i9^3!_!YS&QZz#;_o5k2) z+JRdGRwO0e{z2nb_gKkO24Tvc1ADX}H%FZzk;D#)G^oDNgU_w>g6qn zyav3BTsnsCeNV{H(GOhzE<{hT&FlXiL&R9_e6h(GAbWP{tal)P17bC!p_uxWuwP>p z>EG4XdgrdYU$mu#VXf{1qK$(t6DgB|X7OB4MM5v$5(yO={L-*H9oRqjO2xV--`Zth z?5VBHE*PznQcOw97sWWWZ=W%6mHt+t2uLFrcEPb7v(E$m6HOMMTwt&IB5-fb;CHv+ zLzT0-i&m>sul>N8ag!{Ub;p;9OTMnhf%rrGJXk5BEE~reBVc!!;@4aR-Lgq3xnSX>wZ!FjCTU}$6JXCQYZr3;FnnB8<^(=miqbV z;^*MIydVx;-fwt`CdC+ncg@o7c97o7ZQ+0Yz~f^;O=($)7(NGovK80&!jp0}-RzU- z^6$%%t37pjNk)Gb_;hPvj)3A`8VC%e{9aQ!>&ZF$m-q%hzvh`X^-#O5u`K^-S2Q74 zFNyNL%{j4y#ifeT?rl93z&YpYFHnTOhPO2Zug)8U{+R+;wB`D*_pdNT4S`jzet>+W z)f8#{};3yQp`fhBxQI-pRY4Doo$H!J>xX_6|)rAn4 zr|NFaKYzdD$h7|!*yY~=L%~2Q`l11XbFR;+3OM_=TkTH$o5gDz6kNMaH#jykalqrM zQFNulK`=O}wrP1Md8h`)GR|D<7|p|8rr}PkSrB|7r3FDfxWOTb;0fe+yKIk(^j2F- z7{4g}LA(6-rlm*vjP+!qqt)L&;#!f9wI@DPQd{jGJ_&tZ^QTzuKd0}!0PtwqSdNba zphc}(xeO1Y<@^d*A>~zz@AoDMgXMTgCe7B>Jm%VdHDDO+yu1>Ig@PC(%`s)ka3u%t_J z-vBgX5~{FkQf@3R_P~3{fIeF_+NGLT>wcAZiRi^FGuR~#Y852;KIIZE>-F1C;c`0%{Xwi&QstdhX8Xm{2tInegi*YtxtP2E78*)-uMKLe6rY$#wQ1r$( zVM60Gz}5ZSp%#z7)3eLm4^3SL2=Cx4z9(S0fSi-h#8u4=aDJ?r0aOH0zlO2{l_h@x zP#auj3eY*ijnam+46jkATKMDg7)sIOwzVH7O3ab9t^m+jaJUh#@D(foXof}UhpyHC zDJ8~tIo875fkyXxe|_*uiE!TGdTEPfGXB%RbD3^k=yrvBJY;%-N4XjTdJ5w8%19*wr z7XV#ah=G1AG@^g~Elx(0$U@??yKn*=aN0k-g0A`lANtowu4@EC_5?@Ceabb5q*Yu{rp0!Qd(c@DsM37r&<&S;$Q_(sSRz`MqK?=Iy(#J{M6 zJr`2R3;g|t^GB!-lfd4?_5QYQD8KqH*lXlaCy`{#AsYHvaX%XU?}fQeF%3V_G3Vyl zxY!6zah6SLyBhoTTutAnifR^euQH|pv4?-crN*=?uV4t4d;8m?>bYvK28MaOV42n(&KUqi6=H^{^?K^e~0e=>*;Ob_OkZ-^$qH}VOGzRZNX@I z3+y~j*=e;Dzz463N~pTe4*&vnX3p$%;ZsvMyz1Rg{@c*&#cA?^E}Ef4|4? zd5-6J?!WH4`{?34uk-wT-k;YrHjbqw!Zfluv=``RnWBm=xt6T$H8>fbnJy@g$!1F7 zyWj`0>J`nonIRY+k^)o{iJ4Eur1is)J_F4`H-m?oBQuUy!Mk2l5qv3In?i}fL+6)A zN3_918SjFfZ4Q=owz8_v=OtSM4xvK8@S|S?O!Jvebe%gl7s}?C%0_#iR}VQqs3M=u zrC8S}dmI&6)KfJ#0`0%EOj7Qys;G&(X%-;?F-7z{EWP&{e|RUZNaJJ+Qc~v>m-#wH z2}oA>RTT%DYQ6)~g^BR{9^inP|7fr3RL4Hynbz^L9w<4}u?8OO zbm%#`X&kYvG9Z2*g>);EWzW4`@xzWLYo(2lCpNUc^&SqolQGfMhG(mY)U0iy*rQp) z@rc=k@K&8Whh_Kt+!eoG0BO^ zja-BU5sROsIdpoZ=0`xVt+U#ck)1%kCa5@_pZQ+cZ1~lmAa9|3y9I~5?5)mo1qTms z>%XTGWy|v)JiazD_ep5@AK1$?)->_$y4^v0=c)ZCWIq_z2_+|M#WE~eJRY1pL1-mp zwR*)fGCb@YQbr)7OFPCAP257LK}bQ+*|IoT>1IT~PSMf0`1l-$z(*Ln+&ihr$vSCu zQW{9?N%QbqNLtWKB%S2Ex>X;~JM3*PjUr*`Krsl{)^ zP*;I#fDofhjkkJ5nSU-vLu=eT>IR?H4bT>m(^xw)3!#RnBgv~+x6h2Tbk$yN8SdUw#x4vBDyfAGt~0J4OHU7XTwm2Siz9rsZRpX zc(&6moYcHgPi_uuRLr+WbA(spX%Mq)cZtN4lap^Yx|Rl6i-t!NY7p-^_cW>**zAUA znsdfiUgeY@lWryZ>iLy#e}7ST%%Pbv0dF?OwdOk&L+E$)?-vfwSa-j#O+Oz(Bx_b% zH1jz1^oO~uJBE3OWj|!nCfA>%lU6GeFY0Ya(VPwy-0zs02vu*2sxau7`M95AkAq6e z;bzCk%Q|8rjNkVOG>B?lR!G7h;Z@!;=E|zHE7-k5u1cYao+ctdQ*WPDHaMu6R)5xy zM5@3h&41Lk)%jSDouA-raaUDjq))aFKnVDp$)x!EHrT`W)8hBr3_rj1brF-&t~360 zJ%8KE{d`^e^ScWtkLCH+!%0DC(JysPfE)R^+em|UL8{<#u!jx3>_!m$J(2o*Rr~Bl8a|mU0sb0^YHa8dHKjMEr&$M zT#|!Xzi&`jyaK=9C0rzlMj-s!(BTL*iNvp%j&ykF@F2CYAXc?jn~&kciOp0F*N@ow0r@sFluG*^CB;w~vYdRDQbfo@x;2tYID3e#NgTF9r`OO@aOKiF z&a=_!Vj%#qb@@P^^iLSgC*Sb?Rr}GyNnrk1z$@6Qk7}N=6vrz?YLAWu zeZm#wsf}@sNQ9pW>ZVk+Fr*@-BE6{67*}NY%rucBe<1*wwwzes^-lCmm1sCx1r5GS z9)EoOZW5}Ckq9JAVX@vn-Tee69vT3Y6HzO|dwGiO{Y)D%wADDrX( zZt(jk^ARq+%?5>b)Af;Sh-!9^sSSoinJc-h;zW*TG!mF*WPRtSp@dp#{bA!bqz?B% zr~|P4yq6+ZkPhBOSv#zQDxr#`cSXwCZA%XQOKNJ{T=Q%J=crAkHx0JY%n*eP?_{4i z!SA3AwsMWGeoNRpdibo13zrZA_2F*XP7%6c{;1GfQ+Z4&No}yc?0ZrLo0?pN6FacY zqy$K|@XQaDVP!5AorkNzjZV9eTk0$3&|MXzIFb2@CD{?$Q# zF3BBqp?d0i{Z32g!+qb``#tKDhdp~P6Lqr|3<>J35+aAxixs5)6YDJNI-+-#dUCtQ z=6W1zNet!K8Nw=Hau?+bofRHBBYyQ{OQ*U4vOaG9xS+nCB5JS@SDU>{Lhv({)FDIW z>C_Xke-|At&an1RAUT?t?aZ(Vp@e>zcst1kXbM@{2hXk*V;54^C*?8TXq#gzqXb31 zBQTD+W%R%p5Ufr776zd7aZfn;4g3Lvh2~oB9ZKr+I4fHi5QW8Jihr<~$0g#$*E)P} z_HVi!nAc^o^7VNZ_s2ibGH0Xycs8cpDAA({O^i-N=Wq9p_Z`9_Yszo6Z_j_Lhf0O$ zO^e1deGGfTO739c{m&v3I&@d_AAH&W81HF5iOAjGZY~Y;$wiN;N~-cp!%1^= zx;;gx*f51xxAVj#fDqyZNr8czi$9?Pl%4F?TL`K&GeUU2VK^`>4T$-65{YBbd}&v` zVrwCpsqz@A^hZ7XOAGa+Pr_OMZ)N5wqfhi2hOT}J>fhH^eISpAY2dkFFSE)JBYf)* zn5GxdWIC71?%duvYhuwLVj&$<#2f*+mHU?6ztMd1n!@BSWOYQf#mTmN1YeJMRcm4! z=c32bzkgDiJvQ^{9_LjKM~)lGd_*dd6iawR;*l0Cdr7kM{8@3G4k-KE)kPo4F3=~$ zxzKmXg_h4g)Z_}b(AqvHcK2Za@0!YiI!}ks^7L0Q`U{wP=6V@yxpeNX8V$gI2svXi zNsR+yKff0cVTu7)&p>veXYXbtwn^f)k^P#nGS<0D0+^$3HQo+31WMp9)Q%^yA+40T z{$1YtUBNzN{94(WJg3~Ei9T+B@e*4&lrD8wX<1T=C6trit+Xt-@-pTM<%$fiz==*7 zW?m=x)8e$)CB?TVwhW|nWP4C8`|#~Z=DjC7wn+NtW2I)whyAd3No0RaU?^T>Q>BFY zxlzL9Yv*h`q-9&~0d4Z-<<8ZUAwJ(PkrJm;@UpH=Sp|Wuxs+Of;lw&&cw}7z^mKjo z27EU%5Y9g2m|ZO;NA@OyrOPE8g?Kv`#T(>zhqTjsS%5+~+J{%!39wFTT6a{j&PYzx z!l%bEwQ#H%0k5MD;Y3C$u*bvm;slyw^}#8p-(hpM^&8u0>VJdf^Fk)Sfjcge^s9WH zgVny}B=q_Gcryp)4j-f-({8}tj_C{Cz2zOp3nS!Tm=9R(n1 z3EkUyBGcH3+zgH6Tclfo46c!)C@OJ6S2U`$ui2J+gfpJQc2(yS!HYF2g9q6hkdIyZ z>Su=Ua^4aojQ8e;t75`i79ax%o+V5DO9wzwnX;UlZO@^O@9{V$6)_0)B7kv1=i z$Lc@60^T*!nP}pv(a3QVV`cg)CydkOQ+^b8q~H*tacL}dKWxz$f$9-2ZL=jFDI%JD zumqhB1Pag$=$^ApjkF=hTf|)LMd1V_Wz#~gxh9J=z`pGF{Pp3)o*_?VOc9b&>E@d= z9iD#Q%Gn3rK)HOMru|)`y-WHg&ow=i*PC{vP_=SY*!9N{OaNPNjEPOA>POt4Bpr9zG8R7(v+@-`)~D}9fYm{%c; zh&|rwXoS}4h0%PyRk@i{Ko4ZHbaUSQh5yVC_t2Jd*BH6(VeEC&pNvc4Pe|+P9WyRz z4@w~Y2&^M(yh28WRh37ldP=x{Z(X3j8LNg?n9PUxb}`H8k{@FY4WyiXOS*Z|(Y@}y zXUJJ+B@se1AJ@k-HKJtmjlzBx=}k@>6u-Db0L1j_^ak7;gL=vZOOcxmpthWV1J8ie;~m4 zZ1V`)dv@?tF;>0dFK>_Q;eNF8P&41vVt9q5KoG0EEjyi`q`2GGY0yh&84;NfG!T+= zb3637y>)O7wjA;UUX)%9Gz%zefMa>JGwWAb4>sc0`Sj*h>|h@vQ%^XxSF^4Z88x?k{J93iYmsq?RvTj zM0|H+9Tb7;WFo8bmKW==eOx$p3uKTFRYR2Ib3Lc-6)BQpWFq-=d6if{u88RttnqY& z{InBn#<7#aR=pE^3HE}=eq9Y~4C~t4Pi-z1I~@DF&1mgCA+xw4B;Yeso*f9?k)Wu@2?sxLI{5jgYhzUF;i-%w(OZdX9Q8ybe@k|@8&)U zKACWm1E1Gz`knO7bjm1TIrcqr!)^O4RO>Rkir>1ot|(%(FD-g>Ke^v^b+;s|-Tq8n zU%nVln3ZuNEs@mWFUsqD5Uv_P8#422wc+Rweiw8r0eoeSdF0R-A<*le+)xYWB zR7(!MrT-@KB0}Tc8^oO*{cLrYOBttuB9n~N>KwQop^;0LZ$mRHh4~5Yv*r&rm^%zGPbL;S<+u|=;&mMpNmkFy`&yY zJQtwKmC|enyx@bYNy(MlZ5)=5GPoAY`9opShUrdKf#zsS3g^;j%(B|P-^#`2{4`qQ zO>ZKroaqJ+?sWx$t}XAq&e-$J;l|HRYvAnAMXNL^r=oYFiY~q3mFwW=dvo_CRd+;g zl45p#vGkGsGr4=AsClv8_pk<=kJ_pA^y0w*Kg1>Rk}2Uz+_S3(srA{L9}@6B<)R7q z`%*4kvDWMyx9`fd^R@#wx!*+R#)ky8kVHM4T8QF_-{=3HTd=HoLo_zz3V1?&gq!_2<)Xi89%M$@-+ukJ{kBf{-Z>6DRc;<#8H&_A$ zGJjL#a@^lHMIVPwt*|F(xC?nr%@sx|j>`Z^cmOrZBJ)DHJGC z5hxOb_S(6)u{iieH(UBsemLSyUxpbZ@i9^VT|zkF$>4pq`}Y;|CLYIeT?D|dkX-N9aWo6NZ1umioHRzhu62j%XGCu3YnUq(mW4;~LE87atepGR60 zE@~9Ev|j58?aLqXXfaQu(b##@f^eUG4V~Wl+eTC;Ajn#tVD}*4WXI-*-H;m(Np*tv z53gyk`o04OW+^i_^nDZG!~h@|qmZj_tM?U38zK`JpP80F&l85QFjiFhg?E?1mi}0s zkbCkAAdAZgSU&UuDP;K#PE1x1J>1PI`nknzsF`0sHbSZ~aHoau!(ZmeYC#T;Qyta^hG1`@G1 zev67_u%AW%lKkcT5u5YZ{QR3P^M48pbwh!MT ziz31Dk`Bvt>Hr>7q z-MPNzZ+(j5&d{4@K~>Z8U1#OLk9cx9nywbB^8Gdv3d0Ijw#OmFgzupb9C3{66)At0 zL{cMdo}CZ#b+aVhpWZhX4UgzQMBej&_1XK>w7;l~aBBPM#Tx&9X~5w06`4q22uUb? zK$2CZ^I@tr$scsE>1NOzD>nHk1vUt3WEhc~O}80=E>su$3rNm{%jnxzF2mK&xjpj> zozZir2KLx!3g+X(SczON%GFO z$ozLy$0HbUgpQ0Abwc9PH3NPa_#ET;grwFZyvKd38Or?Q#^b%kIX`G9nR-_lV3bLfU=s~Fg_)YfpLjm;xEWz6L{0dn?P zV^b%d*H9K;12?2bBWLh%XO3OUfBEVbASrJsBnKK%LjYlY5`z9IqX=o|!?~V^D^O@d z_8O>oGc)*>(zwwocBh;V+ut8R5sz$83c}fLL2-|qIK};^3Y!DX5ci~OOx076uc-Nl z4d@qjyEjPTRs!jtSBW&7_RS%NSkU>j;XEW#q(-bTCbq37foVY?sT{?(Wg-oR_3n`#ty%*i#*o=T#NL_PhU!*9usp$9W9(*ZVMgs?+*61=>hQt_w z%V*}3KF8(^1t1j9!4nR-80~DIyvBloHK>>xo4ce?iL)_MW5S# zIz{W7>-B6E6Jju@Ju)MH_<4>EX7{VNw~4Mi#xxZ>)r4yzY7X2sQAvO7r^lLU3u%Wc zwXoJZ=R@t!!KLXxu(D(Dz0;&Y=Z^{h;>V?R?1A)J8#g9=Mavqv+V-0Svm@O^xbs{d z8+z~Hn~x2{ul5dazR5)`Njue&ppA+z&mpUz;{&%?);8Sz6kiW&Y;CM^qZUf1zyPxLroV}yCMH2qzf6?Q*1y`%0#-XU zj#M6mOLnPr=*cziLJEG>z|TRevcVOok-ZO~Hilax{)HFf29bQc@h~tTBtB-Yb$xD! zD3)^Rs->dZT{RAI9_VXjkvnlRrqWYHQgYd@>_$g_Xora8a_#=wM@%^HNg6e)^WE&Z zx<>re(X`-dztLTT*$QNQrPz`xyiX=T`?xSCsg@;Sj*KBLS?jRm;3*J*Y##rQuSsx! zvXv^@yPYRZ@Umy$9F4!sVUU%jRpxOsR}l;YRMVHehboxNMeZf{igw;u5TYpsl!=m{ zY0=!dmglV=KrV^C+=oLng9<7P+vMt9AGo#L{zYXZR8Gp}Yw2{24vC7jf=#`cdC6kU zRHEFO!ONk-s`#B==>%WcUDjbpul@rS7}WLIXfGd=F+$z5HQ=C*$6S?f8X#fJz=Q`} z$cJKfzdx`Gt)_9WSPd01bMouRQ=z0TgzuPI!+B;?$&KCd!Q!=^0nMdDkyaL)t)~Yo zn#rp;6oou6p~h!cIV(1)RXKJShU14qhsdksqtlFN_eraqHd!w2O_a|K89Ee?Km3^s z)+sP}rSaRunOgXEwByUE8$N#c?>^*w48POt#UUkTRvF;XvW#fzko&za2X2y_{%r3> zyPx8ZUwPf0c_c1JPI6>|LM#u7DC?BR8Z~LKR^C%qMoZoQ#jxkq84zJ)8xw2L`0j1k zU0(!yCSwq&Tx#u%+TQb|wij>ec%2Zl8_^$HvN>RrBy?F}N*XS~{84yYiOw)Sp~C@* zkXg0hH;c}@SRy%yJ7nn*q3*Z}0-uqK+Cp|Sxi=$x&4kCgYFqy*%{F#i5>m#2>UG8kx)!lKC6#n0n>m zNyyI>|ENh@&(0nR%b`qAM`A{8`pYBi_5)}t>}Stig!fD1$YaL! z%daVrG}Akh2?lN=WOQ5DsVy_hV(Z^<$w_Zwx*eL>n+hvNw=B&s42;(2j2YEA9{YU_QyoIx0Q!&|@g72j)} zYZ=9HWu=j^`GHTElK*bk)29hatGRW?Z{JxLD(QnLO$iMq4YFsST&E?Nv;|+PRdX3j zot|@)@OWSsmHk5t8^9jCaaUxmlns+dtb%HaygB&evSow2D(dtH#O)Ej5LqKst;vIunC05$l74nE^twaDhC7`fzwL7O&h5&i z43fK^mYpV}uZ)HS5}A3Ki68j&`f!+_BzKLozG-oIJc<6L#z2sC)s(a|w#_66958lE zoCL=S@G?*6-~`T=R>gF^yZs~5eg?!oJ|rw2VS$hiT>AEuVS%*d?YS5yYJ+g>b#7UD z6RtPvR_;h7ftVDx&MBL-cyiOpXG@n0L*GT;ap)%IbX_CI;ZOz2*bOVy@yPUB`l$zd zb1{S0F5y^WWfTE<5A2rz2!q?8>a506=y>+^2$`*yoDAHnp7Bmb<7(`$5(=@3l zS`7OgQItR5-CM&>IG(>4d}{yqs$$3a8>ylbvZrM?GbtjKp!kRs&KYfM!(FxQ6HXCo$)y!k4EK$Y8J8jk`fc~JY~K_O@ic5+wu&RHw9;=X8|p) z;zVj${jT#jAUIs4oDT^dLi{0oNEWIz`(ilAiMn_1ALph1EV7`gpBkIeWHP}{ilN6n z{s++lxd7w^p9{i>?(juMB}xaPzEDoZK@}KlCr9;Mt_QWInEqhrC|Y^jqZ72K;sPos zc`j% zA2qp0OY|VsUeIc%2q_myi~0$3+bJ}zCNq{9$3yq@5M8-JQ|h7rj552BhD+N1DSJFa z)|NtEssF_btcHCD-0`Am)Yqu-de=Bpl^s6_Xd^&{Q~5hfqrpAxvY^!pW+K|aZj-lk z@erCy%#QCze-V+0CFvD~{!G&N|2jZqhO&)T1dB|bz!n}qU+J1^r-@IP?g(rh7q;@0 z9OK08eKdYb+6X~`XuxF-a+3i)T!1S6dJc!b+8g-^gIoPpwR+!gt-jJ}ZMxGili%he z6m-UN`gu2h=#R78*Fss)B(d!&e0&5GPe**&cg3k`J97Bm3DONr@Ims;+X$m=H4f#0 z-;MHxBf(Wur>Kn=O`X#(hbnx@&1SB`pzpNbl%eF(x}ul3s-U=0vT_+RBWcjrC%9zx zR>;;z&^ZeW)>OpPRbd3g4I{5SWD*G{nmg2&8hXZP%3?+sM=a=%Mb4`|qR~d#d&^u; z+Eut+%fk9GZDrls2SqCIBZc&0O?Hmx{VBH4-1FN@{iXHweWG6n0J5LD>3eNEXnWw> z$^w}M8M}i(+K^;Je*@Opt`%2K9=B@1dlmQXk7t2bX6SDHlWu$^i-Co`XNT+=^~@78 zli(B4&M}ktb(qlazb}MUXqtP(d&=@nrvZx+)%9zJDNFrEqv=m*!pSHI59lP+gDiUc z9`Q8xk){X)(N$zLeiCN+{VT2MkTd11cyPi{XqAd%{p}ZOIrpCy{?RPCmBE%tUZuF9 zz`|23x?SD|zS47K z8tGFyM0Ko0{jW3RO(prac7YR@qt&?y+PiV(%35b7D>#_AnnyI889KFSOquR`D7;oH{+x8FWU~EvrUEXWhL%5)R&zb zhfkNCNq88ReOBdmho)72@7_bqwya=Sigb;F)&nhjzbeoC5g=mQ67HTq(uKia=XXMB zu|%(^8F{fzGke&o+n>H9uyHVVdO3Z~s--pe_T_0VgWOoh3ogGLKV13;lt=!W5ejW{ zeofqq1j?I5xRkZ7M%s4{NAT+>zE8+^L<|klDd&$I8-ITWBFz0*qf9FfcZV0HB`XaK z?Z=l*Z#3c=CbS~??nJ|7IZ79${tE)!)3Ai`MuK<&z%qZZ-@2T$$!j`6MWCWc_mNf+ zq^`_#u|(~97o=?s#9=E@Yd-fBe$ZbmP@cfHh*Z_cl{DgX)| z>oHlHq}X}|6h&^w)tawt13{^ry~5+GQFLLw=kde(6Cpp-J$R47GfY3VPsncUBj@wO za)${UybA;Q)t+1hF~Z_XRgNC&Z7@oh&voaF0v#RJjw+{ZC6blqLdFfRsr70 zOnMI?qVrJAf?Z%JxXI!_-*+lh)>K&mJfg}}+U7?A_lRCk7>#7GWE+#x?}7WQ77A&b z5Xx4_o&b$z)!3NRCA%;n`nfP z6hpMX+UNT(OzxY|2w&F<Lg$fMK|H+S--zcg3sG|Wh2q_`L>*51WF7Vo`C>Qgf42SDDfoYZ?Szv)B4;pU zAfj0La{lt3$ORRLHv*@#Kupds4hg7B04baiJJm#9tet!cGAFkDz|2vivGmpmZL4Vs zL@K4j0-W^+5JKeK~O zmr_#5t|CxS!mLuE8rwj;tu()}w2u(m1pI3ban6_twLFYSt^gV=ho^4p9r#pE;G+f* zfnyz%j~>~Hn_dl{8qGEn^zNDn$gENGaH0Nl>b%Tyd7RhIXC)`ziY|_vzbGzpS%4U0 zo*@znp;vtu?VA(-+SV-yT$BS7mcP-C^Yh^@Yaz`49-AhTG22nFTS zMBc?MP3#okkoBRh?M$&bd{W<;@6Nja5I)N1xWk0@0DqVSk^OAQ!H$DJT(pcRjS@g= zW$JOqmtVHBopuTm-yT7OO_^}^{TLWSwm=m0=I57ZwIu98y+Sq9_iC;i#||KFcey9U z`u}lBZP9o%UN5d+2d8%}6furKx{IJIU!MF+Ina?+jv{Utn!Jksy!EFsjfl$tiP9!p z9|DN+NV11iAO^2CktCw6^WUBY_s6JJ`GbFNL8?3f1+!#Tr1ko2mv@ewzu)QSRrj8Y zvqOjIGNQmLf46qN-R4pUo;=u|9#q=Gn(Yq{P_7NI6~Rd0d2QU8w}q*B?O89i)7!E% z^Ai7iP0dlg=f_iZrF(|2*N(c5Tb&$!og`m`X;3<<&Swa3i!QU9&J9H`@*FD?uRvoO;< zYAZ40bb|BoiZ`5?gO#&Nxq@O1{^QIBhncj(! zw(!usbbD_0x;oOf)HLoG2H#6H`8gAm$u{g;+q=OpQpc@CYOixrHd`sQo)jZ(b`}C= zF0?dh$D9d?xEvK-7rY|hRqtZWk)*6)zAVbQ3N)^)&VBUEopN$h-XCyNeE|}cw4#{{LXoT`J<7QOxJM_al3}oH z8auq))uNUT*|DDXUY%*=35+C6*_RG0I59$x5zLY;T)Kg|kMXJE(TiSh^2^LBihksZ zLyjGDdpq26r%q58IFipJ=F?2Fd+D+B$MGYTy;NT0Io*KuTW56`)*9BK0+1H-$VvMx zn#3(f>3hQ2kM}GpP**K=N5{_d!Iv$bog1GianjmicZrkq{(XADeHPXT`~T9B3s9I0 zb0<0?h!BwlKzsP_&esGK(V$yUPEkgG13bSKDco4bkqL`?=K;NyE{qCL>(%1W+6M@( z;JzbFDC!%su(*$c>;Jf2d0E|=lBdpG93+d*1ck%@z6$8@x0+#XN?`J;One9Q=rZh< z{wJll(}-{%jLqC;&eTlPYWo>))d?lCS$auA;7rU4;V_aYC_S|v))=-fpBG~2glscF zmw6)xa$twuE^ccHJsXP(hY1jOL+_~d_bgLGyOL7U(FpP}k_$z_GEDn4! z2d?$O*~mQ*E5zJ!|MrRi@s@?l^ZOe9w{{h>5kLZimh5#yOnVP94!0yQ&%O0uS$=TE zo)IJaVuEj4a1T@xU++CLDxcY}%gUyNiLcrXyU#N9@dcuiE^n3`cAXk}ef_#LQUWbQ z60xe%CnQ;=IuZ!GrAcH#wp59-3b^m`IoPvgg^*X}Xi$c&rZaco5wQL>i22L-^TXiH zE86KX#|azo@@cISZ&|*ad0;lgW?$Ru2(;mBDcf;2kE=)1zH=yG z`(IkB_%MILFyt6|HnoVvJN$++%fIG=S#7Z!ts(4ZY)%~R zt%K^xOmK3wrq3p{d^hb*yf^N``h^wE9U1Bn?TwJs*68bn^*s}e7yYt-Dttba`bARY z-Pn_k$fC3FS^c=O;}Dg*1QVhk67GTc7VBKLPj)_38bfH$2bZoR=PP#ddy6lj7|Yan zrr~gZ;b4E^!k3W!&4ugBK;C+hg_v6EbvP0t9YHqJG$tOdytg($r?NL+imyFK&feA`Qmb#vR;PB37mDo^!mEDe30i{S8f# ze)bdl-!uNR753Iae&=8iNh!@+>>9R5R(588_WaBU^VWaw5-I+g;P{g+Vms@49R1XI zf*)Zt+bo*~`x2p!lD!d{t1rCz7Rm#=g;?21+0yl1qT@BfL_-h%yoQW-?zm`F0{XW% zMHl*0={pbAXb*8pW>ot@uc8M?fWW;K8C(3QAn@NG23r8UL1=V3_0F_ghM*cXC1TW9 zFSroy5oS!N>7x4PrZ>+r2zYbNfcevf@I!?A<@KkLqS1(Z7C)7Qb9_J_NAk}xuuO63JOcKOB7rQ@cwaICeKhsz?Z_jh5Jtc z*n;Q=z0Dz?e};fL-e)A<_w=c+|NKQf8&a7^KKU~sFd?v+5s-sPIfHG(QacONO(*eL zSh(Kb_XZhJHxK1;>+AzqHWQ*%RYKT{th3?wY3>uYV7?8pD*wpQuZ}kN?Rtcr;Jkrm zslLyiZKTB*JkKOB*_x*;Zc@!X5zW$SG3?hG zYS+M0QXy2&oaPYM*nR=%s#!RJVYco`SiG3JM$YH6{vbkR=Kl@)aubhaCBXw8|I2pMQDa_vPN7uK^G4 z{`uBRMIHr30!2akg_b+&_RIrs$}yjLgf9q%{7iwM;e|%So)m+OMjVGh43$!oNxQ2z z!&jcFF`iU}sl{sS#5O~6nA<}OH;aeD)_Vf)y~9`7LpYg7W%TJ7)iL(4MPB$}$fGdq z*28>Y1s7 z;qScfk6Uw7pw02v7sR5XI${AErrfiiA0tcUUYZz1SF zMFliFej#K5{FW;IBNCw5Jf}_Q_qE_Bfrm+el5j6vg7A06L0;!SfqWAm1@Gt!+TJ^( z+VG_w$RFi*Q2(Zc%`E^lzvP+&7EdtN`aJB$|GUF}Vy|6ye~1SAZ>rT_5L*CNIh*Fh zWQz0@C(>V{;E5f)ufp4lj&&^y&V{sP#$ow}z4TB1*^mCMJniNUEq^8kG=h#&iNZu+ z&N6)Cc$}{h6@pU5SEfoNp(F75RIZcIS&aB(x|!T_tKW|3_Lw22>q_MdbI~^iF0y zeLoa#_G&Sg-?Zi(K4@YSpqT&HIR7=ye+aH zAKtOXZ>0N$#xS^meJoA&Bm-^LghfjVlCNu^7I4yL>QhGnVR)a@Bh3|UJ8}Mrmmcc+ zp_EU62UYLm>gf%U0+ zjI;eCfiz}9vhAMXS8>FsXsaCi$GySwFnrF_?o6BP#4N#_7*qf&!Fe(H#Q}cA)XKd3 zp10au83ct}ER*#MBlbTr=ME?TVENA?FqRQ1Bz>-EUb%S?)L%+lV=DPU7t>U?IbZ1Z zqQ`NBc-*~iu%El+33kRml4O5(WU<*g^oAOMxt!iy=9aCvQW`wgAc^9?jU6XYP{8^Af z%Di}kwC;bJ760_g`2U_j4k=ktzp1wX0O_SPc*Hz50W~%3Lj47bzbQJFwwr>I(!Imo zK)+&$ysK@39PJqzCQ5fBUzYSgNqJZ5C_maqmu~W$WfFh)72i4V)8%8fS(TEU#2*_ zv`8ehe&Xl%WWV%2-1~sDo^p5ig)yH;Rc}6^d03!`0Zo~unIMYE=6OiiKUc5Doc;;^ z!53n+pK0SC;2J&Q_VVxO*XXAGDbZ&X!}|?cZtGPU`0e~oxy}8dImwq!pmm29#pXVj zIek==#Il8DnCi*HXC_(GL`aeOr#@-?pM(?-7P4POPl)~7v4J#hEi3Yo?W(ZzB!92l z@vo>iCvxbkLBrNloIe~P5%fI%czk+NP%%rqVpeX_u2XB@`iy?g*;dX%%*CMXJ^~*# z7CU!~hv(!drgZ1s4P@$>N9;@Qf5OPS4B#l~N8Rk0P3BlYuz`6qSUH4hCbaibam+Q_ zmc2qHgOK~1M4cFqH(zgq?n~F-xuG;BiU}}@A!;)NPRA43BYfzx{0!omzI&yqt$O%^ zR^8uucPQFJu1U4RrjsLa_bQeZyZd+ASyA{%U|Nx4Uwugxf`#o^_?!qG6-^T0%_sFx zM!nkYbiZ4;S9S0grw?r#DiKaf$I=^Yc+Q(`AaAR?rpu zrUNRI*^JBT&iYxzm#d#=;FK1fVfS1fn|AYW-Rt8?-a8a?yoa3cNjukk&z25X4VcZ` zou5N&zj6Z*wA@1$f&J!U70+D$G2J7LOWPzdd9QgFT9XEvt>~OEibjQTf-RHSiA()# zN*{W>eIwEJy7u1S)D$zmc#RzR_#24)ljr*Pne2j%;ei20babwXdW(8WQCHcB2A9{R zb2Vl?N~du5spOQz_ATPC6I%0+FGsW}W)0+MXOYj9?{a$2D&#j1!O*|rzTfo`XX@1k zSt`yY<}U?r(r=ycir}lAJin_}9r$*Ml6S8&m&e66I`Mp^)byuMiw@&?rB$-C-j})0 zxeOR~sN7Zx)NU0|w`ZAzt8HVNnovD~Fm1yQq$sjeMx$%Y!Y_$Rwbg6s?l5Gnu1z=DFk|adcad|F~$dh7DnwF@wTx% zKn=xe`ByBh<3&qUN}+--lBklsm-bvHey_97$&8n)apY0wjsD370&0kqUrgFS7?(LK38IwoF- zMTA57u72o=ZS~a3oc96G;(>dsU3K@sQh^2WRh~7OwXo2+wbiEfPnA-GY7HY{N1IKK zS$bnKf4xjHvDAbK^oOPIOy|dXy~G6FhJa&xn{wQtjFogsRy*b3XhV__0R|$lGOgZ^ z7LL#V=>Dd!oB6n(3yDI_BU;Z3DEJUdLvl5dOXdnhHGnms5Pr^XncF* zx}h#ryy?7dCX(^n1y|&qZoJ zWX9^Rn5@Ky&`r0|i456ls2lMWsI*|;gdlUyNgCf{lNIMR8k29zvb44Urg9<&absFSU~C_E=hFX$7F8?2;Cza6&yRMp9Y zWOVXl7Ur94jD}MKr<&DftEb$W6H?znQuJboAYJdl37YM#qVg7|DRRSd3vK^xqm>v% z;^UL%NlAnnT%Re@jx$a&QN`{IZCNfN_kwwyVwFe{BHI&5y$KacH0>? zI_11EV3=YkIn$wPqFNUcUNmJshPzAS@uQnwOTvnc<{zeFQU1=ed49mpmJX#g1)y>C+XmVH z2@U`A>E-N59_o5M@$U(wCE+iB(8ak1FOZe)v(LTxe<-8lqaWVzAS(g# zbLmv;4MnQ_hIPly+PeUjJNsP_oye!aQC~Tjd&>#|y}PnS!4bD7(YGy;o$tonRAjQh zL7qFH$QhabN`G(o9A)E^f;u&Ye!c22M4RKcsJJdI0D1%f!ciF0tDS15zVKEvUNjNQ zYgI=gL-=Wa;OW(334&}}_hO2t0l1!;=CZ{{qbNJGLP zlkJQc?{iab36AnOp?qtjT0A1BU~*h!8rX}zA%qX>UT7zvdCR;y5b^=I)nDe;o>U#< z;)J$uDa{3TOgsZ{6-ad{&6Imvei+t?H^`sWBD!;Y<8qX>$FZXB{BEdyPgm{a@&2ku zR|)4+xKW%HA}z<8Vos&1R64ZrRMbq%B^QtquRmsDxD(*;_i1DNf~E(=@Tt^$Nje97>s@-gvq(OQ~%E z>d&N%g$oPh0=_qte`(_?A{EN(;3XX91>q7x=4N&eyEk-T4US+fHwGqGVwS3J1Hfp+ zBz<)>ENYmY9WYip*V*MdQRsp;-d;R3I((=4&qVX*G?4FG&X{G9c~bEre-U^`8NZ{M z*OtLooEVpSH&vXx-DxPIerA1>LxBrx3O*!xAd=e?G6RgWLCOT@}~^x zr0mP^W}DNmSGviyfu7^-rkYZDQVAo%xnITH3*n=?#Md=Ep$#!JeBEn-euQOtfwsdh z45|0Z&}{4ua4>SKdbnsNR}xM_(+?}x0&_R&T?9xK*4npwSrr*vsR*nqsFJG+{qqAvLIxAP8z9$zJI zuzSpDx_ybQ?wH5jQ%2mTnJnLeqr1ra%xRQXPCK#xLTrC$te#cxL~E#XmHf?$XsQp# zEm0vM!08dPptb{hR9Cepx&x)Sf|O22_Tm{Cu~#mdRV-;Xo{NxAADQXS;MIz`D;e4P zs>F2i>`6d=Jx4tavSfIm^6P~_Fw;!VklYz$K0`)Tu*lQ1E6_hv!(Zexmx)TwFpy9oM>%sDCG@FG=Offe;DMiR;!m#B zQ9eZ<+fNm9JMp_sV5zM9^4Fs!uWgitXPi~iLm4tGGPpIu1pwRl?;Vmp-D&Kr#Itta zzNd%oa`Gl4F%#{BdN}2O!(LiWKxf*Xqe^@NQ^%!p>_qz^^#?3EZY3lTnm@2O8Bt7Y zJzQwR?^wApN8H30w2fP9hn2tfNoL`hpg4QzwjewMShSS0#MN&k`!`O zb<_A1YDMsQEsNjugNcv1371{YuaXGQkNjt34lQWdV_{b|5}tvWA@YyMK4L>(ay}>w z>Mix$DHm=L7Gfw}e6~Tp-`zh)8J$T*7VRJH=RRWRu)LVGZnRWdamhf$o@3H63!lk3 zrKl&ruYrpX`2g1f8Z)3dzGyaFjtO$Jk4tsi?fe#``gusu1U z35A8JJHr9(+3P|YvwtTW|7}c%h-fOes{buXAf7byYJ)52E0I?@VR1*q3Qlou<$>M5 z|8#_Ig*AZJE%tM044jt%UR zg?Si8RmlU=zdzC+=}cbQ&nZkgPVfF-@p6YITu33EN%QY|8p+58EXSiRkE-za753cb z-I&-jRW<0uVfjkz>=r*#zG9;wB=OiF93y_uVHaEYF`Hy+C;3EY!#%6S>NPI>Pfo~W8yV7)7GCd6aPZQwGR&FIp%NK z{Lmik-gGHFHdJw^B9TV%UqpLiGS!YkKtzi*698Q+<-AT(IQK>yR?Z~{@@b1w`xo0{>^`1;>?{hQ) z`5ct?`r)OwDV~~+(zFcCg0k`f7Jp;b#jMS|XVa7E*3eqSS3bZ7$zw{g_|pixS}&Mx zYmdA38o}nm0a>aGgl4clO_i8uDQ+?a%d!wg;`W3pIJaI!BmK68KN8X>h;c~q0H=H8 z#{8s)?yPOMjU&2$EtYWZ^TopkU7^6K6}>Ta2zzlNF;uRoZBUlAyw;@Dx|Qst)x~jp z_v^=^S;0Y8;Wb+|caJ*-PCVd%xpl!QHQUS4hEDKcBDFaz-}Cf)iUFD*ni=MG3Ex;@ z%E32@m4kvHZw`;e)3LCyhKr3O-?-AFa0qPY-~s5**>-7nx%in*949zFa2V5szN%^yJl!VAMN=dKrVeBv7f~lwE zXT4in%|E8)#Ls|%roqV#6{2}(4QF-}s?RO3XE8s`$uSy;M6GOToJFlIZ2AD<0>&fQ z2yDQH+QWO5Z_V4niXSZ7qUdeCEW%p$H1O1ZnHW9nHX}qf=Y;_xYy7gFSS3;BOSE)* z@Zzbk+?0%Xif5{+T(7GBqS4#pR^Mz-IrjJ$d&eI>fUe4mor5cIUT3!CCXQ@HA$who zfRQ}c4Kl3zhthTmZY&j+5GB>>`(C0ry+cU1lt%!$SuuMsZ!}&Q;Z?#4#Y0fNjIZ{- zd>-0BCK%|=k1$&96B`^d$h{u{Lx&)?YhGs-5Y;*{JJ}249O?r|vxtTbs+)RNSRn|i ztb*#II>rrCv?~IYNi+RB>XRSMyC-^DD{I;-Mqnxv&r_N}G57xB9t359C}YzQON&y7 z#*#c2X0GCw#&2?5)zdLI-``PvE(AttWBE|AzuSt^4#ze# zvduI46!#P0an^ggvoLk2#(%4BWaT@6lm}#harUZl4H|SaJ2h@{k~p;l3=iH6e`@b7 zc#WQIlsz^?kFnQ4+g~9{PsK(X#xN-8s0$5+7N}n>PYyYLc>~`roiiK%MDpuSOUN;$ zk$*zHn7$s*b=l3D_*SBt*m?r~_TbM;mADtcqOm9XQW6;NH>jG;u}?S6cmnE&2_mf? zn@*koTyZ4(DY}_OLj?`h^}JEsX?im)&EnjfWEuTR7N``$`74J@psPEifOCI~-okXN zPIjWq({aAZNCzf`9J};jIy5LX^5jGK8RgKjA2>u1s(iBio9iuC-)YYpgw&pef+}W@ z%)Ff%yutIhfl}O!_wf}m0{y*!$#r@;-YJ&Y@z%VF2#a!SH<}iP#6`Ufm4xf|wzkL8 zIr_#X70z07==7r`YSt((GvylTYPrefmA=H>DV~9?4^N9nIGUtzPZJ)7xDnL~Pd>rW zeyKm@VDMw5a}0e-PAK=gsPC&2p&;R9$ZDpd*^An^IwsGnJJkC_aU{ICOI?8*qS%$r z=gVro)4-E>S4i;sfzrPeCns@h8;XNA^xZhik9%e+5$_yJ^w_bEnfscy*N}D~6*G&I z5RD|}^Ud@O4{fFP%8>acb|fiCl+mnJMq@GKajMbXZJzi^T%Zfg=FL7H8cvO&2Yf}6 z^3RMvo&SbrhnmPR9j;;@(}J8Z@S~a2JE)=irrTL1-1sj9kAxE*FwHVuC}IRDwA9V` zd~iIxBCeg9Nqk4bk@&`cBN{-)B?0AggRbSKt&5dr6EFua#CJ`9!l+S` zPsavFrE1zq(=FE9c;j+eV~>&SP?l04uBFgkE2Hs1ay)l@Bf*x}V0mLSGpOJI$$AFq z9nPnduIlmQx+q9R2Zf2Yt9UB%z7P0$FVzNr-VS+NFVd7Je{*OFpp%r<@#G6=cK+1< zq23vmC?vS;K}>2l%u0Hrbjl=I*dv5#^KpSziZV6>HnSad?+lhWTBZ7v#c?%hj9E_g zHI~|7#2GJFlW2mbX)@E=lM8~53RhT)!>dwwIEG4ijMgj(sCI;U<- zK3;pg5yJ@jn-(3_?zC9_0I9`0pNt`VXV=I{3d^gO${A3HHTZ0n7aKLbOaLa&TE2X? zVm2|1y)1(=-kX)`*l?-0>5{!%c&{hIwK31L78=*O+p3<>ajP(sBkFiHtKhNrLvC4@ zKZ7_@+!J4ttA0pN_TQ%UM7KLTU!I7iKdg@=>K@Caic?oR_8XT>?EUv81bCPq=vDv+ zc683uJMrHVON(}0WN`Isv^K5s&)&%p=p8c+z+cxEbG>!7t5b_8U;sqs!W6G z3O9xsxeme%C9(IBb34{YpWPa^8pfC{fK}vEi|&a6g89J)E1NMp{E?j!bGWRq!k!Vj z{M^@uJ7Z=xxpJGNc4`XXiJp&6VK$rJZ2zXIXUxsKZl1-p`HGS3wRBORCDs~eD=>Uz z5|-W`aq^Of8f;pz(=i_jbp!J8QMOKwyI><_kwP2u9K1E!DGDwUpHz%?&nrc0=M^9h zD`(-;dPxPl=?M>YgS~B9??xTQ;HX9ON&Krs3l}Kn4^LOVboYdYQ+rZ{5_gxbj z@v@V^qp^+Qw5BkleqsE8|MLxTXnhA`tyxzDwur8PWp zAT;A6dvgHhaABLb90L>-d+258?YB(7YR;V`bd)G) zkff0+Lp!VTv11-M^D|NPQrj5qg>h%~ZfCwutxJItU#THeeRXNV&CL_~*z)qXBTHjZ z4S97>nxc+!0@g35F%~R_);WI29C0<}+?xLSfcdn#*=Stb=z8mE=p;!;G z02I{PjTZ-k8N=S`Y|UGFWBs2E4c}`)n}O+`TC_D<}qZ$c*040kO1&pmDw%#BppN0~g6Bq#9{?|} zHYhhk%g0*xgN>?Y9?+g!L-FYei9zEfX@>OO7;lXn^SYrnDfEhCN{i~_#SBZYCf=2a)n6sDYiuhF!lGCE~ zBTdqMr(GU&wojW#7?h_w58E)spZ)(X14OYQ%N|We1u^6stO5L!LdfAN$5!hm4JDx= ziN-0qvM}26ihuWuM#^XAMzf#G}sT!`-NjJjq{ z4uinS6p|7;D3D0edieK)8<`RDdh{FwXG}k4j0Toqdvmp1=6l8qVZ)&>3C0(4RLae4 zb9#zcBhzdfM*YnZA|m8#bVDg=4_Jw~`(!kdGDpfoyc#qcNjHrej|n2PGKvEO1z z4eGouaiP$VXZkNejIFyrJ_dg1Ia^AXRTzCfD@KY{?4@#zJ5ghv%y=8va)jJzN;4_L z&(%bFmGfo{|DR(HZef2@SZ!@2;asvc^jxebJr!k4tMjt5a!v)D_q-&lyWIJWVTvR3 zlr*~X#J%w-d%fN8d^?M(U==8KL8QW~br$6g^+dCd0#EusTk(D?Xp6JbFTaBdVlZ#8 z7Cr?uKgN5;@Ph~0S_zPtUNVfEhq0HqEhD6uWoWuzP3@B7U%yftI&#KB=g@Z%=rC2O z_rwVEtAkuhb%*1TQIdamX2bbAZN95mIoPANdHK(ta?}^Se2=Eg$_@3r+1v=F$=6vOl)K-_Hv7+n%VKTvt%ZvD$jnHF?*)akY(#S)}VuAr~gQ?N$6zU7*>;Zq;JW zv$BU~p>3h__r|*a6|wwjO6uifL_P2&589#je8feQW&UwTQ?Fxnz1pmp3u8keHSd>v zsIrm!iI)4T3ex(jVm>AsF;B6{ZG|DgqAnqi>%g4<0QNXTN*Mg98H!%}yC76zoW>M8 z^J>AC(k2x>t#bLR&F)WC3w$P7ApwdI*5ZWK8*}0+-UAs+AN58Y4`w+zvAX$mM{Q1B zeZ41slrd)W}rO|Gaas72B-+v{w*hg!S1^8;$FA z1KU~(m*UbOW4Z0t{UWVYPr9fhz*n)<_x5zs4{ipDeYTFx$jDAwN}M1Ui#G@N48qNj zfUF5R$4}*832=(8sYl@<>_}-Yj&M^CfCtLTuJp9(J-VlOjB@4 z2MCaAENaCq=E;JnPl(*^ZRRZ<_KV*pY>V!TO@MDrEK7F66Pym!#IFrZZU-)ZH3iLP zp4#SrlUy4)8TN27tjvB7v6*QQOHZY<|BDVF51W-Y*B+7PZoK_AU7fAYXa9?oOcAcbxd}R$U2NA zj$X414ju%4EuPv)_)rbzq=w6dP|6${{}l6BIelYdwCtPp5a%+8o_X!Y`efzBR=C07 zsH@!R&K&1RJaH|^T=BCXG1~Bpyu6dT>ADf7%d@?hE-bNBop-LA?*RU+nZ0Z$ZbOFa zA+PpTuUi|jyvN~A=>zwahCjt5Jtjk|aOHQC4kH&Q2@Hx)gXs~}2NzduQ!{MxtVCCN%Q#*$6zRFid9f+3}^GxBM`YyD=%Wx6LVyw^r@(4y@#&oE>QlG=L^^P+BGlHv9| z2cD1@zLi&Hrw-*${CWl0JR0yW(YAwnQD2T+M|6d3T>7+O{S5pMRYZCTl1ZSB5Xv7_ zlGMIm$``4zuh!B^XocHTtlR~=X3FQwDoqLZPzhusqSjuRZ#CSYY-dZiBfXE>1(d6G z+&Eg@kLHD1Z*Tb_4nng-Ek0&bJO=|83UfcFgS#)3?6cr`wwb#C)z6509lZrbOR9Z; z)c_$NHPs8FGDBI>17V4z2E4t&C&pb(*FoMN0%h3jf~al0I@hnaDo^R3ZPLV(^Exlz zJ31}2{BZca?W}vR_@aVn9NScC#Pj`RL3{T^80~Z7-t`S7xyxK$HFzDXkkD$hE?y&7 zyIPvcR#gWjvQJLB59we1OkRx`gEeq5t7a&9%OoJXVij1@V7|+Xx3@FmUJ%|*(z7}T z{JmM0NXTsCqA0Z%Xje@I$x71QikBT##MOc;ZAz8bUNh*v3UH+5)~Bsqnn_Xk4iYLT zJpg6K$JgY&7R_f8Rp}pMt+08QY}`CZ-nq792(sUXKUw2?t?Mh5pBm_GH4?sky}9>s z$PORFml(Qq>pwH}0WP`qQQ2koYYZ7O<3d9x3dFD0+gF@GS5Tt54?pwszlo6o8_cd2 zd4{{k7xS$1>}COi9>=ZDc6{_6TTVDWDkp>8u+C1+*P3Q%>EVCdMT%lXg^d-+%0G_( z0EDw$`&>U&B5bf1zD^N#why>`{K$E_I=yV37*NYV;4xRl?A)N~|3-5N*$8G;&*>FN zbDj9+QaNn}XGT{oAS$B%o@SMDvycTYp~>02304gy9qy9y;_J9E+zOM6$D%Ca0OG;N zj>t*-GwUGpmlE9D@@`rZNDx3Yv1%E8SVA?v80*vYC92Q%uC;;T;-zY7>*Xcjb=Y&= zMrFO;W=W9rdUJmLwVrv|3iORgICSNeZ|HWBuSA_(-HP3UDA_5VQf2|DPOg4=Rz6Xr z>fEqAU&DuRfLHJ(n^s`Pf)S##ADmo1+ORXpxQGe%GxE8%-8il;;IOxS$l=EQv2W|3 zb7YmIHajqoqsTDo$4(S6U;teMpzX1H*zKeHh{5ySUClM4L?C~6UiSr9PD=8`_zBu7T@-ev8V@J@&+h{G zP#c%&@}It?(}Q)dhmXeUga~YmjU_hMMx;w*t9DJE^H+C8^Q%d74%slxI?lkFr0|U9 z8oabkHGqDrixO0K+;FTUI5Bvjj17wr|y_Ph*#G_AZ&vVHzU z%{OQpxr=4)CQXUXd^iYFKY?C;x(KQfJ6YXs-8!VewW;ox;8FAD@68cz7tikx9JO7ZHJMzt+<)6ZPDLI5G^jKAAvcn#JhoW0_jDgjvR6j8 zL-$oT)YQxIT;0Tnj~NaV7x`u(Jl)#^7ZGiwkGSNb7SCs5*_v(SL#V!ka)oX1lGjRO zrQGnk^D>O9-50_)NY%TzEye8r4Z^g0NM2%I(N$325Xh1C`R_ptp~FLzfweJG(A~Le4FKQp?}Akp=_Gp;e?$6vk(!t)q?}B zjEz;^@WZyw&^GSm0Z+C1K}R`NK0}_CV~3=+Uh0vdf#^r1!!X2XM}X%wePJiW-eJ7M zX)Hk-xseE8vX|FNNXt0Ca+=fqzB+WTdG@lYzk5m!%(z3vpC9krJ$kR)dM`FMKN1(8 z+z=tB>Rbr=;Mo?C+265|4=%ZhHGkM`<7vqByd9hbFI%q>~6#GjQff}Bm*-1*{OZ#Ml( zs0?qLI($|=K&H9wb47@jpZFw>xj&g0Y1pKxT^*B9&+Im%na2&R7gfjAcP#9=HOu!cw^6@xyY>hX8-GQj1(4N5x zMUUmCsG(3<>^SQz@F;yNRCl(jJ1@eR>ymn$6(90tNGq=!S^yg==nDk}fuYOJf~Fgb zt#@1$G1A?xQAvi}g3fQ2(>T`;u8n@ob93QQj}zH$B!Xzk(cl&2qi=haob0`@s4eJr z^R~u~wqPg=pq$V#%DJ`jYYyKmpTMSd5HN9NV|8vmB7Ao1#WId&#^ydxkD+nq=E8+b zvGYw|U7gj08eWrw=BHh>P+MiEN6mxDF;R~80nXLIM~sto6p`SpPYr$ts?Zk{40no7 zvmsF;W}9lSf8DBj>RwQfI_3{SPK?O42x7LfrJP|wa;k;K`J89LMmhz%v#7*jL=77+ zzx26~Zx;Qe%$f%x@eZ-yb3lr4XkZ7zJ|p;Zvy1hdzl>6}7hw2TR=KER-FUtmlO>~~ zi2Z4wt~=KLown>+c&xi2<&ZlSdF{ht#>%s zOghT%b10gH-wzsopeMb#VMSUkeC`pRl=iFpwLn#deO+NGB%uAV;k$aFzG9o#OsV`O zD_}Xvk`75H;*L-K-(PPvU;H^4@Y6@@ONZ00r7{Wa{hEVlX=}GPtoF6};JDOs?SZ!! z;$oUShq*}GF(N#T`|&(X9WDBuCP~h0*IL@USP42tc}Lhcc}|80jM2%-H&^I|ot7>k z1eO(-RYJ7gTbc*d&_}uNHnL_YTDMNR%VmyChf?yOa%E2}llT$sIVMo%ub_tWgs+~r zzshiMB0C!Dum-4zKgzz-LvqL@u|grXb&9Y9pA&a7EG(z>(6e_eBuVCdDBQepCS|0s zm**qL366rVv%U7}o)X~PPi>NRP;~MQP1_V=OfBN?j=w2dXw9fcNV?Cc@TQttpS0~a zbOHm+9EzSLb54v9XmXvjo-|$8kC1l72KjIs7_tBX2!{EWnJNV#WnhO}E=ckGw`4V2 zCBq(0?vrnjdypV)d!t>+v5Jt9HcY@&qirSH@dJ;v-0DY)S@SR|io|xDn?W;7beEt} zo^GBsI&(|bE=yj2(+IZ_^?>G(;ZMl1rqb6Oi_*gR7(&7G3$hU@z5PYscMZbtZR^ag z9eIb}>^~+ayUIHXOE}zcC*8A`FQzR+RK{xpLH!w#7MeL&)W9I@V>&HIU@#&Ns1k7Khql-Vr+pOC+^~{bZNA!`~~aB!(~d z;Lb2!{M{+e6=Jit#zDO^+uF#d!Cb2pyHjnsZ|(#1i9a{kdwoSjI74WrWR+aB%{IaE zU&F#ZwA<-X1!sool(cEbkt{mr2+-IEIQcnEMWGw0yH_*C+)pFK8cJbhy0&_mtA_go zBCHH$VUG^gwNtd7n%OV^YD6T9^OVe*EWbdvITzJfh6lP`G{M3Ogho!tBw^} zjUAdFC(t_X;o3H!b~+-to5=c7KiRGtVgx7Yir7ijEJslA+jm}FAQm+mgL|1gw466= z#?pIF^)UR?h+jUX7*~a#9#mvY1=F`GX>h~k_k95mDQAU*u?&-QjBq^=Lo%yArAd8 zgWUwa3fP|dsyY+t+>Kx#@^vme!ivw5n1M5x={Coq8jqllAh#hN&PL}yEQP%0v>uT< zTjbSroG?7hXpQX7YwFg})nbnXdi3b6+;|9^HIhHmD1ZiZ8ZMAMYQM$a>8bp5M8~{# zymhJbJOtdsxN%@iKQz{KdfVxL&ue8S63*^ePEIWgl(_;$R41Cc;LP(+VcD`|sr0M&%y2>IM@zt1` z{FG4Yt-qZPiW2lc*&6)}Zc+ThpoX^jmyZX=Qbgjdp`FwXQnX06zdNN9SEn6mG&2+c zKR$&Hi;kZZB|N6@CMv}9c2+Ys6uq};^yceR%3P!T2Gd1wHnp~KB>8vQIX4m4Q)oq{ z+oyZYlY6Jo0gS%RSt@H8d<+k1^p7)|9y0kaQC{V3a)!kjqH5mv<7BstTbgOl+3k^m zGMCDch;YE`4vN+|sBX93t;MABspu6(; zS;Sdxwq=f3ZWn*}isMQoKk(TB>^Pj&W7nJ5jep%R%I8vk(DWKTXoLD;zhM&>znu#?{qyaa zCRwauf^e6Sm9}Tm;?L3xz%504Bb!fX+qJ^lhldA7nENTc4NiAnGxNr;3;~)YyMcwU z2aU^uN`U-^Z@LT6xRf8SYWbmUWVkNhdGQ1mDWdHSud2c>WDBB=HftF;H3NP`OnXR0 z%;BD@t{iwti)Yl{0PmI?T!Po09y$omXc!S((JFUqsr)i~FPwc_E;M#StMQhaFOfS@>dtmefe@ zPb-KjmfGp@^*I3Y@Ut7wlL6YYVB^m#Z9tyv_;4cwCM|(k6}^2de`Tz@$yU7wd5O)V z_Il#%vEYPa=Y~)Vax8$b<6`#q1i3KgUBk@`^CqN<|EVpLKQslMhx082k!GR?t%Y#> z*{}DL$iPPO=~XdVPx4G_=y6R9J0Kd6w;7ZjZ>8mz(UjHE*|s%WRi8~0z0ra&&vDB6 zzVOtf+z|Kdl{)!0dhy1l%0Og?b2dcqu*6+kP7DZt?>bn~!C%Mm=~F(Act~;J1}L=( zJ%I%H;J)9dy7`cIA@9C}Jgb27v2XSqT4LDt<*!qPN^JK}bIp#gAl=ejU%KB`R`||Q zBX-X+Vx)pVk*`if>)pC?BAe(p8x|MeBKB@~fK0uYE%v@fO^2YHM^X~18Vk69PVD^& z<=HS^0{J|@reyACn}jnDV1{)=`s_&QkyzaXUF)r6L+`w5?FY1@#d@4*E|0fc-%;#_zz!;v z{Np@A}C>Lc7q`=1k`od#~M zSi4E98Ml>1)~&$=zJql5dT*FNJxo&ZbHe#n`FKSJIcxZ{LVfD*e%CFQ@8OQlLq@E0 zuo;1Tcyl=;uH*HoAsIDUw_;D&uAM>)*KRfdIa@OWHD_GY^^~Hyj=Y!|-K1*kQWru^ zHOyBiCZ1g7)4p#=!k&M$ma?qd-^3Cgu9R0-r@_Y76+U+1W4fPM^UmyoQRFrnvC{sIovOq*>e__$Yrv+O$XPndz3h5! zVt?hk@%Tn+!Np^@<746+J<7d@*6Hr~vTDW#x}6i)W8ponkg*lU-9oZQSH$t3f-!qo zng!d!q&fbWlKwF&wM1VubaJAh=k4v{)wZJSME93QBk3uMSskSt$U!rukB=t`=DNqq zkEV?LF1ZKz!{Q9LgQxwACtkzCGd!|&QrI7+93~A9UtSA46LKV9f#GSwZZA$n%Y;L>gOJnj&eI^W7 z`_J`eND+H~9&lCRIuUdPB3CHV@f<|UtbL~6zHdnFdxT6}-lAnbC&%%0`Nrh_(Rqn^ zW|jJrj;s8sYRGK>`lCyHgS%bTKfQeD4--GF9-U9IvI7@~H`}1`l?+hfy(Wn42ci=9 zWnWOqTVvV&%o9Q#oMKvu@Y}f8Z1tM=&|yY+`P1oIoK?!1Y0gZ*lzV%FT<)ZHx-e(* zLQKQ{V)vq>;PU0|QG2<|6qCu8QUXVeo0uc5m4fU@8GR-q9rzgx-X1WXpAK0GKTNzm zQOn`#mqsHbEv9tsLX|DAgY)V?(IaHfuN zCL*h`T+wr-M0z6QDvuegfj?z@zLLEtQ=W4H7MwiMku^Q3BB>)aH#MlOiQe6%M%>_o z8EzNmZvAoaal2`*row=M9@L2`{(aaVum8n$;c`YEy3nt7wK z(**LeUrO^OM>#+Yq$&1H-zJL_N2Z`fteGbl#Yxovi@m=Lt7_~1fMMN&B8YS-sfZ{o zAtCxF>E3KaKw3bgTe_rs^GwR)@p$h0zTR)|^ZdVXEw^jU zHRqV4$8XF9FUL65!9m7wTio_uLHSop3m5Ls!_WM;VSS9wXR)SiZFk!pFcK-(mjWCY zgV7hzrwy@#zHc|gcj>n-MCZBxafY_w4!l}4PcssppLsy)PJ+>N|A67YsG!3y zV3KkX)RsR|z};W<8qSU&do?!DB*F+6ynO)N)1m^vy(jc_F6X@)f|U!4s_%&WcK(6C zBmBV&WU)Dqbss=y=ix~lz)fmt*B+%geZOnFo(6o~i`M7uKI!7TD!*2-HEiKm%1%|5 zlSR31+4!31k-D&nXnXi2l`AMN|4TOn>AB*a$DMSlf|hl5u-X@=v0l5ruTa~#A7X+V zei8ME>dbj>KYG;(`XKvG0eYgW1_k1tU`M{KPh+ENPSyI*!D0o(kDW>aH>&xXG{p`W zcKHTeGuN@>tnSkCGFu|oho`e_*q~ZN0CElC1QJ*LrB26QhPhso#wF##!#AezwfDS} zLkP~*IY7s{{s(j{Qc|FN6k)2fbGl36@R4c4ck*!6vck*&g2}h~7Ed44XrrAeQrPsG z*}PHDly1A~VFuC`pcLLy==Lf&l#n(8&&xoZIvaxIP)o-A;he&w(bK|Y@GMLqAq}44{ggATJ z>&=QJ?p1wa_06mKQvINPw##Hq1qVJ2ba`qdD>__*@C$UQAz57$;T(AY zY#8CA0)cDbf)#HoAvxAkzIzBJ0Z%31@~>TeNHrkRtrZm|DR0ZJ+@bczgy-TH%?;#8 zoAu8G8*?psT-4!&Xe+pQ`}&-BBS>~$!}Libc@y6zUwfu@Rf6Wjb!y>P%!V2YEM6on z1n#c9KSWyT#-juW(VrPUrg&G7xkP>`2ygAn`*-gI z#lI#N7ptT(h$pFG(~6lxC;Et%`A;!^y>>UwIKVkz2}5e^(r>hKh3uUXvKpT$7(V;8 zwO%c2$ba0&>k=lv>-X>fJYXe{kW_Z#7Be)$g-W+(nYOjeTz93 zpiY~}0Rj|KB)g(&AkHto@nJgUsyhx9UThah$`7aVw2f-6kKn+ILucV{f4;v##w!nn z+fWG&bqw>1WaVERKkOX|7X!_`5ii4kJGRt69~ogyqNm5${mS8cL(Hx0sJpKGR`}UJ z>qtMD1D=5y#RHQQ5fM!df$xxZJPbedSNszXjstmGEz9`x&I2MLGayc%UwV;bOK*3Y zXqU5rGljD_{BTmWzhOQy``eudPQhm~cvaP7!%~Y^JEV>s2?}W7w+|`LI^K4se9J-8 zxLD@E%WqjioNi;ohfMZ3%2MOCyYm@NCGf zMgRu_)b1UTiHyu*+Ll`x-7{ZLm~vT4d!W~Ihm?d_^V(4@CT+YglF=frIH0hD zi9CFFssQg!KgV@Gp?CyGV5>VP=+peN6F8WPu(fyyrh1>61nqK|1#=+QyuZ1=2QNw6 z4(Z{M7f2_7WmCy+W|z}#xy2UWbo(zswYvArI8Ec}t6hdOT?$)WDPM1tfhO!w&Hbo8 zCrznWCK`P=l(PpygXnRa#@5C)F5UFsLer3T4}QO3ZEsQ{W5m$dtPCf=v6Pa0JLeA8 zcnLw`a<#I%z5@WGl0PNxAgR*SW_Lnu?WXnaDX$h}p+LsB1f zz1h=f$~s{Aku7tsYP6SvC37Hw&;2V0n(_h`XqkI#;br1$eZh-^qw_Df6*udB_xbjN z=DE8H^&Jg>@l{LKbiZnzoBC=t?v+Ez3(hz&hTIi282k7!!!#EA0B;t7|K;>uv}rx@ z$oV6vfHnnSxRaLGL(x3=^N0nEIwQ3YTVFsc&&uP+MM2B-jF%EVNszPjlz(aw_s9## zyheD#RJHdC(<}H*m-tr(Q$jo)EjF(s+>$%&?wYzw$F>`$CL;w^U$-c$VO-OLo9Ouk zbz7Eyvb;@KNGp*`tJ=8NQu0QxKGd|nJ!FAz!+yEHKM&EkSkdn6(h6N<5Qd;(NTxo! zNu4S$uZD(!OD*{9)M@bBf4T_2clxw8y9X1)f4T2uEHM7557S?RiT~Tc#}7P_1xKx4 z&}n@0UuSakK*vqc5qp<2S@^$C{on!kB|>nH)Qj%itN+7V!2?JeOu}oYOr$#luG+0q zs21I}gty35L^rBoK{!w!_O(bL(*!e@Ss&&xZlcLf4BdQM*cS`D5a{@ zbg~d06c&H=Y~odDjj^=X)67;Q(@wAdiJtKH7+|@KOkWIcU#LB|nC)FHc1pznCcL}0 z9{()mJz-&gmT6Xfjd_fKidm-;^nc@l>o;WmZ&G)ZceQ$L&NiMK^L|a!aZmb#q_hND zi&RS6?nj~*mJ=G}|C#T8(uEWnr%m%LxkodfdXy@g%$ur$a>>S7*k;WSis9^s!O?Zi zY{fb!hclr)&QMf41Mla9|LMZQTkzf)WP86+;H}y$XfVPX&CVM^@juQe{JCIN%Co5Z z5gOuv!JeOQq99uD{igoGGw4TI1RDcaIfv+Y>BA*bNW zxSLI{H5k*Vy!Q(#u3p5!MA7>xDK|dCefqAUCOaz33nylLqLi>#;?kc*U;7C<@i*b! zowbrpdzqvyrBAbB!wt%Ij^8-*GO3F)Fn6CU=h>>cDjBD0jZ;&f${<%$r8tjEN+TQ5 ze=haj^`eZXn!Rl0PYir`XX*)Uh#_cCMB}bs@;9H3@?cQ6Zqs`=(_@HMg8xTO78JuM zH&$gERbD>V8S7fgwvM=EB_~FxapgSWd*sJbjom)m5(jm*k1?+VZ(?DbVi^A>ZwEn3 zxyD*~dvyNwQdD5P*Pj=4s92vU4QEPR&Ej)qbt`8X-O{7~Y(vk3bs7>x@Gkp}$}KJu zb1hQ2uncvK;8LLPADQ19r~8+!Iu-&DL%oxNC}r{UIlhwfYDgPV37B$`2Fu%+976C8dD4=ZCBO~g^ET@Vtb%8~kQV!@Hm@Q@9e zHJxF11OA-`|2O%6CF}oD{x?zn zg`@tDZT|m;Z3JyaX5ugZ)CHIDy8FK2@z_=1Q)mvfbqV+fsJ*W>!ThSAENrbS6jh)j zmfvC?)12@*@?eEe;9R93(lF6}|tL#q>+$;DpfrX!(n<+m5 zr=8y|f`(;kLG8QftL`1|Z)QF|y*KjVrTwuML(XAhoQl+hmHdoeyz#rYKBgxYX5fdr zosP|!<_ftACMvPrVylW_QEWTG4U*H9=?%%r2H7$Ex_JgH2FD|UP}KG8 zPY!=05|=moC8@DJSaQC|LQA*F+^h-O_pOa+r?)g=MSCNu!|L8Ty7V}Es4EfYHNk^w ztn~OsrL*@Oj5#Imb?Wk^$d=uMz!2_<5!J9rzc=zgFPfdvf<889#Q3j}OHvA0oUgkY zKkYNC55)D%3j*oPlr%PGl`?wbg2GQR73UH%6WL+&A03->7NXc!c4GR=*{=spykxi6 zsmI*p4L0ex$qg0@9IA++{e@d84>Su&WG2{^3Lnx-KSO32biMIV*dM(M*ciEn^TxK35&;gR6ipd;O@+P0VO9k&TGCbBMnf!{h ztmXQ*kSS9O(`2^&^v7yI+vW5I=~!9!p^+Y^jT2zZURP2-F*-lCDqnr57)GuskY_tk1TVN18L!v_r6zk2y((BTJQ zQ0b&GQurvryNq`p+r2Zp44Or_pidRDT|I3Xj0p#+6snfse3wJA2reCXt*G0N?*Ckm z9K=g~vd(uBN>MQRDSf(RniIIphc_cV(fW0x9@gE0)X>UmDqflHLtu6Zyor z2EkslZR_Z8_7-Li^P)|q;%QfQGIR1d@bnJWVR02~CtbUxx1c44g5pJ~9f$Jv?=O&n z)*vj30^9K8>%4}Tb_#eMJ1rB{;UltY2N6|zMeu<)eK3aXI%J+gv#pp3j8isiB(3JZ{r!jqQm@Ae@N#k~2{3C^I}XXy)=ytCjA#ln!pKF11!R#qM}XFJoeY~qL<3o=L}_@_dDtIacT=8@K(K$Bkmuv2ht)3_saax9H2vzcUS(4DL9H7CQE z4d|!s5+rDPpFUQjQiN(I;4OP8!^1urkxJR{;FL-)=5{qcijm%F0`Pi-@gt(6>>WA* z?kOc!wevsi58RiCc`LH~Whp6l>OG+DqjkXYIrrVgQbjX7 zdQQPgO72yv(){J5z-QCo+(*w_RXrc}^6k zpqieJ`f!R0UkpDy=^4r*P+R&)S?DtZ72YIpDoy4{xi-ZJD%vVuSjBl@Z!g2a@wHX! zo;depSJ0_!s{W#z^g)j+Curf&>Gl}3=hzo<&MAGgmA8SY;&Hna3tWn4At9=7HQdfK zil990`gxUG{1tKu=reMeS~timlJL3eh5QGG)ZGuBBSEXVj>Qk3_!n4EqC2=ZfB-$N z=ZeP+{s88-emOfkTj1=uvnxZo3R{%A>zD7h9PlZtA-q8hIs!RXl?R!ZciEYZsU2>^ z$owSb7SE*9&>fzO6VT_C4O>BR3pF>sd>ag9Q4$gox<#C=*O-KJvnNIV2mXPZpo5)K zOv@sxsv3YZaXI^HVyY`S<(BcPBj}Hw{^~znGRA~07N_#o{naUW{rM>zy2)`9h&<63 zDrrRvqpn4B<5s#avI46E^IsoXA0@NG?;NC;4VX@B36c~Z>e#$BzrMaiCGCBu=OBa< zSBRVU8M0g%+>BOE=)6&?Gb{8aON`LtfR$5j`18+6pUJOPmnPojbn&77@c8`Vp&m>F zR;9)ee<7S{Ggav`ku!%fLv5s{LTdsFb(jkBwkzbp6<5u!SyyY1gO`4Zsq=g!2H zrd}JS91HH&;J61AIl;@jEruOSADp?TN2otIQw@@nearCfAi20V()?p-l@FVPo#Ojb zyxH+--G%gYj0j&p32mF(Pn}@W~iM|f~48+4z?}9TS`@!gNbu8 z4W-%8k_!r%&>_RKj^=zQkAiqu4h<@wd=j4?>ZWJlY`qo@I}wF0g_^1t?xQ89Jb9X` z1vG+jjOc|Xj?DW!%zK*l6S?~OhO+O(Sp?Xph0@a9xBVyW2u!YXC4n$_rk#`r8Lqh%grN*KM7bE7_vG46IeUK}>SGAoq10V|o^=pQ z!+Q?0^^x1l$9kzjEv~Te?zb#{F7qEv6PaxT9qP9!H8tHEQnKj9<22Ck^5E1?F!e2W z%D}pkD_0mh=w#)CR<9Vh)YBE;*?L}DROGVlKP}Jb6TDuYdk3*(Perw|?Ud^pXKIgt z3~hd-8?6)Ah343<%+nB+?l{B9_jfb*anGF5eNo}|qqUL>2lWHy8x>X)BE3PXYNBM@ z@ByhuWtUn76>#}6UA&)TP=^LLm7%l|*!{X*JX@7|oeE7!F5vNF#Yk@?e7%c?t151$ zK5141&?HJyJ9qDK?;;1sy~yjfwI@wE^A@igH*ddgbfMuuF5*DUg^kI z*}Z8;W#pIkwzu11UNj-QlIt6ExuZV6O;PO2P?gJwb%0lmK@=5E|5{g@rM8UQ5K)&K zqf;5qC$7rXbTQoS0B0{0Vo>jU@%Cu;>djErin`)?XMu+cw{FMkU@`-i!-c5F3{Vd3 zy2E@$sho`@mxElpqqTUs!%^S7cKJjuqpdd>n{J)!_oKxr?VzpLAbqG+-T0EysjKTj zvio4Qp7097bvAKnX#dC9(q}(&28Recfqfg7g-VpT%LoircdZo~5x|S3`aWM^S7e!e zqnkj6gpCZSd{!~|^b8ITE$Dik2IjLrl8kIL#hBT*NXzx{5j)cI(eZIeDTnI11U6Md-SjqxR1qa>%4~Lcw}Nc;67*w+#3R3bH`pm$9HNMh)|89gHIsnVsbbQ} znz=q4x#q<)K9g4JunN!qj#c8=VV{tYM(^ZwS9fui=PDOBZppRJ#?_H-Pm=HpY^_HP z%(sHNH$!nm6|EM21WKP)4Hy9*vv&6WER(r6|DBS?)dTpfaSzb~!!suBD{%KW=NE-{ z;Aghl#4pbn)iU+Zno)v2ew(0D@#BAQ>n5+UH(@I$uNbkFI49@6+DyKX5>jkOX=q8^J<_B+6n*^H{vdUes4-EqtdO zE96^o+)zlOzaUN}}y zP_19zpqbtvmEqlzT(0=er;Dg7MwJ=|3VhAvq4e83l%u$J=#p9*Uz-yxj>W13qbWk3 zifhoq)X8?^VdD{}OvON-~yaB3B~TzC+qCNeXy2<4(emhp0#?Paa(sc z+eANl{hj%F=eoJbsanZ>O44kNX^-$k(OpT{L?M3>U(7%$-xk+K;1Y9=A2J`4u)4J> zU+HNnexwO3{DM@3w5!W1TYnI?)Ek%8w)puGT``y0TM}<4JX83YyHGzeLKt+aUw6u> z&-!)yZhh%K%tX-AuvQGMhe#F>1s?_@AZB%e7{+~~B{BEtoQx)8saO^-M{N&&zfxQE zoy9Kb9HO42P*v5c zyr=;@24zMvcBy%bOlFNddP;}go?SwvF&~rM`7PzKrDmR$)Kq!JC+oG%(yF7D`+km-jP~a_G%%kaMt6ah<2DQq@d|jdIKGHE^@s&lue=8ZjV|O z9g7DTPiL(@*VTKjt#l`60mK(ijT(Gx96jF>S;tZpKePYpfmhF zM$|ija_#ws%{ChK(HV)ZPf@RF)&joW0pYuMg>R_}Y@+%Q;FUh(81$Iylc6 znsA)r7{9`5Z{`|h-;%b=;Kk{-9l^BJG+|1kn4a}soD#+8=f%P+fUxeOq1nmAy%CJu z8C+?(Ps^kU$z%3kQY-G``%uaKL5OwC*w|>)e%Bbz$>8*c39OJZ0rK82RT>r+Jt9CteL(=(_%B@{9 zzHQ1xw)3)l48Sw_%vW2_Z{REAhm_nWcUch)S!y7AL7tp=K0B6+Z!LU5p^&$XFUD=6 z=UTY!4gZ3>TRkR*5hU(AfpK5c(sXl;4c`Xge03Ub(|v)otJvKRAI_d-9>T3~$Em}C z?KVBc?dA-Nrb=Zop%mky+6_qxISWe$)+>m{Ns56s+liAC)poT^TS&9^LDgRZ_qr>WD!SgWT0SX;tOQxHFyhWtCBF3 zEdjOXN9#H+JSpO2R+>*SB@T&kcS9Kmq3r4t&}nKYE~3Oc$oXO4hshYWs}qGyj^gEZ zeBkw{%_rv{spo^!U=3bY+))G?^sUyVKa*gGmM^5HVu}Yl?9Ge!A7-()vPA3>Ds7iNV4f3FiG0 z6~`9Er4881{)0L=Ut53f)6IOz3S}k5?~Atvt(u7zZn1Ud(#y|46ct0{qP;U#LyejJ z9GRGIg-gnJ4fK&<+j7<^%sbrE26;kZYqtBMP$*Qxerr3NCzW}>bfS57`JVR5+sWFG zj-n9z>-f7Np;wr^z~T^lnF4-rI7%1Ob}}tYCvDS> zK{tb(mlHj){m{OBL$G|yEHi0xOfC-awUCe8ZPK*%fmw&??A)}rL%-fZ#Y{?vk%7p% z&8$&IPdS*+bY6SjlV_nX*c~(uQy!2 zVqIa`8vAg7@OOgN9SM59wT`#U2Xj@*M(9?;Yzr&#y~48mcu>=Rqqz(Mq^3LeHLmuq z{BENZm3OZ9oLk(x?}nLPFgx@XR_r=h8ktaOjIY0{x&Ylo3*Wk*w^5)R zr=ONmjP*T!)}?bruD8rw`p)UD$Zv>jRJC+lEUk&gsy*rp5krorg$U7@%7|92xp&0h zz=n7tZ?3wJp~EvZ?9`I-y7-q?TezcEJrmDVwg*%)Yr&MFO2FYp{3F9Gd+~nY!;uFB zcLW5d9ePQB6k#3g0DPq@(|46Kgx3BmbZWmRc28f!d0ufT$lQ50T#q5}VID{EY=9^w zSx==&&!rrbp1^>4n{gp31ShI)625G09-dz~tlEw$Yh zWAS?V+3VcFbLJ*m;b_*;6k%FR@J;vZT_>x-tnf+;VYa|#?NvMbaZ&b?@5?Z}je#xY zrO=|mw>T*AS?hu3*f-hx19UoFzUOpmRCPt~N6c?a)511L<=d>8;2+OtsJhxEWsfn@ z;Xu?WaCl2OZ);g;akQFxa9}XHQdHz~P(=)aPc#`Dv(1mq==IjUPLydjpA@|d@)a79 zkGdWAq$(}?by}dso5gj$e6n}~dxJ8ciV*8VWAEvxK2sqUYBz)%%R7Ae3LdOysF&Ss z?@}P%pNM%AV|?D5g%wmj%BepPNyvB!V@=k?boJ@N>Z)8)umS^ENn-iUlVS;XIo;Hc z>K16Z^Z;6$&XlfoqGT>$^)Gu>M(u{I2ecvD-9?~VGh8d#BSxu-5SQ^rSjYwUAtKmz zNu@_4m)2#TA^|EAO_#^JqfCor{HEXcEZBUSkVx$$p(XXB%t#JKdGYMhwa>E|Qws|Z zMc7rtVNmCh=B3RS`*db5b*U>g8edA;lww>}dDk}geT>a^^%SmGpU$WlPARq-W2qHz z+v5t9lN_X+%qV%BBre6|r8|Ib?h~!P*bMsVAEKvz7^T$3bYP|hjRyqh;GEas8k6HM zkXglgwj9lTpj6T=tx ze5ur-wQPuUBK(+5BOxT^(*u^97Bb7qyVeP5Dg56uD(^s}I_qtGttXgCZ}xU_p^VtD zn*xTX8nd?^qPlqocbw%SHJn$idOC&b{07Z=KJvYmISuc@<8BjL4Za?KJx$mGSmaXsV);NwA% zxkI%^508sjpQDhR%1}JmMoo4tpFp{Jf2n59Qf~2d3Jp!PH@{&^)SabN*z!)IeGQAt zaN<(W(4Jq)Gx9{87YAf|b1yCnai_fteDMH8xhkduIrp@-_Qud)$vf$3zGz;_eW6!C z=8pe7+7o4J*QtSyK9XsV>!K`M)E}_G{BHw`0Y2f9ow#N~5FQ0mu2xzt*I5T3bTc?I z#Aw$yjZ4*qoDQcQXLC>*kXFBW^Ki33h>9eGP*twcX6g%sSezCBj z)`_CXx0e$OY#+Pv?Lp0#ai`Bv?W!bo+#ifWd%ZW@LUmpKscjw4pj*f1J4%Nv4c4-7 z%jGOa69S1udyb@hD6-YdZFKF!Zoc32XDA&8O8iV?`?(oV9g}7T#hj7=b__zb;Rkyo zfS-WbOBP_XX%ve+WE_yV9-q@^I;qHl-%^=>G<8Id@eEb)DP^kRVG%cZg@#V-I+gO> z-DGHvFD<1nZwl0t`-DMJ>(6hI1eNTc6ITB#O>*)Qudbz5KX`j$uQ#tu5 z&)L{ybJ4-I3~^QJsun=iPiBL{{CoE5WBa!$AXDs#1v15EJD5K*_t4L$XbPuJKF9BH!g``e4DB963C@$+TyO}|Z^4o8GDM_$y* z;%?QCCoaoDR9?V2*qTpEYvA6?{|2IX0zSS9OH?Umj#bU_W^Y#|5jdE6!%su^qyjfg zS#9tPRNpQLtnx3|_>;tv$e21_fp#|rtxc3*j}iA*FIcM1U9)%uswrZgQG(hcXwqM) zQs7!)w5Ops0wqk)P#agoaP}ob67WfqgN&vQ$Y?5gXC_DT#^8M_E)F=zQ>;9m{ zx9-MAoban51xQayr_|F>Wv$Zfm;BEwr9ZRNoFj`=Gq@J3^ZA;DZ6_6RHJ$h+401d` z*zmJ1n$BC`v?xjgDH!*~H=0KR_5uhPTlc%P2PXnnirg@-WH2&H8A>EqHi#{tM-5l*rY6c192M15gyWKvZ^rZ$ z^1WrF#Q4_JNz~lj+@Yg%!LzvT#%m?nvk+Z$>D4dJa}l~lymOSZx75{rm<&F#%kYJJ zx0GuazG0==qU8P{a5*xp`Jonj08on8gU9b_TY;Q@z2lujSOi9JB1no_nb_L?EMk2S zR9AO)nQnwp?XM5H(3j%a)}^}57hR>;Gp>5R)i&rx{T9esS{gmHT7x+ZmYtRLW@u;5 zV6?1g0I$HdfChYh;+@CBLd?Vc(w?ILCq-VnCzm||fw(elP@2l#lOSQ-mLS~L`_{*O zd*Rlbg|TEoG!eAxJ73g)bYTo+WHa6!laf;x5O6B%GpJyE#faP$ZUpgm9#%w+cXFmc z)Geb+c;Wd#bF}L*AF#)1h(0YSBxSr+*FOOVB(qp-@NLu(k+FCI|D#@cqKKH;qYq&( zYBGeNahva`451ol;PXFgTd}{l=wpxAOWB-YEhTJ*QI+eX4aZOolE8Leu>kKE z#{V=UnU*Ip%v+%y^FS3fr3orQ5}y!Ro@KZ9yLEtzx4?vKxevPUBqG*7K@G7`chUPH z(7+JnI|w_G)be(9f*~gQ71k^;XANSWqe;Ra41d~J6FK1+YQebAXoG9==3XciD&SFc zfst_kb`Fl8iC>Y`pkp11c3n|fi;_g^13%2k+FCv-5w3cq#o)_Cpr(W{&7Mf6VP$0{ z<)g!Z5bP@6HGQ8{!vKmRgN1TzkZQ2!$2A?GMV|sMEs-&wMz@pC&fx|fJaMO*RwC@$ zp3a@~?--8MKdTx58a&0G{iy@~b|Je8Fa=$F8~jl$lRJZXNmGsK&yHq-3-L4WC2!>~ zW4d~N_wHnrqUDnCB*|4W!{d0HmReR7qOWkA^*k2fBfk3e4Guy8SR-rFhp>K{esy$< z#I6p0hT;fiaSF~x1Qb*Xc6RTtPyAzy;1nbd%mD@poZf_p%4~s46f5um)1z|xGeO7m zApQC)^yI|q{f%IsuSMs$t7*cS`LI4no#uJ;|oQ}h53RAs$S1^Yx0{x}~H z1}U?xjpenPQL%3;FZeiNU${_PNMSCC6#*|zgB+9F5KE24Mr)4UM~ z_`CP-=}kl)u}Hvo4H&k(4;e9T7Zh8Ap^>e6J_EWpS85|1KQM(aK@^Os@Lr*R4<`!f& zq{1}EMa(#5_PtdejlWxAdn@?78JK5-43bY|s{qvWyZLG*jT*g9fo6;;oYVx+ss-3l z<~m13U00^Wi;CBwE~ixRu?y|y9pzNzZk6}(9IaCJ3FITPF{4WH1Wzp61OewK_|kDL zdTX*EDETT}Ycv$}8Lq%r^Mc({D4ljo9t!9lB4%y~054~}YXWp`ZIH6ZGlDOEr^pM! zxTD{;Unc%SbrFCw4#@`eMHT=pKk0%l{M?O1G-+rzeu%MBjjv|*r!g^V73ec4$%>p&UQ}94yAm;7O%K5MC{p9kc61CAV$VCNV4gmO_ z3W}dm__05s=pENVdr$8T+Ml|Gl>%b;%$GWP+CTUhPRptWusFxUB%1|9*viy#?ZgHg zz?E)N=M8?G5&c(j_?5W@pP-{X(=Rks^!^*Go_JY5SABc|&j6sLMklQFqbU8?k^cC6 z1Q4#!o)pDff3f24L;fl%xd2tb_&2Q8f4Un(PYH<3lSrkSf1B>FV;>^9PJPHI`mp^I zAb6tF{`kDVD$unNrfNk0fWH6nG3r?e@YMg0@P8l<;K~2@5#FK!(n%!BJl_bq0IvTx)(eh*mVLJ2Lz`m6-+#vq4@X|I_I=>G=U^^uvnn-r*8uQl zoIix8^lH0uslo&1P>u_kI|6?j4Acu^Af4l?rUgHR{zML=G_ zDDbzZsAWeXBBvOZ&MLV4rer{a%Rz@xNM_0AHWp~4sknM~2QxO697wxGzMwE0=}396 z2N%=ln%3;kH6Ke8TR;4S#2#z^f4JBu%zd5dL$2l1yPq*=;8K@c(f_2S2qgi~20bx^ z^|$384}CJyPt}9T036{Dh({-u=N~uxyz}rvunJA`wCb;5u)jY5G7t`Fe51a~45P^l z9*C+pJvU!!0BTyifmo+mS(;gY>I<6~UEP+K{B(ro11!vTknI-n$xQpxdT^=zP2mf0 zk5*n)y#3wW^Mbhp?M9+N>xm7|NZ`TNO_<$GO_|M9BO0AvbsmMPVDD&w4wWCZzV$Ce zxIqaYa>!>}aBB8|)^uDI2WV0SG^ZfE3$VrMH}5jZy*HRi&~6h+bR9c&eBjd4)VWfqI>cdSa(#%^*Bw zzG%-X>J$vezwORQP+LRR6!8`%pCey~MW1u;D*x&AlQU7DntO`Z_b>B|Lqm5u=SS!y zjnDlUWC;Y{V{v%DZJhcc_`XYI>%_f$bpYt#R=@dk(7*G~?;92u2WW%Yz+m}j%AaVU zpMwxe1KT`ek}P?$yno~SBT~bk18hCRjqFb+)BBgDos1^S4H)ZHUI^Ro^E|oXx7T5U z9BW2gIsX4DT_b2{&xl%-8ve^PPTq3k9k6iEm&K!g-_(;EexuL-5&u6BO7Q<{#8)8U zu=V^8a!B|JIaui1oiiIUH8talK*1Ak1l0Z49)Ks{@%QE!DYT|L1o#inobPB=*zOTOj+74g7MK!%N(A7XlgRF;kpH}B3P-k( zXhr9g>nX%Q_;ts77L1NAC<{&u3UgNaftHoX#P2U+U+u$XB@uEbo9zh%uLX?9lA7A=r%>0Ijs+(|t&o$9k z{TBji-$tW1ppkbxHf(>=Gla3o^RE*g944afp&RjHX${2BRu!JRf&q#b`Fhe@?=Vv%3q`4hHC=;$eU$3 zY2Nzh9tz4~+d69=sBa@PH`fBpMq(A`g0Fh& zOj^S(V86e*j}947`Jj(^yD|Eogg*R5?@?T0B6|zpE!1u(h$HY)~$JZz#<++7-XKM9^9@ zSlFU_W$vV`a=^(-Km#;mmP!8k725O%_^VrA1aJoi_y7z`AS6jc;u(?~l1+D7OQ6_X z9#m1?nk68W?a6%S(a6XtFPcOd&s`M+rzTvaT96T$Y7`b;wtkDsIO-??orpy57>Cl* z@BG=P2~HDqHPu8+&7Zp%Y(x|s2@XK`t8+m-p`I|;8?CpuZ@Iwq>C+pDYA0PetOtEd zOA5!8lwSnxP~zYT#7`r-qFA8L&Z2T*@*{^51TclYgHvZFE86IY>8YlVRa6a&Hs{Gzv4qz0ypM$bh_n zL3-NctiS$o3ihWCI=<*&kd>FGFz$*Xh>YpQk1R6rQDg`e*3=wWeR8!R0k?alVPbP+ zLx3xVu|#h+C3d^{X$0(v%NSOyhjo`G36`|_rOC`a&s|4**rHz$(xb3=OLPB9TAD8} z=2n%!z8eQ;ETys*Y&Lu~H@~gZ_QqwhNxk>tGav?%gG5K43qqKyyDj#1PKub$2Mk$@ zMjHhkk^Ke%pihXHQ2O;hGD_}s0FK|b1mO6#Aue1TzTF3_8#sEIZ)SopZ2k#zms3aW|Y0f@7%)n>=gG0y2`~j#sb6@1LsB~q1&CLq} zu|A7#i!Bap=e_emU?>Im^=&EMK)GKHYaHJ}=CgnwkC+7Q+17=V>tD1p_+P<#Oby%d zNL^_m&GO|LUs=qMVcLOJQ4#vy+~nRVc^-9N!g}-cC`YolfF}uB4_fz zL@D5z^{wlsLzO&tdCEXp>vq<|ZNZZcO===m%6F}&|43Af;0!@cO;Zi~c}`1Ep1LlN zvIW@W0Wt$nAPEViLUz+IVLseOCMY6bO1D3cVoqm zmd0hRZlInpAf#Tw&x<(QXR@##ua9deB>M-pz?ONfLV4<>V0r$IXR%4s;GAhvVhqLI zuM|U-V+-7qb0L;h+XX%8Ntf|%oPiGx4m$9iHyNyi!OFL1W-W&W4G`aae5}Wqx-{^x z#i*X1D6%97s&m%*k1~52Ld$f3SnOXV=&DI~hWbfG-mdeFE?dqH%=&th5RtcD>(NlV z8#2xD49i+b{#wlaM;%u6%lclaw9#8wp~AC<2BJeF3wm!`5mOS7tD%gVBw?CsRmC%} z{p!N__a!W+i3F&~m>uo;D|>rOZrLwBI8$V#sD(a=O;U+YNVFVlB6>B(h!s}|NG9q?UE3DU7={X?&_2aj`h<#j865e)*d zHJ!Umo}Fvxzk*Ko4bzMgr;h*7*2&gFI(D)n*g-D+( zu^&P`H8=KKUq5ruVCjq=v=WR?&tR-$@!c=0yxVo^Cv=(-z2i z%OCC8{NS3N8=q8hZo*xFIlOF(?US=nKtb0$`@U}f%0x31pP^-dw6AE8eg3reqH`;c zjN0q_u|HPX)kht_fUXS?DFCIJfTY>_%*- z1fZF&LZkJMr@3PZU zjk2%U`T^{(w-1LrZ|LGLo@ROa>57hSkXX?GF`C)XIqH3+w+4nx<1-WA11D?V7wWB8Q`Z z(btxwoYT&%f=el?z|x^;+YfHonz88m?p6BEsVV$mBy1*_VHU$VR=O z+;(@H+n5*kU%@zYRh5+q_O}U&YL)jW=lO%>MTc(Zx5=O{C~p zUgZvz?qGT7tsT=}UdrkU0h!g-N8*2R`d@-~cpoT|Zf<5$c>)*UDmI!#Sl?nX5BLBp zmB2nCw%MGElT*Za_T#S8cIDt8ViS2zfOtCJA7H8;3ly)o*KG2eZPDRHaHybzhze&l;U!7a!h7ol1ndM zyja^IWA2%r-Ot_bmP&hC3*a`kKHIr!4t(BrV4&Lm!n8yZyC1o9vFDePvEXSC`VgH` z4E&c3i|a>MQ+o^AYWoszj={B~HNlY!{ZSDUDAc^Yu#}Xd%QOte#Ekg<+@&Iv1xhSn zIrfFnPgFU-sECDsbwr)UwQ?(kY}vh)-&k?C3brL98f$cfYC2*D$N``NH$(4V1WfP_ z7NfeI$k#3|&~9#@atVBIk}xw9i%qVsuyB2|J#3#JHD9@p>pA*L zV#S^y2Vf*cMMXqbi3an&mZm0A(9Su&z5<(gIdLK0|*bQFGyk0($-$Q?b`{HzU5ILN@jU1uA94R z-fKjN_{1$J`ZRq9?$JKDZ4gG6BhW{NTbhiJ+<~S9VK`9D0}&RMhPh0$yir8avKx|F z0X)Exr)#0;6x^eDsMak99#r^7H$w^I5R6&~MMC z?YYN3O%AN&TmnDs2wXD|8WaK!QT1@HiP~oYI`gM&?~kf4U%vhQ|MsqZ_T5q$)(7u> zgAYat{})jgZ*Sue@p7 z{$$Lq^FQ}kWcKZQpR3QcbGHT;rM-XMdv5!IkGit=uUySGKlZWrCv4i#;lWLg9Q)bZ zzpg_I8gStUjevC354TqDPJVsX{Fwjm{e9Wr+Y{L9e-}p0uW0&S5(m{e6qdim6GP4v| z&k8jqz$#t18xJr6?O~0G^TM4vk&_-`&aWQtTDhID8q?qV0K=Qi zlfQ~#0?26rn9>hOa_!l=+pco^`Stm|yW7?Ozvc!ONJlu|+?RVidw#h3PXGF2bN?*T zho``Qq7i?8Uf+B6KF30w>3$-U{U5_)>i6aU?|%QEO?Z8LRDb`vHOF25i`Z=PU&v{~ zea8xTT-W`u$LAhC`~2p1e}Bz9Q~B0(XuC;HD}mu#fZToT8Sz1BPNQv>1Vx8)ys; z(HmT6b-@RV!1Ti-1(WfLmFK`Ke+D!&5JekuiNnaWl8G&0<=Kt3+iJQvv~=ZM8i@ub@Y`*sD6VR^%dyJ zuj&lwR>BNJ^=xAZF!5Y@_5@>X6{<35VZR{m0K=MpEDTsBF)W+_OyJXQ&A9`28D7Q} zZi$Ag^BEX$wi8g|I0zW;L9-_#HKH-{d#=!*aV diff --git a/docs/user_guide/rollback.md b/docs/user_guide/rollback.md index a4e066e5..2c08268b 100644 --- a/docs/user_guide/rollback.md +++ b/docs/user_guide/rollback.md @@ -1,39 +1,116 @@ ### 功能介绍: -节点回退允许流程回退到某个特定的节点重新开始。**该回退会删除目标节点之后所有已执行过的节点信息和数据，让流程表现的是第一次执行的样子。** -需要注意的流程的回退并不是无限制的，需要满足以下条件的节点才允许回退。 -- 只能回退运行中的流程 -- 子流程暂时不支持回退 -- 目标节点的状态必须为已完成状态 -- 并行网关内的节点不允许回退。并行网关包含条件并行网关和并行网关 -- 网关节点不允许回退 -- 条件网关只允许条件网关内已经运行的节点允许回退，未执行到的分支不允许回退。 +## 回滚配置 -节点在回退前会强制失败掉当前运行的节点，只有流程中没有正在运行的节点时才会开始执行回退逻辑。 -节点回退的过程无法中断，因为中断导致的回退失败可能会导致无法通过再次回退重试成功 +### 开启配置项 -针对如下图的流程，蓝色框所标注的节点是允许回退的节点。 +在开始体验回滚之前，我们需要在django_settings 中开启以下的配置: -![rollback.png](..%2Fassets%2Fimg%2Frollback%2Frollback.png) +```python +PIPELINE_ENABLE_ROLLBACK = True +ROLLBACK_QUEUE = "default_test" +PIPELINE_ENABLE_AUTO_EXECUTE_WHEN_ROLL_BACKED = False +``` -### 使用事例: +其中： +- PIPELINE_ENABLE_ROLLBACK 表示开启回滚 +- ROLLBACK_QUEUE: 表示回滚所使用的队列 +- PIPELINE_ENABLE_AUTO_EXECUTE_WHEN_ROLL_BACKED: 是否开启回滚后自动开始，开启时，回滚到目标节点将会自动开始，未开启时流程回到目标节点将会暂停。 -查询可以回退的节点列表: +### Install App +之后需要在 `INSTALLED_APPS` 增加配置: ```python -from pipeline.contrib.rollback import api +INSTALLED_APPS += ( + "pipeline.contrib.rollback", +) +``` -# 获取该pipeline允许回滚的节点列表 -result = api.get_allowed_rollback_node_id_list(pipeline_id) -node_ids = result.data +### 执行 migrate 操作 +```bash +python manage.py migrate rollback ``` -节点的回退使用非常简单，只需要指定pipeline_id和node_id即可，如下所示: -```python +之后回滚的一切便已经就绪了。 + +## 回滚的边界条件 + +### Token 模式: + + +现阶段的回滚的行为受到严格限制，在TOKEN 模式下，回滚将不能随意的指向某个节点。流程回滚时将沿着原路径依次回滚(如果存在子流程，则先回滚子流程，再继续主流程的回滚)。 +在流程回滚时, 节点的状态机如下: + +![rollback.png](..%2Fassets%2Fimg%2Frollback%2Frollback.png) + +以下是回滚的各项边界条件： + +#### 任务状态 + +只有处于 `RUNNING` 和 `ROLL_BACK_FAILED` 状态的任务才允许回滚。当任务处于结束，暂停时，将不允许回滚。 + +#### 任务节点 + +在 token 模式下，**回滚的起始和目标节点的token须保持一致**。同时不允许同 token下 **存在正在运行的节点**。 + +##### 回滚的起点 + +- 回滚开始的节点的状态只支持`FINISHED` 或 `FAILED`, 正在运行的节点将不允许回滚。 +- 回滚开始的节点必须是流程当前正在运行的节点之一，也就是流程的末端节点。 + +##### 回滚的目标节点 + +- 回滚的目标节点只支持`FINISHED` 状态，回滚的目标节点只支持任务类型的节点，网关节点不支持回滚。 + +#### 回滚预约 +- 回滚预约只能预约为RUNNING状态的节点，当该节点结束时，将自动开始回滚。 +- **一个流程同时只能有一个预约回滚的任务** + + +### ANY 模式: + +当为ANY 模式的回滚时，流程可以从任何地方开始，回滚到之前的任意节点上去，此时流程**将不会按照路径调用回滚(不会调用节点的rollback方法)**，而是直接回到目标节点，并删除回滚路径上已经执行过的节点信息，从目标位置开始。 + +#### 任务状态 + +只有处于 `RUNNING` 的任务才允许回滚。当任务处于结束，暂停时，将不允许回滚。 + +#### 任务节点 + +在 any 模式下，回滚的边界条件将少得多，由于 any 状态下的回滚将直接回到目标节点并开始，类似于任务的任意节点跳转。 + +在 any 模式下，回滚开始前**不允许当前流程存在处于 running 状态的节点。** + +##### 回滚的起点 + +- 回滚开始的节点必须是流程当前正在运行的节点，也就是流程的末端节点。 +- -回滚开始的节点的状态只支持`FINISHED` 或 `FAILED`, 正在运行的节点将不再允许回滚。 + +##### 回滚的目标节点 + +- 回滚的目标节点只支持`FINISHED` 状态，回滚的目标节点只支持任务节点类型。 + +#### 回滚预约 +- 回滚预约只能预约为running状态的节点，当该节点结束时，将自动开始回滚。 +- **一个流程同时只能有一个预约回滚的任务** + +#### 回滚的使用: + +``` python from pipeline.contrib.rollback import api -result = api.rollback(pipeline_id, node_id) -if result.result: - pass -``` \ No newline at end of file +# 节点回滚，其中mode 有 TOKEN 和 ANY 两种模式可选 +api.rollback(root_pipeline_id, start_node_id, target_node_id, mode="TOKEN") + +# 回滚预约 +api.reserve_rollback(root_pipeline_id, start_node_id, target_node_id, mode="TOKEN") + +# 取消回滚预约 +api.cancel_reserved_rollback(root_pipeline_id, start_node_id, target_node_id, mode="TOKEN") + + +# 获取本次回滚支持的范围 +api.get_allowed_rollback_node_id_list(root_pipeline_id, start_node_id, mode="TOKEN") + +``` diff --git a/runtime/bamboo-pipeline/pipeline/conf/default_settings.py b/runtime/bamboo-pipeline/pipeline/conf/default_settings.py index 437cfe9d..5f52c1a6 100644 --- a/runtime/bamboo-pipeline/pipeline/conf/default_settings.py +++ b/runtime/bamboo-pipeline/pipeline/conf/default_settings.py @@ -104,3 +104,5 @@ # 是否开启PIPELINE HOOKS 事件通知 ENABLE_PIPELINE_EVENT_SIGNALS = getattr(settings, "ENABLE_PIPELINE_EVENT_SIGNALS", False) + +ROLLBACK_QUEUE = getattr(settings, "ROLLBACK_QUEUE", "rollback") diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/api.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/api.py index d123c1a7..f8deac76 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/rollback/api.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/api.py @@ -10,23 +10,40 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from pipeline.contrib.rollback.handler import RollBackHandler +from pipeline.contrib.rollback.constants import TOKEN +from pipeline.contrib.rollback.handler import RollbackDispatcher from pipeline.contrib.utils import ensure_return_pipeline_contrib_api_result @ensure_return_pipeline_contrib_api_result -def rollback(root_pipeline_id: str, node_id: str): +def rollback( + root_pipeline_id: str, start_node_id: str, target_node_id: str, skip_rollback_nodes: list = None, mode: str = TOKEN +): """ :param root_pipeline_id: pipeline id - :param node_id: 节点 id + :param start_node_id: 开始的 id + :param target_node_id: 开始的 id + :param mode 回滚模式 :return: True or False - - 回退的思路是，先搜索计算出来当前允许跳过的节点，在计算的过程中网关节点会合并成一个节点 - 只允许回退到已经执行过的节点 """ - RollBackHandler(root_pipeline_id, node_id).rollback() + RollbackDispatcher(root_pipeline_id, mode).rollback(start_node_id, target_node_id) + + +@ensure_return_pipeline_contrib_api_result +def reserve_rollback(root_pipeline_id: str, start_node_id: str, target_node_id: str, mode: str = TOKEN): + RollbackDispatcher(root_pipeline_id, mode).reserve_rollback(start_node_id, target_node_id) + + +@ensure_return_pipeline_contrib_api_result +def cancel_reserved_rollback(root_pipeline_id: str, start_node_id: str, target_node_id: str, mode: str = TOKEN): + RollbackDispatcher(root_pipeline_id, mode).cancel_reserved_rollback(start_node_id, target_node_id) + + +@ensure_return_pipeline_contrib_api_result +def retry_rollback_failed_node(root_pipeline_id: str, node_id: str, retry_data: dict = None, mode: str = TOKEN): + RollbackDispatcher(root_pipeline_id, mode).retry_rollback_failed_node(node_id, retry_data) @ensure_return_pipeline_contrib_api_result -def get_allowed_rollback_node_id_list(root_pipeline_id: str): - return RollBackHandler(root_pipeline_id, None).get_allowed_rollback_node_id_list() +def get_allowed_rollback_node_id_list(root_pipeline_id: str, start_node_id: str, mode: str = TOKEN): + return RollbackDispatcher(root_pipeline_id, mode).get_allowed_rollback_node_id_list(start_node_id) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/apps.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/apps.py new file mode 100644 index 00000000..5072026b --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/apps.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.apps import AppConfig + + +class RollbackConfig(AppConfig): + name = "pipeline.contrib.rollback" + verbose_name = "PipelineRollback" + + def ready(self): + pass diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/constants.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/constants.py new file mode 100644 index 00000000..3798671b --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/constants.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# 回滚开始的标识位 +START_FLAG = "START" + +# 回滚结束的标志位 +END_FLAG = "END" + +ANY = "ANY" # 任意跳转模式，此时将不再检查token，可以任意回退到指定节点 +TOKEN = "TOKEN" # TOKEN 跳转模式，只允许跳转到指定的范围的节点 diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/graph.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/graph.py new file mode 100644 index 00000000..8b2863cd --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/graph.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +import copy + +from pipeline.contrib.rollback import constants +from pipeline.core.constants import PE + +from bamboo_engine.utils.graph import RollbackGraph + + +class CycleHandler: + """ + 环处理器，作用是去除拓扑中的环 + """ + + def __init__(self, node_map): + self.node_map = copy.deepcopy(node_map) + + def get_nodes_and_edges(self): + """ + 从node_map 中解析出环和边 + """ + nodes = [] + edges = [] + for node, value in self.node_map.items(): + nodes.append(node) + targets = value["targets"] + for target in targets.values(): + # 过滤掉那些没有执行的分支 + if target not in self.node_map: + continue + edges.append([node, target]) + return nodes, edges + + def has_cycle(self, nodes, edges) -> list: + """ + 判断是否有环，存在环是，将返回一个有效的list + """ + graph = RollbackGraph(nodes, edges) + return graph.get_cycle() + + def delete_edge(self, source, target): + """ + 删除环边 + """ + targets = self.node_map[source]["targets"] + + keys_to_remove = [] + for key, val in targets.items(): + if val == target: + keys_to_remove.append(key) + for key in keys_to_remove: + del targets[key] + + def remove_cycle(self): + while True: + nodes, edges = self.get_nodes_and_edges() + cycles = self.has_cycle(nodes, edges) + if not cycles: + break + source = cycles[-2] + target = cycles[-1] + self.delete_edge(source, target) + return self.node_map + + +class RollbackGraphHandler: + def __init__(self, node_map, start_id, target_id): + self.graph = RollbackGraph() + # 回滚开始的节点 + self.start_id = start_id + # 回滚结束的节点 + self.target_id = target_id + self.graph.add_node(start_id) + self.graph.add_node(target_id) + # 去除自环边 + self.node_map = CycleHandler(node_map).remove_cycle() + # 其他不参与回滚，但是需要被清理的节点，主要是网关节点和子流程节点 + self.others_nodes = [] + + def build(self, node_id, source_id=None): + """ + 使用递归构建用于回滚的图谱，最终会生成一条连线 source_id -> node_id + @param node_id 本次遍历到的节点id + @param source_id 上一个遍历到的节点id + """ + node_detail = self.node_map.get(node_id) + if node_detail is None: + return + node_type = node_detail["type"] + + if node_type not in [PE.ServiceActivity]: + self.others_nodes.append(node_id) + + if node_type == PE.ServiceActivity: + next_node_id = node_detail.get("id") + self.graph.add_node(next_node_id) + if source_id and source_id != next_node_id: + self.graph.add_edge(source_id, next_node_id) + + # 如果遍历到目标节点，则返回 + if node_id == self.start_id: + return + source_id = next_node_id + targets = node_detail.get("targets", {}).values() + elif node_type == PE.SubProcess: + # 处理子流程 + source_id = self.build(node_detail["start_event_id"], source_id) + targets = node_detail.get("targets", {}).values() + elif node_type == PE.ExclusiveGateway: + targets = [target for target in node_detail.get("targets", {}).values() if target in self.node_map.keys()] + else: + targets = node_detail.get("targets", {}).values() + + # 为了避免循环的过程中source_id值被覆盖，需要额外临时存储source_id + temporary_source_id = source_id + for target in targets: + source_id = self.build(target, temporary_source_id) + + return source_id + + def build_rollback_graph(self): + """ + 这里将会从结束的节点往开始的节点进行遍历，之后再反转图 + """ + self.graph.add_node(constants.END_FLAG) + # 未整个流程加上结束节点 + self.graph.add_edge(constants.END_FLAG, self.target_id) + self.graph.add_node(constants.START_FLAG) + self.graph.add_edge(self.start_id, constants.START_FLAG) + self.build(self.target_id, self.target_id) + + return self.graph.reverse(), self.others_nodes diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py index b11fff64..a6394044 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py @@ -14,197 +14,443 @@ import json from django.db import transaction +from django.db.models import Q +from pipeline.conf.default_settings import ROLLBACK_QUEUE from pipeline.contrib.exceptions import RollBackException -from pipeline.core.constants import PE -from pipeline.eri.models import ( - CallbackData, - ExecutionData, - ExecutionHistory, - LogEntry, - Node, - Process, - Schedule, - State, +from pipeline.contrib.rollback import constants +from pipeline.contrib.rollback.constants import ANY, TOKEN +from pipeline.contrib.rollback.graph import RollbackGraphHandler +from pipeline.contrib.rollback.models import ( + RollbackPlan, + RollbackSnapshot, + RollbackToken, ) +from pipeline.contrib.rollback.tasks import any_rollback, token_rollback +from pipeline.core.constants import PE +from pipeline.eri.models import Node, Process, State from pipeline.eri.runtime import BambooDjangoRuntime -from bamboo_engine import api, states +from bamboo_engine import states -class RollBackHandler: - def __init__(self, root_pipeline_id, node_id): - self.root_pipeline_id = root_pipeline_id - self.node_id = node_id - self.runtime = BambooDjangoRuntime() +class RollbackValidator: + @staticmethod + def validate_pipeline(root_pipeline_id): + pipeline_state = State.objects.filter(node_id=root_pipeline_id).first() + if not pipeline_state: + raise RollBackException( + "rollback failed: pipeline state not exist, pipeline_id={}".format(root_pipeline_id) + ) - def _compute_validate_nodes(self, node_id, node_map, nodes=None): - """ - 计算并得到一个允许回调的节点列表。 - 该方法的实现思路如下，从开始节点开始遍历，通过每个节点的 targets 获取到该节点的下一个节点 - - 对于并行网关和条件并行网关将直接跳过 - - 对于分支网关，则会裁剪只保留执行的那条分支 - - node_map 记录了所有已经执行过的节点的信息，当遍历到node_map中不存在的节点时，意味着已经遍历到了当前未执行的节点 - 此时会停止计算 - """ + if pipeline_state.name not in [states.RUNNING, states.ROLL_BACK_FAILED]: + raise RollBackException( + "rollback failed: the task of non-running state is not allowed to roll back, " + "pipeline_id={}, state={}".format(root_pipeline_id, pipeline_state.name) + ) + + @staticmethod + def validate_node(node_id, allow_failed=False): + node = Node.objects.filter(node_id=node_id).first() + if node is None: + raise RollBackException("rollback failed: node not exist, node={}".format(node_id)) + + node_detail = json.loads(node.detail) + if node_detail["type"] not in [PE.ServiceActivity, PE.EmptyStartEvent]: + raise RollBackException("rollback failed: only allows rollback to ServiceActivity type nodes") - if nodes is None: - nodes = [] - node_detail = node_map.get(node_id) - # 当搜索不到时，说明已经扫描了所有已经执行过的节点了，此时直接结束 - if node_detail is None: - return nodes + target_node_state = State.objects.filter(node_id=node_id).first() - if node_detail["type"] == PE.ServiceActivity: - nodes.append(node_id) + if target_node_state is None: + raise RollBackException("rollback failed: node state not exist, node={}".format(node_id)) - # 对于并行网关，无法跳转到任何路径 - if node_detail["type"] in [PE.ParallelGateway, PE.ConditionalParallelGateway]: - targets = [node_detail.get(PE.converge_gateway_id)] - # 对于分支网关内的, 只允许跳转到已经执行过的路径 - elif node_detail["type"] == PE.ExclusiveGateway: - targets = [target for target in node_detail.get("targets", {}).values() if target in node_map.keys()] - else: - targets = node_detail.get("targets", {}).values() + allow_states = [states.FINISHED] + if allow_failed: + allow_states = [states.FINISHED, states.FAILED] + if target_node_state.name not in allow_states: + raise RollBackException( + "rollback failed: only allows rollback to finished node, allowed states {}".format(allow_states) + ) - for target in targets: - # 如果目标节点已经出现在了node中，说明出现了环，跳过该分支 - if target in nodes: - continue - self._compute_validate_nodes(target, node_map, nodes) + @staticmethod + def validate_token(root_pipeline_id, start_node_id, target_node_id): + try: + rollback_token = RollbackToken.objects.get(root_pipeline_id=root_pipeline_id) + except RollbackToken.DoesNotExist: + raise RollBackException( + "rollback failed: pipeline token not exist, pipeline_id={}".format(root_pipeline_id) + ) - return nodes + tokens = json.loads(rollback_token.token) + + start_node_token = tokens.get(start_node_id) + target_node_token = tokens.get(target_node_id) - def _clean_engine_data(self, target_state): + if start_node_token is None or target_node_token is None: + raise RollBackException("rollback failed: token not found, pipeline_id={}".format(root_pipeline_id)) + + if start_node_token != target_node_token: + raise RollBackException( + "rollback failed: start node token must equal target node, pipeline_id={}".format(root_pipeline_id) + ) + + @staticmethod + def validate_node_state_by_token_mode(root_pipeline_id, start_node_id): """ - 执行清理工作 + 使用token模式下的回滚，相同token的节点不允许有正在运行的节点 """ - # 获取当前正在运行的节点 - state_list = State.objects.filter(root_id=self.root_pipeline_id, name=states.RUNNING).exclude( - node_id=self.root_pipeline_id - ) - for state in state_list: - # 强制失败这些节点 - result = api.forced_fail_activity(self.runtime, node_id=state.node_id, ex_data="") - if not result.result: - raise RollBackException( - "rollback failed: forced_fail_activity failed, node_id={}, message={}".format( - target_state.node_id, result.message - ) - ) + try: + rollback_token = RollbackToken.objects.get(root_pipeline_id=root_pipeline_id) + except RollbackToken.DoesNotExist: + raise RollBackException( + "rollback failed: pipeline token not exist, pipeline_id={}".format(root_pipeline_id) + ) - # 之后清理多余的进程信息，只保留主process即可。 - Process.objects.filter(root_pipeline_id=self.root_pipeline_id).exclude(parent_id=-1).delete() + tokens = json.loads(rollback_token.token) + start_token = tokens.get(start_node_id) + if start_token is None: + raise RollBackException("rollback failed: can't find the not token, node_id={}".format(start_node_id)) - # 查询到所有在该节点之后创建的状态信息 - need_clean_node_id_list = list( - State.objects.filter(root_id=self.root_pipeline_id, created_time__gt=target_state.created_time).values_list( - "node_id", flat=True - ) - ) - # 同时清理掉目标节点的信息 - need_clean_node_id_list.append(target_state.node_id) + node_id_list = [] + for node_id, token in node_id_list: + if token == start_token: + node_id_list.append(node_id) - # 清理状态信息 - State.objects.filter(root_id=self.root_pipeline_id, node_id__in=need_clean_node_id_list).delete() - # 清理Schedule 信息 - Schedule.objects.filter(node_id__in=need_clean_node_id_list).delete() - # 清理日志信息 - LogEntry.objects.filter(node_id__in=need_clean_node_id_list).delete() - ExecutionHistory.objects.filter(node_id__in=need_clean_node_id_list).delete() - ExecutionData.objects.filter(node_id__in=need_clean_node_id_list).delete() - CallbackData.objects.filter(node_id__in=need_clean_node_id_list).delete() + if State.objects.filter(node_id__in=node_id_list, name=states.RUNNING).exists(): + raise RollBackException( + "rollback failed: there is currently the same node that the same token is running, node_id={}".format( + start_node_id + ) + ) - def get_allowed_rollback_node_id_list(self): + @staticmethod + def validate_node_state_by_any_mode(root_pipeline_id): + """ + 使用any模式下的回滚, 不允许有正在运行的节点 """ - 获取允许回退的节点id列表 + if ( + State.objects.filter(root_id=root_pipeline_id, name=states.RUNNING) + .exclude(node_id=root_pipeline_id) + .exists() + ): + raise RollBackException("rollback failed: there is currently the some node is running") + + @staticmethod + def validate_start_node_id(root_pipeline_id, start_node_id): """ - # 不需要遍历整颗树，获取到现在已经执行成功的所有列表 + 回滚的开始节点必须是流程的末尾节点 + """ + if not Process.objects.filter(root_pipeline_id=root_pipeline_id, current_node_id=start_node_id).exists(): + raise RollBackException("rollback failed: The node to be rolled back must be the current node!") + + +class BaseRollbackHandler: + mode = None + + def __init__(self, root_pipeline_id): + self.root_pipeline_id = root_pipeline_id + self.runtime = BambooDjangoRuntime() + # 检查pipeline 回滚的合法性 + RollbackValidator.validate_pipeline(root_pipeline_id) + + def get_allowed_rollback_node_id_list(self, start_node_id): + return [] + + def _get_allowed_rollback_node_map(self): + # 不需要遍历整颗树，获取到现在已经执行成功和失败节点的所有列表 finished_node_id_list = ( - State.objects.filter(root_id=self.root_pipeline_id, name=states.FINISHED) + State.objects.filter(root_id=self.root_pipeline_id, name__in=[states.FINISHED, states.FAILED]) .exclude(node_id=self.root_pipeline_id) .values_list("node_id", flat=True) ) + node_detail_list = Node.objects.filter(node_id__in=finished_node_id_list) + # 获取node_id 到 node_detail的映射 + return {n.node_id: json.loads(n.detail) for n in node_detail_list} + + def _reserve(self, start_node_id, target_node_id, reserve_rollback=True): + # 节点预约 需要在 Node 里面 插入 reserve_rollback = True, 为 True的节点执行完将暂停 + RollbackValidator.validate_start_node_id(self.root_pipeline_id, start_node_id) + RollbackValidator.validate_node(target_node_id) + node = Node.objects.filter(node_id=start_node_id).first() + if node is None: + raise RollBackException("reserve rollback failed, the node is not exists, node_id={}".format(start_node_id)) + + state = State.objects.filter(node_id=start_node_id).first() + if state is None: + raise RollBackException( + "reserve rollback failed, the node state is not exists, node_id={}".format(start_node_id) + ) + + # 不在执行中的节点不允许预约 + if state.name != states.RUNNING: + raise RollBackException( + "reserve rollback failed, the node state is not Running, current state={}, node_id={}".format( + state.name, start_node_id + ) + ) - # 获取到除pipeline节点之外第一个被创建的节点，此时是开始节点 + with transaction.atomic(): + if reserve_rollback: + # 一个流程只能同时拥有一个预约任务 + if RollbackPlan.objects.filter(root_pipeline_id=self.root_pipeline_id, is_expired=False).exists(): + raise RollBackException( + "reserve rollback failed, the rollbackPlan, current state={}, node_id={}".format( + state.name, start_node_id + ) + ) + RollbackPlan.objects.create( + root_pipeline_id=self.root_pipeline_id, + start_node_id=start_node_id, + target_node_id=target_node_id, + mode=self.mode, + ) + else: + # 取消回滚，删除所有的任务 + RollbackPlan.objects.filter(root_pipeline_id=self.root_pipeline_id, start_node_id=start_node_id).update( + is_expired=True + ) + + node_detail = json.loads(node.detail) + node_detail["reserve_rollback"] = reserve_rollback + node.detail = json.dumps(node_detail) + node.save() + + def reserve_rollback(self, start_node_id, target_node_id): + """ + 预约回滚 + """ + RollbackValidator.validate_token(self.root_pipeline_id, start_node_id, target_node_id) + self._reserve(start_node_id, target_node_id) + + def cancel_reserved_rollback(self, start_node_id, target_node_id): + """ + 取消预约回滚 + """ + RollbackValidator.validate_token(self.root_pipeline_id, start_node_id, target_node_id) + self._reserve(start_node_id, target_node_id, reserve_rollback=False) + + +class AnyRollbackHandler(BaseRollbackHandler): + mode = ANY + + def get_allowed_rollback_node_id_list(self, start_node_id): + node_map = self._get_allowed_rollback_node_map() start_node_state = ( State.objects.filter(root_id=self.root_pipeline_id) .exclude(node_id=self.root_pipeline_id) .order_by("created_time") .first() ) + target_node_id = start_node_state.node_id + rollback_graph = RollbackGraphHandler(node_map=node_map, start_id=start_node_id, target_id=target_node_id) + graph, _ = rollback_graph.build_rollback_graph() - # 获取到所有当前已经运行完节点的详情 - node_detail_list = Node.objects.filter(node_id__in=finished_node_id_list) - # 获取node_id 到 node_detail的映射 - node_map = {n.node_id: json.loads(n.detail) for n in node_detail_list} + return list(set(graph.nodes) - {constants.START_FLAG, constants.END_FLAG, start_node_id}) - # 计算当前允许跳过的合法的节点 - validate_nodes_list = self._compute_validate_nodes(start_node_state.node_id, node_map) + def retry_rollback_failed_node(self, node_id, retry_data): + """ """ + raise RollBackException("rollback failed: when mode is any, not support retry") - return validate_nodes_list + def reserve_rollback(self, start_node_id, target_node_id): + """ + 预约回滚 + """ + self._reserve(start_node_id, target_node_id) - def rollback(self): - pipeline_state = State.objects.filter(node_id=self.root_pipeline_id).first() - if not pipeline_state: + def cancel_reserved_rollback(self, start_node_id, target_node_id): + """ + 取消预约回滚 + """ + self._reserve(start_node_id, target_node_id, reserve_rollback=False) + + def rollback(self, start_node_id, target_node_id, skip_rollback_nodes=None): + RollbackValidator.validate_node_state_by_any_mode(self.root_pipeline_id) + # 回滚的开始节点运行失败的情况 + RollbackValidator.validate_start_node_id(self.root_pipeline_id, start_node_id) + RollbackValidator.validate_node(start_node_id, allow_failed=True) + RollbackValidator.validate_node(target_node_id) + + node_map = self._get_allowed_rollback_node_map() + rollback_graph = RollbackGraphHandler(node_map=node_map, start_id=start_node_id, target_id=target_node_id) + + graph, other_nodes = rollback_graph.build_rollback_graph() + node_access_record = {node: 0 for node in graph.nodes} + + rollback_snapshot = RollbackSnapshot.objects.create( + root_pipeline_id=self.root_pipeline_id, + graph=json.dumps(graph.as_dict()), + node_access_record=json.dumps(node_access_record), + start_node_id=start_node_id, + target_node_id=target_node_id, + other_nodes=json.dumps(other_nodes), + skip_rollback_nodes=json.dumps([]), + ) + + any_rollback.apply_async( + kwargs={"snapshot_id": rollback_snapshot.id}, + queue=ROLLBACK_QUEUE, + ) + + +class TokenRollbackHandler(BaseRollbackHandler): + mode = TOKEN + + def get_allowed_rollback_node_id_list(self, start_node_id): + """ + 获取允许回滚的节点范围 + 规则：token 一致的节点允许回滚 + """ + try: + rollback_token = RollbackToken.objects.get(root_pipeline_id=self.root_pipeline_id) + except RollbackToken.DoesNotExist: raise RollBackException( - "rollback failed: pipeline state not exist, pipeline_id={}".format(self.root_pipeline_id) + "rollback failed: pipeline token not exist, pipeline_id={}".format(self.root_pipeline_id) ) + node_map = self._get_allowed_rollback_node_map() + service_activity_node_list = [ + node_id for node_id, node_detail in node_map.items() if node_detail["type"] == PE.ServiceActivity + ] - if pipeline_state.name != states.RUNNING: + tokens = json.loads(rollback_token.token) + start_token = tokens.get(start_node_id) + if not start_token: + return [] + + nodes = [] + for node_id, token in tokens.items(): + if start_token == token and node_id != start_node_id and node_id in service_activity_node_list: + nodes.append(node_id) + + return nodes + + def retry_rollback_failed_node(self, node_id, retry_data): + """ + 重试回滚失败的节点 + """ + pipeline_state = State.objects.filter(node_id=self.root_pipeline_id).first() + if pipeline_state.name != states.ROLL_BACK_FAILED: raise RollBackException( - "rollback failed: the task of non-running state is not allowed to roll back, pipeline_id={}".format( - self.root_pipeline_id - ) + "rollback failed: only retry the failed pipeline, current_status={}".format(pipeline_state.name) + ) + node_state = State.objects.filter(node_id=node_id).first() + if node_state.name != states.ROLL_BACK_FAILED: + raise RollBackException( + "rollback failed: only retry the failed node, current_status={}".format(node_state.name) ) - node = Node.objects.filter(node_id=self.node_id).first() - if node is None: - raise RollBackException("rollback failed: node not exist, node={}".format(self.node_id)) + # 获取镜像 + try: + rollback_snapshot = RollbackSnapshot.objects.get(root_pipeline_id=self.root_pipeline_id, is_expired=False) + except RollbackSnapshot.DoesNotExist: + raise RollBackException("rollback failed: the rollback snapshot is not exists, please check") + except RollbackSnapshot.MultipleObjectsReturned: + raise RollBackException("rollback failed: found multi not expired rollback snapshot, please check") + + # 重置pipeline的状态为回滚中 + self.runtime.set_state( + node_id=self.root_pipeline_id, + to_state=states.ROLLING_BACK, + ) - node_detail = json.loads(node.detail) - if node_detail["type"] not in [PE.ServiceActivity, PE.EmptyStartEvent]: - raise RollBackException("rollback failed: only allows rollback to ServiceActivity type nodes") + # 驱动这个任务 + token_rollback.apply_async( + kwargs={ + "snapshot_id": rollback_snapshot.id, + "node_id": node_id, + "retry": True, + "retry_data": retry_data, + }, + queue=ROLLBACK_QUEUE, + ) - target_node_state = State.objects.filter(node_id=self.node_id).first() + def _node_state_is_failed(self, node_id): + """ + 判断该节点是不是失败的状态 + """ + node_state = State.objects.filter(node_id=node_id).first() + if node_state.name == states.FAILED: + return True + return False + + def _get_failed_skip_node_id_list(self, node_id_list): + failed_skip_node_id_list = State.objects.filter( + Q(Q(skip=True) | Q(error_ignored=True)) & Q(node_id__in=node_id_list) + ).values_list("node_id", flat=True) + return failed_skip_node_id_list + + def rollback(self, start_node_id, target_node_id, skip_rollback_nodes=None): + + if skip_rollback_nodes is None: + skip_rollback_nodes = [] + + # 相同token回滚时，不允许有正在运行的节点 + RollbackValidator.validate_node_state_by_token_mode(self.root_pipeline_id, start_node_id) + # 回滚的开始节点运行失败的情况 + RollbackValidator.validate_node(start_node_id, allow_failed=True) + RollbackValidator.validate_node(target_node_id) + RollbackValidator.validate_token(self.root_pipeline_id, start_node_id, target_node_id) + + # 如果开始节点是失败的情况，则跳过该节点的回滚操作 + if self._node_state_is_failed(start_node_id): + skip_rollback_nodes.append(start_node_id) + + node_map = self._get_allowed_rollback_node_map() + rollback_graph = RollbackGraphHandler(node_map=node_map, start_id=start_node_id, target_id=target_node_id) + + runtime = BambooDjangoRuntime() + + graph, other_nodes = rollback_graph.build_rollback_graph() + node_access_record = {node: 0 for node in graph.nodes} + + # 所有失败并跳过的节点不再参与回滚 + failed_skip_node_id_list = self._get_failed_skip_node_id_list(node_map.keys()) + skip_rollback_nodes.extend(list(failed_skip_node_id_list)) + + rollback_snapshot = RollbackSnapshot.objects.create( + root_pipeline_id=self.root_pipeline_id, + graph=json.dumps(graph.as_dict()), + node_access_record=json.dumps(node_access_record), + start_node_id=start_node_id, + target_node_id=target_node_id, + other_nodes=json.dumps(other_nodes), + skip_rollback_nodes=json.dumps(skip_rollback_nodes), + ) - if target_node_state is None: - raise RollBackException("rollback failed: node state not exist, node={}".format(self.node_id)) + runtime.set_state( + node_id=self.root_pipeline_id, + to_state=states.ROLLING_BACK, + ) + # 驱动这个任务 + token_rollback.apply_async( + kwargs={ + "snapshot_id": rollback_snapshot.id, + "node_id": constants.START_FLAG, + "retry": False, + "retry_data": None, + }, + queue=ROLLBACK_QUEUE, + ) - if target_node_state.name != states.FINISHED: - raise RollBackException("rollback failed: only allows rollback to finished node") - validate_nodes_list = self.get_allowed_rollback_node_id_list() +class RollbackDispatcher: + def __init__(self, root_pipeline_id, mode): + if mode == ANY: + self.handler = AnyRollbackHandler(root_pipeline_id) + elif mode == TOKEN: + self.handler = TokenRollbackHandler(root_pipeline_id) + else: + raise RollBackException("rollback failed: not support this mode, please check") - if self.node_id not in validate_nodes_list: - raise RollBackException("rollback failed: node is not allow to rollback, node={}".format(self.node_id)) + def rollback(self, start_node_id: str, target_node_id: str, skip_rollback_nodes: list = None): + self.handler.rollback(start_node_id, target_node_id, skip_rollback_nodes) - with transaction.atomic(): - try: - self._clean_engine_data(target_node_state) - except Exception as e: - raise RollBackException("rollback failed: clean engine data error, error={}".format(str(e))) - - try: - # 将当前住进程的正在运行的节点指向目标ID - main_process = Process.objects.get(root_pipeline_id=self.root_pipeline_id, parent_id=-1) - main_process.current_node_id = self.node_id - main_process.save() - - # 重置该节点的状态信息 - self.runtime.set_state( - node_id=self.node_id, - to_state=states.READY, - is_retry=True, - refresh_version=True, - clear_started_time=True, - clear_archived_time=True, - ) - process_info = self.runtime.get_process_info(main_process.id) - self.runtime.execute( - process_id=process_info.process_id, - node_id=self.node_id, - root_pipeline_id=process_info.root_pipeline_id, - parent_pipeline_id=process_info.top_pipeline_id, - ) - except Exception as e: - raise RollBackException("rollback failed: rollback to node error, error={}".format(str(e))) + def reserve_rollback(self, start_node_id: str, target_node_id: str): + self.handler.reserve_rollback(start_node_id, target_node_id) + + def retry_rollback_failed_node(self, node_id: str, retry_data: dict = None): + self.handler.retry_rollback_failed_node(node_id, retry_data) + + def cancel_reserved_rollback(self, start_node_id: str, target_node_id: str): + self.handler.cancel_reserved_rollback(start_node_id, target_node_id) + + def get_allowed_rollback_node_id_list(self, start_node_id: str): + return self.handler.get_allowed_rollback_node_id_list(start_node_id) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0001_initial.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0001_initial.py new file mode 100644 index 00000000..fd0a18d2 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0001_initial.py @@ -0,0 +1,65 @@ +# Generated by Django 3.2.18 on 2023-10-16 07:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="RollbackNodeSnapshot", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("root_pipeline_id", models.CharField(db_index=True, max_length=64, verbose_name="root pipeline id")), + ("node_id", models.CharField(db_index=True, max_length=64, verbose_name="node_id")), + ("code", models.CharField(max_length=64, verbose_name="node_code")), + ("version", models.CharField(max_length=33, verbose_name="version")), + ("inputs", models.TextField(verbose_name="node inputs")), + ("outputs", models.TextField(verbose_name="node outputs")), + ("context_values", models.TextField(verbose_name="pipeline context values")), + ("rolled_back", models.BooleanField(default=False, verbose_name="whether the node rolls back")), + ], + ), + migrations.CreateModel( + name="RollbackPlan", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("root_pipeline_id", models.CharField(db_index=True, max_length=64, verbose_name="root pipeline id")), + ("start_node_id", models.CharField(db_index=True, max_length=64, verbose_name="start node id")), + ("target_node_id", models.CharField(db_index=True, max_length=64, verbose_name="target_node_id")), + ("mode", models.CharField(default="TOKEN", max_length=32, verbose_name="rollback mode")), + ("is_expired", models.BooleanField(default=False, verbose_name="is expired")), + ], + ), + migrations.CreateModel( + name="RollbackSnapshot", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("root_pipeline_id", models.CharField(db_index=True, max_length=64, verbose_name="root pipeline id")), + ("graph", models.TextField(verbose_name="rollback graph")), + ("node_access_record", models.TextField(verbose_name="node access record")), + ("skip_rollback_nodes", models.TextField(verbose_name="skip rollback nodes")), + ("other_nodes", models.TextField(verbose_name="other nodes")), + ("start_node_id", models.CharField(db_index=True, max_length=64, verbose_name="start node id")), + ("target_node_id", models.CharField(db_index=True, max_length=64, verbose_name="target_node_id")), + ("is_expired", models.BooleanField(db_index=True, default=False, verbose_name="is expired")), + ], + ), + migrations.CreateModel( + name="RollbackToken", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("root_pipeline_id", models.CharField(db_index=True, max_length=64, verbose_name="root pipeline id")), + ("token", models.TextField(verbose_name="token map")), + ( + "is_deleted", + models.BooleanField( + db_index=True, default=False, help_text="is deleted", verbose_name="is deleted" + ), + ), + ], + ), + ] diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/__init__.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py new file mode 100644 index 00000000..b482f0a6 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +from django.db import models +from django.utils.translation import ugettext_lazy as _ +from pipeline.contrib.rollback.constants import TOKEN + + +class RollbackToken(models.Model): + """ + 回滚配置token信息 + """ + + root_pipeline_id = models.CharField(verbose_name="root pipeline id", max_length=64, db_index=True) + token = models.TextField(_("token map"), null=False) + is_deleted = models.BooleanField(_("is deleted"), default=False, help_text=_("is deleted"), db_index=True) + + +class RollbackSnapshot(models.Model): + """ + 节点执行的快照信息 + """ + + root_pipeline_id = models.CharField(verbose_name="root pipeline id", max_length=64, db_index=True) + graph = models.TextField(verbose_name="rollback graph", null=False) + node_access_record = models.TextField(verbose_name="node access record") + skip_rollback_nodes = models.TextField(verbose_name="skip rollback nodes") + other_nodes = models.TextField(verbose_name="other nodes") + start_node_id = models.CharField(verbose_name="start node id", max_length=64, db_index=True) + target_node_id = models.CharField(verbose_name="target_node_id", max_length=64, db_index=True) + is_expired = models.BooleanField(verbose_name="is expired", default=False, db_index=True) + + +class RollbackNodeSnapshot(models.Model): + """ + 节点快照 + """ + + root_pipeline_id = models.CharField(verbose_name="root pipeline id", max_length=64, db_index=True) + node_id = models.CharField(verbose_name="node_id", max_length=64, db_index=True) + code = models.CharField(verbose_name="node_code", max_length=64) + version = models.CharField(verbose_name=_("version"), null=False, max_length=33) + inputs = models.TextField(verbose_name=_("node inputs")) + outputs = models.TextField(verbose_name=_("node outputs")) + context_values = models.TextField(verbose_name=_("pipeline context values")) + rolled_back = models.BooleanField(_("whether the node rolls back"), default=False) + + +class RollbackPlan(models.Model): + root_pipeline_id = models.CharField(verbose_name="root pipeline id", max_length=64, db_index=True) + start_node_id = models.CharField(verbose_name="start node id", max_length=64, db_index=True) + target_node_id = models.CharField(verbose_name="target_node_id", max_length=64, db_index=True) + mode = models.CharField(verbose_name="rollback mode", max_length=32, default=TOKEN) + is_expired = models.BooleanField(verbose_name="is expired", default=False) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/tasks.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/tasks.py new file mode 100644 index 00000000..cabd7c4b --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/tasks.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- +import json +import logging + +from celery import task +from django.conf import settings +from django.db import transaction +from pipeline.conf.default_settings import ROLLBACK_QUEUE +from pipeline.contrib.rollback import constants +from pipeline.contrib.rollback.models import ( + RollbackNodeSnapshot, + RollbackPlan, + RollbackSnapshot, +) +from pipeline.eri.models import CallbackData +from pipeline.eri.models import ExecutionData as DBExecutionData +from pipeline.eri.models import ( + ExecutionHistory, + LogEntry, + Node, + Process, + Schedule, + State, +) +from pipeline.eri.runtime import BambooDjangoRuntime + +from bamboo_engine import states +from bamboo_engine.eri import ExecutionData +from bamboo_engine.utils.graph import RollbackGraph + +logger = logging.getLogger("celery") + + +class RollbackCleaner: + def __init__(self, snapshot): + self.snapshot = snapshot + + def _clear_node_reserve_flag(self, node_id): + node = Node.objects.get(node_id=node_id) + node_detail = json.loads(node.detail) + node_detail["reserve_rollback"] = False + node.detail = json.dumps(node_detail) + node.save() + + def clear_data(self): + # 节点快照需要全部删除，可能会有下一次的回滚 + RollbackNodeSnapshot.objects.filter(root_pipeline_id=self.snapshot.root_pipeline_id).delete() + # 回滚快照需要置为已过期 + RollbackSnapshot.objects.filter(root_pipeline_id=self.snapshot.root_pipeline_id).update(is_expired=True) + # 预约计划需要修改为已过期 + RollbackPlan.objects.filter( + root_pipeline_id=self.snapshot.root_pipeline_id, start_node_id=self.snapshot.start_node_id + ).update(is_expired=True) + # 节点的预约信息需要清理掉 + self._clear_node_reserve_flag(self.snapshot.start_node_id) + # 需要删除该节点的进程信息/非主进程的，防止网关再分支处回滚时，仍然有正在运行的process得不到清理 + Process.objects.filter( + root_pipeline_id=self.snapshot.root_pipeline_id, current_node_id=self.snapshot.start_node_id + ).exclude(parent_id=-1).delete() + + graph = json.loads(self.snapshot.graph) + need_clean_node_id_list = graph["nodes"] + json.loads(self.snapshot.other_nodes) + # 清理状态信息 + State.objects.filter(root_id=self.snapshot.root_pipeline_id, node_id__in=need_clean_node_id_list).delete() + # 清理Schedule 信息 + Schedule.objects.filter(node_id__in=need_clean_node_id_list).delete() + # 清理日志信息 + LogEntry.objects.filter(node_id__in=need_clean_node_id_list).delete() + ExecutionHistory.objects.filter(node_id__in=need_clean_node_id_list).delete() + DBExecutionData.objects.filter(node_id__in=need_clean_node_id_list).delete() + CallbackData.objects.filter(node_id__in=need_clean_node_id_list).delete() + + +class TokenRollbackTaskHandler: + def __init__(self, snapshot_id, node_id, retry, retry_data): + self.snapshot_id = snapshot_id + self.node_id = node_id + self.retry_data = retry_data + self.retry = retry + self.runtime = BambooDjangoRuntime() + + def set_state(self, node_id, state): + logger.info("[TokenRollbackTaskHandler][set_state] set node_id state to {}".format(state)) + # 开始和结束节点直接跳过回滚 + if node_id in [constants.END_FLAG, constants.START_FLAG]: + return + self.runtime.set_state( + node_id=node_id, + to_state=state, + ) + + def execute_rollback(self): + """ + 执行回滚的操作 + """ + if self.node_id in [constants.END_FLAG, constants.START_FLAG]: + return True + + # 获取节点快照，可能会有多多份快照，需要多次回滚 + node_snapshots = RollbackNodeSnapshot.objects.filter(node_id=self.node_id, rolled_back=False).order_by("-id") + for node_snapshot in node_snapshots: + service = self.runtime.get_service(code=node_snapshot.code, version=node_snapshot.version) + data = ExecutionData(inputs=json.loads(node_snapshot.inputs), outputs=json.loads(node_snapshot.outputs)) + parent_data = ExecutionData(inputs=json.loads(node_snapshot.context_values), outputs={}) + result = service.service.rollback(data, parent_data, self.retry_data) + node_snapshot.rolled_back = True + node_snapshot.save() + if not result: + return False + + return True + + def start_pipeline(self, root_pipeline_id, target_node_id): + """ + 启动pipeline + """ + # 将当前住进程的正在运行的节点指向目标ID + main_process = Process.objects.get(root_pipeline_id=root_pipeline_id, parent_id=-1) + main_process.current_node_id = target_node_id + main_process.save() + + # 重置该节点的状态信息 + self.runtime.set_state( + node_id=target_node_id, + to_state=states.READY, + is_retry=True, + refresh_version=True, + clear_started_time=True, + clear_archived_time=True, + ) + + process_info = self.runtime.get_process_info(main_process.id) + # 设置pipeline的状态 + self.runtime.set_state( + node_id=root_pipeline_id, + to_state=states.READY, + ) + + # 如果开启了流程自动回滚，则会开启到目标节点之后自动开始 + if getattr(settings, "PIPELINE_ENABLE_AUTO_EXECUTE_WHEN_ROLL_BACKED", True): + self.runtime.set_state( + node_id=root_pipeline_id, + to_state=states.RUNNING, + ) + else: + # 流程设置为暂停状态，需要用户点击才可以继续开始 + self.runtime.set_state( + node_id=root_pipeline_id, + to_state=states.SUSPENDED, + ) + + self.runtime.execute( + process_id=process_info.process_id, + node_id=target_node_id, + root_pipeline_id=process_info.root_pipeline_id, + parent_pipeline_id=process_info.top_pipeline_id, + ) + + def rollback(self): + with transaction.atomic(): + rollback_snapshot = RollbackSnapshot.objects.select_for_update().get(id=self.snapshot_id, is_expired=False) + node_access_record = json.loads(rollback_snapshot.node_access_record) + # 只有非重试状态下才需要记录访问 + if not self.retry: + node_access_record[self.node_id] += 1 + rollback_snapshot.node_access_record = json.dumps(node_access_record) + rollback_snapshot.save() + + graph = json.loads(rollback_snapshot.graph) + target_node_id = rollback_snapshot.target_node_id + rollback_graph = RollbackGraph(nodes=graph["nodes"], flows=graph["flows"]) + skip_rollback_nodes = json.loads(rollback_snapshot.skip_rollback_nodes) + in_degrees = rollback_graph.in_degrees() + + clearner = RollbackCleaner(rollback_snapshot) + + if node_access_record[self.node_id] >= in_degrees[self.node_id]: + # 对于不需要跳过的节点才会执行具体的回滚行为 + if self.node_id not in skip_rollback_nodes: + try: + # 设置节点状态为回滚中 + self.set_state(self.node_id, states.ROLLING_BACK) + # 执行同步回滚的操作 + result = self.execute_rollback() + except Exception as e: + logger.error( + "[TokenRollbackTaskHandler][rollback] execute rollback error," + "snapshot_id={}, node_id={}, err={}".format(self.snapshot_id, self.node_id, e) + ) + # 节点和流程重置为回滚失败的状态 + self.set_state(rollback_snapshot.root_pipeline_id, states.ROLL_BACK_FAILED) + # 回滚失败的节点将不再向下执行 + self.set_state(self.node_id, states.ROLL_BACK_FAILED) + return + + # 节点回滚成功 + if result: + self.set_state(self.node_id, states.ROLL_BACK_SUCCESS) + else: + logger.info( + "[TokenRollbackTaskHandler][rollback], execute rollback failed, " + "result=False, snapshot_id={}, node_id={}".format(self.snapshot_id, self.node_id) + ) + self.set_state(self.node_id, states.ROLL_BACK_FAILED) + # 回滚失败的节点将不再向下执行 + self.set_state(rollback_snapshot.root_pipeline_id, states.ROLL_BACK_FAILED) + return + + next_node = rollback_graph.next(self.node_id) + if list(next_node)[0] == constants.END_FLAG: + self.set_state(rollback_snapshot.root_pipeline_id, states.ROLL_BACK_SUCCESS) + try: + clearner.clear_data() + self.start_pipeline( + root_pipeline_id=rollback_snapshot.root_pipeline_id, target_node_id=target_node_id + ) + except Exception as e: + logger.error("[TokenRollbackTaskHandler][rollback] start_pipeline failed, err={}".format(e)) + return + return + + for node in next_node: + token_rollback.apply_async( + kwargs={ + "snapshot_id": self.snapshot_id, + "node_id": node, + }, + queue=ROLLBACK_QUEUE, + ) + + +class AnyRollbackHandler: + def __init__(self, snapshot_id): + self.snapshot_id = snapshot_id + self.runtime = BambooDjangoRuntime() + + def start_pipeline(self, root_pipeline_id, target_node_id): + """ + 启动pipeline + """ + + main_process = Process.objects.get(root_pipeline_id=root_pipeline_id, parent_id=-1) + main_process.current_node_id = target_node_id + main_process.save() + + # 重置该节点的状态信息 + self.runtime.set_state( + node_id=target_node_id, + to_state=states.READY, + is_retry=True, + refresh_version=True, + clear_started_time=True, + clear_archived_time=True, + ) + + process_info = self.runtime.get_process_info(main_process.id) + + # 如果PIPELINE_ENABLE_AUTO_EXECUTE_WHEN_ROLL_BACKED为False, 那么则会重制流程为暂停状态 + if not getattr(settings, "PIPELINE_ENABLE_AUTO_EXECUTE_WHEN_ROLL_BACKED", True): + self.runtime.set_state( + node_id=root_pipeline_id, + to_state=states.SUSPENDED, + ) + + self.runtime.execute( + process_id=process_info.process_id, + node_id=target_node_id, + root_pipeline_id=process_info.root_pipeline_id, + parent_pipeline_id=process_info.top_pipeline_id, + ) + + def rollback(self): + with transaction.atomic(): + rollback_snapshot = RollbackSnapshot.objects.get(id=self.snapshot_id, is_expired=False) + clearner = RollbackCleaner(rollback_snapshot) + try: + clearner.clear_data() + self.start_pipeline( + root_pipeline_id=rollback_snapshot.root_pipeline_id, target_node_id=rollback_snapshot.target_node_id + ) + except Exception as e: + logger.error( + "rollback failed: start pipeline, pipeline_id={}, target_node_id={}, error={}".format( + rollback_snapshot.root_pipeline_id, rollback_snapshot.target_node_id, str(e) + ) + ) + raise e + + +@task +def token_rollback(snapshot_id, node_id, retry=False, retry_data=None): + """ + snapshot_id 本次回滚的快照id + node_id 当前要回滚的节点id + """ + TokenRollbackTaskHandler(snapshot_id=snapshot_id, node_id=node_id, retry=retry, retry_data=retry_data).rollback() + + +@task +def any_rollback(snapshot_id): + AnyRollbackHandler(snapshot_id=snapshot_id).rollback() diff --git a/runtime/bamboo-pipeline/pipeline/core/flow/activity/service_activity.py b/runtime/bamboo-pipeline/pipeline/core/flow/activity/service_activity.py index 6cb32fb3..d0076b4b 100644 --- a/runtime/bamboo-pipeline/pipeline/core/flow/activity/service_activity.py +++ b/runtime/bamboo-pipeline/pipeline/core/flow/activity/service_activity.py @@ -100,6 +100,9 @@ def need_run_hook(self): def schedule(self, data, parent_data, callback_data=None): return True + def rollback(self, data, parent_data, rollback_data=None): + return True + def finish_schedule(self): setattr(self, self.schedule_result_attr, True) diff --git a/runtime/bamboo-pipeline/pipeline/engine/tasks.py b/runtime/bamboo-pipeline/pipeline/engine/tasks.py index 956350f8..3a6470b1 100644 --- a/runtime/bamboo-pipeline/pipeline/engine/tasks.py +++ b/runtime/bamboo-pipeline/pipeline/engine/tasks.py @@ -11,27 +11,28 @@ specific language governing permissions and limitations under the License. """ -import logging import datetime -from dateutil.relativedelta import relativedelta +import logging + from celery import task from celery.schedules import crontab from celery.task import periodic_task -from django.db import transaction, connection - +from dateutil.relativedelta import relativedelta +from django.apps import apps +from django.db import connection, transaction from pipeline.conf import default_settings from pipeline.core.pipeline import Pipeline from pipeline.engine import api, signals, states from pipeline.engine.core import runtime, schedule from pipeline.engine.health import zombie from pipeline.engine.models import ( + History, NodeCeleryTask, NodeRelationship, PipelineProcess, ProcessCeleryTask, - Status, ScheduleService, - History, + Status, ) from pipeline.models import PipelineInstance @@ -257,6 +258,15 @@ def _clean_pipeline_instance_data(instance_id, timestamp): delete_pipeline_process = ( "DELETE FROM `engine_pipelineprocess` " "WHERE `engine_pipelineprocess`.`root_pipeline_id` = %s" ) + + delete_rollback_plan = "DELETE FROM `rollback_rollbackplan` WHERE `rollback_rollbackplan`.`root_pipeline_id` = %s" + delete_rollback_snapshot = ( + "DELETE FROM `rollback_rollbacksnapshot` WHERE `rollback_rollbacksnapshot`.`root_pipeline_id` = %s" + ) + delete_rollback_token = ( + "DELETE FROM `rollback_rollbacktoken` WHERE `rollback_rollbacktoken`.`root_pipeline_id` = %s" + ) + with transaction.atomic(): with connection.cursor() as cursor: if pipeline_process_ids: @@ -289,6 +299,15 @@ def _clean_pipeline_instance_data(instance_id, timestamp): _raw_sql_execute(cursor, delete_pipeline_process, [instance_id], timestamp) PipelineInstance.objects.filter(instance_id=instance_id).update(is_expired=True) + try: + apps.get_model("rollback", "RollbackToken") + except Exception: + logger.error("[_clean_pipeline_rollback_data] delete error, the rollback app not installed") + return + _raw_sql_execute(cursor, delete_rollback_plan, [instance_id], timestamp) + _raw_sql_execute(cursor, delete_rollback_snapshot, [instance_id], timestamp) + _raw_sql_execute(cursor, delete_rollback_token, [instance_id], timestamp) + def _sql_log(sql, params, timestamp): if isinstance(params, list): diff --git a/runtime/bamboo-pipeline/pipeline/eri/imp/hooks.py b/runtime/bamboo-pipeline/pipeline/eri/imp/hooks.py index a0be2298..23904fbd 100644 --- a/runtime/bamboo-pipeline/pipeline/eri/imp/hooks.py +++ b/runtime/bamboo-pipeline/pipeline/eri/imp/hooks.py @@ -17,6 +17,8 @@ from pipeline.eri.models import LogEntry from pipeline.eri.signals import pipeline_event +from bamboo_engine.utils.constants import RuntimeSettings + class PipelineEvent: def __init__(self, event_type, data): @@ -64,6 +66,9 @@ def pre_prepare_run_pipeline( ) ) + if self.get_config(RuntimeSettings.PIPELINE_ENABLE_ROLLBACK.value): + self.set_pipeline_token(pipeline) + def post_prepare_run_pipeline( self, pipeline: dict, root_pipeline_data: dict, root_pipeline_context: dict, subprocess_context: dict, **options ): diff --git a/runtime/bamboo-pipeline/pipeline/eri/imp/node.py b/runtime/bamboo-pipeline/pipeline/eri/imp/node.py index aad414d3..01706562 100644 --- a/runtime/bamboo-pipeline/pipeline/eri/imp/node.py +++ b/runtime/bamboo-pipeline/pipeline/eri/imp/node.py @@ -13,24 +13,25 @@ import json +from pipeline.eri.models import Node as DBNode + from bamboo_engine import metrics from bamboo_engine.eri import ( - Node, - NodeType, - ServiceActivity, - SubProcess, - ExclusiveGateway, - ParallelGateway, + Condition, ConditionalParallelGateway, ConvergeGateway, - EmptyStartEvent, + DefaultCondition, EmptyEndEvent, + EmptyStartEvent, + ExclusiveGateway, ExecutableEndEvent, - Condition, DefaultCondition, + Node, + NodeType, + ParallelGateway, + ServiceActivity, + SubProcess, ) -from pipeline.eri.models import Node as DBNode - class NodeMixin: def _get_node(self, node: DBNode): @@ -47,6 +48,7 @@ def _get_node(self, node: DBNode): can_skip=node_detail["can_skip"], name=node_detail.get("name"), can_retry=node_detail["can_retry"], + reserve_rollback=node_detail.get("reserve_rollback", False), ) if node_type == NodeType.ServiceActivity.value: diff --git a/runtime/bamboo-pipeline/pipeline/eri/imp/process.py b/runtime/bamboo-pipeline/pipeline/eri/imp/process.py index 05aef601..730da35d 100644 --- a/runtime/bamboo-pipeline/pipeline/eri/imp/process.py +++ b/runtime/bamboo-pipeline/pipeline/eri/imp/process.py @@ -12,17 +12,17 @@ """ import json -from typing import List, Optional, Dict +from typing import Dict, List, Optional -from django.utils import timezone -from django.db.models import F +from django.conf import settings from django.db import transaction - -from bamboo_engine import metrics -from bamboo_engine.eri import ProcessInfo, SuspendedProcessInfo, DispatchProcess - +from django.db.models import F +from django.utils import timezone from pipeline.eri.models import Process +from bamboo_engine import metrics, states +from bamboo_engine.eri import DispatchProcess, ProcessInfo, SuspendedProcessInfo + class ProcessMixin: def beat(self, process_id: int): @@ -273,6 +273,23 @@ def fork( if not qs: raise Process.DoesNotExist("Process with id({}) does not exist".format(parent_id)) + if getattr(settings, "PIPELINE_ENABLE_ROLLBACK", False): + # 如果开启了回滚，则会自动删除相关的process信息，防止异常 + state = self.get_state(root_pipeline_id) + # 如果流程处在回滚中，才会删除 + if state.name == states.ROLLING_BACK: + for current_node, destination in from_to.items(): + Process.objects.filter( + parent_id=parent_id, + asleep=True, + destination_id=destination, + current_node_id=current_node, + root_pipeline_id=root_pipeline_id, + pipeline_stack=stack_json, + priority=qs[0].priority, + queue=qs[0].queue, + ).delete() + children = [ Process( parent_id=parent_id, diff --git a/runtime/bamboo-pipeline/pipeline/eri/imp/rollback.py b/runtime/bamboo-pipeline/pipeline/eri/imp/rollback.py new file mode 100644 index 00000000..99ec147d --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/eri/imp/rollback.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +import json +import logging + +from django.apps import apps +from django.core.serializers.json import DjangoJSONEncoder + +from bamboo_engine.builder.builder import generate_pipeline_token + +logger = logging.getLogger("bamboo_engine") + + +class RollbackMixin: + def set_pipeline_token(self, pipeline_tree: dict): + """ + 设置pipeline token + """ + try: + # 引用成功说明pipeline rollback 这个 app 是安装过的 + RollbackToken = apps.get_model("rollback", "RollbackToken") + except Exception as e: + logger.error( + "[RollbackMixin][set_pipeline_token] import RollbackToken error, " + "Please check whether the rollback app is installed correctly, err={}".format(e) + ) + return + + root_pipeline_id = pipeline_tree["id"] + node_map = generate_pipeline_token(pipeline_tree) + + RollbackToken.objects.create(root_pipeline_id=root_pipeline_id, token=json.dumps(node_map)) + + def set_node_snapshot(self, root_pipeline_id, node_id, code, version, context_values, inputs, outputs): + """ + 创建一分节点快照 + """ + try: + RollbackNodeSnapshot = apps.get_model("rollback", "RollbackNodeSnapshot") + # 引用成功说明pipeline rollback 这个 app 是安装过的 + except Exception as e: + logger.error( + "[RollbackMixin][set_node_snapshot] import RollbackNodeSnapshot error, " + "Please check whether the rollback app is installed correctly, err={}".format(e) + ) + return + + RollbackNodeSnapshot.objects.create( + root_pipeline_id=root_pipeline_id, + node_id=node_id, + code=code, + version=version, + context_values=json.dumps(context_values, cls=DjangoJSONEncoder), + inputs=json.dumps(inputs, cls=DjangoJSONEncoder), + outputs=json.dumps(outputs, cls=DjangoJSONEncoder), + ) + + def start_rollback(self, root_pipeline_id, node_id): + """ + 新建一个回滚任务 + """ + try: + # 引用成功说明pipeline rollback 这个 app 是安装过的 + from pipeline.contrib.rollback.handler import RollbackDispatcher + + RollbackPlan = apps.get_model("rollback", "RollbackPlan") + except Exception as e: + logger.error( + "[RollbackMixin][set_pipeline_token] import RollbackDispatcher or RollbackPlan error, " + "Please check whether the rollback app is installed correctly, err={}".format(e) + ) + return + + try: + rollback_plan = RollbackPlan.objects.get( + root_pipeline_id=root_pipeline_id, start_node_id=node_id, is_expired=False + ) + handler = RollbackDispatcher(root_pipeline_id=root_pipeline_id, mode=rollback_plan.mode) + handler.rollback(rollback_plan.start_node_id, rollback_plan.target_node_id) + rollback_plan.is_expired = True + rollback_plan.save(update_fields=["is_expired"]) + except Exception as e: + logger.error("[RollbackMixin][start_rollback] start a rollback task error, err={}".format(e)) + return diff --git a/runtime/bamboo-pipeline/pipeline/eri/runtime.py b/runtime/bamboo-pipeline/pipeline/eri/runtime.py index 664157ff..a6e0494b 100644 --- a/runtime/bamboo-pipeline/pipeline/eri/runtime.py +++ b/runtime/bamboo-pipeline/pipeline/eri/runtime.py @@ -29,6 +29,7 @@ from pipeline.eri.imp.node import NodeMixin from pipeline.eri.imp.plugin_manager import PipelinePluginManagerMixin from pipeline.eri.imp.process import ProcessMixin +from pipeline.eri.imp.rollback import RollbackMixin from pipeline.eri.imp.schedule import ScheduleMixin from pipeline.eri.imp.state import StateMixin from pipeline.eri.imp.task import TaskMixin @@ -68,10 +69,10 @@ class BambooDjangoRuntime( InterruptMixin, EventMixin, ConfigMixin, + RollbackMixin, EngineRuntimeInterface, ): - - ERI_SUPPORT_VERSION = 7 + ERI_SUPPORT_VERSION = 8 def __init__(self): try: diff --git a/runtime/bamboo-pipeline/pipeline/tests/contrib/test_graph.py b/runtime/bamboo-pipeline/pipeline/tests/contrib/test_graph.py new file mode 100644 index 00000000..f439b1dc --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/tests/contrib/test_graph.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +from unittest import TestCase + +from pipeline.contrib.rollback.graph import RollbackGraphHandler + + +class TestGraph(TestCase): + def test_build_rollback_graph_with_cycle(self): + node_map = { + "start": { + "id": "start", + "type": "EmptyStartEvent", + "targets": {"lf67ec0280323668ba383bc61c92bdce": "node_1"}, + }, + "node_1": { + "id": "node_1", + "type": "ServiceActivity", + "targets": {"l5c6729c70b83c81bd98eebefe0c46e3": "node_2"}, + }, + "node_2": { + "id": "node_2", + "type": "ServiceActivity", + "targets": {"ld09dcdaaae53cd9868b652fd4b7b074": "node_3"}, + }, + "node_3": { + "id": "node_3", + "type": "ExclusiveGateway", + "targets": {"ld9beef12dd33812bb9b697afd5f2728": "node_4", "lffeab3bdb0139b69ac6978a415e3f54": "node_1"}, + }, + "node_4": { + "id": "node_4", + "type": "ServiceActivity", + "targets": {"l995fa16e367312e99a1f8b54458ed6a": "node_5"}, + }, + "node_5": { + "id": "node_5", + "type": "ServiceActivity", + "targets": {"l802b3f8e60e39518915f85d4c943a18": "node_6"}, + }, + "node_6": { + "id": "node_6", + "type": "ExclusiveGateway", + "targets": {"l8ff0721ec8c3745b6f2183a7006d2c6": "node_7", "l5df5ee5497f3616aec4347c0e5913b8": "node_5"}, + }, + "node_7": {"id": "node_7", "type": "EmptyEndEvent", "targets": {}}, + } + + rollback_graph = RollbackGraphHandler(node_map=node_map, start_id="node_5", target_id="node_1") + graph, other_nodes = rollback_graph.build_rollback_graph() + self.assertListEqual(other_nodes, ["node_3"]) + self.assertListEqual(graph.as_dict()["nodes"], ["node_5", "node_1", "END", "START", "node_2", "node_4"]) + self.assertListEqual( + graph.as_dict()["flows"], + [["node_1", "END"], ["START", "node_5"], ["node_2", "node_1"], ["node_4", "node_2"], ["node_5", "node_4"]], + ) + self.assertListEqual(list(graph.next("START")), ["node_5"]) diff --git a/runtime/bamboo-pipeline/pipeline/tests/contrib/test_rollback.py b/runtime/bamboo-pipeline/pipeline/tests/contrib/test_rollback.py index e52b2982..9fc47f6b 100644 --- a/runtime/bamboo-pipeline/pipeline/tests/contrib/test_rollback.py +++ b/runtime/bamboo-pipeline/pipeline/tests/contrib/test_rollback.py @@ -1,80 +1,87 @@ -# -*- coding: utf-8 -*- -""" -Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community -Edition) available. -Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" +# # -*- coding: utf-8 -*- +# """ +# Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +# Edition) available. +# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +# Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://opensource.org/licenses/MIT +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# """ import json import mock from django.test import TestCase -from django.utils import timezone from mock.mock import MagicMock from pipeline.contrib.rollback import api -from pipeline.contrib.rollback.handler import RollBackHandler +from pipeline.contrib.rollback.models import ( + RollbackPlan, + RollbackSnapshot, + RollbackToken, +) from pipeline.core.constants import PE from pipeline.eri.models import Node, Process, State from bamboo_engine import states from bamboo_engine.utils.string import unique_id -forced_fail_activity_mock = MagicMock() -forced_fail_activity_mock.result = True +token_rollback = MagicMock() +token_rollback.apply_async = MagicMock(return_value=True) class TestRollBackBase(TestCase): - def setUp(self) -> None: - self.started_time = timezone.now() - self.archived_time = timezone.now() - - @mock.patch("bamboo_engine.api.forced_fail_activity", MagicMock(return_value=forced_fail_activity_mock)) - @mock.patch("pipeline.eri.runtime.BambooDjangoRuntime.execute", MagicMock()) + @mock.patch("pipeline.contrib.rollback.handler.token_rollback", MagicMock(return_value=token_rollback)) def test_rollback(self): pipeline_id = unique_id("n") - State.objects.create( - node_id=pipeline_id, - root_id=pipeline_id, - parent_id=pipeline_id, - name=states.FINISHED, - version=unique_id("v"), - started_time=self.started_time, - archived_time=self.archived_time, + pipeline_state = State.objects.create( + node_id=pipeline_id, root_id=pipeline_id, parent_id=pipeline_id, name=states.FAILED, version=unique_id("v") ) - node_id_1 = unique_id("n") - node_id_2 = unique_id("n") - State.objects.create( - node_id=node_id_1, + start_node_id = unique_id("n") + start_state = State.objects.create( + node_id=start_node_id, root_id=pipeline_id, parent_id=pipeline_id, name=states.RUNNING, version=unique_id("v"), - started_time=self.started_time, - archived_time=self.archived_time, ) + target_node_id = unique_id("n") State.objects.create( - node_id=node_id_2, + node_id=target_node_id, root_id=pipeline_id, parent_id=pipeline_id, - name=states.RUNNING, + name=states.FINISHED, version=unique_id("v"), - started_time=self.started_time, - archived_time=self.archived_time, ) - node_id_1_detail = { - "id": "n0be4eaa13413f9184863776255312f1", - "type": PE.ParallelGateway, - "targets": {"l7895e18cd7c33b198d56534ca332227": node_id_2}, - "root_pipeline_id": "n3369d7ce884357f987af1631bda69cb", - "parent_pipeline_id": "n3369d7ce884357f987af1631bda69cb", + result = api.rollback(pipeline_id, start_node_id, target_node_id) + self.assertFalse(result.result) + message = ( + "rollback failed: the task of non-running state is not allowed to roll back," + " pipeline_id={}, state=FAILED".format(pipeline_id) + ) + self.assertEqual(str(result.exc), message) + pipeline_state.name = states.RUNNING + pipeline_state.save() + + token = RollbackToken.objects.create( + root_pipeline_id=pipeline_id, token=json.dumps({target_node_id: "xxx", start_node_id: "xsx"}) + ) + + result = api.rollback(pipeline_id, start_node_id, target_node_id) + self.assertFalse(result.result) + message = "rollback failed: node not exist, node={}".format(start_node_id) + self.assertEqual(str(result.exc), message) + + target_node_detail = { + "id": target_node_id, + "type": PE.ServiceActivity, + "targets": {target_node_id: start_node_id}, + "root_pipeline_id": pipeline_id, + "parent_pipeline_id": pipeline_id, "can_skip": True, "code": "bk_display", "version": "v1.0", @@ -82,14 +89,12 @@ def test_rollback(self): "can_retry": True, } - Node.objects.create(node_id=node_id_1, detail=json.dumps(node_id_1_detail)) - - node_id_2_detail = { - "id": "n0be4eaa13413f9184863776255312f1", - "type": PE.ParallelGateway, - "targets": {"l7895e18cd7c33b198d56534ca332227": unique_id("n")}, - "root_pipeline_id": "n3369d7ce884357f987af1631bda69cb", - "parent_pipeline_id": "n3369d7ce884357f987af1631bda69cb", + start_node_detail = { + "id": start_node_id, + "type": PE.ServiceActivity, + "targets": {}, + "root_pipeline_id": pipeline_id, + "parent_pipeline_id": pipeline_id, "can_skip": True, "code": "bk_display", "version": "v1.0", @@ -97,151 +102,163 @@ def test_rollback(self): "can_retry": True, } - Node.objects.create(node_id=node_id_2, detail=json.dumps(node_id_2_detail)) + Node.objects.create(node_id=target_node_id, detail=json.dumps(target_node_detail)) + Node.objects.create(node_id=start_node_id, detail=json.dumps(start_node_detail)) - # pipeline_id 非running的情况下会异常 - message = "rollback failed: the task of non-running state is not allowed to roll back, pipeline_id={}".format( - pipeline_id - ) - result = api.rollback(pipeline_id, pipeline_id) + result = api.rollback(pipeline_id, start_node_id, target_node_id) self.assertFalse(result.result) + message = "rollback failed: only allows rollback to finished node, allowed states ['FINISHED', 'FAILED']" self.assertEqual(str(result.exc), message) - State.objects.filter(node_id=pipeline_id).update(name=states.RUNNING) - # pipeline_id 非running的情况下会异常 - message = "rollback failed: only allows rollback to ServiceActivity type nodes" - result = api.rollback(pipeline_id, node_id_1) + start_state.name = states.FINISHED + start_state.save() + + result = api.rollback(pipeline_id, start_node_id, target_node_id) self.assertFalse(result.result) + message = "rollback failed: start node token must equal target node, pipeline_id={}".format(pipeline_id) self.assertEqual(str(result.exc), message) - node_id_1_detail["type"] = PE.ServiceActivity - Node.objects.filter(node_id=node_id_1).update(detail=json.dumps(node_id_1_detail)) + token.token = json.dumps({target_node_id: "xxx", start_node_id: "xxx"}) + token.save() + + result = api.rollback(pipeline_id, start_node_id, target_node_id) + self.assertTrue(result.result) + rollback_snapshot = RollbackSnapshot.objects.get(root_pipeline_id=pipeline_id) + self.assertEqual(json.loads(rollback_snapshot.skip_rollback_nodes), []) + self.assertEqual(len(json.loads(rollback_snapshot.graph)["nodes"]), 4) + + pipeline_state.refresh_from_db() + self.assertEqual(pipeline_state.name, states.ROLLING_BACK) - message = "rollback failed: only allows rollback to finished node" - result = api.rollback(pipeline_id, node_id_1) + def test_reserve_rollback(self): + pipeline_id = unique_id("n") + State.objects.create( + node_id=pipeline_id, + root_id=pipeline_id, + parent_id=pipeline_id, + name=states.RUNNING, + version=unique_id("v") + # noqa + ) + + start_node_id = unique_id("n") + start_state = State.objects.create( + node_id=start_node_id, + root_id=pipeline_id, + parent_id=pipeline_id, + name=states.FINISHED, + version=unique_id("v"), + ) + + target_node_id = unique_id("n") + State.objects.create( + node_id=target_node_id, + root_id=pipeline_id, + parent_id=pipeline_id, + name=states.FINISHED, + version=unique_id("v"), + ) + + Process.objects.create(root_pipeline_id=pipeline_id, current_node_id=start_node_id, priority=1) + result = api.reserve_rollback(pipeline_id, start_node_id, target_node_id) self.assertFalse(result.result) + message = "rollback failed: pipeline token not exist, pipeline_id={}".format(pipeline_id) self.assertEqual(str(result.exc), message) - State.objects.filter(node_id=node_id_1).update(name=states.FINISHED) - - p = Process.objects.create( - root_pipeline_id=pipeline_id, - parent_id=-1, - current_node_id=node_id_2, - pipeline_stack=json.dumps([pipeline_id]), - priority=1, + + RollbackToken.objects.create( + root_pipeline_id=pipeline_id, token=json.dumps({target_node_id: "xxx", start_node_id: "xxx"}) ) - result = api.rollback(pipeline_id, node_id_1) - self.assertTrue(result.result) + result = api.reserve_rollback(pipeline_id, start_node_id, target_node_id) + self.assertFalse(result.result) + message = "rollback failed: node not exist, node={}".format(target_node_id) + self.assertEqual(str(result.exc), message) - p.refresh_from_db() - self.assertEqual(p.current_node_id, node_id_1) - # 验证Node2 是不是被删除了 - self.assertFalse(State.objects.filter(node_id=node_id_2).exists()) - - state = State.objects.get(node_id=node_id_1) - self.assertEqual(state.name, states.READY) - - def test_compute_validate_nodes(self): - node_map = { - "node_1": { - "id": "node_1", - "type": "EmptyStartEvent", - "targets": {"n": "node_2"}, - }, - "node_2": { - "id": "node_2", - "type": "ServiceActivity", - "targets": {"n": "node_3"}, - }, - "node_3": { - "id": "node_3", - "type": "ServiceActivity", - "targets": {"n": "node_4"}, - }, - "node_4": { - "id": "node_4", - "type": "ParallelGateway", - "targets": {"n": "node_5", "n1": "node_6"}, - "converge_gateway_id": "node_7", - }, - "node_5": { - "id": "node_5", - "type": "ServiceActivity", - "targets": {"n": "node_7"}, - }, - "node_6": { - "id": "node_6", - "type": "ServiceActivity", - "targets": {"n": "node_7"}, - }, - "node_7": { - "id": "node_7", - "type": "ConvergeGateway", - "targets": {"n": "node_8"}, - }, - "node_8": { - "id": "node_8", - "type": "ExclusiveGateway", - "targets": {"n1": "node_13", "n2": "node_9", "n3": "node_3"}, - }, - "node_9": { - "id": "node_9", - "type": "ServiceActivity", - "targets": {"n": "node_10"}, - }, - "node_10": { - "id": "node_10", - "type": "ExclusiveGateway", - "targets": {"n": "node_11", "n2": "node_12"}, - }, + target_node_detail = { + "id": target_node_id, + "type": PE.ServiceActivity, + "targets": {target_node_id: start_node_id}, + "root_pipeline_id": pipeline_id, + "parent_pipeline_id": pipeline_id, + "can_skip": True, + "code": "bk_display", + "version": "v1.0", + "error_ignorable": True, + "can_retry": True, } - node_id = "node_1" - nodes = RollBackHandler("p", node_map)._compute_validate_nodes(node_id, node_map) - self.assertListEqual(nodes, ["node_2", "node_3", "node_9"]) + start_node_detail = { + "id": start_node_id, + "type": PE.ServiceActivity, + "targets": {}, + "root_pipeline_id": pipeline_id, + "parent_pipeline_id": pipeline_id, + "can_skip": True, + "code": "bk_display", + "version": "v1.0", + "error_ignorable": True, + "can_retry": True, + } - def test_get_allowed_rollback_node_id_list(self): - pipeline_id = unique_id("n") + Node.objects.create(node_id=target_node_id, detail=json.dumps(target_node_detail)) + Node.objects.create(node_id=start_node_id, detail=json.dumps(start_node_detail)) + result = api.reserve_rollback(pipeline_id, start_node_id, target_node_id) + self.assertFalse(result.result) + message = "reserve rollback failed, the node state is not Running, current state=FINISHED, node_id={}".format( + # noqa + start_node_id + ) + self.assertEqual(str(result.exc), message) + + start_state.name = states.RUNNING + start_state.save() + + result = api.reserve_rollback(pipeline_id, start_node_id, target_node_id) + self.assertTrue(result.result) + + plan = RollbackPlan.objects.get(root_pipeline_id=pipeline_id) + + self.assertEqual(plan.start_node_id, start_node_id) + self.assertEqual(plan.target_node_id, target_node_id) + + result = api.cancel_reserved_rollback(pipeline_id, start_node_id, target_node_id) + self.assertTrue(result.result) + + def test_allowed_rollback_node_id_list(self): + pipeline_id = unique_id("n") State.objects.create( node_id=pipeline_id, root_id=pipeline_id, parent_id=pipeline_id, name=states.RUNNING, - version=unique_id("v"), - started_time=self.started_time, - archived_time=self.archived_time, + version=unique_id("v") + # noqa ) - - node_id_1 = unique_id("n") - node_id_2 = unique_id("n") + start_node_id = unique_id("n") State.objects.create( - node_id=node_id_1, + node_id=start_node_id, root_id=pipeline_id, parent_id=pipeline_id, name=states.FINISHED, version=unique_id("v"), - started_time=self.started_time, - archived_time=self.archived_time, ) + target_node_id = unique_id("n") State.objects.create( - node_id=node_id_2, + node_id=target_node_id, root_id=pipeline_id, parent_id=pipeline_id, - name=states.RUNNING, + name=states.FINISHED, version=unique_id("v"), - started_time=self.started_time, - archived_time=self.archived_time, ) - node_id_1_detail = { - "id": "n0be4eaa13413f9184863776255312f1", + target_node_detail = { + "id": target_node_id, "type": PE.ServiceActivity, - "targets": {"l7895e18cd7c33b198d56534ca332227": node_id_2}, - "root_pipeline_id": "n3369d7ce884357f987af1631bda69cb", - "parent_pipeline_id": "n3369d7ce884357f987af1631bda69cb", + "targets": {target_node_id: start_node_id}, + "root_pipeline_id": pipeline_id, + "parent_pipeline_id": pipeline_id, "can_skip": True, "code": "bk_display", "version": "v1.0", @@ -249,14 +266,12 @@ def test_get_allowed_rollback_node_id_list(self): "can_retry": True, } - Node.objects.create(node_id=node_id_1, detail=json.dumps(node_id_1_detail)) - - node_id_2_detail = { - "id": "n0be4eaa13413f9184863776255312f1", + start_node_detail = { + "id": start_node_id, "type": PE.ServiceActivity, - "targets": {"l7895e18cd7c33b198d56534ca332227": unique_id("n")}, - "root_pipeline_id": "n3369d7ce884357f987af1631bda69cb", - "parent_pipeline_id": "n3369d7ce884357f987af1631bda69cb", + "targets": {}, + "root_pipeline_id": pipeline_id, + "parent_pipeline_id": pipeline_id, "can_skip": True, "code": "bk_display", "version": "v1.0", @@ -264,8 +279,62 @@ def test_get_allowed_rollback_node_id_list(self): "can_retry": True, } - Node.objects.create(node_id=node_id_2, detail=json.dumps(node_id_2_detail)) + Node.objects.create(node_id=target_node_id, detail=json.dumps(target_node_detail)) + Node.objects.create(node_id=start_node_id, detail=json.dumps(start_node_detail)) + + RollbackToken.objects.create( + root_pipeline_id=pipeline_id, token=json.dumps({target_node_id: "xxx", start_node_id: "xxx"}) + ) + + result = api.get_allowed_rollback_node_id_list(pipeline_id, start_node_id) + self.assertTrue(result.result) + self.assertEqual(len(result.data), 1) + self.assertEqual(result.data[0], target_node_id) + + @mock.patch("pipeline.contrib.rollback.handler.token_rollback", MagicMock(return_value=token_rollback)) + def test_retry_rollback_failed_node(self): + root_pipeline_id = unique_id("n") + pipeline_state = State.objects.create( + node_id=root_pipeline_id, + root_id=root_pipeline_id, + parent_id=root_pipeline_id, + name=states.RUNNING, + version=unique_id("v"), + ) + node_id = unique_id("n") + node_state = State.objects.create( + node_id=node_id, + root_id=root_pipeline_id, + parent_id=root_pipeline_id, + name=states.FINISHED, + version=unique_id("v"), + ) + result = api.retry_rollback_failed_node(root_pipeline_id, node_id) + self.assertFalse(result.result) + message = "rollback failed: only retry the failed pipeline, current_status=RUNNING" + self.assertEqual(str(result.exc), message) + + pipeline_state.name = states.ROLL_BACK_FAILED + pipeline_state.save() + + result = api.retry_rollback_failed_node(root_pipeline_id, node_id) + self.assertFalse(result.result) + message = "rollback failed: only retry the failed node, current_status=FINISHED" + self.assertEqual(str(result.exc), message) + + node_state.name = states.ROLL_BACK_FAILED + node_state.save() - result = api.get_allowed_rollback_node_id_list(pipeline_id) - self.assertEqual(result.result, True) - self.assertEqual(result.data, [node_id_1]) + result = api.retry_rollback_failed_node(root_pipeline_id, node_id) + self.assertFalse(result.result) + message = "rollback failed: the rollback snapshot is not exists, please check" + self.assertEqual(str(result.exc), message) + + RollbackSnapshot.objects.create( + root_pipeline_id=root_pipeline_id, + graph=json.dumps({}), + node_access_record=json.dumps({}), + ) + + result = api.retry_rollback_failed_node(root_pipeline_id, node_id) + self.assertTrue(result.result) diff --git a/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/control/test_rollback_node.py b/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/control/test_rollback_node.py deleted file mode 100644 index 992fdc99..00000000 --- a/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/control/test_rollback_node.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community -Edition) available. -Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" -import time - -from bamboo_engine import Engine - -from bamboo_engine.builder import * # noqa -from pipeline.engine import states -from pipeline.eri.models import Process, State -from pipeline.eri.runtime import BambooDjangoRuntime - -from pipeline.contrib.rollback import api - - -def test_rollback_sample_pipeline(): - start = EmptyStartEvent() - act_0 = ServiceActivity(component_code="callback_node") - act_1 = ServiceActivity(component_code="callback_node") - end = EmptyEndEvent() - start.extend(act_0).extend(act_1).extend(end) - pipeline = build_tree(start) - runtime = BambooDjangoRuntime() - engine = Engine(runtime) - engine.run_pipeline(pipeline=pipeline, root_pipeline_data={}) - time.sleep(3) - - state = runtime.get_state(act_0.id) - engine.callback(act_0.id, state.version, {"bit": 1}) - pipeline_id = pipeline["id"] - time.sleep(3) - assert State.objects.filter(node_id=act_0.id, name=states.FINISHED).exists() - api.rollback(root_pipeline_id=pipeline_id, node_id=act_0.id) - time.sleep(3) - process = Process.objects.get(root_pipeline_id=pipeline_id, parent_id=-1) - # 此时最新进程被指向了最新的node_id - assert process.current_node_id == act_0.id - # 此时第一个节点重回RUNNING状态 - assert State.objects.filter(node_id=act_0.id, name=states.RUNNING).exists() - - -def test_rollback_pipeline_with_exclusive_gateway(): - """ - -> act_0 - 开始 -> 分支网关 -> 汇聚网关 -> act_2 -> 结束 - -> act_1 - - 当执行到 act_2 时，此时回退到act_0 应当能够再次回到 act_2 - """ - - runtime = BambooDjangoRuntime() - - start = EmptyStartEvent() - eg = ExclusiveGateway( - conditions={0: "True == True", 1: "True == False"} - ) - act_0 = ServiceActivity(component_code="callback_node") - act_1 = ServiceActivity(component_code="callback_node") - act_2 = ServiceActivity(component_code="callback_node") - - cg = ConvergeGateway() - end = EmptyEndEvent() - - start.extend(eg).connect(act_0, act_1).converge(cg).extend(act_2).extend(end) - - pipeline = build_tree(start) - engine = Engine(BambooDjangoRuntime()) - engine.run_pipeline(pipeline=pipeline, root_pipeline_data={}) - time.sleep(3) - - state = runtime.get_state(act_0.id) - engine.callback(act_0.id, state.version, {"bit": 1}) - - time.sleep(3) - pipeline_id = pipeline["id"] - - process = Process.objects.get(root_pipeline_id=pipeline_id, parent_id=-1) - - # 此时执行到了act_2 - assert process.current_node_id == act_2.id - - api.rollback(pipeline_id, act_0.id) - time.sleep(3) - - process.refresh_from_db() - # 此时最新进程被指向了最新的node_id - assert process.current_node_id == act_0.id - # 此时第一个节点重回RUNNING状态 - assert State.objects.filter(node_id=act_0.id, name=states.RUNNING).exists() - - -def test_rollback_pipeline_with_conditional_parallel(): - """ - -> act_1 - 开始 -> act_0 并行网关 -> 汇聚网关 -> act_3 -> 结束 - -> act_2 - - 当执行到 act_2 时，此时回退到act_0 应当能够再次回到 act_2 - """ - - runtime = BambooDjangoRuntime() - - start = EmptyStartEvent() - act_0 = ServiceActivity(component_code="debug_node") - pg = ParallelGateway() - - act_1 = ServiceActivity(component_code="debug_node") - act_2 = ServiceActivity(component_code="debug_node") - cg = ConvergeGateway() - act_3 = ServiceActivity(component_code="callback_node") - end = EmptyEndEvent() - - start.extend(act_0).extend(pg).connect(act_1, act_2).converge(cg).extend(act_3).extend(end) - - pipeline = build_tree(start) - engine = Engine(BambooDjangoRuntime()) - engine.run_pipeline(pipeline=pipeline, root_pipeline_data={}) - - time.sleep(3) - pipeline_id = pipeline["id"] - - process = Process.objects.get(root_pipeline_id=pipeline_id, parent_id=-1) - # 此时执行到了act_2 - assert process.current_node_id == act_3.id - - # 此时回到开始节点 - api.rollback(pipeline_id, act_0.id) - time.sleep(3) - - process.refresh_from_db() - # 此时第二次执行到act_2 - assert process.current_node_id == act_3.id - # 此时第一个节点重回RUNNING状态 - assert State.objects.filter(node_id=act_3.id, name=states.RUNNING).exists() - # callback act_2 此时流程结束 - state = runtime.get_state(act_3.id) - engine.callback(act_3.id, state.version, {"bit": 1}) - time.sleep(3) - - assert State.objects.filter(node_id=pipeline_id, name=states.FINISHED).exists() diff --git a/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py b/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py index ed03f568..3b092716 100755 --- a/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py +++ b/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py @@ -13,13 +13,13 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os +from celery import Celery from pipeline.celery.queues import ScalableQueues # noqa from pipeline.celery.settings import * # noqa from pipeline.eri.celery import queues, step -from celery import Celery -CELERY_QUEUES.extend(queues.CELERY_QUEUES) -CELERY_QUEUES.extend(queues.QueueResolver("api").queues()) +CELERY_QUEUES.extend(queues.CELERY_QUEUES) # noqa +CELERY_QUEUES.extend(queues.QueueResolver("api").queues()) # noqa step.PromServerStep.port = 8002 @@ -55,6 +55,7 @@ "pipeline", "pipeline.log", "pipeline.engine", + "pipeline.contrib.rollback", "pipeline.component_framework", "pipeline.variable_framework", "pipeline.django_signal_valve", @@ -161,7 +162,7 @@ BROKER_URL = "amqp://guest:guest@localhost:5672//" -# BROKER_URL = 'redis://localhost:6379/0' +# BROKER_URL = "redis://localhost:6379/0" PIPELINE_DATA_BACKEND = "pipeline.engine.core.data.redis_backend.RedisDataBackend" diff --git a/tests/builder/__init__.py b/tests/builder/__init__.py new file mode 100644 index 00000000..26a6d1c2 --- /dev/null +++ b/tests/builder/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/tests/builder/test_token.py b/tests/builder/test_token.py new file mode 100644 index 00000000..52a1325c --- /dev/null +++ b/tests/builder/test_token.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- + +from bamboo_engine.builder import ( + ConditionalParallelGateway, + ConvergeGateway, + EmptyEndEvent, + EmptyStartEvent, + ExclusiveGateway, + ParallelGateway, + ServiceActivity, + SubProcess, + build_tree, +) +from bamboo_engine.builder.builder import generate_pipeline_token + + +def get_node_token(tree, name: str, node_map): + # 根据 name 获取对应的token + if name.startswith("act"): + for activity_id, value in tree["activities"].items(): + if value["name"] == name: + return node_map[activity_id] + + if ( + name.startswith("ParallelGateway") + or name.startswith("ExclusiveGateway") + or name.startswith("ConvergeGateway") + or name.startswith("ConditionalParallelGateway") + ): + for gateway_id, value in tree["gateways"].items(): + if value["name"] == name: + return node_map[gateway_id] + + if name.startswith("start_event"): + return node_map[tree["start_event"]["id"]] + + if name.startswith("end_event"): + return node_map[tree["end_event"]["id"]] + + +def test_inject_pipeline_token_normal(): + start = EmptyStartEvent() + act = ServiceActivity(name="act_1", component_code="example_component") + end = EmptyEndEvent() + + start.extend(act).extend(end) + pipeline = build_tree(start) + + node_token_map = generate_pipeline_token(pipeline) + + assert get_node_token(pipeline, "act_1", node_token_map) == get_node_token(pipeline, "start_event", node_token_map) + assert get_node_token(pipeline, "start_event", node_token_map) == get_node_token( + pipeline, "end_event", node_token_map + ) + + +def test_inject_pipeline_token_parallel_gateway(): + start = EmptyStartEvent() + pg = ParallelGateway(name="ParallelGateway") + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + cg = ConvergeGateway(name="ConvergeGateway") + end = EmptyEndEvent() + + start.extend(pg).connect(act_1, act_2, act_3).to(pg).converge(cg).extend(end) + + pipeline = build_tree(start) + node_token_map = generate_pipeline_token(pipeline) + + assert ( + get_node_token(pipeline, "start_event", node_token_map) + == get_node_token(pipeline, "ParallelGateway", node_token_map) + == get_node_token(pipeline, "ConvergeGateway", node_token_map) + == get_node_token(pipeline, "end_event", node_token_map) + != get_node_token(pipeline, "act_1", node_token_map) + ) + + assert ( + get_node_token(pipeline, "act_1", node_token_map) + != get_node_token(pipeline, "act_2", node_token_map) + != get_node_token(pipeline, "act_3", node_token_map) + ) + + +def test_inject_pipeline_token_exclusive_gateway(): + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + eg = ExclusiveGateway(conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0"}, name="ExclusiveGateway") + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + end = EmptyEndEvent() + start.extend(act_1).extend(eg).connect(act_2, act_3).to(eg).converge(end) + + pipeline = build_tree(start) + node_token_map = generate_pipeline_token(pipeline) + + assert ( + get_node_token(pipeline, "start_event", node_token_map) + == get_node_token(pipeline, "act_1", node_token_map) + == get_node_token(pipeline, "ExclusiveGateway", node_token_map) + == get_node_token(pipeline, "end_event", node_token_map) + != get_node_token(pipeline, "act_2", node_token_map) + ) + + assert get_node_token(pipeline, "act_2", node_token_map) == get_node_token(pipeline, "act_3", node_token_map) + + +def test_inject_pipeline_token_conditional_exclusive_gateway(): + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + cpg = ConditionalParallelGateway( + conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0", 2: "${act_1_output} >= 0"}, + name="ConditionalParallelGateway", + ) + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + act_4 = ServiceActivity(component_code="pipe_example_component", name="act_4") + cg = ConvergeGateway(name="ConvergeGateway") + end = EmptyEndEvent() + + start.extend(act_1).extend(cpg).connect(act_2, act_3, act_4).to(cpg).converge(cg).extend(end) + + pipeline = build_tree(start) + node_token_map = generate_pipeline_token(pipeline) + assert ( + get_node_token(pipeline, "start_event", node_token_map) + == get_node_token(pipeline, "act_1", node_token_map) + == get_node_token(pipeline, "ConditionalParallelGateway", node_token_map) + == get_node_token(pipeline, "ConvergeGateway", node_token_map) + == get_node_token(pipeline, "end_event", node_token_map) + != get_node_token(pipeline, "act_3", node_token_map) + ) + + assert ( + get_node_token(pipeline, "act_2", node_token_map) + != get_node_token(pipeline, "act_3", node_token_map) + != get_node_token(pipeline, "act_4", node_token_map) + ) + + +def test_inject_pipeline_token_subprocess(): + def sub_process(name): + subproc_start = EmptyStartEvent() + subproc_act = ServiceActivity(component_code="pipe_example_component", name="act_2") + subproc_end = EmptyEndEvent() + subproc_start.extend(subproc_act).extend(subproc_end) + return SubProcess(start=subproc_start, name=name) + + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + eg = ExclusiveGateway(conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0"}, name="ExclusiveGateway") + + subproc_1 = sub_process(name="act_3") + subproc_2 = sub_process(name="act_4") + end = EmptyEndEvent() + + start.extend(act_1).extend(eg).connect(subproc_1, subproc_2).converge(end) + + pipeline = build_tree(start) + node_token_map = generate_pipeline_token(pipeline) + + assert ( + get_node_token(pipeline, "start_event", node_token_map) + == get_node_token(pipeline, "end_event", node_token_map) + == get_node_token(pipeline, "ExclusiveGateway", node_token_map) + == get_node_token(pipeline, "end_event", node_token_map) + != get_node_token(pipeline, "act_3", node_token_map) + ) + + assert get_node_token(pipeline, "act_3", node_token_map) == get_node_token(pipeline, "act_4", node_token_map) + + subproc_pipeline = pipeline["activities"][subproc_1.id]["pipeline"] + + assert ( + get_node_token(subproc_pipeline, "start_event", node_token_map) + == get_node_token(subproc_pipeline, "end_event", node_token_map) + == get_node_token(subproc_pipeline, "act_2", node_token_map) + ) + + +def test_inject_pipeline_token_with_cycle(): + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + eg = ExclusiveGateway( + conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0", 2: "${act_1_output} >= 0"}, + name="ExclusiveGateway", + ) + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + end = EmptyEndEvent() + start.extend(act_1).extend(eg).connect(act_2, act_3, act_1).to(eg).converge(end) + + pipeline = build_tree(start) + node_token_map = generate_pipeline_token(pipeline) + + assert ( + get_node_token(pipeline, "start_event", node_token_map) + == get_node_token(pipeline, "act_1", node_token_map) + == get_node_token(pipeline, "ExclusiveGateway", node_token_map) + == get_node_token(pipeline, "end_event", node_token_map) + != get_node_token(pipeline, "act_2", node_token_map) + ) + + assert get_node_token(pipeline, "act_2", node_token_map) == get_node_token(pipeline, "act_3", node_token_map) diff --git a/tests/engine/test_engine_execute.py b/tests/engine/test_engine_execute.py index 59cab573..362cfd2a 100644 --- a/tests/engine/test_engine_execute.py +++ b/tests/engine/test_engine_execute.py @@ -96,6 +96,7 @@ def node(node_id): parent_pipeline_id="root", code="", version="", + reserve_rollback=True, error_ignorable=False, ) @@ -380,7 +381,6 @@ def test_execute__rerun_and_have_to_sleep(node_id, pi, interrupter, node, state) def test_execute__have_to_sleep(node_id, pi, interrupter, node, state): - runtime = MagicMock() runtime.get_process_info = MagicMock(return_value=pi) runtime.batch_get_state_name = MagicMock(return_value={"root": states.RUNNING}) @@ -780,7 +780,6 @@ def test_execute__has_dispatch_processes(node_id, pi, interrupter, node, state): def test_execute__have_to_die(node_id, pi, interrupter, node, state): - runtime = MagicMock() runtime.get_process_info = MagicMock(return_value=pi) runtime.batch_get_state_name = MagicMock(return_value={"root": states.RUNNING}) @@ -851,6 +850,81 @@ def test_execute__have_to_die(node_id, pi, interrupter, node, state): assert interrupter.check_point.execute_result is not None +def test_execute__has_reversed_rollback_plan(node_id, pi, interrupter, node, state): + runtime = MagicMock() + runtime.get_process_info = MagicMock(return_value=pi) + runtime.batch_get_state_name = MagicMock(return_value={"root": states.RUNNING}) + runtime.get_node = MagicMock(return_value=node) + runtime.get_config = MagicMock(return_value=True) + runtime.start_rollback = MagicMock(return_value=True) + runtime.get_state_or_none = MagicMock(return_value=None) + runtime.get_state = MagicMock(return_value=state) + runtime.set_state = MagicMock(return_value=state.version) + handler = MagicMock() + handler.execute = MagicMock( + return_value=ExecuteResult( + should_sleep=False, + schedule_ready=False, + schedule_type=None, + schedule_after=-1, + dispatch_processes=[], + next_node_id=None, + should_die=True, + ) + ) + + get_handler = MagicMock(return_value=handler) + + engine = Engine(runtime=runtime) + + with mock.patch( + "bamboo_engine.engine.HandlerFactory.get_handler", + get_handler, + ): + engine.execute(pi.process_id, node_id, pi.root_pipeline_id, pi.top_pipeline_id, interrupter, {}) + + runtime.beat.assert_called_once_with(pi.process_id) + runtime.get_node.assert_called_once_with(node_id) + runtime.get_state_or_none.assert_called_once_with(node_id) + runtime.node_rerun_limit.assert_not_called() + runtime.set_state.assert_called_once_with( + node_id=node.id, + to_state=states.RUNNING, + version=None, + loop=1, + inner_loop=1, + root_id=pi.root_pipeline_id, + parent_id=pi.top_pipeline_id, + set_started_time=True, + reset_skip=False, + reset_retry=False, + reset_error_ignored=False, + refresh_version=False, + ignore_boring_set=False, + ) + runtime.start_rollback.assert_called_once_with(pi.root_pipeline_id, node_id) + runtime.sleep.assert_not_called() + runtime.set_schedule.assert_not_called() + runtime.schedule.assert_not_called() + runtime.execute.assert_not_called() + runtime.die.assert_called_once_with(pi.process_id) + + get_handler.assert_called_once_with(node, runtime, interrupter) + + handler.execute.assert_called_once_with( + process_info=pi, + loop=state.loop, + inner_loop=state.loop, + version=state.version, + recover_point=interrupter.recover_point, + ) + + assert interrupter.check_point.name == ExecuteKeyPoint.EXECUTE_NODE_DONE + assert interrupter.check_point.state_already_exist is False + assert interrupter.check_point.running_node_version == "v" + assert interrupter.check_point.execute_result is not None + + def test_execute__recover_with_state_not_exsit(node_id, pi, interrupter, node, state, recover_point): recover_point.state_already_exist = False recover_point.running_node_version = "set_running_return_version" diff --git a/tests/engine/test_engine_schedule.py b/tests/engine/test_engine_schedule.py index f3cd6ddc..32ca385c 100644 --- a/tests/engine/test_engine_schedule.py +++ b/tests/engine/test_engine_schedule.py @@ -10,7 +10,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - +import copy import mock import pytest @@ -128,7 +128,6 @@ def recover_point(): def test_schedule__lock_get_failed(node_id, schedule_id, state, pi, schedule, interrupter): - runtime = MagicMock() runtime.get_process_info = MagicMock(return_value=pi) runtime.apply_schedule_lock = MagicMock(return_value=False) @@ -470,7 +469,7 @@ def test_schedule__schedule_done(node_id, state, pi, schedule, node, interrupter runtime.get_node.assert_called_once_with(node_id) runtime.get_execution_data.assert_called_once_with(node_id) runtime.set_execution_data.assert_called_once_with(node_id=node.id, data=execution_data) - runtime.get_data_inputs.assert_called_once_with(pi.root_pipeline_id) + runtime.get_data_inputs.assert_has_calls([call("root"), call("root")]) runtime.get_callback_data.assert_not_called() service.hook_dispatch.assert_called_once() handler.schedule.assert_called_once_with( @@ -496,6 +495,71 @@ def test_schedule__schedule_done(node_id, state, pi, schedule, node, interrupter assert interrupter.check_point.lock_released is True +def test_schedule__schedule_done_with_node_reserved_rollback(node_id, state, pi, schedule, node, interrupter): + node = copy.deepcopy(node) + node.reserve_rollback = True + + schedule.type = ScheduleType.POLL + runtime = MagicMock() + runtime.get_process_info = MagicMock(return_value=pi) + runtime.apply_schedule_lock = MagicMock(return_value=True) + runtime.get_schedule = MagicMock(return_value=schedule) + runtime.start_rollback = MagicMock(return_value=True) + runtime.get_config = MagicMock(return_value=True) + runtime.get_state = MagicMock(return_value=state) + runtime.get_node = MagicMock(return_value=node) + runtime.node_finish = MagicMock() + handler = MagicMock() + handler.schedule = MagicMock( + return_value=ScheduleResult( + has_next_schedule=False, + schedule_after=-1, + schedule_done=True, + next_node_id="nid2", + ) + ) + get_handler = MagicMock(return_value=handler) + + engine = Engine(runtime=runtime) + + with mock.patch( + "bamboo_engine.engine.HandlerFactory.get_handler", + get_handler, + ): + engine.schedule(pi.process_id, node_id, schedule.id, interrupter, headers={}) + + runtime.beat.assert_called_once_with(pi.process_id) + runtime.get_process_info.assert_called_once_with(pi.process_id) + runtime.apply_schedule_lock.assert_called_once_with(schedule.id) + runtime.schedule.assert_not_called() + + runtime.start_rollback.assert_called_once_with(pi.root_pipeline_id, node_id) + + runtime.get_schedule.assert_called_once_with(schedule.id) + runtime.node_finish.assert_called_once_with(pi.root_pipeline_id, node.id) + + runtime.get_state.assert_has_calls([call(node_id), call(node_id)]) + runtime.get_node.assert_called_once_with(node_id) + runtime.get_callback_data.assert_not_called() + handler.schedule.assert_called_once_with( + process_info=pi, + loop=state.loop, + inner_loop=state.inner_loop, + schedule=schedule, + callback_data=None, + recover_point=interrupter.recover_point, + ) + runtime.set_next_schedule.assert_not_called() + runtime.finish_schedule.assert_called_once_with(schedule.id) + + assert interrupter.check_point.name == ScheduleKeyPoint.RELEASE_LOCK_DONE + assert interrupter.check_point.version_mismatch is False + assert interrupter.check_point.node_not_running is False + assert interrupter.check_point.lock_get is True + assert interrupter.check_point.schedule_result is not None + assert interrupter.check_point.lock_released is True + + def test_schedule__recover_version_mismatch(node_id, pi, state, schedule, interrupter, recover_point): recover_point.version_mismatch = True interrupter.recover_point = recover_point diff --git a/tests/utils/test_graph.py b/tests/utils/test_graph.py new file mode 100644 index 00000000..599d362a --- /dev/null +++ b/tests/utils/test_graph.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- +from bamboo_engine.utils.graph import RollbackGraph + + +def test_graph(): + graph1 = RollbackGraph([1, 2, 3, 4], [[1, 2], [2, 3], [3, 4]]) + assert not graph1.has_cycle() + assert graph1.get_cycle() == [] + graph2 = RollbackGraph([1, 2, 3, 4], [[1, 2], [2, 3], [3, 4], [4, 1]]) + assert graph2.has_cycle() + assert graph2.get_cycle() == [1, 2, 3, 4, 1] + graph3 = RollbackGraph([1, 2, 3, 4], [[1, 2], [2, 3], [3, 4], [4, 2]]) + assert graph3.has_cycle() + assert graph3.get_cycle() == [2, 3, 4, 2] + graph4 = RollbackGraph( + [ + "n20c4a0601193f268bfa168f1192eacd", + "nef42d10350b3961b53df7af67e16d9b", + "n0ada7b4abe63771a43052eaf188dc4b", + "n0cd3b95c714388bacdf1a486ab432fc", + "n1430047af8537f88710c4bbf3cbfb0f", + "n383748fe27434d582f0ca17af9d968a", + "n51426abd4be3a4691c80a73c3f93b3c", + "n854753a77933562ae72ec87c365f23d", + "n89f083892a731d7b9d7edb0f372006d", + "n8d4568db0ad364692b0387e86a2f1e0", + "n8daedbb02273a0fbc94cc118c90649f", + "n90b7ef55fe839b181879e036b4f8ffe", + "n99817348b4a36a6931854c93eed8c5f", + "na02956eba6f3a36ab9b0af2f2350213", + "nc3d0d49adf530bbaffe53630c184c0a", + "nca50848d1aa340f8c2b4776ce81868d", + "ncab9a48e79d357195dcee68dad3a31f", + "ncb4e013a6a8348bab087cc8500a3876", + "ne1f86f902a23e7fa4a67192e8b38a05", + "ne26def77df1385caa206c64e7e3ea53", + "nf3ebee137c53da28091ad7d140ce00c", + "nfc1dcdd7476393b9a81a988c113e1cf", + "n0197f8f210b3a1b8a7fc2f90e94744e", + "n01fb40259ad3cf285bb11a8bbbe59f2", + "n03f39191e8a32629145ba6a677ed040", + "n03ffc3b9e12316d8be63261cb9dec71", + "n07982b8985139249bca3a046f3a4379", + "n0b9e36e6b633ddb906d2044f658f110", + "n136c4fedebe3eb0ba932495aff6a945", + "n17cdc62c5d43976a413bda8f35634eb", + "n1d48483d8023439ad98d61d156c85fb", + "n26725bdcc0931fab0bc73e7244545ca", + "n2890db24f6c3cd1bbcd6b7d8cf2c045", + "n2ad9caac5b737bd897d4c8844c85f12", + "n2c88d1c1d8b35aebf883cbf259fb6bc", + "n302d25dfc9c369ab13104d5208e7119", + "n31688b7ab44338e9e6cb8dcaf259eef", + "n374443fbdc1313d98ebbe19d535fec2", + "n38c3dd0344a3f86bc7511c454bcdf4c", + "n3934eef90463940a6a9cf4ba2e63b1c", + "n40d5f0ca4bc3dd99c0b264cb186f00f", + "n476ddcb6dd33e2abac43596b08c2bc1", + "n4790f8aa48e335aa712e2af757e180b", + "n48bbfdc912334fc89c4f48c05e8969e", + "n5bef4f4532a382eaf79a0af70b2396b", + "n5ced56bcc863060ac4977755f35a5f5", + "n66a0562670e37648a3e05c243335bff", + "n6dc118cd3f7341d9ef8c97c63e2e9d9", + "n6e9d52e1ea53958a93e5b34022e7037", + "n786694b5ed33295a885b5bcd8c7c1ce", + "n7dccd56c80233469a4609f684ebe457", + "n8492d92ab6a3da48c2b49d6fcb8a479", + "n86a8b1a56f9399f90c4c227594a9d03", + "n8a805c0cd02307bad9f7828880b53dc", + "n8c7e35b0457300d9d6a96a6b1d18329", + "n91fdaed36403d06a07f4afe85e2892c", + "n9335d0718a937f9a39ec5b36d5637fe", + "n9372fb07ad936cba31f3d4e440f395a", + "n9ab96f926d83a93a5d3ebe2888fd343", + "na2a8a54e68033d0a276eb88dbff91c3", + "na493a7b5d5b3cc29f4070a6c4589cb7", + "nadfa68cb2503a39aac6626d6c72484a", + "nae1218ddd2e3448b562bc79dc084401", + "nc012287be793377b975b0230b35d713", + "ncb2e01f0c5336fe82b0e0e496f2612b", + "ncb5843900903b4c8a0a8302474d8c51", + "ncbf4db2c48f3348b2c7081f9e3b363a", + "nd4ee6c3248935ce9239e4bb20a81ab8", + "ndb1cf7af0e2319c9868530d0df8fd93", + "ne36a6858a733430bffa4fec053dc1ab", + "ne7af4a7c3613b3d81fe9e6046425a36", + "ne8035dd8de732758c1cc623f80f2fc8", + "ned91fdb914c35f3a21f320f62d72ffd", + "nf5448b3c66430f4a299d08208d313a6", + "nfaa0756a06f300495fb2e2e45e05ed3", + ], + [ + ["n8d4568db0ad364692b0387e86a2f1e0", "n5bef4f4532a382eaf79a0af70b2396b"], + ["n8daedbb02273a0fbc94cc118c90649f", "nf5448b3c66430f4a299d08208d313a6"], + ["n01fb40259ad3cf285bb11a8bbbe59f2", "ne1f86f902a23e7fa4a67192e8b38a05"], + ["ncab9a48e79d357195dcee68dad3a31f", "n0197f8f210b3a1b8a7fc2f90e94744e"], + ["na493a7b5d5b3cc29f4070a6c4589cb7", "ne1f86f902a23e7fa4a67192e8b38a05"], + ["n89f083892a731d7b9d7edb0f372006d", "n136c4fedebe3eb0ba932495aff6a945"], + ["n51426abd4be3a4691c80a73c3f93b3c", "n9ab96f926d83a93a5d3ebe2888fd343"], + ["n89f083892a731d7b9d7edb0f372006d", "n8492d92ab6a3da48c2b49d6fcb8a479"], + ["n17cdc62c5d43976a413bda8f35634eb", "n6e9d52e1ea53958a93e5b34022e7037"], + ["n476ddcb6dd33e2abac43596b08c2bc1", "ne1f86f902a23e7fa4a67192e8b38a05"], + ["n6dc118cd3f7341d9ef8c97c63e2e9d9", "nfc1dcdd7476393b9a81a988c113e1cf"], + ["n91fdaed36403d06a07f4afe85e2892c", "ncb4e013a6a8348bab087cc8500a3876"], + ["n8a805c0cd02307bad9f7828880b53dc", "n3934eef90463940a6a9cf4ba2e63b1c"], + ["n2890db24f6c3cd1bbcd6b7d8cf2c045", "n0ada7b4abe63771a43052eaf188dc4b"], + ["ned91fdb914c35f3a21f320f62d72ffd", "n383748fe27434d582f0ca17af9d968a"], + ["n89f083892a731d7b9d7edb0f372006d", "n0b9e36e6b633ddb906d2044f658f110"], + ["nc3d0d49adf530bbaffe53630c184c0a", "na493a7b5d5b3cc29f4070a6c4589cb7"], + ["ncb2e01f0c5336fe82b0e0e496f2612b", "nc012287be793377b975b0230b35d713"], + ["n86a8b1a56f9399f90c4c227594a9d03", "nf3ebee137c53da28091ad7d140ce00c"], + ["nc3d0d49adf530bbaffe53630c184c0a", "nadfa68cb2503a39aac6626d6c72484a"], + ["na02956eba6f3a36ab9b0af2f2350213", "na2a8a54e68033d0a276eb88dbff91c3"], + ["n8daedbb02273a0fbc94cc118c90649f", "n07982b8985139249bca3a046f3a4379"], + ["n136c4fedebe3eb0ba932495aff6a945", "nfc1dcdd7476393b9a81a988c113e1cf"], + ["n9372fb07ad936cba31f3d4e440f395a", "n1430047af8537f88710c4bbf3cbfb0f"], + ["n8d4568db0ad364692b0387e86a2f1e0", "n91fdaed36403d06a07f4afe85e2892c"], + ["n854753a77933562ae72ec87c365f23d", "n40d5f0ca4bc3dd99c0b264cb186f00f"], + ["n854753a77933562ae72ec87c365f23d", "n1d48483d8023439ad98d61d156c85fb"], + ["n9ab96f926d83a93a5d3ebe2888fd343", "n383748fe27434d582f0ca17af9d968a"], + ["ne36a6858a733430bffa4fec053dc1ab", "n0cd3b95c714388bacdf1a486ab432fc"], + ["n03ffc3b9e12316d8be63261cb9dec71", "nca50848d1aa340f8c2b4776ce81868d"], + ["ne8035dd8de732758c1cc623f80f2fc8", "n0ada7b4abe63771a43052eaf188dc4b"], + ["n51426abd4be3a4691c80a73c3f93b3c", "ned91fdb914c35f3a21f320f62d72ffd"], + ["nd4ee6c3248935ce9239e4bb20a81ab8", "nfaa0756a06f300495fb2e2e45e05ed3"], + ["n5bef4f4532a382eaf79a0af70b2396b", "ncb4e013a6a8348bab087cc8500a3876"], + ["ne26def77df1385caa206c64e7e3ea53", "n786694b5ed33295a885b5bcd8c7c1ce"], + ["n854753a77933562ae72ec87c365f23d", "ne8035dd8de732758c1cc623f80f2fc8"], + ["n374443fbdc1313d98ebbe19d535fec2", "ndb1cf7af0e2319c9868530d0df8fd93"], + ["nfaa0756a06f300495fb2e2e45e05ed3", "n8c7e35b0457300d9d6a96a6b1d18329"], + ["n90b7ef55fe839b181879e036b4f8ffe", "n26725bdcc0931fab0bc73e7244545ca"], + ["n8d4568db0ad364692b0387e86a2f1e0", "ncb2e01f0c5336fe82b0e0e496f2612b"], + ["ncb5843900903b4c8a0a8302474d8c51", "ncb4e013a6a8348bab087cc8500a3876"], + ["nf5448b3c66430f4a299d08208d313a6", "nf3ebee137c53da28091ad7d140ce00c"], + ["n20c4a0601193f268bfa168f1192eacd", "nd4ee6c3248935ce9239e4bb20a81ab8"], + ["nca50848d1aa340f8c2b4776ce81868d", "nc3d0d49adf530bbaffe53630c184c0a"], + ["na02956eba6f3a36ab9b0af2f2350213", "n03ffc3b9e12316d8be63261cb9dec71"], + ["n7dccd56c80233469a4609f684ebe457", "n8daedbb02273a0fbc94cc118c90649f"], + ["n0ada7b4abe63771a43052eaf188dc4b", "na02956eba6f3a36ab9b0af2f2350213"], + ["n9335d0718a937f9a39ec5b36d5637fe", "n99817348b4a36a6931854c93eed8c5f"], + ["n90b7ef55fe839b181879e036b4f8ffe", "n5ced56bcc863060ac4977755f35a5f5"], + ["ncb4e013a6a8348bab087cc8500a3876", "ne26def77df1385caa206c64e7e3ea53"], + ["na02956eba6f3a36ab9b0af2f2350213", "n4790f8aa48e335aa712e2af757e180b"], + ["nc012287be793377b975b0230b35d713", "ncb4e013a6a8348bab087cc8500a3876"], + ["n8d4568db0ad364692b0387e86a2f1e0", "ncb5843900903b4c8a0a8302474d8c51"], + ["n40d5f0ca4bc3dd99c0b264cb186f00f", "n0ada7b4abe63771a43052eaf188dc4b"], + ["n38c3dd0344a3f86bc7511c454bcdf4c", "n17cdc62c5d43976a413bda8f35634eb"], + ["n6e9d52e1ea53958a93e5b34022e7037", "n90b7ef55fe839b181879e036b4f8ffe"], + ["nf3ebee137c53da28091ad7d140ce00c", "n51426abd4be3a4691c80a73c3f93b3c"], + ["n99817348b4a36a6931854c93eed8c5f", "n89f083892a731d7b9d7edb0f372006d"], + ["n89f083892a731d7b9d7edb0f372006d", "n6dc118cd3f7341d9ef8c97c63e2e9d9"], + ["n8daedbb02273a0fbc94cc118c90649f", "n66a0562670e37648a3e05c243335bff"], + ["nadfa68cb2503a39aac6626d6c72484a", "ne1f86f902a23e7fa4a67192e8b38a05"], + ["n383748fe27434d582f0ca17af9d968a", "nef42d10350b3961b53df7af67e16d9b"], + ["na02956eba6f3a36ab9b0af2f2350213", "n03f39191e8a32629145ba6a677ed040"], + ["nae1218ddd2e3448b562bc79dc084401", "n383748fe27434d582f0ca17af9d968a"], + ["n26725bdcc0931fab0bc73e7244545ca", "n1430047af8537f88710c4bbf3cbfb0f"], + ["n48bbfdc912334fc89c4f48c05e8969e", "n8a805c0cd02307bad9f7828880b53dc"], + ["ne7af4a7c3613b3d81fe9e6046425a36", "ncb4e013a6a8348bab087cc8500a3876"], + ["nfc1dcdd7476393b9a81a988c113e1cf", "n8d4568db0ad364692b0387e86a2f1e0"], + ["n0197f8f210b3a1b8a7fc2f90e94744e", "n99817348b4a36a6931854c93eed8c5f"], + ["n90b7ef55fe839b181879e036b4f8ffe", "n302d25dfc9c369ab13104d5208e7119"], + ["n1d48483d8023439ad98d61d156c85fb", "n0ada7b4abe63771a43052eaf188dc4b"], + ["na2a8a54e68033d0a276eb88dbff91c3", "nca50848d1aa340f8c2b4776ce81868d"], + ["n90b7ef55fe839b181879e036b4f8ffe", "n9372fb07ad936cba31f3d4e440f395a"], + ["ndb1cf7af0e2319c9868530d0df8fd93", "n2ad9caac5b737bd897d4c8844c85f12"], + ["n8492d92ab6a3da48c2b49d6fcb8a479", "nfc1dcdd7476393b9a81a988c113e1cf"], + ["n8d4568db0ad364692b0387e86a2f1e0", "ne7af4a7c3613b3d81fe9e6046425a36"], + ["n302d25dfc9c369ab13104d5208e7119", "n1430047af8537f88710c4bbf3cbfb0f"], + ["n51426abd4be3a4691c80a73c3f93b3c", "n2c88d1c1d8b35aebf883cbf259fb6bc"], + ["n786694b5ed33295a885b5bcd8c7c1ce", "n0cd3b95c714388bacdf1a486ab432fc"], + ["n854753a77933562ae72ec87c365f23d", "n2890db24f6c3cd1bbcd6b7d8cf2c045"], + ["nc3d0d49adf530bbaffe53630c184c0a", "n476ddcb6dd33e2abac43596b08c2bc1"], + ["n2c88d1c1d8b35aebf883cbf259fb6bc", "n383748fe27434d582f0ca17af9d968a"], + ["n0cd3b95c714388bacdf1a486ab432fc", "n854753a77933562ae72ec87c365f23d"], + ["n51426abd4be3a4691c80a73c3f93b3c", "nae1218ddd2e3448b562bc79dc084401"], + ["nc3d0d49adf530bbaffe53630c184c0a", "n01fb40259ad3cf285bb11a8bbbe59f2"], + ["ne1f86f902a23e7fa4a67192e8b38a05", "n374443fbdc1313d98ebbe19d535fec2"], + ["n0b9e36e6b633ddb906d2044f658f110", "nfc1dcdd7476393b9a81a988c113e1cf"], + ["ncab9a48e79d357195dcee68dad3a31f", "ncbf4db2c48f3348b2c7081f9e3b363a"], + ["n8daedbb02273a0fbc94cc118c90649f", "n86a8b1a56f9399f90c4c227594a9d03"], + ["ncbf4db2c48f3348b2c7081f9e3b363a", "n99817348b4a36a6931854c93eed8c5f"], + ["n1430047af8537f88710c4bbf3cbfb0f", "ncab9a48e79d357195dcee68dad3a31f"], + ["n4790f8aa48e335aa712e2af757e180b", "nca50848d1aa340f8c2b4776ce81868d"], + ["ne26def77df1385caa206c64e7e3ea53", "ne36a6858a733430bffa4fec053dc1ab"], + ["ncab9a48e79d357195dcee68dad3a31f", "n31688b7ab44338e9e6cb8dcaf259eef"], + ["n07982b8985139249bca3a046f3a4379", "nf3ebee137c53da28091ad7d140ce00c"], + ["n66a0562670e37648a3e05c243335bff", "nf3ebee137c53da28091ad7d140ce00c"], + ["n03f39191e8a32629145ba6a677ed040", "nca50848d1aa340f8c2b4776ce81868d"], + ["n8c7e35b0457300d9d6a96a6b1d18329", "n38c3dd0344a3f86bc7511c454bcdf4c"], + ["n5ced56bcc863060ac4977755f35a5f5", "n1430047af8537f88710c4bbf3cbfb0f"], + ["n2ad9caac5b737bd897d4c8844c85f12", "n48bbfdc912334fc89c4f48c05e8969e"], + ["n31688b7ab44338e9e6cb8dcaf259eef", "n99817348b4a36a6931854c93eed8c5f"], + ["n3934eef90463940a6a9cf4ba2e63b1c", "n7dccd56c80233469a4609f684ebe457"], + ["ncab9a48e79d357195dcee68dad3a31f", "n9335d0718a937f9a39ec5b36d5637fe"], + ], + ) + assert not graph4.has_cycle() + assert graph4.get_cycle() == [] + graph5 = RollbackGraph([1, 2, 3, 4, 5], [[1, 2], [2, 3], [2, 4], [4, 5], [5, 2]]) + assert graph5.has_cycle() + assert graph5.get_cycle() == [2, 4, 5, 2] + + graph6 = graph5.reverse() + assert graph6.next(2), {1, 5} From e97fbddb647bbb0a9fa7a11af11f995e98a8099c Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Wed, 18 Oct 2023 15:36:36 +0800 Subject: [PATCH 02/24] minor: release bamboo-engine 2.9.0rc1 --- bamboo_engine/__version__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bamboo_engine/__version__.py b/bamboo_engine/__version__.py index 0dd3e0fe..8cc71537 100644 --- a/bamboo_engine/__version__.py +++ b/bamboo_engine/__version__.py @@ -11,4 +11,4 @@ specific language governing permissions and limitations under the License. """ -__version__ = "2.8.1" +__version__ = "2.9.0rc1" diff --git a/pyproject.toml b/pyproject.toml index 97eb088b..942bb580 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-engine" -version = "2.8.1" +version = "2.9.0rc1" description = "Bamboo-engine is a general-purpose workflow engine" authors = ["homholueng "] license = "MIT" From a7facd6459be199459301659269122d3874cabba Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Wed, 18 Oct 2023 15:58:34 +0800 Subject: [PATCH 03/24] minor: release bamboo-engine 3.28.0rc1 --- runtime/bamboo-pipeline/pipeline/__init__.py | 2 +- runtime/bamboo-pipeline/poetry.lock | 4 ++-- runtime/bamboo-pipeline/pyproject.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/bamboo-pipeline/pipeline/__init__.py b/runtime/bamboo-pipeline/pipeline/__init__.py index ad561a56..5d67aa6c 100644 --- a/runtime/bamboo-pipeline/pipeline/__init__.py +++ b/runtime/bamboo-pipeline/pipeline/__init__.py @@ -13,4 +13,4 @@ default_app_config = "pipeline.apps.PipelineConfig" -__version__ = "3.27.0" +__version__ = "3.28.0rc1" diff --git a/runtime/bamboo-pipeline/poetry.lock b/runtime/bamboo-pipeline/poetry.lock index 61cc5dbb..2db464a1 100644 --- a/runtime/bamboo-pipeline/poetry.lock +++ b/runtime/bamboo-pipeline/poetry.lock @@ -57,7 +57,7 @@ tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "c [[package]] name = "bamboo-engine" -version = "2.8.1" +version = "2.9.0rc1" description = "Bamboo-engine is a general-purpose workflow engine" category = "main" optional = false @@ -735,7 +735,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">= 3.6, < 4" -content-hash = "dd378de3f4e8e58678c31bc9c3898ee1b5d89765bfcda3c1de20c027f7058bd4" +content-hash = "debbfa59d92f30b7b448e2c897088be3fc1ad32db4ee3471631268923ebe40fa" [metadata.files] amqp = [] diff --git a/runtime/bamboo-pipeline/pyproject.toml b/runtime/bamboo-pipeline/pyproject.toml index 30a63da7..ec47b631 100644 --- a/runtime/bamboo-pipeline/pyproject.toml +++ b/runtime/bamboo-pipeline/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-pipeline" -version = "3.27.0" +version = "3.28.0rc1" description = "runtime for bamboo-engine base on Django and Celery" authors = ["homholueng "] license = "MIT" @@ -16,7 +16,7 @@ requests = "^2.22.0" django-celery-beat = "^2.1.0" Mako = "^1.1.4" pytz = "2019.3" -bamboo-engine = "2.8.1" +bamboo-engine = "2.9.0rc1" jsonschema = "^2.5.1" ujson = "4.1.*" pyparsing = "^2.2.0" From 6efaf054811cc7ac39b192c47a50c0e8ccb5c0ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E6=95=B0?= <33194175+hanshuaikang@users.noreply.github.com> Date: Fri, 20 Oct 2023 19:40:37 +0800 Subject: [PATCH 04/24] =?UTF-8?q?feature:=20=E6=B5=81=E7=A8=8B=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E4=BB=8E=E6=8C=87=E5=AE=9A=E4=BD=8D=E7=BD=AE=E5=BC=80?= =?UTF-8?q?=E5=A7=8B=20(#180)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feature: 流程支持从指定位置开始 * minor: code review 补充文档 * minor: github PR check 新增 develop 分支检查 --- .github/workflows/pr_check.yml | 2 +- README.md | 1 + bamboo_engine/engine.py | 6 +- bamboo_engine/exceptions.py | 5 + bamboo_engine/validator/__init__.py | 6 +- bamboo_engine/validator/api.py | 19 ++- bamboo_engine/validator/utils.py | 36 ++++++ .../run_pipeline.png | Bin 0 -> 47128 bytes ..._the_pipeline_at_the_specified_location.md | 31 +++++ .../tests/control/test_run_pipeline.py | 49 ++++++++ tests/validator/test_validate_start_node.py | 112 ++++++++++++++++++ 11 files changed, 258 insertions(+), 9 deletions(-) create mode 100644 docs/assets/img/start_the_pipeline_at_the_specified_location/run_pipeline.png create mode 100644 docs/user_guide/start_the_pipeline_at_the_specified_location.md create mode 100644 runtime/bamboo-pipeline/test/eri_imp_test_use/tests/control/test_run_pipeline.py create mode 100644 tests/validator/test_validate_start_node.py diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index a0063b37..40139e1e 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -2,7 +2,7 @@ name: PR check on: pull_request: - branches: [ master ] + branches: [master, develop] jobs: engine-lint: diff --git a/README.md b/README.md index da01162d..9affdb31 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ bamboo-engine 是一个通用的流程引擎，他可以解析，执行，调度 - [重置某个异常节点的输出](./docs/user_guide/update_node_output.md) - [设置](./docs/user_guide/settings.md) - [增强包 - 节点超时功能](./docs/user_guide/node_timeout_introduction.md) + - [流程从指定位置开始](./docs/user_guide/start_the_pipeline_at_the_specified_location.md) ## 整体设计 diff --git a/bamboo_engine/engine.py b/bamboo_engine/engine.py index c7d5ed7b..b8fdd66b 100644 --- a/bamboo_engine/engine.py +++ b/bamboo_engine/engine.py @@ -116,6 +116,10 @@ def run_pipeline( cycle_tolerate = options.get("cycle_tolerate", False) validator.validate_and_process_pipeline(pipeline, cycle_tolerate) + start_node_id = options.get("start_node_id", pipeline["start_event"]["id"]) + # 如果起始位置不是开始节点，则需要进行额外校验 + validator.validate_pipeline_start_node(pipeline, start_node_id) + self.runtime.pre_prepare_run_pipeline( pipeline, root_pipeline_data, root_pipeline_context, subprocess_context, **options ) @@ -127,7 +131,7 @@ def run_pipeline( # execute from start event self.runtime.execute( process_id=process_id, - node_id=pipeline["start_event"]["id"], + node_id=start_node_id, root_pipeline_id=pipeline["id"], parent_pipeline_id=pipeline["id"], ) diff --git a/bamboo_engine/exceptions.py b/bamboo_engine/exceptions.py index 6a198162..39d4fe7d 100644 --- a/bamboo_engine/exceptions.py +++ b/bamboo_engine/exceptions.py @@ -11,6 +11,7 @@ specific language governing permissions and limitations under the License. """ + # 异常定义模块 @@ -38,6 +39,10 @@ class TreeInvalidException(EngineException): pass +class StartPositionInvalidException(EngineException): + pass + + class ConnectionValidateError(TreeInvalidException): def __init__(self, failed_nodes, detail, *args): self.failed_nodes = failed_nodes diff --git a/bamboo_engine/validator/__init__.py b/bamboo_engine/validator/__init__.py index 4d01c48f..8474434a 100644 --- a/bamboo_engine/validator/__init__.py +++ b/bamboo_engine/validator/__init__.py @@ -11,4 +11,8 @@ specific language governing permissions and limitations under the License. """ -from .api import validate_and_process_pipeline # noqa +from .api import ( # noqa + get_allowed_start_node_ids, + validate_and_process_pipeline, + validate_pipeline_start_node, +) diff --git a/bamboo_engine/validator/api.py b/bamboo_engine/validator/api.py index 170b62dc..43bae0e6 100644 --- a/bamboo_engine/validator/api.py +++ b/bamboo_engine/validator/api.py @@ -11,16 +11,23 @@ specific language governing permissions and limitations under the License. """ -from bamboo_engine.eri import NodeType from bamboo_engine import exceptions +from bamboo_engine.eri import NodeType from . import rules -from .connection import ( - validate_graph_connection, - validate_graph_without_circle, -) +from .connection import validate_graph_connection, validate_graph_without_circle from .gateway import validate_gateways, validate_stream -from .utils import format_pipeline_tree_io_to_list +from .utils import format_pipeline_tree_io_to_list, get_allowed_start_node_ids + + +def validate_pipeline_start_node(pipeline: dict, node_id: str): + # 当开始位置位于开始节点时,则直接返回 + if node_id == pipeline["start_event"]["id"]: + return + + allowed_start_node_ids = get_allowed_start_node_ids(pipeline) + if node_id not in allowed_start_node_ids: + raise exceptions.StartPositionInvalidException("this node_id is not allowed as a starting node") def validate_and_process_pipeline(pipeline: dict, cycle_tolerate=False): diff --git a/bamboo_engine/validator/utils.py b/bamboo_engine/validator/utils.py index c7f20cfe..dc586709 100644 --- a/bamboo_engine/validator/utils.py +++ b/bamboo_engine/validator/utils.py @@ -88,3 +88,39 @@ def get_nodes_dict(data): node["target"] = [data["flows"][outgoing]["target"] for outgoing in node["outgoing"]] return nodes + + +def _compute_pipeline_main_nodes(node_id, node_dict): + """ + 计算流程中的主线节点，遇到并行网关/分支并行网关/子流程，则会跳过 + 最后计算出来主干分支所允许开始的节点范围 + """ + nodes = [] + node_detail = node_dict[node_id] + node_type = node_detail["type"] + if node_type in [ + "EmptyStartEvent", + "ServiceActivity", + "ExclusiveGateway", + "ParallelGateway", + "ConditionalParallelGateway", + ]: + nodes.append(node_id) + + if node_type in ["EmptyStartEvent", "ServiceActivity", "ExclusiveGateway", "ConvergeGateway", "SubProcess"]: + next_nodes = node_detail.get("target", []) + for next_node_id in next_nodes: + nodes += _compute_pipeline_main_nodes(next_node_id, node_dict) + elif node_type in ["ParallelGateway", "ConditionalParallelGateway"]: + next_node_id = node_detail["converge_gateway_id"] + nodes += _compute_pipeline_main_nodes(next_node_id, node_dict) + + return nodes + + +def get_allowed_start_node_ids(pipeline_tree): + start_event_id = pipeline_tree["start_event"]["id"] + node_dict = get_nodes_dict(pipeline_tree) + # 流程的开始位置只允许出现在主干，子流程/并行网关内的节点不允许作为起始位置 + allowed_start_node_ids = _compute_pipeline_main_nodes(start_event_id, node_dict) + return allowed_start_node_ids diff --git a/docs/assets/img/start_the_pipeline_at_the_specified_location/run_pipeline.png b/docs/assets/img/start_the_pipeline_at_the_specified_location/run_pipeline.png new file mode 100644 index 0000000000000000000000000000000000000000..e85e75761eb4b7e273b1ce4bea0b72e6f6ef4c01 GIT binary patch literal 47128 zcmeFZby!wg*FFje2qH=e0)jNs-QC?tmvnbGg3{gH-Q6MGEl7uSJ#@!exVL-bcfR*` z&iV6P*ZT)ti-)!5nsdxC?s1QpkH55}5c~^_7hqsu@FK$evS47)lwe@sr7+OICxfvR z&A=auMtpqIB7A%}($*IGMy7gTU?0Nm!bU_PWKg=fhNSmi^OR(CkRq!hQOFc{&N&*)1Zo$^8CwP7(5@IAe@47TB(8^Jb}Z7#({cYq4P%Ada^Jl4_b`i-kmWs zy#`jrt3*(NR*SU$B$u#EHwEUQkN@{)iDEG)xPc*foH0eUSMt^+Lz~5e){FwK66~D; zDk+!o%t^b4XRj#HZ}xVfo!mUWsVPUe2xrFmzW9RDL2&LjE_@p)Rdr0V_xUFRs(o4} zOrn0on&sOM2nJf8aX(XAy_AtvyT7`4P#YIru}aTvH8{L#yc+48EJ zLG0$Lpyw#M_GdZc#dps|WL%ADum|Q*SA+~1xZCc+ZQu) zD0rU#^d4Xo2sHL*|9J=;Ell8!n))~NkpE$-k86NYJAC;cXMyACX$IW!V?$vS;s3B? z9LsNX|7jtQW7iC83$OI6`$bXQL#Q(2&NIUNTpqPiv-0JGh@~Oq)4PT-CLqlWNvLO8f1J9JK#{9Ks zYC&ODRaMUHT;tv{GT{-aT&g(|$);~s3Z1JC7wv+phSYrjp;rj55RjCgK`9eDPvP?| zpI3Jd(Mp2P;0i$z_c$4X0%1 zINZzHVDBVx|Db`Hhl z*e}=*{VX=qj0F|j%f9-D5qma6D*bHNxbU9(XA6OMa(lwq+S>M>@4H;Xs~9q64oylF z7n@a>^_Xw(?462^@bU5z5Y4(l`YOK2dCFS5g*_3`bRZ=dp3EBuK@CC(?nQP>OZt*z zPvRMcLXi?_V(ng7rT%_kJ2GJeottRu73lI5o@f-2P>SQkrI!6-@y;lp?yTwZSaYpL zN`u%azfW5mZG3Tzr1b90jD(WJW`OI*u+~}A-LT5lJQkBGI9hRTp_ne{zT*2&H(WzYbMSF{tnk5a^*>oUjCg?)q$tpe} zf94&h>-t!|vCs8L$jBU%q`m*@glC&4pcK$+D+GT|gw|_kM`gDiY9sr8Z-y@E3jtB* zfGFL#b}M?ev!wT_yGk9WR9HeAmv*&JiPX(=1z?G@=p`5 zoNkishMPKO-sYt&u2)r8E90}%r}6Oq3(sws5aMsv;RX8t)Ou%FAKxz`g)9n|MbZlD z*5P*Nr`s=-%WrwFESGR=)4rsmG%AVm8_yFl?T%VYcK)4d}s_BIy&yfl8dilC-3wGv;D0PJw1I$7HP?oa4Rho z%qx(S;N71S5~K=?O2P@2kQo0lN?hwmCcDb^#(-Y0j2ukXP=diur zHT0?8Wf~YxcHw}gpx04QkaFMrg3V})XqxP7tbagXiE~b55ORM0s^0$WjRF|Y>jPSF zBzW?zQyF}byYD|g6z1p8VCa?W&6$%qMkGaTF?TsUK?y;^HoribBP*Iev}9RX>$-}> z`dBxg8Fkw{&jRJ{IEqMeFk-JrTq^haU13o{O^(;j)>he6?N-Er%qzldy(wE6O{Znb zF+^mHsvqVMgbmPL!m=J`ju;>8}RxyB=YpFr)41&sbDj8&_M{$mh zjv`D2+q$~CoK5Azl@kq9<7-N@1&E)WY6JuXMBU!b=%3E(7D{V=&-`*xGT_pjMgWY_ z<^q8W^`$Fl{Vau;@K2bM1WzV4?)Yk!LDh9F#+p4Dur4;(fEYeZ?%{j}zj+hddV)qsJ!RPHB@kY}l>)!RYS!uTmDSLg$wi-;iLbb9 zay^OQ>ffJVT6FB$<$)7|!jP`hKZ~9!q1qjLp|)#1Ir1pkHc3G7^>Q0|RG&u2IBAeX zot?dJVoSrCn$%!lJG?){{xp>rXE5bkXS!|;Xg`@YW^x0W(#$d5UL9Pb*J$3jaWqv^I&h)D zM)DjS^0Q*woyG9CUG5RYKQulSW-ChKWa#w$^+&}et(-twx$w>|5z~1e_uYwYURRbk z{+Zfta{pO=&=3o)($`&Znd7gmy-|mSU0nXN5bS<3&Nyb*5caRljQ8NDv zv91-DS76@Ohj>o6f+Tn2Nb%B=nC+b7tVkCE^JD%(^uP=3t%;;KKV$#kWE?WM3sE<9 z2W|oy8XltC@{g>8_YCXn5%+?TMGIx$8|<&E`m80K3rz1U9a1Y+8%PD!M&|K*Bt}=G zir5rGOOkEJ#!s?VmL-i$Om=HaIEk6H-h(4=?F5NMP1OfxxsB7`R0YQ1P(wqI4|72G z%6t8bD*s$j*Kx0TEkqojJSSXTEB{6p`OFIk$*{pPTi)%=WjsC!mA^Ts@EbYzGF}{r z)9p_T`lZ6AcnI|lzPhd#a7g2e4z`{W3Bq&;Zxy}H2mP(DTOWj2k>$FpvE)PAlwc6* z-ooka{Go4ugUyy^o48)oUDqz*t!;jEE}i~o`jR@)hSk-ZR(&4>$>?Mn6E%7jMInBl zfi;mCBn(v2_4R03Rn?{3$<9v4K598OBUbCG(d&cj)Z2O|Ss5nDDizHBn-TC zeHzf$uS6oE0kIL|6Z%8`2>JQ>m6VCBdKp={*}dnG$l?svuM)tswaCVaj8}Y z-{=KI3!+vq+)|w^=^vP>1-pe;l$@-NgpT+X1|k#H&}^5Qil2{QzDbu57eN?ejE(6W zl0v*JV{c_iM9BUDPCGOQlp847FBzN0V>%BhA}JcNx_Xc%0;-^D*mCtB?5}!N>T(vm zn5);l&m@wr4B1$gr*v>R7(4~HvBZ86Fqljyhi8-9f54Eqp0mqE6nekPg3P@Y5w%8e zmCI%kJ`~C|XRz;oim4DB8y!jxTLHqA<&DgWaFKL6ett zcBuUfYCNI!2N#AE8RFY?Fgbx>gph%!01f8l1^Gg3M{g)Z&N`>!Q<`bg<6_YB%PV$n zP2)L9%=xEn5CJR{zCDv7Sdki09V%$K!IX8FNai%t){%=e{ZS^dGZ=VxFDq;Pxk#yn z9}gctS^0kLQ|FhHH5^P!dH?cW8M5D`LA4=%l|8FrYx`0LWfKMuvkYXl1Gh9}oJ4N?pu;1IN7c*NCO* z_1D?(YmZb38p-luh+SH8_)m`t^HPL+8utUs!~?PM4H*F&hlVu~ao?oBzd{UhHiLE3 zWl<98CX}6r{x{dA8}xw}`ViM4ZF!Kk{W(`xmBpnE*U)t8^4^_2UH&CbW19U;R~PYJ zrK&0_Hv_a8KQV+yE3o<$&kV1c0Uz4;w-1G++_PmegnMdYkUpN2og7Id8QK}`>oJ9! zeS1mtERIFI^jhg84x$xmHbyD$ST%4RAyzo8-oPbyO3jn$D-%bCyYz+iqQNI|A1-J4&S~y3p%dO|* zQ;RDK#{7;gIJ{Btq`;9bhB!W{8 z;AMv)V1YI+J>jxX7Y^q2367vkNX$-ssjG&KwUIwJaMot4N;UqUi3drh6=f)&zmXpp zex{d6=2wVliKxWRBz9~-AfkZbq0m@X=HKQA`PPaCkCF08lmSRbkV@LML4in*=#*rO zSjr+yRdI6w{(U@tIxg<;$-gg9ODoILxNkWk;0X*zw$|fDG9z%0Zkg=h0G_mIF0Xf^4IHFyb0LyXZ zCL>#D`7fNMW(QaIOcB8VM1Dh=+U%arZh3>$$B?-wDmAUF*>KmkFIH@m;OpaA2!`*h#_&TBc;HqaD;`tv ziPizIq=9(Z^HnD!lcdFFiS+7j>Ae5`v-LK8ppD53c=&K9y76_n>A_mmovEfr1`p;{ zj`**A?V_^(wlAx{?Q8eqf=}NpV`o;s_GIrDB@0pmcB*&34(ZS3RdX9C#x~tMz^NZHs_HCS+w*yDvYgJBc$VPfFgdDP>DA7@H>_ zGpTV`n%!i7{>xOpQng{Abuj&W@ziFy+YBHu@;;E~*R;T>Tn5E|e#9zpsEwh)Fb37Y ztbw2aOmZXap~C5~Ut=`k15TPPBBH%zw5E1qJk0;<>2%hXP{MrF<5DfLl zD2I0>P=5&261h&>C}6(Axqgnu9^o*jjm*OIRNpEJ@IjOiB1dDg>eCM8`ys=WCJlKU zzLJe=sAQ!Ye80>Ruz!5Yrk}zje<=Rn^8v^E%T(=4nbu~rQ`wt&%+-^rm|S&3e7^cH zOBrc2fBh8}FJl3P&P6K#wf{hoUvu;VLmbUd)fX2zn7~x`-ulxled(&BPIwuAJ;e?{ z`eAQ7&mUuGS$cW%tkn+o>1{D(ii* zHDX;ef`wEGLQez+j2eQU{f`m=>*Pjw(b*YyHQ30yF7A|VI#Z)cN;U$W4xugxz!QYj z|Fk`%@?UWOe_AVytXBzTxu9xQO;tch+vpgYx^viHl9`5h_+dE+iNCYXc2OL0<%Eue zFX$hKq4Aw>P_yFmKjLC;1ei%CdQmsGW_pHdVLm?Z8S|*AsTwd) zy^Ci?W~N-fWGap3I#O%`*0Ni!i#>u;rF|t<0xpnzM8d+NII=2_NJ=tvo5}b@58)*P zW1E0;I$&`Sx$t~ot-EleccGR%h88(%G%6aKIG1c^y>Q?@>X)$@>BNhR^J&CHY#akWkc%rYK4js(>LQ^9?jDNc->`pwCt16 z7Y4)Y^#yiRrGZJ&JrOaXs4H(#XHSoanpzUSfFNWQACPnpaNPMaFdk@R`jcsF}0sIGuvX<#UU3LajfVzc8JtxdOWJTr|Avp_f+ z_}mM_w2ZQ>*_!>#Pmjuh1H_jwS&br^PjvaM4*0Z@F_F}8%zRS5&iv=2Qp(Pt5}0sl z*g%*_Wg@Ncgar2c;0!33pmJ_k-XvI+?!GL4yonr%2=C``gSJ-i)>UbvMc@;-k;I9} ze_1l1=30J$-~S3D!1z?Cdm zA2`d|RKj}t#Mwb)6wxLms%8z`Qgs($QcD{f8%Oy0!N9)WOm%q1Vr9PfD~oXzxGDBY z#y^QVGa>|w%P1#gN8?6&PlZPU=9J2^YaFFs@9g-V&Jx;7=0($ScehMT+d7Z}J#PJ| z^CUkB2Ejunt)W3kNT{dq5KGBI^}!o;_KL;kSoa`FnU;)7Q4pzWYi|z)UV3Xfkm{aC z!*x4!_%2q%{i6nT&$~^A(%jRZsqCbxwfj*q3YFb$^`$j6_EO3$U)$ndh%@0+(bA0< zv%bbA4GVddGC$N8~x#y>On|`z&&tVy6qQF%tQw_E%0zh>>LQ|;^UGTP(MPOV!@%U4cLsZe>ww6mA04% z&(_@|7Z%@5-yeT{V_J8WvkSfdFx_IEs^++|3Vy zP>nxVwR^HONO{`GkqL?^imeYvWkjUJqMMrDnW*FG{xYdOD!}XAU-+Ru@y=RgUV@c4 z)j4&J8l28QyTy}Pl~+fUcLu6;pnBF5OiHX#@2008C;@O8c3a3REscGDKYc;4GtawN zLi}*HgdGLsQ#aEmHslUk-LIXly!sx#JztoRWRa@yYI_UY$={>2TRoYWzX78bgvk1x z8Ok;|jSfgxIMt}=4-Hyj8$5f|t+p1x9`{@<4AGx#XAO#>>DDVX0)o!D3b*^IyucXs zEmLb#(;|il?X&M3UV*`hH6r}FK6Ifv_;hp~KdOXt;i@4jCvY&>OGsKR;k2$Q}y6`?x>x z)5;2&VEEQek5{~^Q{3+p24kx6bB7h6+H`Y2;qnA$zEX@jA9U$N#WN+Xxqv5Y5R=jt zNC~t$AEU#*2Hi-p_V~SRUY^<1hzbg_Rc>-jccS@W_!+NwssX(L3zIEfD*hdJn<7NW zvZ_$7lehQ_6)zzECt+3~(9>JzG2!k(Jaq87z80iAu5_7|-WLNd3ZcLmYyL2?!3 zmVNh=tPPo=<$f~c59V`9mNtt^HXwMU~6RdoBUBgxI2KMa7@1_G=qrWr@I; zf8_C%OY3TaK#lnvF4gThNv+N=ve|+%dM05HnxVqNN=8COB{DMVqy5wlDI)HNjP)(n z(J{Gq>Hu|ZLp=I4dO&dDT@UdhA1^;IfqXk&dr^KyfaG*=gXs%yEo!fi8jfGpcZHoc zox*h%gf=!uV&cUFALv$9?xkyYcQCR_S|oRNF@{h~-^eN}QzQ-_gt1f_B&6;}rK9q3 zLPEd6v84Hci2lTTJThmr%Z=Jr6V&k%dVG9sLOQyI8P5wYC91hvP=v?*x#D`ffd{eX z)iD}B+NGp~M7WruI{Ag`N@0cRHWaPdEXvi&>z2(XNhS^V^^Yean{>zPR13TINm!3s z8w&v>Tx6xCbw7q1qMD?}vn|}32OjA~o9peOIB#$3TbWn32>AJHBh*>`{Pf)G6YN%l z+SII?es^_H5dBG|`UiOZyGo$IC9I)QqHAz=o2;h<6^ZVp0g>K(J|plno`pmx^JAHG)O6u&^Yd^x=86m!VyE`cY z*yq4O0{^e80Im)LxQ9=TXmtcbRMS5F?52o_q*F(w?5z!vil$ zfve>anbGtxnNkvMy0RH+=UtoGEgv@3L(2RCRlQVLUQUN7{Or@|cyeP>U_rL~@!BbH zqi-AzE!gl#E4e`G8Ort9YM@6Ln&x$;jiyk2vsMVpu!)A$b`O`+r1(&+oQjHSvZk4t z+cR_w&x6vJQtQ?)!5d)*`v<~b(iMnIjAeeAa=7#iPOuNr+mmWYu-FagwH#%Ki18Z;>i{Bez&Jh7# z-lSX)RBxeAa04({G#q8EbEJ*}?I(n90D@lzYEIXC8fqZuLX3^cWN1DkoIWwqSBDd_ zUceq2U=?z5eN1c%l1Zz7)OnMh*<;IN?jg!rv0rbHQkHO=tEgEkSikMZC{cR14v#!& z7@KI(Fg*Kyi%avS-_3gEOCX2-)d+}-VA}mwr{a1iR2ZPqu2+Q*+Ql-qi%xLt8=E)~ z_NXv)CVguglF@EAufzld2)*FgmdLq`FumZo>SgGW|IneoVh%KLtnASrlg+PuLSCNq zWXbWqd6R6(c-Qk9%gTJWJM{f^{k-i@yQwPo5M;o5O~%K^wf76Yt#k;@3##3ot~>y_ zUfs%r9;~dw`|N6kaAOgsJRx!4-a!`a!EY8PKg@^EOuM@ka(>QhH16+CN2-R zI}l($yYtiVSS&{!57#ODPH#CbN*~VCl)dvh`ZZ;pMTsO5K5$*GbR4fN-c6P3iBW}> zx}9;FBtGCQheEL0>6w}a0YzGo%qvlG0?U=qi!n;G^ARqVy$m6?Kfvy9xn08mcdPWQ z{%p{^z35wh{+v?9KD{!!yu5r$IN-bD1tcY-`79A)xSQsnzP`R=(mBhF5Lj z_)y0*bgQeFnjYLCsP7`*-*0_l(Vv3q8%jRt?REfy37o~ChS4?wZ{;N-8k%pQSt;-R z$55K{;&aTu#`4bph3fY8Q2(SOCnAK0!+*HkmR5R?5BFsQH3lKtN^Rp4=D|o18XQgRpg@I*B@zf%sIF z9}^G1-uXi=t+G;1R7~u~HRE975B18GCzgD=I0; zXt>|*F9+ayXL1n;O|8_b~mfJ7)DmAMEKhUggxB%T@2*~Nnb5J9NU?w z!*MKq>Yrdo(`=elCc=E23pE@Ndu=pIEj@_?A9E^3eX!OURciX;L&wTkDDRpeu1)_+ zp_ile#xEk5Sx&ZC{X|9XjiUsZwyJNAcs#kw1y&dwr4m8t!zu}jOO1*xWOpOT9S4)i zyczGql?KQE8|TL^O~oHJYhCL17l1XF+le%=q6(0sy`5Z)qt1w|>%)IZdH zoSyXG4r+RfeAZUVFQj5P%9~mYC5f$|T7Z>;V5t~J*d(;7UP_Rom_iLYlj&rC`@IQ8c`5AaJo~R~y zYzilt%PUmn3D3(kjDVN4;K|scLMmI8UG%y<_9xp1buajZgy326cu$yRRg434L^n|M&%4mdH4kE8XAz?dy3|)MXBEw{mt8{BPIC8u?LAx*NE);L0KtwU#uU{|Ns6 zRYl;+Qxo>mzEeQR*C&-SJnSoLD{p%FlN*GU>TI+0L=AY6-ah&C(Kd8GEGM=Js1_33 z-i!F(%ncwZsB(N=NGE+F?Iz+;y#V86mlS!A1F~_}2UpM=agqNq%-st|4ok%Gd4lbw zy9|D;`2~yYq+d(@>sQ-*NM3RL%vXC6K^`|vK-rYXsOgy1!;sO8{}-J}3+3k9rwJdf z;p}mWH89!n{>ms2ooIk!)G4XfEfRAYdqbc zny~AHU9UgYrU?j%Hf*6iJ&e$bv}G9U7uiFQ)vV6T%0{>z{yf`mnAa$keyFs8N9QlC z&SR+*AnNoHvsF8d`$W*$6(CNlsjy>N$i#n|34TCAD%-XfatGm+FyuVme2ho zUM}K%IhL?uEALI4pU0UK%T1eu*{n*!Uj-^p1FxK~w8~|c)tixwEVFNxDs$v{WfKS_ zQ&=)$%w{gX61=k3Q&P&Ss&)|Ocl?3F!V=te2z5F$ZV&|MOeXT-j#T)o%-1xj;uDj! zCM}b0J13A7f({-}lJi>cpjc@uvaee_>}qg!I`)UneowKD3u-(9d5zzw-h(kkVNT;8 zh+zdUg{I^+aYkJ=PmQ2!h_PaLVit@DhBj9&W`<@XOAcnE^a;Ol7ayTl*grb?CRFU3 zKds4H7fir1667_^mN1-K&eI9Y%iwXGASR@vW}2WYK0@!`%B2MdKHWOqaPKya8l3qX z*{Q_=3pD0gBp~5%Q#gM|9uX3X;yyzB88@nTt`@`U1Qi|XaMGw2ADpgZEe!4C+qw@eh1o* z%9CeT3zbW;>VRu`cEiZlYCIfsL9+K_jN|we`N25Eq3>&TX}lc)>)JXRc7*p0<5`~x z7L1*fKvx2CN-#6VJVY`U$K7lA-5Gs2FR(XS!GU1S<+gDD3v-HTOGyR|xLCA)ATgSe zRkQ6St!cmuV?v0tRb%%&7X=s^B{;Wkl78>&GQ z=Sq&LF++!6O^C`d{m9~eon#CVlcy36HQ#Xl)j=^#k4UuuWlho@HxN zF?poa!aY@m`%oc}uQm#90n!yQ)u3iPRmR(){)hO+7P7h87K%oPCSs#V*(?(iFQ2XW zYLo0;e`5ZFCOKYh?_GgpOaeBTw_as*xdc~7ATcGo==HIb1YENTeeIvQ5c~UB*0-JL zB&5Yr4GlT&E@O*}ddbu5gnA1XoSdA3ITGU0N%`nV*w~ku<}hlfH;$ItMNm6CJK5K! z!IKR1!mh4O{zOE=YDt@|X(-7bVgULpA|_U-n27%74GB;H)|;K|Q>bq#5P}1#$11Y% zesafevjH*H27H}W(qiTkVmyGB_+4|vgyG+xC5QEJ6wEj`j}2o4-MiQnHB9S$Mm3u_ z$+}OF673MOI2J|tc}w^{dZ6?=h(4~3kIyaGbm27Td$NC|zNnH?9jv%*oRY|eSXE36 zk<*=2YuKTmFf2S=q?^lBKg0AA*-WJM`uY+F!PN;>j1ydAj{<*>c-V|hOz*K+No1|Y zLvRwi8m`>Mdr;2x*#zE3;Xo8kl5P4MQ)f%&*f2D3OgLK+KiJ#)oLRPmS_t2go*(D0 zz3_V^`_ezl8kvCVb8#_B7#={ht&Q~Z=1>+x4n6jMut>WhDc^Cr7x}sr&?^nZqnhST zAGR)GR7IAxeJs^BIR5C>HT-|<)di<&vsuD=_wo``;<+0sbwnzDDx+Wf-v-@Q3WcO` z8BK71Zq<6=&8VSiy%R-C5_+H8Y3i&=+H!K3cWs+fBwSz+h7K4`Xj=7Hy(V z5mVEQN(m$M$fUVO&IxZf!lSy)^sUF6H-{*Dik;M2DSUyY#f~n!x?oT;}Z`Qdhe6jR7B>aLSMdbARZch~Yc1!PNH#Vpui;9f+N*eLUn{u4@Xu=gW-&~!T z;6XuYXAttWqy)i0W^m_T#hb)83h$WL@u>`hu!~Xr?p{uECezFDnl0JPW5}#*{WQEc zv}#s(h6}P_$3Y_>3@6clkPe``ZR$f$0q((BEWvGtMMxbZ7KM!Uaha*g1=_9{oUxf% zg+}|`PPhfjKyvFAQeO&l$2BRJ+c>L+iW-iG<56p}(`jS}=>E=ODi(W^fv)8C2iEXO z10s^S#>WL~v5VLl(9WiJh&fvQ-WI)rGRt)RyHCjA9;no*jqf*Xjqc~?xJ_=3BG)TS zEYhVKvs)Ew_c@97v63U`2+L5-h%HwpgnZkdBQdK;e>gi^;LBoP4`kX^E&9K;6QW2d z7wb5LSDM96$+n@SVpNZ(2ZGy70^v_5r=v*3H&ksN>p{>`)BYFp)*QXj# zovCNVF&UI9Ake+_tT{@loQ9(09}7lW>`=B@UrN1EOYSBb+B zB`vrvrsTwxR8XrBKY!1x4MjnpDT&S?J>JSrNYpK+ub)6S2Q=yF2jnZ!wd()B=v4Pc z4RKC!KhRUI(kkif1>4z*%Q9=7m$+^hH zEvwJz-Vz8Je%cE$@{5in|y z{mG{t?(&M$D#aRF8apE?933;>oj#7rMuIAPOqnQ>NgwR79o8JM56fh9?rH6>3f70| zVvALXoW&>5D=!`R`F%)S{26F|}V*NRs}{|-_P9l)`|mxa~SHzuyZBN}*}@8NfpPP7TsCyZF_zz>2=9>}K` zZBzLZmibgwFD3_+XIQLfgg6OH?^bALTCPC&B#ZkP)Ae%;l4ah&>Mp~9E~y0CaPyu2jOt8l3S)D}Nl?5lEn|e1xKKzkk~+OhQ^ZuUS?|CQb}5l#u?7U1MrKyp$_oNQVqu^~ zkDigfZ`LkRFB>T9O%$P zgeWgZE|T1KuKX4=JIA&r&9hzSpgI>u$H3R#ap!`v(%ZiLb6Dyb1P=~f!AvpO^r|R6?ITK6e4ppd$qUI+kgwth8WSlXn&;Q{7 z!CGrzBg=K?#TeY-eCT2r62js45yXn$u>}~!GdQ2P?OfW5oO&ng_&%zde!D+ZUO=s-#<;-Xe24AeX>QBUVvK%aOce19XngCP0gBu1r>c zXc&B;kB*8_o)B2LOWre&>1U6K6;_AX<5*A(?CSI!0bz#mLT%eR|hwvKW}; zb_ReW15zOpH%{N6;Pik42(ktmdl#{`qG4T|XD?aysg^2R?=nv&Cq|n0<`fdPh- zlKP%8yg`gNCxh1r0GlvJZ49yH^AE6Ia$sZW4GHmk`@wKUrDpdWrz>Qus&wGJOlDp0 zCItHh{=?PT)lfP_E8^Sjk?z^B+Cyz#wxb!9{aZC=x& zy0^Edp{QsRqxfAJ%|}?c6>ZV|M6cpH0B(L*(|OFb{tP*`4(p`KX8)rYf9uzXcV?DV zMjxhs(1hM3YO~dZF+w$YO3u)8dDNU`!(GtFm0s2{`v>o84E7*%xAe7++^(7!p)+G_ zZ_j##7~${?*b_V+c>qqA$Ph1i0}|(CKA>wpEp8Kb4{@Et6aAgaRP@sMLOhf!+^D%c zIXRgirw-)6@dbz*a!-;O?J7*{YkqAFgs@hKvWLBId`5Fkj%3JtE?gECxo3F}`up=7 zH^s{8IpX~>+Y`CHGdnhc;k)l$u&3(cI0^ZXeiHWvmX?Iav&Mh-_7gdCgnTTc-M-pw~P|`I`xpGIn zPZN;+!}nr7akuCWGSY9z!(&l)6*V4u9&JY;Xj)|9{`x8p^A64AGcs%8fd8s2Noiih zYz{S9X>PT}S!Z-CIW73tiHPWM@!Zq(PQeDI(9Y$$0`h_y#;aKZYN?4{n2zb23>3s9E*#0_Wz5 z>5PkYnQv}%a&p!A2dD;C?Q(%4^V`2L{MXeFP3}#mk)dQ#V#z2`7njD$Z!4cxXzh0R z0Y9RHkB~@RLv>1K9GIPvBc~l%F@!epIe17t2NiOUI1Qf)sIRo3yd#UKsIX>nYiM^5 zerFa4m@0Rx_zqUps19({L00A?8Hvcz_@}8ZrADXLd7B%d;^i+X(^cBrYP&8xGV$oe zKkf|lGhx};kM=O9j#NwbUnw;mPs2~$qk)=cuMf7lLKtmr9pftdC!iQP1ee=$n7-=& z*;5MW76>A8XnyGvZaR{KIef=spM~Nyjx+m@2>^~w_WA5a&oD8Eh^_ug>Ut4d=B0J2 z_^!4zDhwoo4$5-ku7^NNh*%6O3AW`R{mQg@d;!6)s+p9wTnsGVit=%X?}X`8@JEro z0=a>-`|Ceo|FO`CBut%JHKgv|qPm}*J1ku3%Tm8U3w0UNkd;kTe^!xEF`MvgasSux zSnG-?*NkX*0)>WVQ4?&F17~Li;~7a7#Ou+>KBX_n2~Z@lsT&((4*G&& zWEV6n_Rrk_!t?pU(yehiyz>4AlJfQ^CzLJO+z=Hd7VR7K6>PniTaptjsjyWwc1kwS zSU2uZlm#-$<2K!+Jk<_}mZ?UXRWqB*I!40L5J=id5Q?~jB*A`VQs2XD4P5EcQ3!&f z;=@S|3U@!B6S{d&z9Lg^i+N1!Lw@EKF^aEdu01EDA;B#}-L#*_LB{m+_YZf3jcm9k z3Y8O(h>s%L&WF{f2vQ*PJo%R67Qtl_P_11>$Q5B%RmW!1!xr3U~v&M#8N<9TplFnUM;oUUTUtm%uw@hpSagFi!;6X z(*q~9j4$PBM((&-lQ8o=7gBG+&kI|3FIje^zWxLr)cMJCULq2n9eAVV%1Tf zu-Msc$N8q%R%G#FJ6)Gf&Ay=-3nzL@lfAhHriw*8d20YEaP5 zETCSrpl@($?;KKrdK|(_i07~0C@qK!2fgl1>jIh!;?EMbhl<`W$T1TVcZrH9N5@kZ zfEI?tr6R;)h9irDbp1mD$^a;ELd6Bb*mCCHh69cR3sQ>3fgP6Bex3 z9P{ZcWgL%#ZjUT5*RBhxBEFpD3;>IzIa6p*bS{CPCGwD5sJNehHVY)lpt{Z=DQ?dX zW?EM;5GD);&HXnQ%^e8JW_6}F7w7tK-#Mv99jifoUf=Ak9x;7PE-#|I?H_Owatm4h zqJg{x8~e9yszHTdlSOrR6Zz9pZKJJ?*mJz0o`Z!w8czFSPs)+)V%$mjIvg%}k%XLF zp3$7`Lufn$Ikv}!#Z=x)cx~~NIAgbLq6_h2=I=!&GA2`Hmt8g%{fpiUnF?wv0RaQ+ z{sv27I|` znyfh#1zl?o_^N;Emjk7WcAPgEw+q;Dh`($%L+L*6Qg) z@RpTvq}@aCLa@W?TAK;Nh0etce;z(8t}Ex>0REj~O7N5}GnJ{sizkf%kvo3+7(9-hQje|`vLdz z)cRgnCj0g(nNK0Jq}94+O+w7hL<{FRm8DhVcc55ql@UsReqn~rz6FU8>gpp z)&`|((iNKB> z7dKMHsOv9$e*XTB`}Fw?gAu>%!|_e>jY1zEaL8n5_CPbF#Oa4%+Y+SX!QFnym~ zW_kt?bfBn7g|;pg$qNgCo4Bio^Ua? zWNx*g^99nBvAF?HEcHi^;ApACqa!-1##u_RIQ3Z-+**Rw_ah zJ*)(R9^2o*z?a}&o~^s`s$EQ4W3<71U527;@i@-|2Z!7v866$vh``~5nWEZpd)_|e zTWBy6XBgbDLCR8Ks^8-GiEs8?i1qCh38sRACvR7M8kyG~ye#1yc{2Kty^OO2^S5mI;zcT0bqHj!15MGG=SnqH6jmA1E$Xd z9s&VG@rE~j0v^8i3Eu~&`Waaur!D`HDQEyoN+jTo86QrTC#>2Q>CR)RX^I`C8eU?2 zauN>I))KByAmnnR#|~nS~`ZV&DrDEIzo!G12i&nNrZ&bl5Eoc$NHh z2F#Ks3}thVvdzZgf{p*?Rz~Yvk+_qInb$iq&fW4W8-pwned8jO7~9svvp9cmm`gA2 z%k7lc7W7mq8)s=>T7tyHBqEA}Qb;sNNehLV(@LNfRZuCS4;}8cNqm!8)}PPPBYG{P zzU>=u@>Tj}a`SpLaFbK2Y~{#JnB1UzFN#MaNP!64kchls!hSn`BW`hy(>aKo4Wyxt zciRLSHuqri zqWz5{vbmLT{3p(EOoT4QV3i;TePv&p>0~Y)2B;a~ZrJGN-+;O0(EVt_?8kjA-ucY&H zU^n=(9y6&page9aP9Br-W{oa6&F=&6O_IG%Ki#ExULT(Vfv(M0i75^Do0QWC9t zbJVC-?$<%K})~|ieKL(CgR-G?V zXWkT;+(7s5|F!ZLz{-cmD1{DrPa=SwvxI`ac3UR-zxevgs5rN!3l!c#fb#k{yk*x_ndvcd&m95VDO`xB~@$Itg2ZS zr~b)Vq9btavBOo|+p=XO2#GSgmH*98ulz0Ih8(m+6`ie{(QW7dslm<`@T8srLf(r%c| zc0trWEA8r$VyBKD#oB3U3K`;s8m*DcS=Y-l=|EpENC*igus^jNKHnqV68yc*Q>!mk zmiJ9zpDT99#VfYYAsi^_OZ{uCy>o?i(OTl!y}gL0=9qyK@kq|n*rP|(-F$eL`$nLR z`uMGoaP!@xLU~ef<^omf$Pg~j7juC}2r<(=UVb=ntaeLXkoe0#8@LL$T`J?oBMlT! z{d&^YI>BFpg8H}05TG)Rux}Ytd?|O=)c5v!KxX}X_kF5T7Vq+B0;|)zOt+!#G264- z^`4mSzTuYK5uiH==npPLzkfqX3n1(lesDaf8#UWbBez2tSihTmUidcYxwfkhP1nHN z zh$Z2RJ%X#Gp&L&r$nJ|m_2%EkFgR=x7D9CDI8wrAX=`&lRd&8(KG2#PdBr^ngmaLW zPwQ4-zG4CnJ}T061nnwz;lp44r?`ZWEC0eQF*UikyS>8z^*Gqrrsv4lzVPWIg7+Wk zNwzf1_j@PVUDPvxMyuFZi$}}c*|H_Zn3$MYPLr0r0$1NA+a_kz$%$-sv9GXI;Q)dw zpM5x`%C*i8jgJlZdTug>;;vK@io2(4HVLT$JR0V5Zsgw$8`oNcqezq3w^zuK)482m za!LLar+(HJnoDnTzUCu3oJ0yOBL5kMjD-=Jj|CP8wb<;FkPr@^@iH9Da$!CDL5B6S z_>2eIGr4Q4Ni`93-?#A8&Y7h6)X(`ZQyDk@j3>yEsW}~ke={K82s-C%*H~Q!wPMCE zyee(8xUJcvo=ThJ^3+7x=zbPM)2MB5J4k% z0RD|pTjffb$IVD0a&jQIdI5T zF@6=*Ua;-mfxCq`YJm>V=gGeqy|d;Z_1fYrWpm4qsfMY`s(Nn5w;7uztHkB(0`GaU z)Q!oVL_uo5qcHkJAK|V5bf?uFa+O-l@7&=icSB*d8t!~O&2e5H*unuq@fUir-i-C7 zRg%0{M@X>!_e0{3Q`_e-%5^t5ctj>lvQn1JXo1&Q%N?pbBN9R6kB92b+xs33j|f$3 zD+t8leh~;cITW`xBbh+u!4(Wu+rqr>qCMHN+FSnz)We%K(`+=IzVdj~)1T~6{s!1S zul?;;XCuDaM_dP&dP~z51r)?l)0VS#<1Z2M64YYTlXt37Dq))&{b{`1S{-K#X0ySJ z&Wnt4DAh`5uvX?+boN&@g2Gev ze%48V%nmAETH?LE_+X%K?dY{$rtmz;Nn89l&kW3S)bF+-8gj7QzE;hOSob5h{6a6{WlLUyy76-!yqX8z#?G$5 zoe8%3g9l~1q#o+RpQqv<2!;e$ggS2+oVqr5W&?}*nq`a4-whwM0p&18w;FQru3>Qi z)z+nn0>M?j5j$fvgbK%?B3r_S5v7E9u#072l^`Va~Af{U5d| zp+i>Xzv@>@17lxQB>^tI{RpZ2NrC}R%P=deCwr5hl8ICptzMEX*BC_~;Y>w%?ZpdA z)+tI@642pX!7(#IVB4yw9M@9VHr}REi&HK$s>-4LXzp}(JE9c%z$-u3GX8E`SHG`$ zaG$1hcuBB&+@Wa9t&bShTR!}kzyG3rtyY+bgg7K#v%S!Jl4k$wc+%?p$NKklZx#s~ zW#X>ElV$e~QCe80zYc~4_$Oekb;GT8`pdg%n_WUSj$5buZb?~z8bkRkf3vIMem3v~ zPnASPSwf2InQ9aU zJ7Q>a6(Q?$QT>h-GDL;M%i<->n5TsP7J%vk@&j)heT{`P*LwhbW9}V*Y{_OV$8TBS zb<;+~ag3~(#-~Q9&|Wij6Yuj$_6y^;^j}{9X$-~}KuHuJQN=Q4IX!@0E7aiQ!5~KV zKGW5^qP4E+H<^rLHV@^3Yy0%uTR6RFU3he!6}98_ zox zoQLqMspfmgzfWKJuBt>?wT1n(PyQWn4SH#}00y2k4+`FA89tTPZfp=<68Trv*y|UF zLAZ4%EWz!ift2pqXlqc5@2eq*t)5|3QGUKC)3cnK&)~gVq$utv?|*oxJ~73z7q zDF|CN1H{QqwT=+o!mFz-zdwgBNFf&S`;z!V`T=zK@=V$N8QGUr;16FAkiTFfKM!C% zT4~Z45Ifp$fB2}rIwQrCz9`kQax3rHQC`cQsfFU{%gc^K8<~*adN~k7*=IUe(T-gS z-V^;2oEm%*0qiJ1akL_MvDs&TR0YwO_^RaU)m#2DycZW$7O5i@)S8EU{d%G}<5ItU z$}XZ+B9z-PFW|k>53uX(IlbbQp231?2>>II;NY8en~Bx}kD}?h z)jpV$-RNBXWQoJnhB#@HkPWsqc<3wmVvWdZ;3Um`&9F1MI$|VZ2fHf}gh2n4kIt)h z(ZUn-IfbE*{>9;Fn$CJ=X0l=#7gTx_Kho#o%unB0C7M{aZ1stlk40=JgIy}a>lz*! z{a`uZxGgrd-8?67N!Z3RlrIFK!F{t_FrX1MVx`UO#IJ(!N-Zy#V2B~wTMLQGThZ5v zVD0alU#Io7IFIYX-m>X#(dOh$it%q43_{O~)<4+Tq?7XL3YMvQeToOtT>VF7lvcLk zK{zU&w33!uVwCaRwY0>2)IY_BT$1GC1s@J;#dkH4GU)@iB-Sw+i~!gjFN zM@eoYY?0Z!ffDqt6o#J96tMPIU~TpqMzn}b!#5s|)d0SaJv;^@>k|s2xrFNhUD*_p z5#VfE__Z3d!Za?2*seg2fWPnJIPGN$`qvU_PoB8smE2&^N{yJxr( zc1Hnl~6Kz0hIOc(B|&=j~T?*z7w%bx@8skG+i%}2(6tF8FINBM0Ug?3N=K7G2gcG^%WUFK20PW}1iF_Sgt zVWQL~n_5)3CXqm1CN6%$;0(c7bWuMG3JOZ%L^sl4(h#Ri0;*RNO)a{4vhG#&vVi~M zqP9ueP@Km#o3kYNR({6<;S)V@(o;dX^8(bsNh@$51y&dPT{}5%m{wd!NxABT`7$lO zajRn~?D4kxF+cnH`3_=IHag<*vSslZMPceXawtbsbIkaE9Tw} zWUh`2z&8(bcifwj!636i>@wGbFA9qwU=LOcg8(?Z;n-zwt;ro`-IW+GfWmsFfTYVKOVKU4Ww=I>THm#G1-e{!{eSQp1HpO!d2#^)^(kawfEzX?!fg+4H^+p`l zGVUK1tNg0T;+y;O@)+lnXa5T>`o3VwHg+*s8-hzpN(IIY2POJY>bQ}yG@K&+{h5D0SRR+l zwav$t%grU#_XyeJ$j*A}(was}2`~U$%x!Y2g7u9 z4zR|m#;4Z-_C2inEa|F~mEL2e$3$(MqCzV$9k-5!VRt|b>8=MxM958CSV#=NvprK@yaxAZB!H^n;Em8473gtH7woKZLg&L602kUE7N`?kO^7 z$!BOedU%%aJm;%%nq%t)C2sQbCGF!pm?v+0hCZW7G!nK|u~~0^#?+4nBhKoW`GwzZ{$ zHng=i{a~Y_&FdiReBC6s;_RGRyH$ow z39LlDLO4?JW2c}&DctLl+cxRAUJTYt5(c>k7SDPpTMQ5Fi=tlsqZtM%HsJ7t?xD$D zV!M3YOJs1p-kE7sAK0d-qifk7iVJL`ipd?-#{wZ6dJ2VxY0rKFT98A)y#$ zUM-7Zp4QGS%IsrdJ7UxE;fhcEDknnQsnc|F3NArXkrAB?HFdtm$C_Wd2w>H&m9dz@ zV2h*A1YjK<9O1AAX$MlcQQ)v{^DQIp-&h#U5)r6;w)-1eEYhsUDLCS6 z-b#n7$5}*?E{;Jsem#5*D$maT?s(c*&>zVkoTs6jHmowllg{eV6i&oCt|=dGClHxe z{o&D}1ieKzFA~MsoGVb9L)bJwWlpq^bInTvPPH-uQA5wX>~XXFN_D+NyFyPtamC);6O|w_d!a2D87T?0!EmjN*D!Zf+o${JxJz9)wPwNH?c%A`e}*AYedt z4)ZI_q*TC~SVjabDjNcA8cG|42x*}8y{j-SPhV8v9qsLjU;i6_`p+VIoekK4IQAOk zVYRyND!!1_?6oBq4g_bG;dHnb zRGcN=eS&CYWOO!(ra%~+&X>&4`7n~pX4-Gqd>*Njk}=2wx57OB>gvA1hIi{KO2G0? zcaKbJH_bay*LOv2MPGyYtB?jNLh&1F5Z7axc@mfVTZxCeBu=x9pQp0|uZBBdiFQMe z4-Y8G{MH3tm)A*vY^e!)M6R!R$9?+EU%>*u2n2nFfJe`4t>Zq&H?cr2mEwHYfXcwD zT6AM`W2={hfbiOOwM`8lq;I7cfQaCw)HU^??(#WkIIG9SN z3xCBft?vut-u!XgaTPO>yAd03Q0IyBd5}c_o;r)8Uyr%mbCR&SYgY{S?jz3tp3Pc8*`RzgIh3 z>%2=(6L5*u<+}1(sBvjUHD^pDU%ltwB)xZXhN+!z2ND=?fSD4K5=fR{CtoC@X2&{R z&hk>mlfU?3An@^vT3erpATjAH;yB>cRS5^Eh$6Hob=}>^rcHBwOD^15V7+oNaq!wv zHeYc#O{U3y{pT4KcnjA#kf~+h*kdN%ec|m73vC*<6Vz7J_T>u0q6Nz>BVi+cLQuJ!h)OFAP*@ zWDWd~RfuqLOmJ@p`}|o1wl?BOrv{>AbBibRCpp9}5Dy$L&dE_;c<19fp<>hP^3OF! ze3pr!n}-D{-^uTfDcN5nbDDe>7ribtVfF@=*5&u?UKK0th77dic97Wac{#A$ysL$H zbCRnN(TSi#GA-iZPYMHUz#7XUJkD*ltqHra@Ilq z&cg5Rx}KnH@nFqY5l|+g&(&L%ZFH^i^OFNwAQ)2b8{cqp8en5@P-|kKxg5B6%b+`y z6b54YJgy%d59K~y#q(YE+leA^8=K^AQ{q7+<>h04{zNe0s-Jtj<4k7r_$(`EQu6X= z)A8hAh_$qS=$}&v<&_vX_6ZV*}pFLYJ7meKh(I#QLHI zBBiJITn}G)3GXcJQy|u; zJBW~<-SPZv`)7zQ-s?(i??`#KRiCg6=-iy<4i)4TRz6&QCa-oiI&wIE zP?gi}>WxTIR-ioizHUlM>E~4Nno?uvM1tmkLH72gaNOOFUyYcqLcHvP6^!=vhgHV?( zTsb1BqTM3A?ea+6gli7UOja7?UXHcXJUf+1yXxX%2n3U&m?cKish9NQP_SoboxqF# zR7Bb$rU=fscxsN>ka9X^c{O|KG%BQ(t%=Xh(Oft9Wms^~()N8LoT<;d$yPDUy3Jra z2enHYPyZ~XLO7sbkGOq+R%-OKkF_(ptvZCIao;<)Dl6I|w9#h;yabeXfD%?-sw=@v z8Ssq|9TSs_XM}J2qaF9{L2_&%SQ7|79?wfNiQ;@JwcFJlozC-=G1OpjnfKCQYTBW8 z!Io|tvQMI2Nr-%ipy&BLL;P#D*PX8rn7o?J=t$sJ7_tnAms&MOH0DiL2lrE~OnCjn zj%=_h{!wL1M}YHguR3*5VsrS^oj0_irtdB1Z4-Npx_s3&wq1~VjLYv858bjgVAiL% zp0kZ!s2zPr1MAq@;O;$9o}kxs$W9^I+*H6HtTw?gNxikjpd*4CK?DvusaWb|rtc}9 zN!vrQ3XPPSTKy?&@Pwt0TEpvga=5TgZ5jY7umurKpR1vh&Ub3{z%CO?XQdw>EZH7o z(aP_=Z}SGHi*dG75R-;1H+%cLp4~@mqk;I=c!CfS4rsCh0FFm22mYmv90JoKeUAh3 z@8B-r_7sPfoM!J&OZcXeTdxR`7E%9mZ|e-Yc0GY|57==k)r^(yK1OQoqCq!in!_SQ zxR-<1p1#mIw@c4$mA-zzW8Cg%iWFAn`6ARefqca&7$Owtb6#InCFvr3#PIY8r{%6j zw!5$DavWVdo%u%Hi(@#9pX^6LYm&*=kbkH6aUR{Sv|g12E7qSSaXuJ%4fE)qlsknx z+>f{bq+g2P!?Azi#dea6sXr>8R}sxtMMXtoHlPs<_`hM*QnFR^kL1za5|M)=oO&N^ z-*<$~z^?*KOh`mX3^K<;5m|{M2%1+(QcCj|?HfahXA1nFl-a(_0DLhk$>zH@I68Vd zr8vw!NfECy;K?cw2lIPA3NO@{1%s;#CRdIy?ST5&1f|@d;x0=;%BYH#$kDdFdLCU1 zdFM8v<9K~+T+K(j3gF!s7z3#~iS@jPvL3r5(|`TLf&K+|h+iePE;2{o?h8V}M2)Xc zy>wZqQo8wGe`J{rPffK(AW1&`qKZYP)$5ubub+_0q;Fc`Vz$#)aG}jCRW^-llyJxZ zYcJc=kWsr%GZAFK^f8UKhf7W2hqOo+CfsVv@#AIhkZTdMnP?XS+!+63=%@ML5if(xG$(=_t z(|0sq4DI;Dgn<^9wf@sh+SC;a@0+}ScaCd&P0h{b`4$R`Y)}RK8y-7$`w6u<2e)$b zgpcrWg$U2FtRU^*+8?7)b#aTc#L`+6rda-yU*JQXr?Jnr&W1;2Rr*!qVBNrB)OcjH ze>U0HH~4;K4_uM)RcFldJ3#X!cv*W7ZiTRcr@!K(ad=*$C%-QjAk4;UIbLn0l5gKG zp;hzVI1Qdz;&9wgkO;A{K!X3MR^yPQ~?ewk3;>h5>- zQH^;j7Ba-J=X(Isl5Q+r?qr^Jl!ZtQ0^n%wI4ojH^}lMhIM+{n82ap}#pU(8C=EdP z>>txAz(%4vo^2~rKL$0KnsyFb4#44!iLE@ob_sD_|4_vMUwlAkSlSgxGwUB%TM!Yi z=ZQ5G$b4y&)fvvYH?8jU;`RNpeETPK#60zBUbkLup7od5q;I>~ZlrhC z1$y5_=yO91Iop8PSb<6IEhD;(+=2C}SX`jixLm^R{ zqED)9P?hj_Em*^Gx^5xZHLL!Os*I60_NGOYAJLoy+^ZH5HyU zHV!`ee=Ca6qRlhg8@&(0dOFADcEOJQnAGF|E*#-drbC~ewfe|&r_Q-T&}Qygs1AbtUuh7biG;Ska>rr5SS zY&u4|xSASNjz|aZtS45Ag5n3&dwLPMcmnEnY!gE4!P3T;$^UtKI~c~b*MA&XR&PPg zut^9M)^idHru{=nv>93OJ|45gpgThkK-Q#^KF!3|o9u)G*V_VN3C2)OOF=;o_XeJ!e_D zhw|M|so7`x?dEZeI@R(uSG$C_i?_jr(03!HB_$oZftVK2FYIfCY@Ex~+Ou?La%GV~ zJUqz-2`XbCgmgYDyqx#+z1rsgq;GhUxoGT-kUkCmpPvW%KX~E5MdbVd^49*@+mnu} zjiA#Vej_hM7gC9WP`a+3Pd}Z}uElmfJQcCAmw`Q1C>LwjaA1L3H_p3+jXt$2m&zs# z!^Pr^oa+`TS3kEvC!SZpB7CY~*(9GG%(_Q}qE;|LTy zlVI6UPTx6tUL7XI#!g1+hC02diV$*3FLi`i(9tb3DC?NnK=%XJTSTNH8DiF3x|MUp`yh$9gcM5E0j=B0x_{>QW?3U5 z>pGW%n*VEJzeD}V#NK6fu!w;vY?HW|F52*6SV_?7Hp@cHvjj&K%uJwr^sAbaXS`G~ zagOW29D`YJvWgjG9DSiEXx#D8(g>eVpyQW~t&`5w@qURF$HyKcZs;%_HYqlv)&kPZ zcw>F95j;{}aZ)GzmffTKnx>%VERo~3nbY=Zp}u8`Tq!i8B3EJdgV_Dvm|fVB8LO3O z0cx-8yNZ>?XzOzQZ}2Zp%%0(eGiG=u{A|C@Q5tOG%t-i}U?8d1>0EEy`ACC+wJaCFcQa@9mGviSBPTV77}a7ao)IwMX+5FzX+qRMnnnj=ZSjg$}K^SfYrTHmH5G6UiE z#(I||>ips2W{J)C_lwQ8{e9x*gIK5SyRU!ZF;`tZJOZ#vkzv*6DshPiKhQ1oT zdNoy|nkVcB$kHLotT4X7#eJ)`Pj2+aD%J%XZWVw0DzT%_i^*Z_Ry&NVl&=H4=BbmM zNog0xBPhsVpIu@#Y-w~+XW=0q=J(gy?XiT8PBD*7mW##j6l&~mnWYcHsTMZpGl5-g zaoE+D)}`(<-X!}*HW%4}!tl34{_xz!@{@YXy7V`j%&@~$dT(WwXxTN35cm_6JcWUatgYFIaOiyZ)Sw^1KM3&&iZBkQ^whu6J;?OPUn^ zta!BSb^X55GjF>+&*3@jtMu``jPE0(TP@`lwl~bJca^h*qNUf98c`44R!H2jPTXg zYCF5pnLQg+BN6p=2H&nkMZ8G`G<@?%37W54DT1*)`6APi zf9DD*uA)e}KNz7b)XJUt{EimHR{R(YC}!GQbT4LWx8vlR-mA~FtK>iP_MABXff->Q zz_!mh3hUPKwu=~$J#WT#TD#SaA{A+aWM}?&FCD-DY zGMA-Ez9g%om`YBnrkGMm+57sa7HrNR0A(;$RcFTaTu)JVRBcGx$aFIFn}_?2YkOkh z=6oL zM`sgDaPaWcW*b!VW_e#t`s5p&n=~4(j?zuHre@|Q2v{q(6F$dDU7;0q5_~yWW8ksR zR=N1$E7dbUpv-fda|A-u5Y0jZ)a0F?(Q7rx31Pj=CuiIx9f^}$sx2QtT5^3a@n&6L z2)fn zLr{*$zts8Qt#Re(VqiF)^JpCg1)F^UBSOkWnIeN!={yrASXRziX6DR%3Zl zWYk+&)Fg^Qphm$uF@cTuYN5%I**bUnRHxwpVt?|3x>P)jCpzo=y^4mO#O~UK_72DO z(+q+Zku2LQ+Za5C{o2_$CH^~J6t3Mgd{Dj9bzf|Xk3-lk43nQ~EeDiX@`h?hodUm#Ha+wGr`yP^{(!Im9I7%G>_G z@Tq=((uj`RAVFACVif0R9py&L!=s?o)X7)j3m8bzwu3X&JC zj;5)u&iK-*1A`r)z$L4|{fmiCBTnJRKRyS|_^hJbWpKqTMSqOT|2A)t?9+?8P(t<_ z3h->S3OitQU4&vXcLxX7*w|P`fH=omdP1u}*{770WpVf`eqDg@>5m)hFyG?zUKHg% zP9r*_P4b8ce^q`rvFPHgAoQw^WtZV;cB4*bt&n<|HV9DhF5eopZAG0H6PJ?n%0bBF zvQm9hRU~AEtUF)NP1hJElr_W__OG@6Bbx)P`W?=Eolo^0tbVOOs6NbeDKTNT@U?w7$M=fx!@#~#`6#8km(O%I=%9)`Kv7FEX< z;4Yf%U2m0n{{vtCQ<{{e0+%}LOJ~67rGg!I+yDLGmAL*jVxJfiQ-{mM2zJuAJKI6N zK|4ck-$*oNaINV|$zwkeyLN_?2om?heI-F&LxZ|p(5tt}-894=t|eXNX{4mMR0;dv z_&sY)cajT9B%uEw;AuY53SqQ%cF9+3+LM1{U^bM$F|dBC)N_mjTAKRyEwz&46XOSBD>Vkg*$W59vx#aPHfPO&jLgdK-}CAq13k6!oiZ-Q zkLlcE7$JC109Z{S?uLx&2>^46_}v|nACp|C@k7y@&uW^_eydd0rPxiu^XW;Dmj;L4 zOl7XU|16X%ZdzhEEV{Yq(iXTDAL{~$fIB?93B7iQSK|Bk7M#dE<8QVyQ;V>}k5pce z7h03K$)I?%ZViV+lL=aDzc)%|*(2Q(wEt43e;QJz|4@A)4@BAc2&G-3-N)ZUL)oq& zE4vZRjBsk8K&I5h{$&~4xKg`2P)LMAq7=PQb`p%1k3u<;oHr{dxP%>S3R%Ukt_}gD z`KUeJnq~V}5MR^LVepGO+kK&y(HOjerLH_@y4Y4xA&INn*%mw)WF&b$WtFtt9H624hfZt zfAgV_1ed4(H$l~ufC<03;^rmm!~1^%KOh7l1GFzEOfGc)M;PYdZL~{~W{0+$W@I{J zwaF=6EfyL?Xg=FSjLk`eoG2duu$wyH;(;QIc9@xoE`IN$9?gef1?6^Nis|{|o!H<| z==RO=UE>XNOj&zL;b&qpdNz9P$s4SMiNX=pZo&4~l;Uz~{CLkB{7Za(*zjLWxBB{u z++Yzh2RwHJNDTt3ji1#upB9Mel(8;#q`{KKr-cz#mW}oI^_&luOr8J#c=NS_tZZG% z*EXL?TWHsN=DKYj1^z4T++}^WZt<8WPK@_SRNbUh$^74k3}N->M+!sHzWCO04l08; zz9FNu|8>{U@BzccF60A}>tJZ5Ad5iDa6*Dv?}bUcj3@eC#wADg2Nd z>8hXC?jh49{0O58GSx{;-mWy3>@eCmO*KisH_+L6cW})1>8C08O9nd_CokzA|r+FI)wl|^M*i-#xEbO$NCZwM?k%5`3>a*bhpxFdI* z<<=Oo#Q8|F)B?rgH=Tkgb%!e`IfM{1mxvnY%-E|>#4zC-O+$=o%+4b*kwDsmnn>8r ztLFP9P{>`X{!kmuQ;n$~2#AtT@@KOc|NU&PPr+rs5kB0vdHE8bQA3wSIVhiY=(3$w z6p7K9cv(<*CnJH5_cA@YQE-9Y3=hNZ=H}gBfl&NsL2sb==FF4g{`z$H)H)c5^^SYN zj!?ZQ;vt}l3Nnvy{W~JbY}L#S5U5%+Z$?B4mi#OuM9K9|HYazJ#HREuD4`O+ojOjw zQ^>h8Vt@aJo7`DQcr0Neoeq!$LuAU)Zr=hjC!)SBJT7xCEiGhQPfc>psz7urvCZ|r znj0stffhgJt2M8nPm1UGd>0qS#U&M%&d!qd(YlVKEY}N%?AsF>iBdjB9=Z2rPe6g6 zDT_Aw;bdhuoSTukV}lt*?R#)Z32LrpvG=6eY_rDD?J*za)W^}J5S1}IHwP?JJIv8K+ka|Qy08E?=C@IWj6@;GpTJY}ly!=(K&!6uX#08|bgCpb< z{>$OL5CRw@ny5X7U%$SeEu48rFu!xvl+JZ&&xs=9q)@xyF6|t!Y{)W4os$VCdF^&BQ?(kHdL|Khy54~B+oT-Fpv4QU0m zhqJqXkF;x1$Z9E2r!OTQ6P)(e*p-&Y8NZ6{Cz$^tEU-Mu@1BGI(|2;Ni;d zZ#2Cz{}An2P{bNcyrIG&tQU0(RG+;>E5C3$7a`Zs;r&@?U%(dmW4 z!Xa{f152B~?`wgOsA|c<0xUvQ^c{k1d$Mo(v?yKb)i%zJt zhgLJB1uP9F`~0CVl<(Dst3jqdYwRjW|9M(Ulq6E>)vMvS?miEpL;T?}&e=7SjAJoD z+MFBYI(p#H)hIBZF2H~P%|%-K-|jJt+SLjj-g{ggr^nX$Z8eKF&wpiq?`8eJErp=^ z1MFa4o_(b%zJ`m~rS17J=xzMx&lFFD`fo1wi}g#zD3TBKB+t=`i_oKgY-A-yVTC6( ztp@0m_JDn|jt{Lf-}1esMYW=)?})m&fZAWa`86=xoc&9RzS3wGtO5Gj z7c_LwnfZofx(yMFzg@GW#F|)-jfzW>nfv3nQ_lw_j$oz#x;$iOU=*fU-yzrkjuo*` z%4(>PU%fF6`?TyMx|7MNYujPmM#{~RCuvn_!41yQ)c3W^Js1>NFeW?ra}*Hn|G2hn z8T;#7G@5~Py9!p+e0U25=6b7mlRt}<PtvWY-qb^K`0jKv?QA^2ki;Qrd zkQ9GY|G|@?5*|-0_$&fTIP7xYp=|{*R7N_S{!7s8<3p9F_uI~ML~+bLCWG?PM4lPX ziu$~$iT{|@>Px`EzLIc!>b(2sn7ix{M5ez?^57B734Xmu4OLMB3^&@(r$|RvmmCrv z8M?1Xmi9%+4GsN(w}8=+mQ zC@#n0*PSkSB|);CA2sujhXjUxMxh;Z#r}I#vJYkSGQ&U-v^tkIucl_uZeLF9rBK#A zT!NZfjoX^f=Fhsi>C2B@M93G>UEgy9V-uAHK{3`Z$vn-HbvZu9>w=H$QP7dAT=)L* z>wf`*LjTOY{I|1R=Y$JEalppGp`MxHsPVpK?kwiS6Z^~m6766iJ)b70@+Ci~JW#%J zIOfOx0`le=|AmV9;}>&{IVZgPqrl314dZyKUe1fxlmDNGQHcYKv>J~{7JTW8xZO|wl2YWamqebI zT0c3Z(5f@e^tzpNz$RNf!>tBq{?8X@ZLhVh4rKoOn}xcv;fm?n^J;RGYih6tfvkUi zRhHuOkLcL^@v4f7(I!^u>E4n+{73hJ9Z7}?2aUHtulTDQVkl}@ewCy*YFL>*{tYS- z5j=G{$VtatBXzk&X1&(TGlGmju+_WZ?tU%R44Sd2TuQS$OUJ?@ByN5YTe=(ho@Rfx z4^Tt^jG&EneWfc9DkwiqnE20o{patWK#I__t3{z*zVIvcR&>H>8X!wn<1{Bwtcc@c zx1=t(D;5v#A}1hWcw?3MO2r@b$G75^mPPm1t>!BDfM%X~q=eP=oW$g9=S z8lCeg;f^-}NK#&2=K~R~^{;i&{pFFTXJ-ajtX)K}E|b?i}_?aq-4_l%JNsN|gp7UxKujH2>``Y~g;95R;M3 zQgPmS&)LEtgJ%T=mx*6QIp{1epK-hLX;@9m$O3~XdkYyVm+C#{I+d0n#Lp-^ny;}F zMAvg-&=`>4`d3vcqF`Yq5psm5k>Q7CXE!m4%zQcEQ7g%6SeY6vi#zKCF(s^L3GF3m zlZW%V=PAnZ8CYMHDER;uu9ywp+{bm?ck`PA2?hC3uFrId^?aSmqv28i9EwK$+ai+c z@yKW3@yJ^_ZhFa1sQB%F5Q5(a1gjEi<3lzn_g_Y(B4#S)+Cz-*@R3#c8Q5kNrEgSp zmFdm$w@Z-315u6V&$H!)h5fg7IWJm}6@}4Y8|6Psk!mh|5$2i>HP7Yzli3Sy%)h_I zgR{Z?8XEd8l98U-a+9dT_Zz0F>8ru3`_IZE2w=r5Neq%A6;&z__`;B#YhzL$L$^pV z1y|oUn2OKfF{<_JjEt!7-?-#&1UIjjMe{~0l(}Q}Xawnm$6o5`eAC5K5kDb7lH}m?j*Vn%vcSD z4EB;{3%l_^44|aHV4Wu?H%87wLkxNH_aG6B(oMRAgao2)Mo1UR6J)DO^&_JZNAi{# z8q61xTg*s{UuRe2@|4ceW&M~k79aeXOJV&mDv7sJ*``S8SMe(T%a=yj*q_EcMi&+3 zF!yNunXgIgADFIW7PgYH?|-FYw>Z@zB;vZLxHO^v=NN%91Z-r~#C`o@Wr1x*x)k4l z>E53nbS5>`0bPG0Bc2ps#Avz0%9_{tD->k(_@_<))2pb)Rh7*gjiGw93YJ;e){eiG z@6<)ViNcV_?r1CuPOnxgfIdB}7bXpX1iF{rYXMeyX{>~y4E|dxVYz8CT-+_FSb-O? zslxumFt^NhPpD@;4K1w;3xWKHEa5B}I0ByAPzRN8h3Y+v%F z3fyNhpS#HQJc_1^qiwSN(<2A)2v6qjqn`~7WgOttaqYJ>S$V90mcZ?=#Y|ZJaCiIu ztD=SGnqs{YuGRG@fo-OnUU&p4QJ|=^lTFbMgzBud{v*tc>VH|sE_FDQzVO&N1Mktd z=dH)yrhM;OpXvf{ZppnqgCwm$A766a3mr9oB#Ucm3VJgPckx^O zPv8=a5QH@p?E<7Pjf7(kmKqHvr;wa=_TGP5qz5MC#y<6)cv%#c*ut*^FHT(7sNU_? z{Apb8Ob|NJw4BB^n>+!4BQEd@eEt>DydI~714|Yi9NfIYaehYT;jEGC$ae;bb6oiU zwRh$JP}=3DZ40{ zvJ{bRB$6Q_@43hGyiDo+2VQ>gxsAD(^S#fx&ULQy{a)8%Kw?6|DUWJO(44N8R>b|H zASa-0yki(g5aVAi6c!b}m?owvwj_GyYK1mPEuNJ#Bflc&>OTxym^0C?L(5s2l= z$^~3%Z8P3G_oOq#JTQy@wsvtzw@Cr^*PGou($a=?{7dcbcXth*`_lWZV6Xb(sPR5H z{2_?~QLI%Ldl*EhBx@D?3dy?eUD9=;b8uh4++VJ^5^2ClHr)HkS-rB2@-Hd_C7n%o z{JQv=`_c~#DIstJR%HeE-aU$Rz5^=(m4AtE)S4$u=GPhb0V zImCE#bTlR__mOTi0n^#m$pk738Y8jv4O1sH-?aH;8s9lu8HfhHFl&aF-ajrJNn=l$RpyO<0)y%dh_%O5%P=sK9^c1}2wTk9Gc7|YFDPfEL6 z_>yE1;C}BrD6G=W2>jjgS*lcc5s!P(L-*iai z7|hPfYWWUWg}eIHOpZi@0)YJVnYZ?eSXRcf^ROe>$s${avM&!m^lWwhna!;i6KIdU?a|}cL@HEy?h`htEN7yL z#g>onlZ3b}YWw1ol5+WdOAB43mHI3gw(o2=A8|*Q z%)ZJb={u^!Qn~)!6&qAi%*`Am@}|j-{HY@=N1-xWl8wt7GWy=1@~! zEccIm-+e6e_|?uNRttb&?#p zfigy9a2acyA!uxjZV8<^ z$g+v~+u3H~U8B&LgEGD?aC_tBpGywF=F6NJPz(SvCJ`jk#+x-pi@?=9J#=R%Mm8tc zdglmldhZiey(7B&iwCF?v;HKsb0qnd=7Oo28IMxs{ICiL&U#2bBFoq>37oDJ>y+y} zI}s)kLjgNVaxFIB4jx@2WP4;R!U0wqCN@7u8d#fc|zk(z4es|h<#uJhKj?;&D zP_BmvU9D*-O`xH&o2Z7Es0jHwf8)7J^#WJFe^+EOxbSd!_-%E(Mag?P_J^LG5S^ZF zDmG+0zj{85RoT?ElVLxX)I$EN1)|M)j{e? zExu2EA*;ZqI&bXbbW2AZTg<7PXIsFfbin%e=R5M770oxeI0S8lsi3NA%PzaDZ02$M zmZSpFBDpK$XFDPCA{$Z4+q9F)&!@LG6h@B|%-vbXCz<`-&1|} zaUiv;m=~yT!MFHPCmnO};LVaE?q~vG=wqK6`~;$MwB@hAwK#QFB~=tFEvT^XB!1b+ zjThSC4ygu(8mHz|LKfC*)&Wjnc!k|?pt$<0D|Rn8vxE|cBuKg!Aw3I5Z*0QXAMMBK zKkmhiSTvH6m3{8x@-!#l0pw+ZTxp@qc(*fT0!^rK1+n&%q0`e_nV3EPo!-!5&}PcM zyLT}Gck*OPz(vY&b91qDgq3hv@SOI_Bhtb4wBiYw>r_mIt|qnh4T*%&*N=Vv;)T5O zFi?tk9Jn1Z{&JpZEiCxjzpg}%y%Q|T3?%7V$Z(0oEW`))gK*skPS$Owa?Etg= zXf9!-HCgQE)Q!=PH@%zMkntWpz76vusUAbMFvvKfE9vmO1Ny5CgpRs};%~4?{P2n` z8{D>CD`R5K-38CJqZ=D^%{4XTRLyWz!O2;-SrhNuYxDH{n41*4Ugmz~5@&s5jb@ba ziiskVU9nZcC5Gj|1kCsFA`kaf6@1H76z}Vi{F|03t;Ts(n-i zJ9JYGwfZzYC!;*%{Jk3_71H+uea&2J|1}zeuC)zLZH(}uaU}jWT>o81A))vVY!7zT znEZAW;s9`arFst& zyJ%N2DmxIWIGGA<>i!lzUxDRb0($Z}99!*^{<#Yup2jWjs$hf62l|r9eGs{D<#fg` zy<<0467$eD6O{I(*9g1daK!1_rxm_fc1{jSE#h{TnnVU*v7)4ADzhi0L0(30ec-;1 zEGn*uq-o#GQzzI$%~{Jyi$VM1Ok*nm*;6DvNU4ClmV%)zrLnfvwyXf+Sr;)3RAl@& z0I8pa*g$Wb;FOd2hcAF{z0(8}0IK_v(Vy?s&kb3Y%#=LgKi!B9q-qiLJQ3%cne^v7 zeUAX|yV}J`{C_+!3Ut4Bw^HH%NU7Fo*Lfi@JN_R*^X^y%LZ(UF+xCN=TMC#<6h#b- zr9!nfbLkW|x5bnsY^HaI14JS(YbUHT`tfYJyJeo`JAE366A^YJmb?VE`r`ASOO8yE zNLhHk@dD80v1SNlOe zb93=-Wn|0tbDN~~Em_Z&X&G_%*WbAr;JaGBD z3JEtdDQ!@<>bK_p z-FnY^U<@!QAWVNPTe5lrkdM+MK9lT#SoIgkR{vxGu9T*nk1Ft&!v_~7r i(BiS`i+J99y3C;LQ;OT-GeV&QKYCjFhl(_8!~O$d<8p@p literal 0 HcmV?d00001 diff --git a/docs/user_guide/start_the_pipeline_at_the_specified_location.md b/docs/user_guide/start_the_pipeline_at_the_specified_location.md new file mode 100644 index 00000000..9e745880 --- /dev/null +++ b/docs/user_guide/start_the_pipeline_at_the_specified_location.md @@ -0,0 +1,31 @@ +# 指定流程从某个位置开始执行 + +默认的 run_pipeline_api 只允许流程从开始节点开始，再某些特殊的场景下，用户可能会新起一个任务，并期望从指定的位置开始。 +因此run_pipeline 支持了该特性，不过需要注意的是，该功能是受限的，这意味着你不能选择流程内的任意一个位置开始流程。 + +使用方式: + +```python +from pipeline.eri.runtime import BambooDjangoRuntime +from bamboo_engine import api + +pipeline = {} +# 可以使用root_pipeline_context的方式补充缺失的上下文信息 +api.run_pipeline(runtime=BambooDjangoRuntime(), + pipeline=pipeline, + start_node_id="xxxxx", + root_pipeline_context={}) +``` + +使用范围: + +start_node_id 的指定需要遵循如下规则: + +- 只允许开始节点和位于流程中的主干节点和分支网关内的节点进行回滚，不允许并行网关内的节点作为开始的起始位置，当分支网关处于并行网关内时，该分支网关内的节点也无法作为开始的起始位置。 +- 位于主流程上的并行网关/条件并行网关/条件网关 允许作为起始节点，汇聚网关不允许作为流程的开始节点。 +- 子流程节点不允许作为流程的开始节点 +- 结束节点不允许作为流程的开始节点 + +下图红框内的节点表示允许作为起始位置的节点。 + +![run_pipeline.png](..%2Fassets%2Fimg%2Fstart_the_pipeline_at_the_specified_location%2Frun_pipeline.png) \ No newline at end of file diff --git a/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/control/test_run_pipeline.py b/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/control/test_run_pipeline.py new file mode 100644 index 00000000..63e29be6 --- /dev/null +++ b/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/control/test_run_pipeline.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +import time + +import pytest +from pipeline.eri.models import State +from pipeline.eri.runtime import BambooDjangoRuntime + +from bamboo_engine.builder import ( + EmptyEndEvent, + EmptyStartEvent, + ServiceActivity, + build_tree, +) +from bamboo_engine.engine import Engine + + +def test_run_pipeline_with_start_node_id(): + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="callback_node") + end = EmptyEndEvent() + + start.extend(act_1).extend(end) + + pipeline = build_tree(start) + runtime = BambooDjangoRuntime() + engine = Engine(runtime) + engine.run_pipeline(pipeline=pipeline, root_pipeline_data={}, start_node_id=act_1.id) + + time.sleep(3) + + with pytest.raises(State.DoesNotExist): + # 由于直接跳过了开始节点，此时应该抛异常 + runtime.get_state(start.id) + + state = runtime.get_state(act_1.id) + + assert state.name == "RUNNING" + + engine.callback(act_1.id, state.version, {}) + + time.sleep(2) + + state = runtime.get_state(act_1.id) + + assert state.name == "FINISHED" + + pipeline_state = runtime.get_state(pipeline["id"]) + + assert pipeline_state.name == "FINISHED" diff --git a/tests/validator/test_validate_start_node.py b/tests/validator/test_validate_start_node.py new file mode 100644 index 00000000..c289ac6d --- /dev/null +++ b/tests/validator/test_validate_start_node.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +import pytest + +from bamboo_engine.builder import ( + ConditionalParallelGateway, + ConvergeGateway, + EmptyEndEvent, + EmptyStartEvent, + ExclusiveGateway, + ParallelGateway, + ServiceActivity, + build_tree, +) +from bamboo_engine.exceptions import StartPositionInvalidException +from bamboo_engine.validator import ( + get_allowed_start_node_ids, + validate_pipeline_start_node, +) +from bamboo_engine.validator.gateway import validate_gateways + + +def test_get_allowed_start_node_ids_by_parallel_gateway(): + """ + 并行网关内的节点将会被忽略 + """ + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + pg = ParallelGateway() + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + cg = ConvergeGateway() + end = EmptyEndEvent() + start.extend(act_1).extend(pg).connect(act_2, act_3).to(pg).converge(cg).extend(end) + pipeline = build_tree(start) + # 需要使用 validate_gateways 匹配网关对应的汇聚节点 + validate_gateways(pipeline) + allowed_start_node_ids = get_allowed_start_node_ids(pipeline) + + assert len(allowed_start_node_ids) == 3 + assert allowed_start_node_ids == [start.id, act_1.id, pg.id] + + +def test_get_allowed_start_node_ids_by_exclusive_gateway(): + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + eg = ExclusiveGateway(conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0"}, name="act_2 or act_3") + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + end = EmptyEndEvent() + + start.extend(act_1).extend(eg).connect(act_2, act_3).to(eg).converge(end) + pipeline = build_tree(start) + validate_gateways(pipeline) + allowed_start_node_ids = get_allowed_start_node_ids(pipeline) + + assert len(allowed_start_node_ids) == 5 + assert allowed_start_node_ids == [start.id, act_1.id, eg.id, act_2.id, act_3.id] + + +def test_get_allowed_start_node_ids_by_condition_parallel_gateway(): + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + cpg = ConditionalParallelGateway( + conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0", 2: "${act_1_output} >= 0"}, + name="[act_2] or [act_3 and act_4]", + ) + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + act_4 = ServiceActivity(component_code="pipe_example_component", name="act_4") + cg = ConvergeGateway() + end = EmptyEndEvent() + start.extend(act_1).extend(cpg).connect(act_2, act_3, act_4).to(cpg).converge(cg).extend(end) + + pipeline = build_tree(start) + validate_gateways(pipeline) + allowed_start_node_ids = get_allowed_start_node_ids(pipeline) + + assert len(allowed_start_node_ids) == 3 + assert allowed_start_node_ids == [start.id, act_1.id, cpg.id] + + +def test_get_allowed_start_node_ids_by_normal(): + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + end = EmptyEndEvent() + start.extend(act_1).extend(act_2).extend(end) + + pipeline = build_tree(start) + validate_gateways(pipeline) + allowed_start_node_ids = get_allowed_start_node_ids(pipeline) + + assert len(allowed_start_node_ids) == 3 + assert allowed_start_node_ids == [start.id, act_1.id, act_2.id] + + +def test_validate_pipeline_start_node(): + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + eg = ExclusiveGateway(conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0"}, name="act_2 or act_3") + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + end = EmptyEndEvent() + + start.extend(act_1).extend(eg).connect(act_2, act_3).to(eg).converge(end) + pipeline = build_tree(start) + validate_gateways(pipeline) + + with pytest.raises(StartPositionInvalidException): + validate_pipeline_start_node(pipeline, end.id) + + validate_pipeline_start_node(pipeline, act_1.id) From bdd6a094e216a8880d734955c9da2495159c5120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E6=95=B0?= <33194175+hanshuaikang@users.noreply.github.com> Date: Tue, 24 Oct 2023 14:43:27 +0800 Subject: [PATCH 05/24] =?UTF-8?q?bugfix:=20=E4=BF=AE=E5=A4=8D=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=BF=AB=E7=85=A7=E6=9F=90=E4=BA=9B=E5=80=BC=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E6=AD=A3=E7=A1=AE=E5=BA=8F=E5=88=97=E5=8C=96=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98=20(#183)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bugfix: 修复节点快照某些值无法正确序列化的问题 * minor: any 模式下的回滚遵循token校验 * minor: any 模式增加token校验 --- .../pipeline/contrib/rollback/fields.py | 31 +++++++ .../pipeline/contrib/rollback/handler.py | 82 ++++++------------- .../migrations/0002_auto_20231020_1234.py | 29 +++++++ .../pipeline/contrib/rollback/models.py | 7 +- .../pipeline/eri/imp/rollback.py | 7 +- 5 files changed, 94 insertions(+), 62 deletions(-) create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/rollback/fields.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0002_auto_20231020_1234.py diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/fields.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/fields.py new file mode 100644 index 00000000..8f6462c8 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/fields.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +import codecs +import json +import pickle + +from django.db.models import TextField + + +class SerializerField(TextField): + """ + 特定的序列化类，用于兼容json和pickle两种序列化数据 + """ + + def to_python(self, value): + try: + return json.loads(value) + except Exception: + return pickle.loads(codecs.decode(value.encode(), "base64")) + + def from_db_value(self, value, expression, connection, context=None): + try: + return json.loads(value) + except Exception: + return pickle.loads(codecs.decode(value.encode(), "base64")) + + def get_prep_value(self, value): + try: + return json.dumps(value) + except TypeError: + return codecs.encode(pickle.dumps(value), "base64").decode() + pass diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py index a6394044..691cff2d 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py @@ -153,7 +153,32 @@ def __init__(self, root_pipeline_id): RollbackValidator.validate_pipeline(root_pipeline_id) def get_allowed_rollback_node_id_list(self, start_node_id): - return [] + """ + 获取允许回滚的节点范围 + 规则：token 一致的节点允许回滚 + """ + try: + rollback_token = RollbackToken.objects.get(root_pipeline_id=self.root_pipeline_id) + except RollbackToken.DoesNotExist: + raise RollBackException( + "rollback failed: pipeline token not exist, pipeline_id={}".format(self.root_pipeline_id) + ) + node_map = self._get_allowed_rollback_node_map() + service_activity_node_list = [ + node_id for node_id, node_detail in node_map.items() if node_detail["type"] == PE.ServiceActivity + ] + + tokens = json.loads(rollback_token.token) + start_token = tokens.get(start_node_id) + if not start_token: + return [] + + nodes = [] + for node_id, token in tokens.items(): + if start_token == token and node_id != start_node_id and node_id in service_activity_node_list: + nodes.append(node_id) + + return nodes def _get_allowed_rollback_node_map(self): # 不需要遍历整颗树，获取到现在已经执行成功和失败节点的所有列表 @@ -232,42 +257,17 @@ def cancel_reserved_rollback(self, start_node_id, target_node_id): class AnyRollbackHandler(BaseRollbackHandler): mode = ANY - def get_allowed_rollback_node_id_list(self, start_node_id): - node_map = self._get_allowed_rollback_node_map() - start_node_state = ( - State.objects.filter(root_id=self.root_pipeline_id) - .exclude(node_id=self.root_pipeline_id) - .order_by("created_time") - .first() - ) - target_node_id = start_node_state.node_id - rollback_graph = RollbackGraphHandler(node_map=node_map, start_id=start_node_id, target_id=target_node_id) - graph, _ = rollback_graph.build_rollback_graph() - - return list(set(graph.nodes) - {constants.START_FLAG, constants.END_FLAG, start_node_id}) - def retry_rollback_failed_node(self, node_id, retry_data): """ """ raise RollBackException("rollback failed: when mode is any, not support retry") - def reserve_rollback(self, start_node_id, target_node_id): - """ - 预约回滚 - """ - self._reserve(start_node_id, target_node_id) - - def cancel_reserved_rollback(self, start_node_id, target_node_id): - """ - 取消预约回滚 - """ - self._reserve(start_node_id, target_node_id, reserve_rollback=False) - def rollback(self, start_node_id, target_node_id, skip_rollback_nodes=None): RollbackValidator.validate_node_state_by_any_mode(self.root_pipeline_id) # 回滚的开始节点运行失败的情况 RollbackValidator.validate_start_node_id(self.root_pipeline_id, start_node_id) RollbackValidator.validate_node(start_node_id, allow_failed=True) RollbackValidator.validate_node(target_node_id) + RollbackValidator.validate_token(self.root_pipeline_id, start_node_id, target_node_id) node_map = self._get_allowed_rollback_node_map() rollback_graph = RollbackGraphHandler(node_map=node_map, start_id=start_node_id, target_id=target_node_id) @@ -294,34 +294,6 @@ def rollback(self, start_node_id, target_node_id, skip_rollback_nodes=None): class TokenRollbackHandler(BaseRollbackHandler): mode = TOKEN - def get_allowed_rollback_node_id_list(self, start_node_id): - """ - 获取允许回滚的节点范围 - 规则：token 一致的节点允许回滚 - """ - try: - rollback_token = RollbackToken.objects.get(root_pipeline_id=self.root_pipeline_id) - except RollbackToken.DoesNotExist: - raise RollBackException( - "rollback failed: pipeline token not exist, pipeline_id={}".format(self.root_pipeline_id) - ) - node_map = self._get_allowed_rollback_node_map() - service_activity_node_list = [ - node_id for node_id, node_detail in node_map.items() if node_detail["type"] == PE.ServiceActivity - ] - - tokens = json.loads(rollback_token.token) - start_token = tokens.get(start_node_id) - if not start_token: - return [] - - nodes = [] - for node_id, token in tokens.items(): - if start_token == token and node_id != start_node_id and node_id in service_activity_node_list: - nodes.append(node_id) - - return nodes - def retry_rollback_failed_node(self, node_id, retry_data): """ 重试回滚失败的节点 diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0002_auto_20231020_1234.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0002_auto_20231020_1234.py new file mode 100644 index 00000000..ae217a0f --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0002_auto_20231020_1234.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.18 on 2023-10-20 12:34 + +import pipeline.contrib.rollback.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("rollback", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="rollbacknodesnapshot", + name="context_values", + field=pipeline.contrib.rollback.fields.SerializerField(verbose_name="pipeline context values"), + ), + migrations.AlterField( + model_name="rollbacknodesnapshot", + name="inputs", + field=pipeline.contrib.rollback.fields.SerializerField(verbose_name="node inputs"), + ), + migrations.AlterField( + model_name="rollbacknodesnapshot", + name="outputs", + field=pipeline.contrib.rollback.fields.SerializerField(verbose_name="node outputs"), + ), + ] diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py index b482f0a6..ab452cfc 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py @@ -2,6 +2,7 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from pipeline.contrib.rollback.constants import TOKEN +from pipeline.contrib.rollback.fields import SerializerField class RollbackToken(models.Model): @@ -38,9 +39,9 @@ class RollbackNodeSnapshot(models.Model): node_id = models.CharField(verbose_name="node_id", max_length=64, db_index=True) code = models.CharField(verbose_name="node_code", max_length=64) version = models.CharField(verbose_name=_("version"), null=False, max_length=33) - inputs = models.TextField(verbose_name=_("node inputs")) - outputs = models.TextField(verbose_name=_("node outputs")) - context_values = models.TextField(verbose_name=_("pipeline context values")) + inputs = SerializerField(verbose_name=_("node inputs")) + outputs = SerializerField(verbose_name=_("node outputs")) + context_values = SerializerField(verbose_name=_("pipeline context values")) rolled_back = models.BooleanField(_("whether the node rolls back"), default=False) diff --git a/runtime/bamboo-pipeline/pipeline/eri/imp/rollback.py b/runtime/bamboo-pipeline/pipeline/eri/imp/rollback.py index 99ec147d..ea1a15a9 100644 --- a/runtime/bamboo-pipeline/pipeline/eri/imp/rollback.py +++ b/runtime/bamboo-pipeline/pipeline/eri/imp/rollback.py @@ -3,7 +3,6 @@ import logging from django.apps import apps -from django.core.serializers.json import DjangoJSONEncoder from bamboo_engine.builder.builder import generate_pipeline_token @@ -49,9 +48,9 @@ def set_node_snapshot(self, root_pipeline_id, node_id, code, version, context_va node_id=node_id, code=code, version=version, - context_values=json.dumps(context_values, cls=DjangoJSONEncoder), - inputs=json.dumps(inputs, cls=DjangoJSONEncoder), - outputs=json.dumps(outputs, cls=DjangoJSONEncoder), + context_values=context_values, + inputs=inputs, + outputs=outputs, ) def start_rollback(self, root_pipeline_id, node_id): From 9f172ecc3c42ea915789339bcdef66e74d6115ce Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Tue, 24 Oct 2023 14:49:51 +0800 Subject: [PATCH 06/24] minor: release bamboo-engine 2.10.0rc1 --- bamboo_engine/__version__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bamboo_engine/__version__.py b/bamboo_engine/__version__.py index 8cc71537..df03c081 100644 --- a/bamboo_engine/__version__.py +++ b/bamboo_engine/__version__.py @@ -11,4 +11,4 @@ specific language governing permissions and limitations under the License. """ -__version__ = "2.9.0rc1" +__version__ = "2.10.0rc1" diff --git a/pyproject.toml b/pyproject.toml index 942bb580..14e1fdc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-engine" -version = "2.9.0rc1" +version = "2.10.0rc1" description = "Bamboo-engine is a general-purpose workflow engine" authors = ["homholueng "] license = "MIT" From 6af6d5e218be2952e7ad3b88392548d19a939a9b Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Tue, 24 Oct 2023 15:31:30 +0800 Subject: [PATCH 07/24] minor: release bamboo-pipeline 3.29.0rc1 --- runtime/bamboo-pipeline/pipeline/__init__.py | 2 +- runtime/bamboo-pipeline/poetry.lock | 2 +- runtime/bamboo-pipeline/pyproject.toml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/bamboo-pipeline/pipeline/__init__.py b/runtime/bamboo-pipeline/pipeline/__init__.py index 5d67aa6c..1cbfc175 100644 --- a/runtime/bamboo-pipeline/pipeline/__init__.py +++ b/runtime/bamboo-pipeline/pipeline/__init__.py @@ -13,4 +13,4 @@ default_app_config = "pipeline.apps.PipelineConfig" -__version__ = "3.28.0rc1" +__version__ = "3.29.0rc1" diff --git a/runtime/bamboo-pipeline/poetry.lock b/runtime/bamboo-pipeline/poetry.lock index 2db464a1..9dbfa3ec 100644 --- a/runtime/bamboo-pipeline/poetry.lock +++ b/runtime/bamboo-pipeline/poetry.lock @@ -57,7 +57,7 @@ tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "c [[package]] name = "bamboo-engine" -version = "2.9.0rc1" +version = "2.10.0rc1" description = "Bamboo-engine is a general-purpose workflow engine" category = "main" optional = false diff --git a/runtime/bamboo-pipeline/pyproject.toml b/runtime/bamboo-pipeline/pyproject.toml index ec47b631..b6ff4a2e 100644 --- a/runtime/bamboo-pipeline/pyproject.toml +++ b/runtime/bamboo-pipeline/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-pipeline" -version = "3.28.0rc1" +version = "3.29.0rc1" description = "runtime for bamboo-engine base on Django and Celery" authors = ["homholueng "] license = "MIT" @@ -16,7 +16,7 @@ requests = "^2.22.0" django-celery-beat = "^2.1.0" Mako = "^1.1.4" pytz = "2019.3" -bamboo-engine = "2.9.0rc1" +bamboo-engine = "2.10.0rc1" jsonschema = "^2.5.1" ujson = "4.1.*" pyparsing = "^2.2.0" From afc1c28ce8d7fe557480e3887adee41e4643084c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E6=95=B0?= <33194175+hanshuaikang@users.noreply.github.com> Date: Tue, 31 Oct 2023 17:45:42 +0800 Subject: [PATCH 08/24] =?UTF-8?q?bugfix:=20=E4=BF=AE=E5=A4=8D=E7=8E=AF?= =?UTF-8?q?=E9=97=AE=E9=A2=98=20&&=20=E6=9E=84=E5=BB=BAtoken=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=8F=AF=E6=89=A7=E8=A1=8C=E7=BB=93=E6=9D=9F=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=88=A4=E5=AE=9A=20(#188)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bugfix: 构建token新增可执行结束节点判定 * bugfix: 修复环问题 --- bamboo_engine/builder/builder.py | 44 +++++++++++++++++++------------- tests/builder/test_token.py | 35 +++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/bamboo_engine/builder/builder.py b/bamboo_engine/builder/builder.py index 26a7e84c..4c10b0d1 100644 --- a/bamboo_engine/builder/builder.py +++ b/bamboo_engine/builder/builder.py @@ -101,6 +101,10 @@ def _get_next_node(node, pipeline_tree): out_goings = node["outgoing"] + # 说明曾经去除过环，此时没有out_goings + if out_goings == "": + return [] + # 当只有一个输出时, if not isinstance(out_goings, list): out_goings = [out_goings] @@ -211,7 +215,7 @@ def inject_pipeline_token(node, pipeline_tree, node_token_map, token): if node["type"] in ["ParallelGateway", "ExclusiveGateway", "ConditionalParallelGateway"]: next_nodes = _get_next_node(node, pipeline_tree) node_token = unique_id("t") - target_node = None + target_nodes = {} for next_node in next_nodes: # 分支网关各个分支token相同 node_token_map[next_node["id"]] = node_token @@ -220,23 +224,23 @@ def inject_pipeline_token(node, pipeline_tree, node_token_map, token): node_token = unique_id("t") node_token_map[next_node["id"]] = node_token - # 如果是网关，沿着路径向内搜索，最终遇到对应的分支网关会返回 + # 如果是并行网关，沿着路径向内搜索，最终遇到对应的汇聚网关会返回 target_node = inject_pipeline_token(next_node, pipeline_tree, node_token_map, node_token) - - if target_node is None: - return - - # 汇聚网关可以直连结束节点，所以可能会存在找不到对应的汇聚网关的情况 - if target_node["type"] == "EmptyEndEvent": + if target_node: + target_nodes[target_node["id"]] = target_node + + for target_node in target_nodes.values(): + # 汇聚网关可以直连结束节点，所以可能会存在找不到对应的汇聚网关的情况 + if target_node["type"] in ["EmptyEndEvent", "ExecutableEndEvent"]: + node_token_map[target_node["id"]] = token + continue + # 汇聚网关的token等于对应的网关的token node_token_map[target_node["id"]] = token - return - # 汇聚网关的token等于对应的网关的token - node_token_map[target_node["id"]] = token - # 到汇聚网关之后，此时继续向下遍历 - next_node = _get_next_node(target_node, pipeline_tree)[0] - # 汇聚网关只会有一个出度 - node_token_map[next_node["id"]] = token - return inject_pipeline_token(next_node, pipeline_tree, node_token_map, token) + # 到汇聚网关之后，此时继续向下遍历 + next_node = _get_next_node(target_node, pipeline_tree)[0] + # 汇聚网关只会有一个出度 + node_token_map[next_node["id"]] = token + inject_pipeline_token(next_node, pipeline_tree, node_token_map, token) # 如果是汇聚网关，并且id等于converge_id，说明此时遍历在某个单元 if node["type"] == "ConvergeGateway": @@ -244,12 +248,16 @@ def inject_pipeline_token(node, pipeline_tree, node_token_map, token): # 如果是普通的节点，说明只有一个出度，此时直接向下遍历就好 if node["type"] in ["ServiceActivity", "EmptyStartEvent"]: - next_node = _get_next_node(node, pipeline_tree)[0] + next_node_list = _get_next_node(node, pipeline_tree) + # 此时有可能遇到一个去环的节点，该节点没有 + if not next_node_list: + return + next_node = next_node_list[0] node_token_map[next_node["id"]] = token return inject_pipeline_token(next_node, pipeline_tree, node_token_map, token) # 如果遇到结束节点，直接返回 - if node["type"] == "EmptyEndEvent": + if node["type"] in ["EmptyEndEvent", "ExecutableEndEvent"]: return node if node["type"] == "SubProcess": diff --git a/tests/builder/test_token.py b/tests/builder/test_token.py index 52a1325c..54f43f69 100644 --- a/tests/builder/test_token.py +++ b/tests/builder/test_token.py @@ -45,7 +45,6 @@ def test_inject_pipeline_token_normal(): start.extend(act).extend(end) pipeline = build_tree(start) - node_token_map = generate_pipeline_token(pipeline) assert get_node_token(pipeline, "act_1", node_token_map) == get_node_token(pipeline, "start_event", node_token_map) @@ -54,6 +53,39 @@ def test_inject_pipeline_token_normal(): ) +def test_inject_pipeline_token_with_complex_cycle(): + start = EmptyStartEvent() + end = EmptyEndEvent() + + write_document = ServiceActivity(name="act_1", component_code="example_component") + review_document = ServiceActivity(name="act_2", component_code="example_component") + rework_document = ServiceActivity(name="act_3", component_code="example_component") + + release_document = ServiceActivity(name="act_4", component_code="example_component") + gateway = ExclusiveGateway( + conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0", 2: "${act_1_output} >= 0"}, + name="ExclusiveGateway", + ) + + start.extend(write_document).extend(review_document).extend(gateway).connect( + release_document, end, rework_document + ).to(release_document).extend(end) + rework_document.extend(review_document) + pipeline = build_tree(start) + node_token_map = generate_pipeline_token(pipeline) + + assert ( + get_node_token(pipeline, "start_event", node_token_map) + == get_node_token(pipeline, "act_1", node_token_map) + == get_node_token(pipeline, "act_2", node_token_map) + == get_node_token(pipeline, "ExclusiveGateway", node_token_map) + == get_node_token(pipeline, "end_event", node_token_map) + != get_node_token(pipeline, "act_3", node_token_map) + ) + + assert get_node_token(pipeline, "act_4", node_token_map) == get_node_token(pipeline, "act_3", node_token_map) + + def test_inject_pipeline_token_parallel_gateway(): start = EmptyStartEvent() pg = ParallelGateway(name="ParallelGateway") @@ -67,7 +99,6 @@ def test_inject_pipeline_token_parallel_gateway(): pipeline = build_tree(start) node_token_map = generate_pipeline_token(pipeline) - assert ( get_node_token(pipeline, "start_event", node_token_map) == get_node_token(pipeline, "ParallelGateway", node_token_map) From ce53e06de95e107a0f48b933a8d3d03df6f1bf84 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Tue, 31 Oct 2023 20:28:32 +0800 Subject: [PATCH 09/24] minor: release 2.10.0rc2 --- bamboo_engine/__version__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bamboo_engine/__version__.py b/bamboo_engine/__version__.py index df03c081..45b0db57 100644 --- a/bamboo_engine/__version__.py +++ b/bamboo_engine/__version__.py @@ -11,4 +11,4 @@ specific language governing permissions and limitations under the License. """ -__version__ = "2.10.0rc1" +__version__ = "2.10.0rc2" diff --git a/pyproject.toml b/pyproject.toml index 14e1fdc6..1b4c725a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-engine" -version = "2.10.0rc1" +version = "2.10.0rc2" description = "Bamboo-engine is a general-purpose workflow engine" authors = ["homholueng "] license = "MIT" From e404cff591d9472ac72926cac71ff5675ad556e1 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Wed, 1 Nov 2023 14:38:31 +0800 Subject: [PATCH 10/24] minor: release 3.29.0rc2 --- runtime/bamboo-pipeline/pipeline/__init__.py | 2 +- runtime/bamboo-pipeline/poetry.lock | 4 ++-- runtime/bamboo-pipeline/pyproject.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/bamboo-pipeline/pipeline/__init__.py b/runtime/bamboo-pipeline/pipeline/__init__.py index 1cbfc175..5f9a3c3d 100644 --- a/runtime/bamboo-pipeline/pipeline/__init__.py +++ b/runtime/bamboo-pipeline/pipeline/__init__.py @@ -13,4 +13,4 @@ default_app_config = "pipeline.apps.PipelineConfig" -__version__ = "3.29.0rc1" +__version__ = "3.29.0rc2" diff --git a/runtime/bamboo-pipeline/poetry.lock b/runtime/bamboo-pipeline/poetry.lock index 9dbfa3ec..1e5bea1a 100644 --- a/runtime/bamboo-pipeline/poetry.lock +++ b/runtime/bamboo-pipeline/poetry.lock @@ -57,7 +57,7 @@ tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "c [[package]] name = "bamboo-engine" -version = "2.10.0rc1" +version = "2.10.0rc2" description = "Bamboo-engine is a general-purpose workflow engine" category = "main" optional = false @@ -735,7 +735,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">= 3.6, < 4" -content-hash = "debbfa59d92f30b7b448e2c897088be3fc1ad32db4ee3471631268923ebe40fa" +content-hash = "1f6465e86e2ab50a9018e202165ee8fda95461a31e0c05bf972fd2e240b6f6ba" [metadata.files] amqp = [] diff --git a/runtime/bamboo-pipeline/pyproject.toml b/runtime/bamboo-pipeline/pyproject.toml index b6ff4a2e..995a0ae1 100644 --- a/runtime/bamboo-pipeline/pyproject.toml +++ b/runtime/bamboo-pipeline/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-pipeline" -version = "3.29.0rc1" +version = "3.29.0rc2" description = "runtime for bamboo-engine base on Django and Celery" authors = ["homholueng "] license = "MIT" @@ -16,7 +16,7 @@ requests = "^2.22.0" django-celery-beat = "^2.1.0" Mako = "^1.1.4" pytz = "2019.3" -bamboo-engine = "2.10.0rc1" +bamboo-engine = "2.10.0rc2" jsonschema = "^2.5.1" ujson = "4.1.*" pyparsing = "^2.2.0" From b05b2c1e3289c3117b50859e38fd34e5a5e89269 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Wed, 8 Nov 2023 22:04:11 +0800 Subject: [PATCH 11/24] =?UTF-8?q?minor:=20=E6=96=B0=E5=A2=9Eapi=EF=BC=8C?= =?UTF-8?q?=E8=BF=94=E5=9B=9E=E5=BD=93=E6=8C=87=E5=AE=9A=E5=BC=80=E5=A7=8B?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E6=97=B6=EF=BC=8C=E8=A2=AB=E8=B7=B3=E8=BF=87?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E7=9A=84=E8=8A=82=E7=82=B9=E5=88=97=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bamboo_engine/validator/api.py | 44 +++++++- bamboo_engine/validator/utils.py | 29 +++-- ..._the_pipeline_at_the_specified_location.md | 105 +++++++++++++++++- tests/validator/test_validate_start_node.py | 23 +++- 4 files changed, 187 insertions(+), 14 deletions(-) diff --git a/bamboo_engine/validator/api.py b/bamboo_engine/validator/api.py index 43bae0e6..e5243a95 100644 --- a/bamboo_engine/validator/api.py +++ b/bamboo_engine/validator/api.py @@ -10,6 +10,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import copy from bamboo_engine import exceptions from bamboo_engine.eri import NodeType @@ -17,7 +18,12 @@ from . import rules from .connection import validate_graph_connection, validate_graph_without_circle from .gateway import validate_gateways, validate_stream -from .utils import format_pipeline_tree_io_to_list, get_allowed_start_node_ids +from .utils import ( + compute_pipeline_main_nodes, + compute_pipeline_skip_executed_map, + format_pipeline_tree_io_to_list, + get_nodes_dict, +) def validate_pipeline_start_node(pipeline: dict, node_id: str): @@ -30,6 +36,42 @@ def validate_pipeline_start_node(pipeline: dict, node_id: str): raise exceptions.StartPositionInvalidException("this node_id is not allowed as a starting node") +def get_skipped_execute_node_ids(pipeline_tree, start_node_id, validate=True): + if validate and start_node_id not in get_allowed_start_node_ids(pipeline_tree): + raise Exception("the start_node_id is not legal, please check") + start_event_id = pipeline_tree["start_event"]["id"] + node_dict = get_nodes_dict(pipeline_tree) + # 流程的开始位置只允许出现在主干，子流程/并行网关内的节点不允许作为起始位置 + will_skipped_nodes = compute_pipeline_skip_executed_map(start_event_id, node_dict, start_node_id) + return list(will_skipped_nodes) + + +def get_allowed_start_node_ids(pipeline_tree): + # 检查该流程是否已经经过汇聚网关填充 + def check_converge_gateway(): + gateways = pipeline_tree["gateways"] + if not gateways: + return True + # 经过填充的网关会有converge_gateway_id 字段 + for gateway in gateways.values(): + if ( + gateway["type"] in ["ParallelGateway", "ConditionalParallelGateway"] + and "converge_gateway_id" not in gateway + ): + return False + + return True + + if check_converge_gateway(): + pipeline_tree = copy.deepcopy(pipeline_tree) + validate_gateways(pipeline_tree) + start_event_id = pipeline_tree["start_event"]["id"] + node_dict = get_nodes_dict(pipeline_tree) + # 流程的开始位置只允许出现在主干，子流程/并行网关内的节点不允许作为起始位置 + allowed_start_node_ids = compute_pipeline_main_nodes(start_event_id, node_dict) + return allowed_start_node_ids + + def validate_and_process_pipeline(pipeline: dict, cycle_tolerate=False): for subproc in [act for act in pipeline["activities"].values() if act["type"] == NodeType.SubProcess.value]: validate_and_process_pipeline(subproc["pipeline"], cycle_tolerate) diff --git a/bamboo_engine/validator/utils.py b/bamboo_engine/validator/utils.py index dc586709..10f97b6c 100644 --- a/bamboo_engine/validator/utils.py +++ b/bamboo_engine/validator/utils.py @@ -90,7 +90,7 @@ def get_nodes_dict(data): return nodes -def _compute_pipeline_main_nodes(node_id, node_dict): +def compute_pipeline_main_nodes(node_id, node_dict): """ 计算流程中的主线节点，遇到并行网关/分支并行网关/子流程，则会跳过 最后计算出来主干分支所允许开始的节点范围 @@ -110,17 +110,28 @@ def _compute_pipeline_main_nodes(node_id, node_dict): if node_type in ["EmptyStartEvent", "ServiceActivity", "ExclusiveGateway", "ConvergeGateway", "SubProcess"]: next_nodes = node_detail.get("target", []) for next_node_id in next_nodes: - nodes += _compute_pipeline_main_nodes(next_node_id, node_dict) + nodes += compute_pipeline_main_nodes(next_node_id, node_dict) elif node_type in ["ParallelGateway", "ConditionalParallelGateway"]: next_node_id = node_detail["converge_gateway_id"] - nodes += _compute_pipeline_main_nodes(next_node_id, node_dict) + nodes += compute_pipeline_main_nodes(next_node_id, node_dict) return nodes -def get_allowed_start_node_ids(pipeline_tree): - start_event_id = pipeline_tree["start_event"]["id"] - node_dict = get_nodes_dict(pipeline_tree) - # 流程的开始位置只允许出现在主干，子流程/并行网关内的节点不允许作为起始位置 - allowed_start_node_ids = _compute_pipeline_main_nodes(start_event_id, node_dict) - return allowed_start_node_ids +def compute_pipeline_skip_executed_map(node_id, node_dict, start_node_id): + nodes = [node_id] + if node_id == start_node_id: + return nodes + node_detail = node_dict[node_id] + next_nodes = node_detail.get("target", []) + if node_detail["type"] in ["ExclusiveGateway"]: + for next_node_id in next_nodes: + node_ids = compute_pipeline_skip_executed_map(next_node_id, node_dict, start_node_id) + # 如果开始的位置在分支网关内，只处理该分支 + if start_node_id in node_ids: + nodes += node_ids + else: + for next_node_id in next_nodes: + nodes += compute_pipeline_skip_executed_map(next_node_id, node_dict, start_node_id) + + return set(nodes) - {start_node_id} diff --git a/docs/user_guide/start_the_pipeline_at_the_specified_location.md b/docs/user_guide/start_the_pipeline_at_the_specified_location.md index 9e745880..34245ebf 100644 --- a/docs/user_guide/start_the_pipeline_at_the_specified_location.md +++ b/docs/user_guide/start_the_pipeline_at_the_specified_location.md @@ -12,8 +12,8 @@ from bamboo_engine import api pipeline = {} # 可以使用root_pipeline_context的方式补充缺失的上下文信息 api.run_pipeline(runtime=BambooDjangoRuntime(), - pipeline=pipeline, - start_node_id="xxxxx", + pipeline=pipeline, + start_node_id="xxxxx", root_pipeline_context={}) ``` @@ -28,4 +28,103 @@ start_node_id 的指定需要遵循如下规则: 下图红框内的节点表示允许作为起始位置的节点。 -![run_pipeline.png](..%2Fassets%2Fimg%2Fstart_the_pipeline_at_the_specified_location%2Frun_pipeline.png) \ No newline at end of file +![run_pipeline.png](..%2Fassets%2Fimg%2Fstart_the_pipeline_at_the_specified_location%2Frun_pipeline.png) + +其他工具方法: + +1. 获取某个流程所允许的回滚范围 + +```python + +from bamboo_engine.builder import ( + ConditionalParallelGateway, + ConvergeGateway, + EmptyEndEvent, + EmptyStartEvent, + ServiceActivity, + build_tree, +) + +from bamboo_engine.validator.api import get_allowed_start_node_ids + +start = EmptyStartEvent() +act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") +cpg = ConditionalParallelGateway( + conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0", 2: "${act_1_output} >= 0"}, + name="[act_2] or [act_3 and act_4]", +) +act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") +act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") +act_4 = ServiceActivity(component_code="pipe_example_component", name="act_4") +cg = ConvergeGateway() +end = EmptyEndEvent() +start.extend(act_1).extend(cpg).connect(act_2, act_3, act_4).to(cpg).converge(cg).extend(end) + +pipeline = build_tree(start) +allowed_start_node_ids = get_allowed_start_node_ids(pipeline) +``` + +2. 检查某个节点是否可作为开始节点: + +```python +from bamboo_engine.builder import ( + ConditionalParallelGateway, + ConvergeGateway, + EmptyEndEvent, + EmptyStartEvent, + ServiceActivity, + build_tree, +) + +from bamboo_engine.validator.api import validate_pipeline_start_node + +start = EmptyStartEvent() +act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") +cpg = ConditionalParallelGateway( + conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0", 2: "${act_1_output} >= 0"}, + name="[act_2] or [act_3 and act_4]", +) +act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") +act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") +act_4 = ServiceActivity(component_code="pipe_example_component", name="act_4") +cg = ConvergeGateway() +end = EmptyEndEvent() +start.extend(act_1).extend(cpg).connect(act_2, act_3, act_4).to(cpg).converge(cg).extend(end) + +pipeline = build_tree(start) +validate_pipeline_start_node(pipeline, act_2.id) +``` + +2.当开始节点为某个节点时，流程被跳过执行的节点列表: + +```python +from bamboo_engine.builder import ( + ConditionalParallelGateway, + ConvergeGateway, + EmptyEndEvent, + EmptyStartEvent, + ServiceActivity, + build_tree, +) + +from bamboo_engine.validator.api import get_skipped_execute_node_ids + +start = EmptyStartEvent() +act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") +cpg = ConditionalParallelGateway( + conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0", 2: "${act_1_output} >= 0"}, + name="[act_2] or [act_3 and act_4]", +) +act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") +act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") +act_4 = ServiceActivity(component_code="pipe_example_component", name="act_4") +cg = ConvergeGateway() +end = EmptyEndEvent() +start.extend(act_1).extend(cpg).connect(act_2, act_3, act_4).to(cpg).converge(cg).extend(end) + +pipeline = build_tree(start) + +# validate = True 将会校验节点合法性 +skipped_execute_node_ids = get_skipped_execute_node_ids(pipeline, act_2.id, validate=True) + +``` \ No newline at end of file diff --git a/tests/validator/test_validate_start_node.py b/tests/validator/test_validate_start_node.py index c289ac6d..efb2743f 100644 --- a/tests/validator/test_validate_start_node.py +++ b/tests/validator/test_validate_start_node.py @@ -12,8 +12,9 @@ build_tree, ) from bamboo_engine.exceptions import StartPositionInvalidException -from bamboo_engine.validator import ( +from bamboo_engine.validator.api import ( get_allowed_start_node_ids, + get_skipped_execute_node_ids, validate_pipeline_start_node, ) from bamboo_engine.validator.gateway import validate_gateways @@ -39,6 +40,10 @@ def test_get_allowed_start_node_ids_by_parallel_gateway(): assert len(allowed_start_node_ids) == 3 assert allowed_start_node_ids == [start.id, act_1.id, pg.id] + skipped_execute_node_ids = get_skipped_execute_node_ids(pipeline, pg.id) + assert len(skipped_execute_node_ids) == 2 + assert set(skipped_execute_node_ids) == {start.id, act_1.id} + def test_get_allowed_start_node_ids_by_exclusive_gateway(): start = EmptyStartEvent() @@ -56,6 +61,18 @@ def test_get_allowed_start_node_ids_by_exclusive_gateway(): assert len(allowed_start_node_ids) == 5 assert allowed_start_node_ids == [start.id, act_1.id, eg.id, act_2.id, act_3.id] + skipped_execute_node_ids = get_skipped_execute_node_ids(pipeline, eg.id) + assert len(skipped_execute_node_ids) == 2 + assert set(skipped_execute_node_ids) == {start.id, act_1.id} + + skipped_execute_node_ids = get_skipped_execute_node_ids(pipeline, act_2.id) + assert len(skipped_execute_node_ids) == 3 + assert set(skipped_execute_node_ids) == {start.id, act_1.id, eg.id} + + skipped_execute_node_ids = get_skipped_execute_node_ids(pipeline, act_3.id) + assert len(skipped_execute_node_ids) == 3 + assert set(skipped_execute_node_ids) == {start.id, act_1.id, eg.id} + def test_get_allowed_start_node_ids_by_condition_parallel_gateway(): start = EmptyStartEvent() @@ -78,6 +95,10 @@ def test_get_allowed_start_node_ids_by_condition_parallel_gateway(): assert len(allowed_start_node_ids) == 3 assert allowed_start_node_ids == [start.id, act_1.id, cpg.id] + skipped_execute_node_ids = get_skipped_execute_node_ids(pipeline, cpg.id) + assert len(skipped_execute_node_ids) == 2 + assert set(skipped_execute_node_ids) == {start.id, act_1.id} + def test_get_allowed_start_node_ids_by_normal(): start = EmptyStartEvent() From e1485c07ef6bc769dfb69b3d2e6f387bc9080096 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Thu, 9 Nov 2023 11:41:11 +0800 Subject: [PATCH 12/24] minor: code review --- bamboo_engine/validator/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bamboo_engine/validator/api.py b/bamboo_engine/validator/api.py index e5243a95..e4edcf60 100644 --- a/bamboo_engine/validator/api.py +++ b/bamboo_engine/validator/api.py @@ -40,6 +40,11 @@ def get_skipped_execute_node_ids(pipeline_tree, start_node_id, validate=True): if validate and start_node_id not in get_allowed_start_node_ids(pipeline_tree): raise Exception("the start_node_id is not legal, please check") start_event_id = pipeline_tree["start_event"]["id"] + + # 如果开始节点 = start_node_id， 说明要从开始节点开始执行，此时没有任何节点被跳过 + if start_node_id == start_event_id: + return [] + node_dict = get_nodes_dict(pipeline_tree) # 流程的开始位置只允许出现在主干，子流程/并行网关内的节点不允许作为起始位置 will_skipped_nodes = compute_pipeline_skip_executed_map(start_event_id, node_dict, start_node_id) From 75cf42b7657f944af45ce1614c028aa3ca0e280b Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Thu, 9 Nov 2023 15:10:31 +0800 Subject: [PATCH 13/24] minor: release 2.10.0rc3 --- bamboo_engine/__version__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bamboo_engine/__version__.py b/bamboo_engine/__version__.py index 45b0db57..39cf9317 100644 --- a/bamboo_engine/__version__.py +++ b/bamboo_engine/__version__.py @@ -11,4 +11,4 @@ specific language governing permissions and limitations under the License. """ -__version__ = "2.10.0rc2" +__version__ = "2.10.0rc3" diff --git a/pyproject.toml b/pyproject.toml index 1b4c725a..721157f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-engine" -version = "2.10.0rc2" +version = "2.10.0rc3" description = "Bamboo-engine is a general-purpose workflow engine" authors = ["homholueng "] license = "MIT" From 0af03b1410d11ee100838e68f5c1aa073f8244c1 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Thu, 9 Nov 2023 20:09:06 +0800 Subject: [PATCH 14/24] minor: release pipeline 3.29.0rc3 --- runtime/bamboo-pipeline/pipeline/__init__.py | 2 +- runtime/bamboo-pipeline/poetry.lock | 4 ++-- runtime/bamboo-pipeline/pyproject.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/bamboo-pipeline/pipeline/__init__.py b/runtime/bamboo-pipeline/pipeline/__init__.py index 5f9a3c3d..92dc81e9 100644 --- a/runtime/bamboo-pipeline/pipeline/__init__.py +++ b/runtime/bamboo-pipeline/pipeline/__init__.py @@ -13,4 +13,4 @@ default_app_config = "pipeline.apps.PipelineConfig" -__version__ = "3.29.0rc2" +__version__ = "3.29.0rc3" diff --git a/runtime/bamboo-pipeline/poetry.lock b/runtime/bamboo-pipeline/poetry.lock index 1e5bea1a..13734303 100644 --- a/runtime/bamboo-pipeline/poetry.lock +++ b/runtime/bamboo-pipeline/poetry.lock @@ -57,7 +57,7 @@ tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "c [[package]] name = "bamboo-engine" -version = "2.10.0rc2" +version = "2.10.0rc3" description = "Bamboo-engine is a general-purpose workflow engine" category = "main" optional = false @@ -735,7 +735,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">= 3.6, < 4" -content-hash = "1f6465e86e2ab50a9018e202165ee8fda95461a31e0c05bf972fd2e240b6f6ba" +content-hash = "3026f8497b07ea1593e3d03d656f777d5262d8a1726f5bbdf3395e633b6bfeb8" [metadata.files] amqp = [] diff --git a/runtime/bamboo-pipeline/pyproject.toml b/runtime/bamboo-pipeline/pyproject.toml index 995a0ae1..3f15a31b 100644 --- a/runtime/bamboo-pipeline/pyproject.toml +++ b/runtime/bamboo-pipeline/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-pipeline" -version = "3.29.0rc2" +version = "3.29.0rc3" description = "runtime for bamboo-engine base on Django and Celery" authors = ["homholueng "] license = "MIT" @@ -16,7 +16,7 @@ requests = "^2.22.0" django-celery-beat = "^2.1.0" Mako = "^1.1.4" pytz = "2019.3" -bamboo-engine = "2.10.0rc2" +bamboo-engine = "2.10.0rc3" jsonschema = "^2.5.1" ujson = "4.1.*" pyparsing = "^2.2.0" From a6c9c211ca70423a780d3dab1b9a96df73bba335 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Thu, 16 Nov 2023 21:03:13 +0800 Subject: [PATCH 15/24] =?UTF-8?q?minor:=20=E6=96=B0=E5=A2=9E=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E5=B7=B2=E6=9C=89recursive=5Freplace=5Fid=EF=BC=8C=20?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E8=BF=94=E5=9B=9Emap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bamboo-pipeline/pipeline/parser/utils.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/runtime/bamboo-pipeline/pipeline/parser/utils.py b/runtime/bamboo-pipeline/pipeline/parser/utils.py index 9b1f7203..669dbe01 100644 --- a/runtime/bamboo-pipeline/pipeline/parser/utils.py +++ b/runtime/bamboo-pipeline/pipeline/parser/utils.py @@ -13,23 +13,35 @@ import logging -from pipeline.utils.uniqid import node_uniqid, line_uniqid from pipeline.core.constants import PE from pipeline.exceptions import NodeNotExistException +from pipeline.utils.uniqid import line_uniqid, node_uniqid logger = logging.getLogger("root") BRANCH_SELECT_GATEWAYS = {PE.ExclusiveGateway, PE.ConditionalParallelGateway} -def recursive_replace_id(pipeline_data): +def _recursive_replace_id_with_node_map(pipeline_data, subprocess_id=None): + """ + 替换pipeline_id 并返回 对应的 node_map 映射 + """ pipeline_data[PE.id] = node_uniqid() - replace_all_id(pipeline_data) + node_map = {} + replace_result_map = replace_all_id(pipeline_data) + pipeline_id = subprocess_id or pipeline_data[PE.id] + node_map[pipeline_id] = replace_result_map activities = pipeline_data[PE.activities] for act_id, act in list(activities.items()): if act[PE.type] == PE.SubProcess: - recursive_replace_id(act[PE.pipeline]) + replace_result_map = _recursive_replace_id_with_node_map(act[PE.pipeline], act_id) act[PE.pipeline][PE.id] = act_id + node_map[pipeline_id].setdefault("subprocess", {}).update(replace_result_map) + return node_map + + +def recursive_replace_id(pipeline_data): + return _recursive_replace_id_with_node_map(pipeline_data) def replace_all_id(pipeline_data): From fff2f73b0fb958126718441f06abb10a359b3364 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Fri, 17 Nov 2023 12:01:34 +0800 Subject: [PATCH 16/24] =?UTF-8?q?test:=20=E8=A1=A5=E5=85=85replace?= =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pipeline/tests/parser/__init__.py | 1 + .../pipeline/tests/parser/test_replace.py | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 runtime/bamboo-pipeline/pipeline/tests/parser/__init__.py create mode 100644 runtime/bamboo-pipeline/pipeline/tests/parser/test_replace.py diff --git a/runtime/bamboo-pipeline/pipeline/tests/parser/__init__.py b/runtime/bamboo-pipeline/pipeline/tests/parser/__init__.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/tests/parser/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/runtime/bamboo-pipeline/pipeline/tests/parser/test_replace.py b/runtime/bamboo-pipeline/pipeline/tests/parser/test_replace.py new file mode 100644 index 00000000..b7830800 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/tests/parser/test_replace.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +from django.test import TestCase +from pipeline.parser.utils import recursive_replace_id, replace_all_id + +from bamboo_engine.builder import ( + ConvergeGateway, + Data, + EmptyEndEvent, + EmptyStartEvent, + ExclusiveGateway, + ParallelGateway, + ServiceActivity, + SubProcess, + Var, + build_tree, + builder, +) + + +class ReplaceTests(TestCase): + def test_replace_all_id(self): + start = EmptyStartEvent() + act = ServiceActivity(component_code="example_component") + end = EmptyEndEvent() + start.extend(act).extend(end) + pipeline = builder.build_tree(start) + node_map = replace_all_id(pipeline) + self.assertIsInstance(node_map, dict) + self.assertIn(pipeline["start_event"]["id"], node_map["start_event"][start.id]) + self.assertIn(pipeline["end_event"]["id"], node_map["end_event"][end.id]) + self.assertEqual(list(pipeline["activities"].keys())[0], node_map["activities"][act.id]) + + def test_replace_all_id_gateway(self): + start = EmptyStartEvent() + pg = ParallelGateway() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + cg = ConvergeGateway() + end = EmptyEndEvent() + + start.extend(pg).connect(act_1, act_2, act_3).to(pg).converge(cg).extend(end) + pipeline = build_tree(start) + node_map = replace_all_id(pipeline) + + self.assertIn(pg.id, node_map["gateways"].keys()) + self.assertIn(cg.id, node_map["gateways"].keys()) + + self.assertIn(node_map["gateways"][pg.id], pipeline["gateways"].keys()) + self.assertIn(node_map["gateways"][cg.id], pipeline["gateways"].keys()) + + def test_recursive_replace_id(self): + start = EmptyStartEvent() + pg = ParallelGateway() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + act_2 = ServiceActivity(component_code="pipe_example_component", name="act_2") + act_3 = ServiceActivity(component_code="pipe_example_component", name="act_3") + cg = ConvergeGateway() + end = EmptyEndEvent() + start.extend(pg).connect(act_1, act_2, act_3).to(pg).converge(cg).extend(end) + pipeline = build_tree(start) + node_map = recursive_replace_id(pipeline) + self.assertIn(pg.id, node_map[pipeline["id"]]["gateways"].keys()) + self.assertIn(cg.id, node_map[pipeline["id"]]["gateways"].keys()) + self.assertIn(node_map[pipeline["id"]]["gateways"][pg.id], pipeline["gateways"].keys()) + self.assertIn(node_map[pipeline["id"]]["gateways"][cg.id], pipeline["gateways"].keys()) + self.assertIn(act_1.id, node_map[pipeline["id"]]["activities"].keys()) + + def test_recursive_replace_id_with_subprocess(self): + def sub_process(data): + subproc_start = EmptyStartEvent() + subproc_act = ServiceActivity(component_code="pipe_example_component", name="sub_act") + subproc_end = EmptyEndEvent() + + subproc_start.extend(subproc_act).extend(subproc_end) + + subproc_act.component.inputs.sub_input = Var(type=Var.SPLICE, value="${sub_input}") + + return SubProcess(start=subproc_start, data=data) + + start = EmptyStartEvent() + act_1 = ServiceActivity(component_code="pipe_example_component", name="act_1") + eg = ExclusiveGateway(conditions={0: "${act_1_output} < 0", 1: "${act_1_output} >= 0"}, name="act_2 or act_3") + + sub_pipeline_data_1 = Data(inputs={"${sub_input}": Var(type=Var.PLAIN, value=1)}) + subproc_1 = sub_process(sub_pipeline_data_1) + + sub_pipeline_data_2 = Data(inputs={"${sub_input}": Var(type=Var.PLAIN, value=2)}) + subproc_2 = sub_process(sub_pipeline_data_2) + end = EmptyEndEvent() + + start.extend(act_1).extend(eg).connect(subproc_1, subproc_2).converge(end) + + pipeline = build_tree(start) + node_map = recursive_replace_id(pipeline) + + self.assertEqual(len(node_map[pipeline["id"]]["subprocess"].keys()), 2) + self.assertIn( + node_map[pipeline["id"]]["activities"][subproc_1.id], node_map[pipeline["id"]]["subprocess"].keys() + ) + self.assertIn( + node_map[pipeline["id"]]["activities"][subproc_2.id], node_map[pipeline["id"]]["subprocess"].keys() + ) From 5a0f0d061e67a440aa3b1cd219252eac850905a3 Mon Sep 17 00:00:00 2001 From: crayon <42019787+ZhuoZhuoCrayon@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:42:42 +0800 Subject: [PATCH 17/24] =?UTF-8?q?feature:=20=E8=8A=82=E7=82=B9=E8=AE=A1?= =?UTF-8?q?=E6=97=B6=E5=99=A8=E8=BE=B9=E7=95=8C=E4=BA=8B=E4=BB=B6=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20(closed=20#189)=20(#190)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feature: 节点计时器边界事件支持 (closed #189) * minor: poetry fix * sprintfix: 场景测试修复 * optimization: 支持开发接入事件 & 超时节点记录自删除 & Action 使用 Metaclass 优化定义 * docs: 计时器边界事件使用文档刷新 --- .../node_timer_event_introduction.md | 125 ++++++++ .../commands/start_node_timeout_process.py | 10 +- .../contrib/node_timer_event/__init__.py | 12 + .../contrib/node_timer_event/adapter.py | 138 +++++++++ .../contrib/node_timer_event/admin.py | 26 ++ .../pipeline/contrib/node_timer_event/api.py | 54 ++++ .../pipeline/contrib/node_timer_event/apps.py | 31 ++ .../contrib/node_timer_event/constants.py | 35 +++ .../contrib/node_timer_event/handlers.py | 166 ++++++++++ .../node_timer_event/management/__init__.py | 12 + .../management/commands/__init__.py | 12 + .../start_node_timer_event_process.py | 74 +++++ .../migrations/0001_initial.py | 40 +++ .../node_timer_event/migrations/__init__.py | 12 + .../contrib/node_timer_event/models.py | 128 ++++++++ .../contrib/node_timer_event/settings.py | 71 +++++ .../node_timer_event/signals/__init__.py | 12 + .../node_timer_event/signals/handlers.py | 80 +++++ .../contrib/node_timer_event/tasks.py | 110 +++++++ .../contrib/node_timer_event/types.py | 25 ++ .../contrib/node_timer_event/utils.py | 64 ++++ .../pipeline/eri/imp/process.py | 2 +- .../tests/contrib/test_node_timer_event.py | 288 ++++++++++++++++++ runtime/bamboo-pipeline/poetry.lock | 12 + runtime/bamboo-pipeline/pyproject.toml | 1 + .../test/pipeline_sdk_use/settings.py | 1 + 26 files changed, 1534 insertions(+), 7 deletions(-) create mode 100644 docs/user_guide/node_timer_event_introduction.md create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/__init__.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/adapter.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/admin.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/api.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/apps.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/constants.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/handlers.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/__init__.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/commands/__init__.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/commands/start_node_timer_event_process.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/migrations/0001_initial.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/migrations/__init__.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/models.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/settings.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/signals/__init__.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/signals/handlers.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/tasks.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/types.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/utils.py create mode 100644 runtime/bamboo-pipeline/pipeline/tests/contrib/test_node_timer_event.py diff --git a/docs/user_guide/node_timer_event_introduction.md b/docs/user_guide/node_timer_event_introduction.md new file mode 100644 index 00000000..d2ef878c --- /dev/null +++ b/docs/user_guide/node_timer_event_introduction.md @@ -0,0 +1,125 @@ +# 增强包 - 节点计时器边界事件 + +## 特性 + +- 支持节点边界事件扫描与处理 +- 支持自定义节点计时器边界事件处理方式 +- 支持自定义节点计时器边界事件相关消息队列 + +## 启动配置 + +0. 该功能依赖 `mq`、`Redis` 和 `DB` 三个服务，需要在启动时保证这三个服务已经启动。 +1. 在 Django 项目配置文件的 INSTALLED_APPS 中添加 `pipeline.contrib.node_timer_event` 应用。 +2. 执行 `python manage.py migrate` 命令，创建数据库表。 +3. 启动计时器到期扫描进程，执行 `python manage.py start_node_timer_event_process` 命令。 +4. 启动对应的 worker 进程，执行 `python manage.py celery worker -l info -c 4` 命令。 + +## 接口 + +目前，pipeline.contrib.node_timer_event 模块提供了以下接口或扩展： + +1. Action + + SDK 内置了两个 Action 提供「节点超时」处理能力 + - `bamboo_engine_forced_fail`: 节点超时强制失败 + - `bamboo_engine_forced_fail_and_skip`: 节点超时强制失败并跳过 + + SDK 也提供了比较友好的自定义 Action 扩展和接入能力，用于定义业务层计时器边界事件的处理动作，例如定义一个名为 `example` + 的 Action + + ```python + import logging + + from pipeline.core.data.base import DataObject + from pipeline.contrib.node_timer_event.handlers import BaseAction + + logger = logging.getLogger(__name__) + + class ExampleAction(BaseAction): + def do(self, data: DataObject, parent_data: DataObject, *args, **kwargs) -> bool: + logger.info("[Action] example do: data -> %s, parent_data -> %s", data, parent_data) + return True + + class Meta: + action_name = "example" + + ``` + +2. apply_node_timer_event_configs + 该接口用于在 pipeline_tree 中应用节点计时器边界事件，接口定义如下： + ```python + def apply_node_timer_event_configs(pipeline_tree: dict, configs: dict): + """ + 在 pipeline_tree 中应用节点时间事件配置 + :param pipeline_tree: pipeline_tree + :param configs: 节点时间时间配置 + """ + ``` + 例如，创建一个节点运行 10 min 后启动的计时器边界事件，事件处理动作为步骤 1. 定义的 `example` 的计时器边界事件配置，可以这样写： + ```python + pipeline_tree = {} # 此处省略 pipeline_tree 的创建过程 + configs = {"node_id": [{"enable": True, "action": "example", "timer_type": "time_duration", "defined": "PT10M"}]} + new_pipeline_tree = apply_node_timer_event_configs(pipeline_tree, configs) + ``` + + 节点计时器边界事件配置中 + - enable 代表是否启用该节点的计时器事件配置 + - action 表示计时器触发时执行的动作 + - defined 代表计时器定义 + - timer_type 表示计时器类型 + - defined & timer_type 更多配置方式，请参考文末「附录」 + +3. batch_create_node_timeout_config + 该接口用于批量创建节点计时器边界事件，接口定义如下： + ```python + def batch_create_node_timer_event_config(root_pipeline_id: str, pipeline_tree: dict): + """ + 批量创建节点计时器边界事件配置 + :param root_pipeline_id: pipeline root ID + :param pipeline_tree: pipeline_tree + :return: 节点计时器边界事件配置数据插入结果 + """ + ``` + 插入结果示例: + ``` python + { + "result": True, # 是否操作成功, True 时关注 data，False 时关注 message + "data": [...], # NodeTimerEventConfig Model objects + "message": "" + } + ``` + +## 自定义 + +节点计时器边界事件模块的自定义配置 Django Settings 来实现，配置项和默认值如下： + +``` python +from pipeline.contrib.node_timer_event.handlers import node_timeout_handler + +PIPELINE_NODE_TIMER_EVENT_KEY_PREFIX = "bamboo:v1:node_timer_event" # Redis key 前缀，用于记录正在执行的节点，命名示例: {app_code}:{app_env}:{module}:node_timer_event +PIPELINE_NODE_TIMER_EVENT_HANDLE_QUEUE = None # 节点计时器边界事件处理队列名称, 用于处理计时器边界事件， 需要 worker 接收该队列消息，默认为 None，即使用 celery 默认队列 +PIPELINE_NODE_TIMER_EVENT_DISPATCH_QUEUE = None # 节点计时器边界事件分发队列名称, 用于记录计时器边界事件， 需要 worker 接收该队列消息，默认为 None，即使用 celery 默认队列 +PIPELINE_NODE_TIMER_EVENT_EXECUTING_POOL = "bamboo:v1:node_timer_event:executing_node_pool" # 执行节点池名称，用于记录正在执行的节点，需要保证 Redis key 唯一，命名示例: {app_code}:{app_env}:{module}:executing_node_pool +PIPELINE_NODE_TIMER_EVENT_POOL_SCAN_INTERVAL = 1 # 节点池扫描间隔，间隔越小，边界事件触发时间越精准，相应的事件处理的 workload 负载也会提升，默认为 1 s +PIPELINE_NODE_TIMER_EVENT_MAX_EXPIRE_TIME = 60 * 60 * 24 * 15 # 最长过期时间，兜底删除 Redis 冗余数据，默认为 15 Days，请根据业务场景调整 +PIPELINE_NODE_TIMER_EVENT_ADAPTER_CLASS = "pipeline.contrib.node_timer_event.adapter.NodeTimerEventAdapter" # 边界事件处理适配器，默认为 `pipeline.contrib.node_timer_event.adapter.NodeTimerEventAdapter` +``` + +## 使用样例 + +假设当前开发者已经准备好了对应的 pipeline_tree 和对应的节点计时器边界事件配置，那么在进行项目配置并启动对应的进程后，可以按照以下步骤进行处理： + +1. 调用 apply_node_timer_event_configs 接口，将节点计时器边界事件配置应用到 pipeline_tree 中 +2. 调用 batch_create_node_timeout_config 接口，将节点计时器边界事件配置插入到数据库中 +3. 启动 pipeline 运行，等待动计时器到期扫描进程处理到期边界事件，验证时请确认节点执行完成时间大于设置的计时器到期时间 +4. 查看节点计时器边界事件处理结果是否符合预期 + +## 附录 + +### 支持的计时器定义 + +| 计时器类型 | 配置值 | 描述 | `defined` 样例 | 备注 | +|---------------------|-----------------|----------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------| +| 时间日期（Time date） | `time_date` | ISO 8601 组合日期和时间格式 | `2019-10-01T12:00:00Z` - UTC 时间
`2019-10-02T08:09:40+02:00` - UTC 加上两小时时区偏移
`2019-10-02T08:09:40+02:00[Europe/Berlin]` - UTC 加上柏林两小时时区偏移 | | +| 持续时间（Time duration） | `time_duration` | ISO 8601 持续时间格式，模式：`P(n)Y(n)M(n)DT(n)H(n)M(n)S` | `PT15S` - 15 秒
`PT1H30M` - 1 小时 30 分钟
`P14D` - 14 天
`P14DT1H30M` - 14 天 1 小时 30 分钟 | `P` - 持续事件标识
`Y` - 年
`M` - 月
`D` - 天
`T` - 时间分量开始标识
`H` - 小时
`M` - 分钟
`S` - 秒 | +| 时间周期（Time cycle） | `time_cycle` | ISO 8601 重复间隔格式，包含重复次数模式：`R(n)` 及持续时间模式：`P(n)Y(n)M(n)DT(n)H(n)M(n)S` | `R5/PT10S` - 每10秒一次，最多五次
`R1/P1D` - 每天一次，最多一次 | | \ No newline at end of file diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timeout/management/commands/start_node_timeout_process.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timeout/management/commands/start_node_timeout_process.py index 6ab1fdbb..3a5513a4 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/node_timeout/management/commands/start_node_timeout_process.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timeout/management/commands/start_node_timeout_process.py @@ -12,17 +12,15 @@ """ import datetime import json +import logging import signal import time -import logging -from django.conf import settings from django.core.management import BaseCommand from django.db import connections - +from pipeline.contrib.node_timeout.models import TimeoutNodesRecord from pipeline.contrib.node_timeout.settings import node_timeout_settings from pipeline.contrib.node_timeout.tasks import dispatch_timeout_nodes -from pipeline.contrib.node_timeout.models import TimeoutNodesRecord logger = logging.getLogger("root") @@ -33,8 +31,8 @@ class Command(BaseCommand): def handle(self, *args, **options): signal.signal(signal.SIGTERM, self._graceful_exit) - redis_inst = settings.redis_inst - nodes_pool = settings.EXECUTING_NODE_POOL + redis_inst = node_timeout_settings.redis_inst + nodes_pool = node_timeout_settings.executing_pool while not self.has_killed: try: start = time.time() diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/__init__.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/__init__.py new file mode 100644 index 00000000..26a6d1c2 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/adapter.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/adapter.py new file mode 100644 index 00000000..ca0d5683 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/adapter.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import abc +import datetime +import json +import logging +import re +from typing import Dict, List, Optional, Union + +from pipeline.contrib.node_timer_event.models import NodeTimerEventConfig +from pipeline.contrib.node_timer_event.settings import node_timer_event_settings +from pipeline.contrib.node_timer_event.types import TimerEvent, TimerEvents +from pipeline.contrib.node_timer_event.utils import parse_timer_defined + +logger = logging.getLogger(__name__) + +EVENT_KEY_PATTERN = re.compile(r".*node:(?P.+):version:(?P.+):index:(?P\d+)") + + +class NodeTimerEventBaseAdapter(abc.ABC): + + node_id: str = None + version: str = None + root_pipeline_id: Optional[str] = None + events: Optional[TimerEvents] = None + index__event_map: Optional[Dict[int, TimerEvent]] = None + + def __init__(self, node_id: str, version: str): + self.node_id: str = node_id + self.version: str = version + + def is_ready(self) -> bool: + """适配器是否就绪""" + if not self.events: + return False + return True + + def fetch_keys_to_be_rem(self) -> List[str]: + """ + 获取需要被移除的事件 Key + :return: + """ + return [self.get_event_key(event) for event in self.events] + + def get_event_key(self, event: TimerEvent) -> str: + """ + 获取事件 Key + :param event: + :return: + """ + + # zset 没有按字符串匹配模式批量删除 key 的支持，使用 key 的命名采用已检索的信息进行拼接 + # 之前想把 loop 也维护进去，发觉移除操作非常麻烦，故采用 incr 的方式，单独维护每个事件事件的触发次数 + key_prefix: str = f"{node_timer_event_settings.key_prefix}:node:{self.node_id}:version:{self.version}" + return f"{key_prefix}:index:{event['index']}" + + @classmethod + def get_next_expired_time(cls, event: TimerEvent, start: Optional[datetime.datetime] = None) -> float: + """ + 获取时间事件下一次过期时间 + :param event: 事件详情 + :param start: 开始时间，默认取 datetime.now() + :return: + """ + return parse_timer_defined(event["timer_type"], event["defined"], start=start or datetime.datetime.now())[ + "timestamp" + ] + + def add_to_pool(self, redis_inst, event: TimerEvent): + + key: str = self.get_event_key(event) + + expired_time: float = self.get_next_expired_time(event) + + # TODO 考虑 incr & zadd 合并，使用 lua 封装成原子操作 + loop: int = int(redis_inst.incr(key, 1)) + redis_inst.expire(key, node_timer_event_settings.max_expire_time) + if loop > event["repetitions"]: + logger.info( + "[add_to_pool] No need to add: node -> %s, version -> %s, loop -> %s, event -> %s", + self.node_id, + self.version, + loop, + event, + ) + return + + redis_inst.zadd(node_timer_event_settings.executing_pool, mapping={key: expired_time}, nx=True) + + logger.info( + "[add_to_pool] add event to redis: " + "node_id -> %s, version -> %s, event -> %s, key -> %s, expired_time -> %s", + self.node_id, + self.version, + event, + key, + expired_time, + ) + + @classmethod + def parse_event_key(cls, key: str) -> Dict[str, Union[str, int]]: + match = EVENT_KEY_PATTERN.match(key) + if match: + key_info: Dict[str, Union[str, int]] = match.groupdict() + # to int + key_info["index"] = int(key_info["index"]) + + return key_info + + else: + raise ValueError(f"invalid key -> {key}") + + +class NodeTimerEventAdapter(NodeTimerEventBaseAdapter): + def __init__(self, node_id: str, version: str): + super().__init__(node_id, version) + + node_timer_event_config: NodeTimerEventConfig = NodeTimerEventConfig.objects.filter( + node_id=self.node_id + ).first() + + if not node_timer_event_config: + return + + self.root_pipeline_id: str = node_timer_event_config.root_pipeline_id + self.events: TimerEvents = json.loads(node_timer_event_config.events) + self.index__event_map: Dict[int, TimerEvent] = {event["index"]: event for event in self.events} diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/admin.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/admin.py new file mode 100644 index 00000000..9fa6711b --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/admin.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.contrib import admin +from pipeline.contrib.node_timer_event import models + + +@admin.register(models.NodeTimerEventConfig) +class NodeTimerEventConfigAdmin(admin.ModelAdmin): + list_display = ["root_pipeline_id", "node_id", "events"] + search_fields = ["root_pipeline_id__exact", "node_id__exact"] + + +@admin.register(models.ExpiredNodesRecord) +class ExpiredNodesRecordAdmin(admin.ModelAdmin): + list_display = ["id", "nodes"] diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/api.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/api.py new file mode 100644 index 00000000..26ce1e01 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/api.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import copy +from typing import Any, Dict, List + +from pipeline.contrib.node_timer_event.models import NodeTimerEventConfig +from pipeline.contrib.utils import ensure_return_pipeline_contrib_api_result +from pipeline.core.constants import PE + + +@ensure_return_pipeline_contrib_api_result +def apply_node_timer_event_configs(pipeline_tree: Dict[str, Any], configs: Dict[str, List[Dict[str, Any]]]): + """ + 在 pipeline_tree 中应用节点计时器边界事件配置 + :param pipeline_tree: pipeline_tree + :param configs: 节点计时器边界事件配置 + e.g. {"node_id": [{"enable": True, "action": "forced_fail", "timer_type": "time_duration", "defined": "PT10M"}]} + """ + new_pipeline_tree = copy.deepcopy(pipeline_tree) + for act_id, act in pipeline_tree[PE.activities].items(): + if act["type"] == PE.SubProcess: + apply_node_timer_event_configs(act[PE.pipeline], configs) + elif act["type"] == PE.ServiceActivity and act_id in configs: + act.setdefault("events", {})["timer_events"] = [ + { + "enable": config["enable"], + "timer_type": config["timer_type"], + "action": config["action"], + "defined": config["defined"], + } + for config in configs[act_id] + ] + return new_pipeline_tree + + +@ensure_return_pipeline_contrib_api_result +def batch_create_node_timer_event_config(root_pipeline_id: str, pipeline_tree: Dict[str, Any]): + """ + 批量创建节点时间事件配置 + :param root_pipeline_id: pipeline root ID + :param pipeline_tree: pipeline_tree + :return: 节点计时器边界事件配置数据插入结果，e.g. {"result": True, "data": objs, "message": ""} + """ + return NodeTimerEventConfig.objects.batch_create_node_timer_event_config(root_pipeline_id, pipeline_tree) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/apps.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/apps.py new file mode 100644 index 00000000..2f7e43ee --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/apps.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.apps import AppConfig +from django.conf import settings + + +class NodeTimerEventConfig(AppConfig): + name = "pipeline.contrib.node_timer_event" + verbose_name = "PipelineNodeTimerEvent" + + def ready(self): + from pipeline.contrib.node_timer_event.signals.handlers import ( # noqa + bamboo_engine_eri_node_state_handler, + ) + from pipeline.contrib.node_timer_event.tasks import ( # noqa + dispatch_expired_nodes, + execute_node_timer_event_action, + ) + + if not hasattr(settings, "redis_inst"): + raise Exception("Django Settings should have redis_inst attribute") diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/constants.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/constants.py new file mode 100644 index 00000000..97afb78f --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/constants.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from enum import Enum + + +class TimerType(Enum): + + # 时间日期（Time date）ISO 8601 组合日期和时间格式 + # 2019-10-01T12:00:00Z - UTC 时间 + # 2019-10-02T08:09:40+02:00 - UTC 加上两小时时区偏移 + # 2019-10-02T08:09:40+02:00[Europe/Berlin] - UTC 加上柏林两小时时区偏移 + TIME_DATE = "time_date" + + # 时间周期（Time cycle）ISO 8601 重复间隔格式，包含重复次数模式：R(n) 及持续时间模式：P(n)Y(n)M(n)DT(n)H(n)M(n)S + # R5/PT10S - 每10秒一次，最多五次 + # R/P1D - 每天，无限 + TIME_CYCLE = "time_cycle" + + # 持续时间（Time duration）ISO 8601 持续时间格式，模式：P(n)Y(n)M(n)DT(n)H(n)M(n)S + # PT15S - 15 秒 + # PT1H30M - 1 小时 30 分钟 + # P14D - 14 天 + # P14DT1H30M - 14 天 1 小时 30 分钟 + TIME_DURATION = "time_duration" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/handlers.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/handlers.py new file mode 100644 index 00000000..1725b686 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/handlers.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from abc import abstractmethod +from typing import Any, Dict, Tuple + +from pipeline.core.data.base import DataObject +from pipeline.eri.runtime import BambooDjangoRuntime + +from bamboo_engine import api as bamboo_engine_api +from bamboo_engine.eri import ExecutionData + +logger = logging.getLogger(__name__) + + +class ActionManager: + __hub = {} + + @classmethod + def register_invocation_cls(cls, invocation_cls): + action_name = invocation_cls.Meta.action_name + existed_invocation_cls = cls.__hub.get(action_name) + if existed_invocation_cls: + raise RuntimeError( + "func register error, {}'s action_name {} conflict with {}".format( + existed_invocation_cls, action_name, invocation_cls + ) + ) + + cls.__hub[action_name] = invocation_cls + + @classmethod + def clear(cls): + cls.__hub = {} + + @classmethod + def get_action(cls, root_pipeline_id: str, node_id: str, version: str, action_name: str) -> "BaseAction": + """ + 获取 Action 实例 + :param root_pipeline_id: 根节点 ID + :param node_id: 节点 ID + :param version: 节点版本 + :param action_name: Action 名称 + :return: + """ + if action_name not in cls.__hub: + raise ValueError("{} not found".format(action_name)) + return cls.__hub[action_name](root_pipeline_id, node_id, version) + + +class ActionMeta(type): + """ + Metaclass for FEEL function invocation + """ + + def __new__(cls, name, bases, dct): + # ensure initialization is only performed for subclasses of Plugin + parents = [b for b in bases if isinstance(b, ActionMeta)] + if not parents: + return super().__new__(cls, name, bases, dct) + + new_cls = super().__new__(cls, name, bases, dct) + + # meta validation + meta_obj = getattr(new_cls, "Meta", None) + if not meta_obj: + raise AttributeError("Meta class is required") + + action_name = getattr(meta_obj, "action_name", None) + if not action_name: + raise AttributeError("action_name is required in Meta") + + # register func + ActionManager.register_invocation_cls(new_cls) + + return new_cls + + +class BaseAction(metaclass=ActionMeta): + def __init__(self, root_pipeline_id: str, node_id: str, version: str): + self.root_pipeline_id = root_pipeline_id + self.node_id = node_id + self.version = version + + @classmethod + def get_execution_data(cls, root_pipeline_id: str, node_id: str) -> Tuple[DataObject, DataObject]: + runtime: BambooDjangoRuntime = BambooDjangoRuntime() + data: ExecutionData = runtime.get_execution_data(node_id) + root_pipeline_inputs: Dict[str, Any] = { + key: di.value for key, di in runtime.get_data_inputs(root_pipeline_id).items() + } + root_pipeline_data: ExecutionData = ExecutionData(inputs=root_pipeline_inputs, outputs={}) + + data_obj: DataObject = DataObject(inputs=data.inputs, outputs=data.outputs) + parent_data_obj: DataObject = DataObject(inputs=root_pipeline_data.inputs, outputs=root_pipeline_data.outputs) + return data_obj, parent_data_obj + + def notify(self, *args, **kwargs) -> bool: + data, parent_data = self.get_execution_data(self.root_pipeline_id, self.node_id) + return self.do(data, parent_data, *args, **kwargs) + + @abstractmethod + def do(self, data: DataObject, parent_data: DataObject, *args, **kwargs) -> bool: + raise NotImplementedError + + +class ExampleAction(BaseAction): + def do(self, data: DataObject, parent_data: DataObject, *args, **kwargs) -> bool: + logger.info("[Action] example do: data -> %s, parent_data -> %s", data, parent_data) + return True + + class Meta: + action_name = "example" + + +class ForcedFailAction(BaseAction): + + TIMEOUT_NODE_OPERATOR = "bamboo_engine" + + def do(self, data: DataObject, parent_data: DataObject, *args, **kwargs) -> bool: + logger.info("[Action(bamboo_engine_forced_fail)] do: data -> %s, parent_data -> %s", data, parent_data) + result = bamboo_engine_api.forced_fail_activity( + runtime=BambooDjangoRuntime(), + node_id=self.node_id, + ex_data="forced fail by {}".format(self.TIMEOUT_NODE_OPERATOR), + send_post_set_state_signal=kwargs.get("send_post_set_state_signal", True), + ) + return result.result + + class Meta: + action_name = "bamboo_engine_forced_fail" + + +class ForcedFailAndSkipAction(BaseAction): + + TIMEOUT_NODE_OPERATOR = "bamboo_engine" + + def do(self, data: DataObject, parent_data: DataObject, *args, **kwargs) -> bool: + logger.info("[Action(bamboo_engine_forced_fail_and_skip)] do: data -> %s, parent_data -> %s", data, parent_data) + result = bamboo_engine_api.forced_fail_activity( + runtime=BambooDjangoRuntime(), + node_id=self.node_id, + ex_data="forced fail by {}".format(self.TIMEOUT_NODE_OPERATOR), + send_post_set_state_signal=kwargs.get("send_post_set_state_signal", True), + ) + if result.result: + result = bamboo_engine_api.skip_node( + runtime=BambooDjangoRuntime(), + node_id=self.node_id, + ex_data="forced skip by {}".format(self.TIMEOUT_NODE_OPERATOR), + send_post_set_state_signal=kwargs.get("send_post_set_state_signal", True), + ) + return result.result + + class Meta: + action_name = "bamboo_engine_forced_fail_and_skip" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/__init__.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/__init__.py new file mode 100644 index 00000000..40097292 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/commands/__init__.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/commands/__init__.py new file mode 100644 index 00000000..40097292 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/commands/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/commands/start_node_timer_event_process.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/commands/start_node_timer_event_process.py new file mode 100644 index 00000000..f02e3092 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/management/commands/start_node_timer_event_process.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import datetime +import json +import logging +import signal +import time + +from django.core.management import BaseCommand +from django.db import connections +from pipeline.contrib.node_timer_event.models import ExpiredNodesRecord +from pipeline.contrib.node_timer_event.settings import node_timer_event_settings +from pipeline.contrib.node_timer_event.tasks import dispatch_expired_nodes + +logger = logging.getLogger("root") + + +class Command(BaseCommand): + help = "scanning expired nodes and dispatch them to celery task" + has_killed = False + + def handle(self, *args, **options): + signal.signal(signal.SIGTERM, self._graceful_exit) + redis_inst = node_timer_event_settings.redis_inst + nodes_pool = node_timer_event_settings.executing_pool + while not self.has_killed: + try: + start = time.time() + self._pop_expired_nodes(redis_inst, nodes_pool) + end = time.time() + logger.info(f"[node_timeout_process] time consuming: {end-start}") + except Exception as e: + logger.exception(e) + time.sleep(node_timer_event_settings.pool_scan_interval) + + def _graceful_exit(self, *args): + self.has_killed = True + + def _pop_expired_nodes(self, redis_inst, nodes_pool) -> list: + now = datetime.datetime.now().timestamp() + expired_nodes = [ + node.decode("utf-8") if isinstance(node, bytes) else node + for node in redis_inst.zrangebyscore(nodes_pool, "-inf", now) + ] + if expired_nodes: + self.record_expired_nodes(expired_nodes) + redis_inst.zrem(nodes_pool, *expired_nodes) + return expired_nodes + + @staticmethod + def record_expired_nodes(timeout_nodes: list): + # 处理因为过长时间没有访问导致的链接失效问题 + for conn in connections.all(): + conn.close_if_unusable_or_obsolete() + + record = ExpiredNodesRecord.objects.create(nodes=json.dumps(timeout_nodes)) + if node_timer_event_settings.dispatch_queue is None: + dispatch_expired_nodes.apply_async(kwargs={"record_id": record.id}) + else: + dispatch_expired_nodes.apply_async( + kwargs={"record_id": record.id}, + queue=node_timer_event_settings.dispatch_queue, + routing_key=node_timer_event_settings.dispatch_queue, + ) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/migrations/0001_initial.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/migrations/0001_initial.py new file mode 100644 index 00000000..381aea45 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/migrations/0001_initial.py @@ -0,0 +1,40 @@ +# Generated by Django 3.2.18 on 2023-10-25 02:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="ExpiredNodesRecord", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False, verbose_name="ID")), + ("nodes", models.TextField(verbose_name="到期节点信息")), + ], + options={ + "verbose_name": "到期节点数据记录 ExpiredNodesRecord", + "verbose_name_plural": "到期节点数据记录 ExpiredNodesRecord", + }, + ), + migrations.CreateModel( + name="NodeTimerEventConfig", + fields=[ + ("root_pipeline_id", models.CharField(max_length=64, verbose_name="root pipeline id")), + ( + "node_id", + models.CharField(max_length=64, primary_key=True, serialize=False, verbose_name="task node id"), + ), + ("events", models.TextField(default="[]", verbose_name="timer events")), + ], + options={ + "verbose_name": "节点时间事件配置 NodeTimerEventConfig", + "verbose_name_plural": "节点时间事件配置 NodeTimerEventConfig", + "index_together": {("root_pipeline_id", "node_id")}, + }, + ), + ] diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/migrations/__init__.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/migrations/__init__.py new file mode 100644 index 00000000..40097292 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/migrations/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/models.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/models.py new file mode 100644 index 00000000..0075ee9f --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/models.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import json +import logging +import re +import typing +from typing import Any, Dict, List, Optional + +from django.db import models +from django.utils.translation import ugettext_lazy as _ +from pipeline.contrib.node_timer_event.types import TimeDefined +from pipeline.contrib.node_timer_event.utils import parse_timer_defined +from pipeline.core.constants import PE + +logger = logging.getLogger(__name__) + + +EVENT_KEY_PATTERN = re.compile(r".*node:(?P.+):version:(?P.+):index:(?P\d+)") + + +class NodeTimerEventConfigManager(models.Manager): + def parse_node_timer_event_configs(self, pipeline_tree: Dict[str, Any]) -> Dict[str, Any]: + """解析事件时间""" + configs: List[Dict[str, Any]] = [] + for act_id, act in pipeline_tree[PE.activities].items(): + if act["type"] == PE.SubProcess: + result = self.parse_node_timer_event_configs(act[PE.pipeline]) + if not result["result"]: + return result + configs.extend(result["data"]) + elif act["type"] == PE.ServiceActivity: + index: int = 1 + treated_timer_events: List[Dict[str, Any]] = [] + timer_events: List[Dict[str, Any]] = (act.get("events") or {}).get("timer_events") or [] + for timer_event in timer_events: + enable: bool = timer_event.get("enable") or False + if not enable: + continue + + timer_type: Optional[str] = timer_event.get("timer_type") + defined: Optional[str] = timer_event.get("defined") + + try: + timer_defined: TimeDefined = parse_timer_defined(timer_type, defined) + except Exception: + # 对于不符合格式要求的情况，记录日志并跳过 + logger.exception( + "[parse_node_timer_event_configs] parse timer_defined failed: " + "node_id -> %s, timer_type -> %s, defined -> %s", + act_id, + timer_type, + defined, + ) + continue + + treated_timer_events.append( + { + "index": index, + "action": timer_event.get("action"), + "timer_type": timer_type, + "repetitions": timer_defined["repetitions"], + "defined": defined, + } + ) + + index += 1 + + if treated_timer_events: + configs.append({"node_id": act_id, "events": treated_timer_events}) + + return {"result": True, "data": configs, "message": ""} + + def batch_create_node_timer_event_config(self, root_pipeline_id: str, pipeline_tree: dict): + """批量创建节点超时配置""" + + config_parse_result: Dict[str, Any] = self.parse_node_timer_event_configs(pipeline_tree) + # 这里忽略解析失败的情况，保证即使解析失败也能正常创建任务 + if not config_parse_result["result"]: + logger.error( + f"[batch_create_node_timer_event_config] parse node timer event config " + f'failed: {config_parse_result["result"]}' + ) + return config_parse_result + + configs: List[Dict[str, Any]] = config_parse_result["data"] + config_objs: typing.List[NodeTimerEventConfig] = [ + NodeTimerEventConfig( + root_pipeline_id=root_pipeline_id, node_id=config["node_id"], events=json.dumps(config["events"]) + ) + for config in configs + ] + objs = self.bulk_create(config_objs, batch_size=1000) + return {"result": True, "data": objs, "message": ""} + + +class NodeTimerEventConfig(models.Model): + root_pipeline_id = models.CharField(verbose_name="root pipeline id", max_length=64) + node_id = models.CharField(verbose_name="task node id", max_length=64, primary_key=True) + events = models.TextField(verbose_name="timer events", default="[]") + + objects = NodeTimerEventConfigManager() + + class Meta: + verbose_name = _("节点时间事件配置 NodeTimerEventConfig") + verbose_name_plural = _("节点时间事件配置 NodeTimerEventConfig") + index_together = [("root_pipeline_id", "node_id")] + + def get_events(self) -> List[Dict[str, Any]]: + return json.loads(self.events) + + +class ExpiredNodesRecord(models.Model): + id = models.BigAutoField(verbose_name="ID", primary_key=True) + nodes = models.TextField(verbose_name="到期节点信息") + + class Meta: + verbose_name = _("到期节点数据记录 ExpiredNodesRecord") + verbose_name_plural = _("到期节点数据记录 ExpiredNodesRecord") diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/settings.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/settings.py new file mode 100644 index 00000000..4eac23ff --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/settings.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from importlib import import_module + +from django.conf import settings +from pipeline.contrib.node_timer_event import types + + +def get_import_path(cls: types.T) -> str: + return f"{cls.__module__}.{cls.__name__}" + + +def import_string(dotted_path: str): + """ + Import a dotted module path and return the attribute/class designated by the + last name in the path. Raise ImportError if the import failed. + """ + try: + module_path, class_name = dotted_path.rsplit(".", 1) + except ValueError as err: + raise ImportError(f"{dotted_path} doesn't look like a module path") from err + + module = import_module(module_path) + + try: + return getattr(module, class_name) + except AttributeError as err: + raise ImportError(f'Module "{module_path}" does not define a "{class_name}" attribute/class') from err + + +class NodeTimerEventSettngs: + PREFIX = "PIPELINE_NODE_TIMER_EVENT" + DEFAULT_SETTINGS = { + # # Redis key 前缀，用于记录正在执行的节点，命名示例: {app_code}:{app_env}:{module}:node_timer_event + # v1 表示 node_timer_event 的版本，预留以隔离 + "key_prefix": "bamboo:v1:node_timer_event", + # 节点计时器边界事件处理队列名称, 用于处理计时器边界事件， 需要 worker 接收该队列消息，默认为 None，即使用 celery 默认队列 + "dispatch_queue": None, + # 节点计时器边界事件分发队列名称, 用于记录计时器边界事件， 需要 worker 接收该队列消息，默认为 None，即使用 celery 默认队列 + "handle_queue": None, + # 执行节点池名称，用于记录正在执行的节点，需要保证 Redis key 唯一，命名示例: {app_code}:{app_env}:{module}:executing_node_pool + "executing_pool": "bamboo:v1:node_timer_event:executing_node_pool", + # 节点池扫描间隔，间隔越小，边界事件触发时间越精准，相应的事件处理的 workload 负载也会提升 + "pool_scan_interval": 1, + # 最长过期时间，兜底删除 Redis 冗余数据，默认为 15 Days，请根据业务场景调整 + "max_expire_time": 60 * 60 * 24 * 15, + # 边界事件处理适配器，默认为 `pipeline.contrib.node_timer_event.adapter.NodeTimerEventAdapter` + "adapter_class": "pipeline.contrib.node_timer_event.adapter.NodeTimerEventAdapter", + } + + def __getattr__(self, item: str): + if item == "redis_inst": + return settings.redis_inst + if item == "adapter_class": + return import_string(getattr(settings, f"{self.PREFIX}_{item.upper()}", self.DEFAULT_SETTINGS.get(item))) + + return getattr(settings, f"{self.PREFIX}_{item.upper()}", self.DEFAULT_SETTINGS.get(item)) + + +node_timer_event_settings = NodeTimerEventSettngs() diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/signals/__init__.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/signals/__init__.py new file mode 100644 index 00000000..26a6d1c2 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/signals/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/signals/handlers.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/signals/handlers.py new file mode 100644 index 00000000..4b3c4d56 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/signals/handlers.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List, Optional, Type + +from django.dispatch import receiver +from pipeline.contrib.node_timer_event.adapter import NodeTimerEventBaseAdapter +from pipeline.contrib.node_timer_event.settings import node_timer_event_settings +from pipeline.eri.signals import post_set_state + +from bamboo_engine import states as bamboo_engine_states + +logger = logging.getLogger(__name__) + + +def _node_timer_event_info_update(redis_inst, to_state: str, node_id: str, version: str): + + adapter: Optional[NodeTimerEventBaseAdapter] = None + + if to_state in [ + bamboo_engine_states.RUNNING, + bamboo_engine_states.FAILED, + bamboo_engine_states.FINISHED, + bamboo_engine_states.SUSPENDED, + ]: + + adapter_class: Type[NodeTimerEventBaseAdapter] = node_timer_event_settings.adapter_class + adapter: NodeTimerEventBaseAdapter = adapter_class(node_id=node_id, version=version) + + if not adapter.is_ready(): + logger.info( + "[node_timer_event_info_update] node_timer_event_config not exist and skipped: " + "node_id -> %s, version -> %s", + node_id, + version, + ) + return + + logger.info( + "[node_timer_event_info_update] load node_timer_event_config: node_id -> %s, version -> %s, events -> %s", + node_id, + version, + adapter.events, + ) + + if to_state == bamboo_engine_states.RUNNING: + # 遍历节点时间事件，丢进待调度节点池 + for event in adapter.events: + adapter.add_to_pool(redis_inst, event) + + elif to_state in [bamboo_engine_states.FAILED, bamboo_engine_states.FINISHED, bamboo_engine_states.SUSPENDED]: + keys: List[str] = adapter.fetch_keys_to_be_rem() + redis_inst.zrem(node_timer_event_settings.executing_pool, *keys) + redis_inst.delete(*keys) + logger.info( + "[node_timer_event_info_update] removed events from redis: " + "node_id -> %s, version -> %s, events -> %s, keys -> %s", + node_id, + version, + adapter.events, + keys, + ) + + +@receiver(post_set_state) +def bamboo_engine_eri_node_state_handler(sender, node_id, to_state, version, root_id, parent_id, loop, **kwargs): + try: + _node_timer_event_info_update(node_timer_event_settings.redis_inst, to_state, node_id, version) + except Exception: + logger.exception("node_timeout_info_update error") diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/tasks.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/tasks.py new file mode 100644 index 00000000..44970a5a --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/tasks.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import json +import logging +from typing import Any, Dict, List, Type, Union + +from celery import task +from pipeline.contrib.node_timer_event.adapter import NodeTimerEventBaseAdapter +from pipeline.contrib.node_timer_event.handlers import ActionManager +from pipeline.contrib.node_timer_event.models import ExpiredNodesRecord +from pipeline.contrib.node_timer_event.settings import node_timer_event_settings +from pipeline.eri.models import Process, State + +logger = logging.getLogger("celery") + + +@task(acks_late=True) +def dispatch_expired_nodes(record_id: int): + record: ExpiredNodesRecord = ExpiredNodesRecord.objects.get(id=record_id) + node_keys: List[str] = json.loads(record.nodes) + logger.info("[dispatch_expired_nodes] record -> %s, nodes -> %s", record_id, node_keys) + + adapter_class: Type[NodeTimerEventBaseAdapter] = node_timer_event_settings.adapter_class + + for node_key in node_keys: + try: + key_info: Dict[str, Union[str, int]] = adapter_class.parse_event_key(node_key) + except ValueError: + logger.warning( + "[dispatch_expired_nodes] failed to parse key, skipped: record -> %s, node_key -> %s", + record_id, + node_key, + ) + continue + + index: int = key_info["index"] + node_id: str = key_info["node_id"] + version: str = key_info["version"] + + if node_timer_event_settings.handle_queue is None: + execute_node_timer_event_action.apply_async(kwargs={"node_id": node_id, "version": version, "index": index}) + else: + execute_node_timer_event_action.apply_async( + kwargs={"node_id": node_id, "version": version, "index": index}, + queue=node_timer_event_settings.handle_queue, + routing_key=node_timer_event_settings.handle_queue, + ) + + logger.info("[dispatch_expired_nodes] dispatch finished: record -> %s, nodes -> %s", record_id, node_keys) + # 删除临时记录 + record.delete() + logger.info("[dispatch_expired_nodes] record deleted: record -> %s", record_id) + + +@task(ignore_result=True) +def execute_node_timer_event_action(node_id: str, version: str, index: int): + + adapter_class: Type[NodeTimerEventBaseAdapter] = node_timer_event_settings.adapter_class + adapter: NodeTimerEventBaseAdapter = adapter_class(node_id=node_id, version=version) + if not adapter.is_ready() or (adapter.index__event_map and index not in adapter.index__event_map): + message: str = ( + f"[execute_node_timer_event_action] no timer config: " + f"node_id -> {node_id}, version -> {version}, index -> {index}" + ) + logger.exception(message) + return {"result": False, "message": message, "data": None} + + event: Dict[str, Any] = adapter.index__event_map[index] + + # 判断当前节点是否符合策略执行要求 + is_process_current_node: bool = Process.objects.filter( + root_pipeline_id=adapter.root_pipeline_id, current_node_id=node_id + ).exists() + is_node_match = State.objects.filter(node_id=node_id, version=version).exists() + if not (is_node_match and is_process_current_node): + message = ( + f"[execute_node_timer_event_action] node {node_id} with version {version} " + f"in pipeline {adapter.root_pipeline_id} has been passed." + ) + logger.error(message) + return {"result": False, "message": message, "data": None} + + # 计算事件下一次触发事件并丢进待调度节点池 + adapter.add_to_pool(node_timer_event_settings.redis_inst, event) + + try: + is_success: bool = ActionManager.get_action( + adapter.root_pipeline_id, node_id, version, event["action"] + ).notify() + logger.info( + f"[execute_node_timer_event_action] node {node_id} with version {version} in pipeline " + f"{adapter.root_pipeline_id} action result is: {is_success}." + ) + return {"result": is_success, "data": None} + except Exception as e: + logger.exception( + f"[execute_node_timer_event_action] node {node_id} with version {version} in pipeline " + f"{adapter.root_pipeline_id} error is: {e}." + ) + return {"result": False, "data": None, "message": str(e)} diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/types.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/types.py new file mode 100644 index 00000000..092524ff --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/types.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + + +import typing + +T = typing.TypeVar("T") + +TimeDefined = typing.Dict[str, typing.Any] + + +TimerEvent = typing.Dict[str, typing.Any] + + +TimerEvents = typing.List[TimerEvent] diff --git a/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/utils.py b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/utils.py new file mode 100644 index 00000000..31c4c3e2 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/node_timer_event/utils.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + + +import datetime +import re +from typing import Any, Callable, Dict, Optional + +import isodate +from pipeline.contrib.node_timer_event.constants import TimerType +from pipeline.contrib.node_timer_event.types import TimeDefined + +TIME_CYCLE_DEFINED_PATTERN = re.compile(r"(^R\d+)/") + + +def handle_result(timestamp: float, repetitions: int = 1, *args, **kwargs) -> TimeDefined: + return {"timestamp": timestamp, "repetitions": repetitions} + + +def handle_timedate(defined: str, start: Optional[datetime.datetime] = None, *args, **kwargs) -> TimeDefined: + return handle_result(timestamp=isodate.parse_datetime(defined).timestamp()) + + +def handle_time_duration(defined: str, start: Optional[datetime.datetime] = None, *args, **kwargs) -> TimeDefined: + start = start or datetime.datetime.now() + return handle_result(timestamp=(start + isodate.parse_duration(defined)).timestamp()) + + +def handle_time_cycle(defined: str, start: Optional[datetime.datetime] = None, *args, **kwargs) -> TimeDefined: + repeat_match = TIME_CYCLE_DEFINED_PATTERN.match(defined) + if repeat_match: + repetitions: int = int(repeat_match.group(1)[1:]) + duration_string = TIME_CYCLE_DEFINED_PATTERN.sub("", defined) + else: + repetitions: int = 1 + duration_string = defined + + return handle_result(timestamp=handle_time_duration(duration_string, start)["timestamp"], repetitions=repetitions) + + +TIMER_TYPE_ROUTES: Dict[str, Callable[[str, Optional[datetime.datetime], Any, Any], TimeDefined]] = { + TimerType.TIME_DURATION.value: handle_time_duration, + TimerType.TIME_CYCLE.value: handle_time_cycle, + TimerType.TIME_DATE.value: handle_timedate, +} + + +def parse_timer_defined( + timer_type: str, defined: str, start: Optional[datetime.datetime] = None, *args, **kwargs +) -> TimeDefined: + if timer_type not in TIMER_TYPE_ROUTES: + raise ValueError(f"Unsupported timer_type -> {timer_type}") + + return TIMER_TYPE_ROUTES[timer_type](defined, start, *args, **kwargs) diff --git a/runtime/bamboo-pipeline/pipeline/eri/imp/process.py b/runtime/bamboo-pipeline/pipeline/eri/imp/process.py index 730da35d..4e703dd6 100644 --- a/runtime/bamboo-pipeline/pipeline/eri/imp/process.py +++ b/runtime/bamboo-pipeline/pipeline/eri/imp/process.py @@ -175,7 +175,7 @@ def get_sleep_process_info_with_current_node_id(self, node_id: str) -> Optional[ parent_id=qs[0].parent_id, ) - def get_process_id_with_current_node_id(self, node_id: str) -> Optional[str]: + def get_process_id_with_current_node_id(self, node_id: str) -> Optional[int]: """ 获取当前节点 ID 为 node_id 且存活的进程 ID diff --git a/runtime/bamboo-pipeline/pipeline/tests/contrib/test_node_timer_event.py b/runtime/bamboo-pipeline/pipeline/tests/contrib/test_node_timer_event.py new file mode 100644 index 00000000..93cc0f4f --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/tests/contrib/test_node_timer_event.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import copy +import datetime +import json +from typing import Any, Dict, List + +from django.test import TestCase +from mock import MagicMock, call, patch +from pipeline.contrib.node_timer_event import constants, types, utils +from pipeline.contrib.node_timer_event.models import ( + ExpiredNodesRecord, + NodeTimerEventConfig, +) +from pipeline.contrib.node_timer_event.tasks import ( + dispatch_expired_nodes, + execute_node_timer_event_action, +) +from pipeline.eri.models import Process, State + +from bamboo_engine.eri import DataInput, ExecutionData + + +class ParseTimerDefinedTestCase(TestCase): + + TIME_FORMAT: str = "%Y-%m-%d %H:%M:%S" + + def test_time_cycle(self): + + start: datetime.datetime = datetime.datetime.strptime("2022-01-01 00:00:00", self.TIME_FORMAT) + cases: List[Dict[str, Any]] = [ + { + "defined": "R5/PT10S", + "repetitions": 5, + "timestamp": (start + datetime.timedelta(seconds=10)).timestamp(), + }, + {"defined": "R1/P1D", "repetitions": 1, "timestamp": (start + datetime.timedelta(days=1)).timestamp()}, + ] + + for case in cases: + time_defined: types.TimeDefined = utils.parse_timer_defined( + timer_type=constants.TimerType.TIME_CYCLE.value, defined=case["defined"], start=start + ) + self.assertEqual(time_defined["repetitions"], case["repetitions"]) + self.assertEqual(time_defined["timestamp"], case["timestamp"]) + + def test_time_duration(self): + start: datetime.datetime = datetime.datetime.strptime("2022-01-01 00:00:00", self.TIME_FORMAT) + cases: List[Dict[str, Any]] = [ + { + "defined": "P14DT1H30M", + "timestamp": (start + datetime.timedelta(days=14, hours=1, minutes=30)).timestamp(), + }, + {"defined": "P14D", "timestamp": (start + datetime.timedelta(days=14)).timestamp()}, + { + "defined": "P14DT1H30M", + "timestamp": (start + datetime.timedelta(days=14, hours=1, minutes=30)).timestamp(), + }, + {"defined": "PT15S", "timestamp": (start + datetime.timedelta(seconds=15)).timestamp()}, + ] + + for case in cases: + time_defined: types.TimeDefined = utils.parse_timer_defined( + timer_type=constants.TimerType.TIME_DURATION.value, defined=case["defined"], start=start + ) + self.assertEqual(time_defined["repetitions"], 1) + self.assertEqual(time_defined["timestamp"], case["timestamp"]) + + def test_time_date(self): + start: datetime.datetime = datetime.datetime.strptime("2022-01-01 00:00:00", self.TIME_FORMAT) + cases: List[Dict[str, Any]] = [ + {"defined": "2019-10-01T12:00:00Z", "timestamp": 1569931200.0}, + {"defined": "2019-10-02T08:09:40+02:00", "timestamp": 1569996580.0}, + {"defined": "2019-10-02T08:09:40+02:00[Europe/Berlin]", "timestamp": 1569996580.0}, + ] + + for case in cases: + time_defined: types.TimeDefined = utils.parse_timer_defined( + timer_type=constants.TimerType.TIME_DATE.value, defined=case["defined"], start=start + ) + self.assertEqual(time_defined["repetitions"], 1) + self.assertEqual(time_defined["timestamp"], case["timestamp"]) + + +class NodeTimerEventTestCase(TestCase): + def setUp(self): + self.node_id = "node_id" + self.version = "version" + self.action = "example" + self.root_pipeline_id = "root_pipeline_id" + self.pipeline_tree = {} + self.timer_events = [ + { + "index": 1, + "action": self.action, + "timer_type": constants.TimerType.TIME_CYCLE.value, + "repetitions": 5, + "defined": "R5/PT10S", + }, + { + "index": 2, + "action": self.action, + "timer_type": constants.TimerType.TIME_DATE.value, + "repetitions": 1, + "defined": "2019-10-01T12:00:00Z", + }, + ] + self.timer_events_in_tree = [ + { + "enable": True, + "action": self.action, + "timer_type": constants.TimerType.TIME_CYCLE.value, + "defined": "R5/PT10S", + }, + { + "enable": True, + "action": self.action, + "timer_type": constants.TimerType.TIME_DATE.value, + "defined": "2019-10-01T12:00:00Z", + }, + ] + runtime = MagicMock() + runtime.get_execution_data = MagicMock( + return_value=ExecutionData({"key": "value", "from": "node"}, {"key": "value"}) + ) + runtime.get_data_inputs = MagicMock(return_value={"key": DataInput(need_render=False, value=1)}) + self.runtime = runtime + self.mock_runtime = MagicMock(return_value=runtime) + + def test_dispatch_expired_nodes(self): + mock_execute_node_timer_event_strategy = MagicMock() + mock_execute_node_timer_event_strategy.apply_async = MagicMock() + with patch( + "pipeline.contrib.node_timer_event.tasks.execute_node_timer_event_action", + mock_execute_node_timer_event_strategy, + ): + ExpiredNodesRecord.objects.create( + id=1, + nodes=json.dumps( + [ + "bamboo:v1:node_timer_event:node:node1:version:version1:index:1", + "bamboo:v1:node_timer_event:node:node2:version:version2:index:1", + ] + ), + ) + + dispatch_expired_nodes(record_id=1) + mock_execute_node_timer_event_strategy.apply_async.assert_has_calls( + [ + call(kwargs={"node_id": "node1", "version": "version1", "index": 1}), + call(kwargs={"node_id": "node2", "version": "version2", "index": 1}), + ] + ) + + self.assertFalse(ExpiredNodesRecord.objects.filter(id=1).exists()) + + def execute_node_timeout_action_success_test_helper(self, index: int): + NodeTimerEventConfig.objects.create( + root_pipeline_id=self.root_pipeline_id, node_id=self.node_id, events=json.dumps(self.timer_events) + ) + Process.objects.create(root_pipeline_id=self.root_pipeline_id, current_node_id=self.node_id, priority=1) + State.objects.create(node_id=self.node_id, name="name", version=self.version) + + redis_inst = MagicMock() + redis_inst.incr = MagicMock(return_value=b"2") + redis_inst.zadd = MagicMock(return_value=b"1") + + key: str = f"bamboo:v1:node_timer_event:node:{self.node_id}:version:{self.version}:index:{index}" + with patch("pipeline.contrib.node_timer_event.handlers.BambooDjangoRuntime", self.mock_runtime): + with patch("pipeline.contrib.node_timer_event.adapter.node_timer_event_settings.redis_inst", redis_inst): + result = execute_node_timer_event_action(self.node_id, self.version, index=index) + self.assertEqual(result["result"], True) + self.runtime.get_execution_data.assert_called_once_with(self.node_id) + self.runtime.get_data_inputs.assert_called_once_with(self.root_pipeline_id) + redis_inst.incr.assert_called_once_with(key, 1) + if index == 1: + redis_inst.zadd.assert_called_once() + else: + redis_inst.zadd.assert_not_called() + + def test_execute_node_timer_event_action_success__time_cycle(self): + """测试时间循环：成功调度时，投递下一个节点""" + self.execute_node_timeout_action_success_test_helper(index=1) + + def test_execute_node_timer_event_action_success__time_date(self): + """测试具体时间日期：无需进行下次调度""" + self.execute_node_timeout_action_success_test_helper(index=2) + + def test_execute_node_timer_event_action_not_current_node(self): + NodeTimerEventConfig.objects.create( + root_pipeline_id=self.root_pipeline_id, node_id=self.node_id, events=json.dumps(self.timer_events) + ) + Process.objects.create(root_pipeline_id=self.root_pipeline_id, current_node_id="next_node", priority=1) + State.objects.create(node_id=self.node_id, name="name", version=self.version) + with patch("pipeline.contrib.node_timer_event.handlers.BambooDjangoRuntime", self.mock_runtime): + result = execute_node_timer_event_action(self.node_id, self.version, index=1) + self.assertEqual(result["result"], False) + self.runtime.get_data_inputs.assert_not_called() + self.runtime.get_execution_data.assert_not_called() + + def test_execute_node_timer_event_action_not_current_version(self): + NodeTimerEventConfig.objects.create( + root_pipeline_id=self.root_pipeline_id, node_id=self.node_id, events=json.dumps(self.timer_events) + ) + Process.objects.create(root_pipeline_id=self.root_pipeline_id, current_node_id=self.node_id, priority=1) + State.objects.create(node_id=self.node_id, name="name", version="ano_version") + + with patch("pipeline.contrib.node_timer_event.handlers.BambooDjangoRuntime", self.mock_runtime): + result = execute_node_timer_event_action(self.node_id, self.version, index=1) + self.assertEqual(result["result"], False) + self.runtime.get_data_inputs.assert_not_called() + self.runtime.get_execution_data.assert_not_called() + + def test_parse_node_timer_event_configs_success(self): + pipeline_tree = { + "activities": { + "act_1": {"type": "ServiceActivity", "events": {"timer_events": self.timer_events_in_tree}}, + "act_2": {"type": "ServiceActivity", "events": {"timer_events": self.timer_events_in_tree}}, + } + } + + parse_result = NodeTimerEventConfig.objects.parse_node_timer_event_configs(pipeline_tree) + self.assertEqual(parse_result["result"], True) + self.assertEqual( + parse_result["data"], + [{"node_id": "act_1", "events": self.timer_events}, {"node_id": "act_2", "events": self.timer_events}], + ) + + def test_parse_node_timer_event_configs_fail_and_ignore(self): + + timer_events_in_tree_act_1 = copy.deepcopy(self.timer_events_in_tree) + timer_events_in_tree_act_1[1]["defined"] = "invalid defined" + + timer_events_in_tree_act_2 = copy.deepcopy(self.timer_events_in_tree) + timer_events_in_tree_act_2[1]["timer_type"] = "invalid timer_type" + + pipeline_tree = { + "activities": { + "act_1": {"type": "ServiceActivity", "events": {"timer_events": timer_events_in_tree_act_1}}, + "act_2": {"type": "ServiceActivity", "events": {"timer_events": timer_events_in_tree_act_2}}, + } + } + parse_result = NodeTimerEventConfig.objects.parse_node_timer_event_configs(pipeline_tree) + self.assertEqual(parse_result["result"], True) + self.assertEqual( + parse_result["data"], + [ + {"node_id": "act_1", "events": [self.timer_events[0]]}, + {"node_id": "act_2", "events": [self.timer_events[0]]}, + ], + ) + + def test_batch_create_node_timer_config_success(self): + config_parse_result = { + "result": True, + "data": [ + {"node_id": "act_1", "events": self.timer_events}, + {"node_id": "act_2", "events": self.timer_events}, + ], + "message": "", + } + with patch( + "pipeline.contrib.node_timer_event.models.NodeTimerEventConfig.objects.parse_node_timer_event_configs", + MagicMock(return_value=config_parse_result), + ): + NodeTimerEventConfig.objects.batch_create_node_timer_event_config(self.root_pipeline_id, self.pipeline_tree) + config_count = len(NodeTimerEventConfig.objects.all()) + self.assertEqual(config_count, 2) + + def test_batch_create_node_timer_config_fail(self): + config_parse_result = {"result": False, "data": "", "message": "test fail"} + with patch( + "pipeline.contrib.node_timer_event.models.NodeTimerEventConfig.objects.parse_node_timer_event_configs", + MagicMock(return_value=config_parse_result), + ): + NodeTimerEventConfig.objects.batch_create_node_timer_event_config(self.root_pipeline_id, self.pipeline_tree) + config_count = NodeTimerEventConfig.objects.count() + self.assertEqual(config_count, 0) diff --git a/runtime/bamboo-pipeline/poetry.lock b/runtime/bamboo-pipeline/poetry.lock index 13734303..2e441a21 100644 --- a/runtime/bamboo-pipeline/poetry.lock +++ b/runtime/bamboo-pipeline/poetry.lock @@ -343,6 +343,17 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = "*" + [[package]] name = "jmespath" version = "0.10.0" @@ -762,6 +773,7 @@ factory-boy = [] faker = [] idna = [] importlib-metadata = [] +isodate = [] iniconfig = [] jmespath = [] jsonschema = [] diff --git a/runtime/bamboo-pipeline/pyproject.toml b/runtime/bamboo-pipeline/pyproject.toml index 3f15a31b..85f9feeb 100644 --- a/runtime/bamboo-pipeline/pyproject.toml +++ b/runtime/bamboo-pipeline/pyproject.toml @@ -26,6 +26,7 @@ django-timezone-field = "^4.0" Werkzeug = "^1.0.0" prometheus-client = "^0.9.0" boto3 = "^1.9.130" +isodate = "^0.6.1" [tool.poetry.dev-dependencies] pytest = "^6.2.2" diff --git a/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py b/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py index 3b092716..3288539e 100755 --- a/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py +++ b/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py @@ -56,6 +56,7 @@ "pipeline.log", "pipeline.engine", "pipeline.contrib.rollback", + "pipeline.contrib.node_timer_event", "pipeline.component_framework", "pipeline.variable_framework", "pipeline.django_signal_valve", From 0aa6dbc106fca6f608e74bf09fb48bff29cf20da Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Mon, 20 Nov 2023 16:02:20 +0800 Subject: [PATCH 18/24] =?UTF-8?q?minor:=20=E5=8E=BB=E9=99=A4any=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E4=B8=8B=E5=BD=93=E5=89=8D=E6=AD=A3=E5=9C=A8=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E8=8A=82=E7=82=B9=E7=9A=84=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pipeline/contrib/rollback/handler.py | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py index 691cff2d..de5240e4 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/handler.py @@ -94,7 +94,7 @@ def validate_token(root_pipeline_id, start_node_id, target_node_id): ) @staticmethod - def validate_node_state_by_token_mode(root_pipeline_id, start_node_id): + def validate_node_state(root_pipeline_id, start_node_id): """ 使用token模式下的回滚，相同token的节点不允许有正在运行的节点 """ @@ -122,18 +122,6 @@ def validate_node_state_by_token_mode(root_pipeline_id, start_node_id): ) ) - @staticmethod - def validate_node_state_by_any_mode(root_pipeline_id): - """ - 使用any模式下的回滚, 不允许有正在运行的节点 - """ - if ( - State.objects.filter(root_id=root_pipeline_id, name=states.RUNNING) - .exclude(node_id=root_pipeline_id) - .exists() - ): - raise RollBackException("rollback failed: there is currently the some node is running") - @staticmethod def validate_start_node_id(root_pipeline_id, start_node_id): """ @@ -262,12 +250,12 @@ def retry_rollback_failed_node(self, node_id, retry_data): raise RollBackException("rollback failed: when mode is any, not support retry") def rollback(self, start_node_id, target_node_id, skip_rollback_nodes=None): - RollbackValidator.validate_node_state_by_any_mode(self.root_pipeline_id) - # 回滚的开始节点运行失败的情况 RollbackValidator.validate_start_node_id(self.root_pipeline_id, start_node_id) RollbackValidator.validate_node(start_node_id, allow_failed=True) RollbackValidator.validate_node(target_node_id) RollbackValidator.validate_token(self.root_pipeline_id, start_node_id, target_node_id) + # 相同token回滚时，不允许同一token路径上有正在运行的节点 + RollbackValidator.validate_node_state(self.root_pipeline_id, start_node_id) node_map = self._get_allowed_rollback_node_map() rollback_graph = RollbackGraphHandler(node_map=node_map, start_id=start_node_id, target_id=target_node_id) @@ -354,11 +342,11 @@ def rollback(self, start_node_id, target_node_id, skip_rollback_nodes=None): if skip_rollback_nodes is None: skip_rollback_nodes = [] - # 相同token回滚时，不允许有正在运行的节点 - RollbackValidator.validate_node_state_by_token_mode(self.root_pipeline_id, start_node_id) # 回滚的开始节点运行失败的情况 RollbackValidator.validate_node(start_node_id, allow_failed=True) RollbackValidator.validate_node(target_node_id) + # 相同token回滚时，不允许同一token路径上有正在运行的节点 + RollbackValidator.validate_node_state(self.root_pipeline_id, start_node_id) RollbackValidator.validate_token(self.root_pipeline_id, start_node_id, target_node_id) # 如果开始节点是失败的情况，则跳过该节点的回滚操作 From 9563da6670a5118fbf59f44ca74fd9901c18383b Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Mon, 20 Nov 2023 16:05:47 +0800 Subject: [PATCH 19/24] minor: release bamboo-pipeline 3.29.0rc4 --- runtime/bamboo-pipeline/pipeline/__init__.py | 2 +- runtime/bamboo-pipeline/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/bamboo-pipeline/pipeline/__init__.py b/runtime/bamboo-pipeline/pipeline/__init__.py index 92dc81e9..a3f486e8 100644 --- a/runtime/bamboo-pipeline/pipeline/__init__.py +++ b/runtime/bamboo-pipeline/pipeline/__init__.py @@ -13,4 +13,4 @@ default_app_config = "pipeline.apps.PipelineConfig" -__version__ = "3.29.0rc3" +__version__ = "3.29.0rc4" diff --git a/runtime/bamboo-pipeline/pyproject.toml b/runtime/bamboo-pipeline/pyproject.toml index 85f9feeb..5bb0db3f 100644 --- a/runtime/bamboo-pipeline/pyproject.toml +++ b/runtime/bamboo-pipeline/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-pipeline" -version = "3.29.0rc3" +version = "3.29.0rc4" description = "runtime for bamboo-engine base on Django and Celery" authors = ["homholueng "] license = "MIT" From 5755dcbc342058310103e2d016c9e44d12f13d30 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Mon, 20 Nov 2023 16:39:33 +0800 Subject: [PATCH 20/24] =?UTF-8?q?bugfix:=20=E4=BF=AE=E5=A4=8D=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E5=9B=9E=E6=BB=9A=E6=97=B6=E5=8F=8D=E5=BA=8F=E5=88=97?= =?UTF-8?q?=E5=8C=96=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- runtime/bamboo-pipeline/pipeline/contrib/rollback/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/tasks.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/tasks.py index cabd7c4b..259df0cb 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/rollback/tasks.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/tasks.py @@ -100,8 +100,8 @@ def execute_rollback(self): node_snapshots = RollbackNodeSnapshot.objects.filter(node_id=self.node_id, rolled_back=False).order_by("-id") for node_snapshot in node_snapshots: service = self.runtime.get_service(code=node_snapshot.code, version=node_snapshot.version) - data = ExecutionData(inputs=json.loads(node_snapshot.inputs), outputs=json.loads(node_snapshot.outputs)) - parent_data = ExecutionData(inputs=json.loads(node_snapshot.context_values), outputs={}) + data = ExecutionData(inputs=node_snapshot.inputs, outputs=node_snapshot.outputs) + parent_data = ExecutionData(inputs=node_snapshot.context_values, outputs={}) result = service.service.rollback(data, parent_data, self.retry_data) node_snapshot.rolled_back = True node_snapshot.save() From 8e75c5975a99d4a1ad9e9cf0b1dfb88f0fc678a4 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Mon, 20 Nov 2023 17:33:44 +0800 Subject: [PATCH 21/24] minor: release bamboo-engine 2.10.0rc4 --- bamboo_engine/__version__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bamboo_engine/__version__.py b/bamboo_engine/__version__.py index 39cf9317..aeff28ea 100644 --- a/bamboo_engine/__version__.py +++ b/bamboo_engine/__version__.py @@ -11,4 +11,4 @@ specific language governing permissions and limitations under the License. """ -__version__ = "2.10.0rc3" +__version__ = "2.10.0rc4" diff --git a/pyproject.toml b/pyproject.toml index 721157f2..64a610ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bamboo-engine" -version = "2.10.0rc3" +version = "2.10.0rc4" description = "Bamboo-engine is a general-purpose workflow engine" authors = ["homholueng "] license = "MIT" From 14cd46fb78f87c6ba180daea51b7730b2fc18cf6 Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Mon, 20 Nov 2023 17:54:19 +0800 Subject: [PATCH 22/24] minor: release bamboo-pipeline 3.29.0rc4 --- runtime/bamboo-pipeline/poetry.lock | 6 +++--- runtime/bamboo-pipeline/pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/bamboo-pipeline/poetry.lock b/runtime/bamboo-pipeline/poetry.lock index 2e441a21..1c575f02 100644 --- a/runtime/bamboo-pipeline/poetry.lock +++ b/runtime/bamboo-pipeline/poetry.lock @@ -57,7 +57,7 @@ tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "c [[package]] name = "bamboo-engine" -version = "2.10.0rc3" +version = "2.10.0rc4" description = "Bamboo-engine is a general-purpose workflow engine" category = "main" optional = false @@ -746,7 +746,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">= 3.6, < 4" -content-hash = "3026f8497b07ea1593e3d03d656f777d5262d8a1726f5bbdf3395e633b6bfeb8" +content-hash = "468798c3a0ae9c7014244e15919d01134dca7ed1be76c3d7d12f8acf985c6b4c" [metadata.files] amqp = [] @@ -773,8 +773,8 @@ factory-boy = [] faker = [] idna = [] importlib-metadata = [] -isodate = [] iniconfig = [] +isodate = [] jmespath = [] jsonschema = [] kombu = [] diff --git a/runtime/bamboo-pipeline/pyproject.toml b/runtime/bamboo-pipeline/pyproject.toml index 5bb0db3f..05688181 100644 --- a/runtime/bamboo-pipeline/pyproject.toml +++ b/runtime/bamboo-pipeline/pyproject.toml @@ -16,7 +16,7 @@ requests = "^2.22.0" django-celery-beat = "^2.1.0" Mako = "^1.1.4" pytz = "2019.3" -bamboo-engine = "2.10.0rc3" +bamboo-engine = "2.10.0rc4" jsonschema = "^2.5.1" ujson = "4.1.*" pyparsing = "^2.2.0" From fe41f9ca49d9bbd26130de8ce5a2b13b10655eaa Mon Sep 17 00:00:00 2001 From: hanshuaikang <1758504262@qq.com> Date: Mon, 27 Nov 2023 19:37:47 +0800 Subject: [PATCH 23/24] =?UTF-8?q?bugfix:=20=E8=B0=83=E6=95=B4=E5=88=86?= =?UTF-8?q?=E6=94=AF=E7=BD=91=E5=85=B3token=E5=88=86=E9=85=8D=E7=AE=97?= =?UTF-8?q?=E6=B3=95=EF=BC=8C=E5=88=86=E6=94=AF=E7=BD=91=E5=85=B3=E5=86=85?= =?UTF-8?q?=E4=B8=8E=E4=B8=BB=E6=B5=81=E7=A8=8Btoken=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bamboo_engine/builder/builder.py | 2 +- tests/builder/test_token.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bamboo_engine/builder/builder.py b/bamboo_engine/builder/builder.py index 4c10b0d1..30d44041 100644 --- a/bamboo_engine/builder/builder.py +++ b/bamboo_engine/builder/builder.py @@ -214,10 +214,10 @@ def inject_pipeline_token(node, pipeline_tree, node_token_map, token): # 如果是网关 if node["type"] in ["ParallelGateway", "ExclusiveGateway", "ConditionalParallelGateway"]: next_nodes = _get_next_node(node, pipeline_tree) - node_token = unique_id("t") target_nodes = {} for next_node in next_nodes: # 分支网关各个分支token相同 + node_token = token node_token_map[next_node["id"]] = node_token # 并行网关token不同 if node["type"] in ["ParallelGateway", "ConditionalParallelGateway"]: diff --git a/tests/builder/test_token.py b/tests/builder/test_token.py index 54f43f69..acec08fd 100644 --- a/tests/builder/test_token.py +++ b/tests/builder/test_token.py @@ -80,7 +80,7 @@ def test_inject_pipeline_token_with_complex_cycle(): == get_node_token(pipeline, "act_2", node_token_map) == get_node_token(pipeline, "ExclusiveGateway", node_token_map) == get_node_token(pipeline, "end_event", node_token_map) - != get_node_token(pipeline, "act_3", node_token_map) + == get_node_token(pipeline, "act_3", node_token_map) ) assert get_node_token(pipeline, "act_4", node_token_map) == get_node_token(pipeline, "act_3", node_token_map) @@ -131,7 +131,7 @@ def test_inject_pipeline_token_exclusive_gateway(): == get_node_token(pipeline, "act_1", node_token_map) == get_node_token(pipeline, "ExclusiveGateway", node_token_map) == get_node_token(pipeline, "end_event", node_token_map) - != get_node_token(pipeline, "act_2", node_token_map) + == get_node_token(pipeline, "act_2", node_token_map) ) assert get_node_token(pipeline, "act_2", node_token_map) == get_node_token(pipeline, "act_3", node_token_map) @@ -196,7 +196,7 @@ def sub_process(name): == get_node_token(pipeline, "end_event", node_token_map) == get_node_token(pipeline, "ExclusiveGateway", node_token_map) == get_node_token(pipeline, "end_event", node_token_map) - != get_node_token(pipeline, "act_3", node_token_map) + == get_node_token(pipeline, "act_3", node_token_map) ) assert get_node_token(pipeline, "act_3", node_token_map) == get_node_token(pipeline, "act_4", node_token_map) @@ -230,7 +230,7 @@ def test_inject_pipeline_token_with_cycle(): == get_node_token(pipeline, "act_1", node_token_map) == get_node_token(pipeline, "ExclusiveGateway", node_token_map) == get_node_token(pipeline, "end_event", node_token_map) - != get_node_token(pipeline, "act_2", node_token_map) + == get_node_token(pipeline, "act_2", node_token_map) ) assert get_node_token(pipeline, "act_2", node_token_map) == get_node_token(pipeline, "act_3", node_token_map) From 94efa96455f1a7ddb8d0c91cc4d4e7dd22226722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A9=E6=95=B0?= <33194175+hanshuaikang@users.noreply.github.com> Date: Tue, 5 Dec 2023 21:28:00 +0800 Subject: [PATCH 24/24] =?UTF-8?q?feature:=20=E5=8D=95=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E6=96=B9=E6=A1=88=20(#203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feature: 支持单节点执行方案 * test: 修复单元测试 * test: 修复单元测试卡住的问题 * test: 修复单元测试 * test: 修复集成测试 * test: 增加测试 test_run_plugin_with_schedule 的等待时间 * minor: code review * bugfix: 默认 service 注入 logger --- .../pipeline/conf/default_settings.py | 1 + .../pipeline/contrib/exceptions.py | 4 + .../pipeline/contrib/{rollback => }/fields.py | 0 .../contrib/plugin_execute/__init__.py | 12 ++ .../pipeline/contrib/plugin_execute/api.py | 41 +++++ .../pipeline/contrib/plugin_execute/apps.py | 21 +++ .../contrib/plugin_execute/contants.py | 19 ++ .../contrib/plugin_execute/handler.py | 169 ++++++++++++++++++ .../plugin_execute/migrations/0001_initial.py | 38 ++++ .../plugin_execute/migrations/__init__.py | 12 ++ .../pipeline/contrib/plugin_execute/models.py | 81 +++++++++ .../pipeline/contrib/plugin_execute/tasks.py | 151 ++++++++++++++++ .../migrations/0002_auto_20231020_1234.py | 8 +- .../pipeline/contrib/rollback/models.py | 2 +- .../tests/contrib/test_plugin_execute.py | 138 ++++++++++++++ .../tests/plugin_execute/__init__.py | 12 ++ .../plugin_execute/test_plugin_execute.py | 83 +++++++++ .../test/pipeline_sdk_use/settings.py | 8 +- .../components/collections/atom.py | 34 +++- 19 files changed, 822 insertions(+), 12 deletions(-) rename runtime/bamboo-pipeline/pipeline/contrib/{rollback => }/fields.py (100%) create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/__init__.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/api.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/apps.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/contants.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/handler.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/migrations/0001_initial.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/migrations/__init__.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/models.py create mode 100644 runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/tasks.py create mode 100644 runtime/bamboo-pipeline/pipeline/tests/contrib/test_plugin_execute.py create mode 100644 runtime/bamboo-pipeline/test/eri_imp_test_use/tests/plugin_execute/__init__.py create mode 100644 runtime/bamboo-pipeline/test/eri_imp_test_use/tests/plugin_execute/test_plugin_execute.py diff --git a/runtime/bamboo-pipeline/pipeline/conf/default_settings.py b/runtime/bamboo-pipeline/pipeline/conf/default_settings.py index 5f52c1a6..eda27c28 100644 --- a/runtime/bamboo-pipeline/pipeline/conf/default_settings.py +++ b/runtime/bamboo-pipeline/pipeline/conf/default_settings.py @@ -106,3 +106,4 @@ ENABLE_PIPELINE_EVENT_SIGNALS = getattr(settings, "ENABLE_PIPELINE_EVENT_SIGNALS", False) ROLLBACK_QUEUE = getattr(settings, "ROLLBACK_QUEUE", "rollback") +PLUGIN_EXECUTE_QUEUE = getattr(settings, "PLUGIN_EXECUTE_QUEUE", "plugin_execute") diff --git a/runtime/bamboo-pipeline/pipeline/contrib/exceptions.py b/runtime/bamboo-pipeline/pipeline/contrib/exceptions.py index 186ea958..a769bc4d 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/exceptions.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/exceptions.py @@ -20,3 +20,7 @@ class RollBackException(PipelineException): class UpdatePipelineContextException(PipelineException): pass + + +class PluginExecuteException(PipelineException): + pass diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/fields.py b/runtime/bamboo-pipeline/pipeline/contrib/fields.py similarity index 100% rename from runtime/bamboo-pipeline/pipeline/contrib/rollback/fields.py rename to runtime/bamboo-pipeline/pipeline/contrib/fields.py diff --git a/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/__init__.py b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/__init__.py new file mode 100644 index 00000000..26a6d1c2 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/api.py b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/api.py new file mode 100644 index 00000000..1a03fc63 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/api.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from pipeline.contrib.plugin_execute.handler import PluginExecuteHandler +from pipeline.contrib.utils import ensure_return_pipeline_contrib_api_result + + +@ensure_return_pipeline_contrib_api_result +def run(component_code: str, version: str, inputs: dict, contexts: dict, runtime_attr: dict = None): + task_id = PluginExecuteHandler.run(component_code, version, inputs, contexts, runtime_attr) + return task_id + + +@ensure_return_pipeline_contrib_api_result +def get_state(task_id: int): + return PluginExecuteHandler.get_state(task_id) + + +@ensure_return_pipeline_contrib_api_result +def callback(task_id: int, callback_data: dict = None): + PluginExecuteHandler.callback(task_id, callback_data) + + +@ensure_return_pipeline_contrib_api_result +def forced_fail(task_id): + PluginExecuteHandler.forced_fail(task_id) + + +@ensure_return_pipeline_contrib_api_result +def retry(task_id: int, inputs: dict = None, context: dict = None, runtime_attr: dict = None): + PluginExecuteHandler.retry_node(task_id, inputs, context, runtime_attr) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/apps.py b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/apps.py new file mode 100644 index 00000000..8bbefc32 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/apps.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.apps import AppConfig + + +class RollbackConfig(AppConfig): + name = "pipeline.contrib.plugin_execute" + verbose_name = "PipelinePluginExecute" + + def ready(self): + from pipeline.contrib.plugin_execute import tasks # noqa diff --git a/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/contants.py b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/contants.py new file mode 100644 index 00000000..008c9397 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/contants.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + + +class State: + READY = "READY" + RUNNING = "RUNNING" + FINISHED = "FINISHED" + FAILED = "FAILED" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/handler.py b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/handler.py new file mode 100644 index 00000000..7eaae4d9 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/handler.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from pipeline.conf.default_settings import PLUGIN_EXECUTE_QUEUE +from pipeline.contrib.exceptions import PluginExecuteException +from pipeline.contrib.plugin_execute.contants import State +from pipeline.contrib.plugin_execute.models import PluginExecuteTask, get_schedule_lock +from pipeline.contrib.plugin_execute.tasks import execute, schedule + +logger = logging.getLogger("celery") + + +def _retry_once(action: callable): + try: + action() + except Exception: + try: + action() + except Exception as e: + raise e + + +class PluginExecuteHandler: + @classmethod + def run(cls, component_code: str, version: str, inputs: dict, contexts: dict, runtime_attrs: dict = None): + if runtime_attrs is None: + runtime_attrs = {} + + if not (isinstance(inputs, dict) and isinstance(contexts, dict) and isinstance(runtime_attrs, dict)): + raise PluginExecuteException("[plugin_execute_run] error, the inputs, contexts, runtime_attrs must be dict") + plugin_execute_task = PluginExecuteTask.objects.create( + state=State.READY, + inputs=inputs, + version=version, + component_code=component_code, + contexts=contexts, + runtime_attrs=runtime_attrs, + ) + + def action(): + # 发送执行任务 + execute.apply_async( + kwargs={"task_id": plugin_execute_task.id}, queue=PLUGIN_EXECUTE_QUEUE, ignore_result=True + ) + logger.info( + "[plugin_execute_run] send execute task, plugin_execute_task_id = {}".format(plugin_execute_task.id) + ) + + try: + _retry_once(action=action) + except Exception as e: + # 如果任务启动出现异常，则删除任务 + plugin_execute_task.delete() + raise e + + return plugin_execute_task.id + + @classmethod + def get_state(cls, task_id): + """ + 获取任务状态 + @param task_id: + @return: + """ + # 直接抛出异常让上层去捕获 + plugin_execute_task = PluginExecuteTask.objects.get(id=task_id) + return { + "task_id": plugin_execute_task.id, + "state": plugin_execute_task.state, + "component_code": plugin_execute_task.component_code, + "version": plugin_execute_task.version, + "invoke_count": plugin_execute_task.invoke_count, + "inputs": plugin_execute_task.inputs, + "outputs": plugin_execute_task.outputs, + "contexts": plugin_execute_task.contexts, + "runtime_attrs": plugin_execute_task.runtime_attrs, + "create_at": plugin_execute_task.created_at, + "finish_at": plugin_execute_task.finish_at, + } + + @classmethod + def forced_fail(cls, task_id): + plugin_execute_task = PluginExecuteTask.objects.get(id=task_id) + if plugin_execute_task.state != State.RUNNING: + raise PluginExecuteException( + "[forced_fail] error, the plugin_execute_task.state is not RUNNING, state={}".format( + plugin_execute_task.state + ) + ) + # 插件状态改成 FAILED, 在schdule会自动停止 + plugin_execute_task.state = State.FAILED + plugin_execute_task.save() + + @classmethod + def callback(cls, task_id: int, callback_data: dict = None): + + if callback_data is None: + callback_data = {} + + if not isinstance(callback_data, dict): + raise PluginExecuteException("[plugin_execute_callback] error, the callback must be dict") + + plugin_execute_task = PluginExecuteTask.objects.get(id=task_id) + if plugin_execute_task.state != State.RUNNING: + raise PluginExecuteException( + "[callback] error, the plugin_execute_task.state is not RUNNING, state={}".format( + plugin_execute_task.state + ) + ) + + def action(): + # 需要加锁，防止流程处在回调的过程中 + with get_schedule_lock(task_id) as locked: + if not locked: + raise PluginExecuteException("[plugin_execute_callback] error, it`s have callback task, please try") + plugin_execute_task.callback_data = callback_data + plugin_execute_task.save() + schedule.apply_async( + kwargs={"task_id": plugin_execute_task.id}, queue=PLUGIN_EXECUTE_QUEUE, ignore_result=True + ) + logger.info("[plugin_execute_callback] send callback task, plugin_execute_task_id = {}".format(task_id)) + + _retry_once(action=action) + + @classmethod + def retry_node(cls, task_id: int, inputs: dict = None, contexts: dict = None, runtime_attrs: dict = None): + + plugin_execute_task = PluginExecuteTask.objects.get(id=task_id) + if plugin_execute_task.state != State.FAILED: + raise PluginExecuteException( + "[retry_node] error, the plugin_execute_task.state is not FAILED, state={}".format( + plugin_execute_task.state + ) + ) + + if contexts and isinstance(contexts, dict): + plugin_execute_task.contexts = contexts + if inputs and isinstance(inputs, dict): + plugin_execute_task.inputs = inputs + if runtime_attrs and isinstance(runtime_attrs, dict): + plugin_execute_task.runtime_attrs = runtime_attrs + + plugin_execute_task.state = State.READY + plugin_execute_task.inputs = inputs + plugin_execute_task.invoke_count += 1 + # 清空输出和callback_data + plugin_execute_task.outputs = {} + plugin_execute_task.callback_data = {} + plugin_execute_task.save() + + def action(): + execute.apply_async(kwargs={"task_id": plugin_execute_task.id}, queue=PLUGIN_EXECUTE_QUEUE) + logger.info( + "[plugin_execute_retry_node] send retry_node task, plugin_execute_task_id = {}".format(task_id) + ) + + _retry_once(action=action) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/migrations/0001_initial.py b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/migrations/0001_initial.py new file mode 100644 index 00000000..969e3ae8 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/migrations/0001_initial.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.18 on 2023-11-28 11:49 + +import pipeline.contrib.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="PluginExecuteTask", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("state", models.CharField(max_length=64, verbose_name="状态名")), + ("invoke_count", models.IntegerField(default=1, verbose_name="invoke count")), + ("component_code", models.CharField(db_index=True, max_length=255, verbose_name="组件编码")), + ("version", models.CharField(default="legacy", max_length=255, verbose_name="插件版本")), + ("inputs", pipeline.contrib.fields.SerializerField(default={}, verbose_name="node inputs")), + ("outputs", pipeline.contrib.fields.SerializerField(default={}, verbose_name="node outputs")), + ("callback_data", pipeline.contrib.fields.SerializerField(default={}, verbose_name="callback data")), + ( + "contexts", + pipeline.contrib.fields.SerializerField(default={}, verbose_name="pipeline context values"), + ), + ("runtime_attrs", pipeline.contrib.fields.SerializerField(default={}, verbose_name="runtime attr")), + ("scheduling", models.BooleanField(db_index=True, default=False, verbose_name="是否正在调度")), + ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="create time")), + ("finish_at", models.DateTimeField(null=True, verbose_name="finish time")), + ], + ), + migrations.AddIndex( + model_name="pluginexecutetask", + index=models.Index(fields=["id", "scheduling"], name="plugin_exec_id_e7c7b2_idx"), + ), + ] diff --git a/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/migrations/__init__.py b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/migrations/__init__.py new file mode 100644 index 00000000..26a6d1c2 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/migrations/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/models.py b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/models.py new file mode 100644 index 00000000..d7bd1353 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/models.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.db import models +from django.utils.translation import ugettext_lazy as _ +from pipeline.contrib.fields import SerializerField + + +class ScheduleManger(models.Manager): + def apply_schedule_lock(self, task_id: int) -> bool: + """ + 获取 Schedule 对象的调度锁，返回是否成功获取锁 + + :return: True or False + """ + return self.filter(id=task_id, scheduling=False).update(scheduling=True) == 1 + + def release_schedule_lock(self, task_id: int) -> None: + """ + 释放指定 Schedule 的调度锁 + :return: + """ + self.filter(id=task_id, scheduling=True).update(scheduling=False) + + +class ScheduleLock(object): + def __init__(self, task_id: int): + self.task_id = task_id + self.locked = False + + def __enter__(self): + self.locked = PluginExecuteTask.objects.apply_schedule_lock(self.task_id) + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.locked: + PluginExecuteTask.objects.release_schedule_lock(self.task_id) + + +def get_schedule_lock(task_id: int) -> ScheduleLock: + """ + 获取 schedule lock 的 context 对象 + :param task_id: + :return: + """ + return ScheduleLock(task_id) + + +class PluginExecuteTask(models.Model): + """ + 单节点执行任务 + """ + + state = models.CharField(_("状态名"), null=False, max_length=64) + invoke_count = models.IntegerField("invoke count", default=1) + component_code = models.CharField(_("组件编码"), max_length=255, db_index=True) + version = models.CharField(_("插件版本"), max_length=255, default="legacy") + inputs = SerializerField(verbose_name=_("node inputs"), default={}) + outputs = SerializerField(verbose_name=_("node outputs"), default={}) + callback_data = SerializerField(verbose_name=_("callback data"), default={}) + contexts = SerializerField(verbose_name=_("pipeline context values"), default={}) + runtime_attrs = SerializerField(verbose_name=_("runtime attr"), default={}) + scheduling = models.BooleanField("是否正在调度", default=False, db_index=True) + created_at = models.DateTimeField("create time", auto_now_add=True) + finish_at = models.DateTimeField("finish time", null=True) + + objects = ScheduleManger() + + class Meta: + indexes = [models.Index(fields=["id", "scheduling"])] diff --git a/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/tasks.py b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/tasks.py new file mode 100644 index 00000000..b1898431 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/contrib/plugin_execute/tasks.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging +import traceback + +from celery import task +from django.utils import timezone +from pipeline.component_framework.library import ComponentLibrary +from pipeline.conf.default_settings import PLUGIN_EXECUTE_QUEUE +from pipeline.contrib.plugin_execute.contants import State +from pipeline.contrib.plugin_execute.models import PluginExecuteTask +from pipeline.core.data.base import DataObject + +logger = logging.getLogger("celery") + + +@task +def execute(task_id): + try: + plugin_execute_task = PluginExecuteTask.objects.get(id=task_id) + except PluginExecuteTask.DoesNotExist: + logger.exception("[plugin_execute] execute error, task not exist, task_id={}".format(task_id)) + return + + # 更新插件的状态 + plugin_execute_task.state = State.RUNNING + plugin_execute_task.save(update_fields=["state"]) + + # 封装data + data = DataObject(inputs=plugin_execute_task.inputs, outputs={}) + parent_data = DataObject(inputs=plugin_execute_task.contexts, outputs={}) + + try: + # 获取 component + comp_cls = ComponentLibrary.get_component_class(plugin_execute_task.component_code, plugin_execute_task.version) + # 获取service + service = comp_cls.bound_service(name=plugin_execute_task.runtime_attrs.get("name", None)) + + # 封装运行时 + service.setup_runtime_attrs(**plugin_execute_task.runtime_attrs, logger=logger) + execute_success = service.execute(data, parent_data) + # 在 pipeline 中，如果插件返回为None,则表示成功 + if execute_success is None: + execute_success = True + plugin_execute_task.outputs = data.outputs + plugin_execute_task.save() + except Exception as e: + # 处理异常情况 + ex_data = traceback.format_exc() + data.outputs.ex_data = ex_data + logger.exception("[plugin_execute] plugin execute failed, err={}".format(e)) + plugin_execute_task.outputs = data.outputs + plugin_execute_task.state = State.FAILED + plugin_execute_task.save() + return + + # 单纯的执行失败, 更新状态和输出信息 + if not execute_success: + plugin_execute_task.state = State.FAILED + plugin_execute_task.save() + return + + # 执行成功, 需要判断是否需要调度 + need_schedule = service.need_schedule() + if not need_schedule: + plugin_execute_task.state = State.FINISHED + plugin_execute_task.finish_at = timezone.now() + plugin_execute_task.save() + return + + # 需要调度，则调度自身 + if service.interval: + schedule.apply_async( + kwargs={"task_id": task_id}, + queue=PLUGIN_EXECUTE_QUEUE, + countdown=service.interval.next(), + ignore_result=True, + ) + + +@task +def schedule(task_id): + try: + plugin_execute_task = PluginExecuteTask.objects.get(id=task_id) + except PluginExecuteTask.DoesNotExist: + logger.exception("[plugin_execute] schedule error, task not exist, task_id={}".format(task_id)) + return + + # 只有处于运行状态的节点才允许被调度 + if plugin_execute_task.state != State.RUNNING: + logger.exception("[plugin_execute] schedule error, task not exist, task_id={}".format(task_id)) + return + + data = DataObject(inputs=plugin_execute_task.inputs, outputs=plugin_execute_task.outputs) + parent_data = DataObject(inputs=plugin_execute_task.contexts, outputs={}) + + try: + comp_cls = ComponentLibrary.get_component_class(plugin_execute_task.component_code, plugin_execute_task.version) + # 获取service + service = comp_cls.bound_service(name=plugin_execute_task.runtime_attrs.get("name", None)) + # 封装运行时 + service.setup_runtime_attrs(**plugin_execute_task.runtime_attrs, logger=logger) + schedule_success = service.schedule( + data=data, parent_data=parent_data, callback_data=plugin_execute_task.callback_data + ) + # 在 pipeline 中，如果插件返回为None,则表示成功 + if schedule_success is None: + schedule_success = True + plugin_execute_task.outputs = data.outputs + plugin_execute_task.save() + except Exception as e: + # 处理异常情况 + ex_data = traceback.format_exc() + data.outputs.ex_data = ex_data + logger.exception("[plugin_execute] plugin execute failed, err={}".format(e)) + plugin_execute_task.outputs = data.outputs + plugin_execute_task.state = State.FAILED + plugin_execute_task.save() + return + + if not schedule_success: + plugin_execute_task.state = State.FAILED + plugin_execute_task.save() + return + + if service.is_schedule_finished(): + plugin_execute_task.state = State.FINISHED + plugin_execute_task.finish_at = timezone.now() + plugin_execute_task.save() + return + + # 还需要下一次的调度 + # 需要调度，则调度自身 + if service.interval: + schedule.apply_async( + kwargs={"task_id": task_id}, + queue=PLUGIN_EXECUTE_QUEUE, + countdown=service.interval.next(), + ignore_result=True, + ) diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0002_auto_20231020_1234.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0002_auto_20231020_1234.py index ae217a0f..26b00ac0 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0002_auto_20231020_1234.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/migrations/0002_auto_20231020_1234.py @@ -1,6 +1,6 @@ # Generated by Django 3.2.18 on 2023-10-20 12:34 -import pipeline.contrib.rollback.fields +import pipeline.contrib.fields from django.db import migrations @@ -14,16 +14,16 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="rollbacknodesnapshot", name="context_values", - field=pipeline.contrib.rollback.fields.SerializerField(verbose_name="pipeline context values"), + field=pipeline.contrib.fields.SerializerField(verbose_name="pipeline context values"), ), migrations.AlterField( model_name="rollbacknodesnapshot", name="inputs", - field=pipeline.contrib.rollback.fields.SerializerField(verbose_name="node inputs"), + field=pipeline.contrib.fields.SerializerField(verbose_name="node inputs"), ), migrations.AlterField( model_name="rollbacknodesnapshot", name="outputs", - field=pipeline.contrib.rollback.fields.SerializerField(verbose_name="node outputs"), + field=pipeline.contrib.fields.SerializerField(verbose_name="node outputs"), ), ] diff --git a/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py b/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py index ab452cfc..cefac0b9 100644 --- a/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py +++ b/runtime/bamboo-pipeline/pipeline/contrib/rollback/models.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- from django.db import models from django.utils.translation import ugettext_lazy as _ +from pipeline.contrib.fields import SerializerField from pipeline.contrib.rollback.constants import TOKEN -from pipeline.contrib.rollback.fields import SerializerField class RollbackToken(models.Model): diff --git a/runtime/bamboo-pipeline/pipeline/tests/contrib/test_plugin_execute.py b/runtime/bamboo-pipeline/pipeline/tests/contrib/test_plugin_execute.py new file mode 100644 index 00000000..1bb3e0f0 --- /dev/null +++ b/runtime/bamboo-pipeline/pipeline/tests/contrib/test_plugin_execute.py @@ -0,0 +1,138 @@ +# # -*- coding: utf-8 -*- +# """ +# Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +# Edition) available. +# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +# Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://opensource.org/licenses/MIT +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# """ + +from unittest import TestCase + +from mock.mock import MagicMock +from pipeline.component_framework.library import ComponentLibrary +from pipeline.contrib.plugin_execute import api +from pipeline.contrib.plugin_execute.models import PluginExecuteTask +from pipeline.contrib.plugin_execute.tasks import execute, schedule +from pipeline.tests import mock +from pipeline_test_use.components.collections.atom import ( + DebugCallbackComponent, + InterruptDummyExecuteComponent, +) + +mock_execute = MagicMock() +mock_execute.apply_async = MagicMock(return_value=True) + +mock_schedule = MagicMock() +mock_schedule.apply_async = MagicMock(return_value=True) + + +class TestPluginExecuteBase(TestCase): + @mock.patch("pipeline.contrib.plugin_execute.handler.execute", MagicMock(return_value=mock_execute)) + def test_run(self): + task_id = api.run("debug_callback_node", "legacy", {"hello": "world"}, {"hello": "world"}).data + task = PluginExecuteTask.objects.get(id=task_id) + self.assertEqual(task.state, "READY") + self.assertDictEqual(task.callback_data, {}) + self.assertDictEqual(task.contexts, {"hello": "world"}) + self.assertDictEqual(task.inputs, {"hello": "world"}) + + @mock.patch("pipeline.contrib.plugin_execute.handler.execute", MagicMock(return_value=mock_execute)) + def test_get_state(self): + task_id = api.run("debug_callback_node", "legacy", {"hello": "world"}, {"hello": "world"}).data + state = api.get_state(task_id).data + self.assertEqual(state["state"], "READY") + self.assertDictEqual(state["inputs"], {"hello": "world"}) + self.assertDictEqual(state["contexts"], {"hello": "world"}) + + @mock.patch("pipeline.contrib.plugin_execute.handler.execute", MagicMock(return_value=mock_execute)) + def test_retry(self): + task_id = api.run("debug_callback_node", "legacy", {"hello": "world"}, {"hello": "world"}).data + task = PluginExecuteTask.objects.get(id=task_id) + result = api.retry(task_id, {}) + + self.assertFalse(result.result) + + task.state = "FAILED" + task.save() + + result = api.retry(task_id, {"hello": "tim"}, {"hello": "jav"}) + self.assertEqual(result.result, True) + + task.refresh_from_db() + + self.assertEqual(task.state, "READY") + self.assertDictEqual(task.inputs, {"hello": "tim"}) + self.assertDictEqual(task.contexts, {"hello": "jav"}) + + @mock.patch("pipeline.contrib.plugin_execute.handler.schedule", MagicMock(return_value=mock_schedule)) + @mock.patch("pipeline.contrib.plugin_execute.handler.execute", MagicMock(return_value=mock_execute)) + def test_callback(self): + task_id = api.run("debug_callback_node", "legacy", {"hello": "world"}, {"hello": "world"}).data + task = PluginExecuteTask.objects.get(id=task_id) + result = api.retry(task_id, {}) + + self.assertFalse(result.result) + + task.state = "RUNNING" + task.save() + + result = api.callback(task_id, {"hello": "sandri"}) + self.assertEqual(result.result, True) + + task.refresh_from_db() + self.assertDictEqual(task.callback_data, {"hello": "sandri"}) + + @mock.patch("pipeline.contrib.plugin_execute.handler.execute", MagicMock(return_value=mock_execute)) + def test_force_fail(self): + task_id = api.run("debug_callback_node", "legacy", {"hello": "world"}, {"hello": "world"}).data + task = PluginExecuteTask.objects.get(id=task_id) + result = api.forced_fail(task_id) + + self.assertFalse(result.result) + + task.state = "RUNNING" + task.save() + + result = api.forced_fail(task_id) + self.assertEqual(result.result, True) + + task.refresh_from_db() + self.assertEqual(task.state, "FAILED") + + def test_execute_task(self): + task = PluginExecuteTask.objects.create( + state="READY", + inputs={"time": 1}, + version="legacy", + component_code="interrupt_dummy_exec_node", + contexts={}, + runtime_attrs={}, + ) + ComponentLibrary.register_component("interrupt_dummy_exec_node", "legacy", InterruptDummyExecuteComponent) + execute(task.id) + task.refresh_from_db() + self.assertEqual(task.state, "FINISHED") + self.assertDictEqual(task.outputs, {"execute_count": 1}) + + def test_schedule_task(self): + task = PluginExecuteTask.objects.create( + state="READY", + inputs={}, + version="legacy", + component_code="debug_callback_node", + contexts={}, + runtime_attrs={}, + ) + ComponentLibrary.register_component("debug_callback_node", "legacy", DebugCallbackComponent) + task = PluginExecuteTask.objects.get(id=task.id) + task.callback_data = {"bit": 1} + task.save() + execute(task.id) + schedule(task.id) + task.refresh_from_db() + self.assertEqual(task.state, "FINISHED") diff --git a/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/plugin_execute/__init__.py b/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/plugin_execute/__init__.py new file mode 100644 index 00000000..26a6d1c2 --- /dev/null +++ b/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/plugin_execute/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/plugin_execute/test_plugin_execute.py b/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/plugin_execute/test_plugin_execute.py new file mode 100644 index 00000000..f45a4702 --- /dev/null +++ b/runtime/bamboo-pipeline/test/eri_imp_test_use/tests/plugin_execute/test_plugin_execute.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +import time + +from pipeline.contrib.plugin_execute import api + + +def test_run_plugin_no_schedule(): + # 测试execute的情况 + task_id = api.run("debug_no_schedule_node", "legacy", {}, {}).data + state = api.get_state(task_id).data + assert state["state"] == "READY" + time.sleep(2) + state = api.get_state(task_id).data + assert state["state"] == "FINISHED" + + +def test_run_plugin_with_schedule(): + # 测试schedule的情况 + task_id = api.run("schedule_node", "legacy", {"count": 1}, {}).data + state = api.get_state(task_id).data + assert state["state"] == "READY" + time.sleep(30) + state = api.get_state(task_id).data + assert state["state"] == "FINISHED" + assert state["outputs"]["count"] == 5 + + +def test_run_plugin_with_callback(): + # 测试callback的情况 + task_id = api.run("hook_callback_node", "legacy", {}, {}).data + state = api.get_state(task_id).data + assert state["state"] == "READY" + time.sleep(5) + state = api.get_state(task_id).data + assert state["state"] == "RUNNING" + + api.callback(task_id, {"bit": 0}) + time.sleep(10) + + state = api.get_state(task_id).data + assert state["state"] == "FAILED" + + api.retry(task_id, inputs={}) + time.sleep(5) + state = api.get_state(task_id).data + assert state["state"] == "RUNNING" + + api.callback(task_id, {"bit": 1}) + time.sleep(5) + state = api.get_state(task_id).data + assert state["state"] == "RUNNING" + + +def test_run_plugin_with_callback_success(): + task_id = api.run("debug_callback_node", "legacy", {}, {}).data + state = api.get_state(task_id).data + + assert state["state"] == "READY" + time.sleep(5) + state = api.get_state(task_id).data + assert state["state"] == "RUNNING" + + api.callback(task_id, {"bit": 1}) + time.sleep(10) + + state = api.get_state(task_id).data + assert state["state"] == "FINISHED" + + +def test_run_plugin_with_force_fail(): + task_id = api.run("debug_callback_node", "legacy", {}, {}).data + state = api.get_state(task_id).data + + assert state["state"] == "READY" + time.sleep(5) + state = api.get_state(task_id).data + assert state["state"] == "RUNNING" + + api.forced_fail(task_id) + time.sleep(3) + + state = api.get_state(task_id).data + assert state["state"] == "FAILED" diff --git a/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py b/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py index 3288539e..3d7e8ed3 100755 --- a/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py +++ b/runtime/bamboo-pipeline/test/pipeline_sdk_use/settings.py @@ -21,7 +21,6 @@ CELERY_QUEUES.extend(queues.CELERY_QUEUES) # noqa CELERY_QUEUES.extend(queues.QueueResolver("api").queues()) # noqa - step.PromServerStep.port = 8002 app = Celery("proj") app.config_from_object("django.conf:settings") @@ -55,13 +54,14 @@ "pipeline", "pipeline.log", "pipeline.engine", - "pipeline.contrib.rollback", "pipeline.contrib.node_timer_event", "pipeline.component_framework", "pipeline.variable_framework", "pipeline.django_signal_valve", "pipeline.contrib.periodic_task", "pipeline.contrib.node_timeout", + "pipeline.contrib.rollback", + "pipeline.contrib.plugin_execute", "django_celery_beat", "pipeline_test_use", "variable_app", @@ -156,7 +156,6 @@ STATIC_URL = "/static/" - ENABLE_EXAMPLE_COMPONENTS = True BROKER_VHOST = "/" @@ -181,3 +180,6 @@ # } # } # ] + + +PLUGIN_EXECUTE_QUEUE = "default" diff --git a/runtime/bamboo-pipeline/test/pipeline_test_use/components/collections/atom.py b/runtime/bamboo-pipeline/test/pipeline_test_use/components/collections/atom.py index e5fdba4e..b2ab9dc2 100755 --- a/runtime/bamboo-pipeline/test/pipeline_test_use/components/collections/atom.py +++ b/runtime/bamboo-pipeline/test/pipeline_test_use/components/collections/atom.py @@ -18,8 +18,9 @@ class HookMixin: __need_run_hook__ = True def recorder(self, hook: HookType, data, parent_data, callback_data=None): - self.logger.info("hook_debug_node({}) id: {}".format(hook.value, self.id)) - self.logger.info("hook_debug_node({}) root_pipeline_id: {}".format(hook.value, self.root_pipeline_id)) + if hasattr(hook.value, "id"): + self.logger.info("hook_debug_node({}) id: {}".format(hook.value, self.id)) + self.logger.info("hook_debug_node({}) root_pipeline_id: {}".format(hook.value, self.root_pipeline_id)) logger.info("hook_debug_node hook(%s) data %s ", hook.value, pprint.pformat(data.inputs)) logger.info("hook_debug_node hook(%s) parent data %s ", hook.value, pprint.pformat(parent_data.inputs)) logger.info("hook_debug_node hook(%s) output data %s ", hook.value, pprint.pformat(data.outputs)) @@ -441,6 +442,32 @@ class CallbackComponent(Component): form = "index.html" +class DebugCallbackService(Service): + __need_schedule__ = True + interval = None + + def execute(self, data, parent_data): + return True + + def schedule(self, data, parent_data, callback_data=None): + if callback_data: + if int(callback_data.get("bit", 1)) == 1: + self.finish_schedule() + return True + + return False + + def outputs_format(self): + return [] + + +class DebugCallbackComponent(Component): + name = "callback component" + code = "debug_callback_node" + bound_service = DebugCallbackService + form = "index.html" + + class HookCallbackService(HookMixin, CallbackService): pass @@ -469,7 +496,6 @@ def schedule(self, data, parent_data, callback_data=None): logger.info("[{}]: callback_data={}".format(_scheduled_times, callback_data)) if callback_data: if int(callback_data.get("bit", 0)) == 0: - print("hahacai") return False _scheduled_times += 1 @@ -646,7 +672,7 @@ def outputs_format(self): return [] -class InterruptScheduleComponent(Component): +class InterruptRaiseScheduleComponent(Component): name = "debug 组件" code = "interrupt_raise_test" bound_service = InterruptRaiseService