From 13c583320786694a1874422253bf037bc59eefe7 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sun, 23 Aug 2020 10:47:53 +0300 Subject: [PATCH 01/57] [ADDED] Zeebe client + inherit from object --- zeebepy/client/__init__.py | 0 zeebepy/client/client.py | 2 ++ zeebepy/client/client_test.py | 0 zeebepy/task/task_context.py | 2 +- zeebepy/task/task_status_controller.py | 2 +- 5 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 zeebepy/client/__init__.py create mode 100644 zeebepy/client/client.py create mode 100644 zeebepy/client/client_test.py diff --git a/zeebepy/client/__init__.py b/zeebepy/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/zeebepy/client/client.py b/zeebepy/client/client.py new file mode 100644 index 00000000..3f6a2f5b --- /dev/null +++ b/zeebepy/client/client.py @@ -0,0 +1,2 @@ +class ZeebeClient(object): + pass diff --git a/zeebepy/client/client_test.py b/zeebepy/client/client_test.py new file mode 100644 index 00000000..e69de29b diff --git a/zeebepy/task/task_context.py b/zeebepy/task/task_context.py index 2e822cd3..2720e58f 100644 --- a/zeebepy/task/task_context.py +++ b/zeebepy/task/task_context.py @@ -1,7 +1,7 @@ from typing import Dict -class TaskContext: +class TaskContext(object): def __init__(self, key: int, _type: str, workflow_instance_key: int, bpmn_process_id: str, workflow_definition_version: int, workflow_key: int, element_id: str, element_instance_key: int, custom_headers: Dict, worker: str, retries: int, deadline: int, variables: Dict): diff --git a/zeebepy/task/task_status_controller.py b/zeebepy/task/task_status_controller.py index 707e69ca..bff2e52c 100644 --- a/zeebepy/task/task_status_controller.py +++ b/zeebepy/task/task_status_controller.py @@ -2,7 +2,7 @@ from zeebepy.task.task_context import TaskContext -class TaskStatusController: +class TaskStatusController(object): def __init__(self, context: TaskContext, zeebe_adapter: ZeebeAdapter): self.zeebe_adapter = zeebe_adapter self.context = context From a51c65628d8537e37bb8cd94baaafdd0a4da4564 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sun, 23 Aug 2020 10:48:32 +0300 Subject: [PATCH 02/57] [FIXED] inherit from object --- zeebepy/grpc_internals/zeebe_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeebepy/grpc_internals/zeebe_adapter.py b/zeebepy/grpc_internals/zeebe_adapter.py index 39269b63..595835f9 100644 --- a/zeebepy/grpc_internals/zeebe_adapter.py +++ b/zeebepy/grpc_internals/zeebe_adapter.py @@ -9,7 +9,7 @@ from zeebepy.task.task_context import TaskContext -class ZeebeAdapter: +class ZeebeAdapter(object): def __init__(self, hostname: str = None, port: int = None, channel: grpc.Channel = None, **kwargs): self._connection_uri = f'{hostname}:{port}' or os.getenv('ZEEBE_ADDRESS') or 'localhost:26500' self._channel = channel or grpc.insecure_channel(self._connection_uri) From 152f38deac73a526bc7dcf8ab03d7a671d953b88 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sun, 23 Aug 2020 11:02:59 +0300 Subject: [PATCH 03/57] Basic func --- zeebepy/client/client.py | 19 ++++++++++++++++++- zeebepy/client/client_test.py | 21 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/zeebepy/client/client.py b/zeebepy/client/client.py index 3f6a2f5b..744685d9 100644 --- a/zeebepy/client/client.py +++ b/zeebepy/client/client.py @@ -1,2 +1,19 @@ +from typing import Dict, List + +from zeebepy.grpc_internals.zeebe_adapter import ZeebeAdapter + + class ZeebeClient(object): - pass + def __init__(self, hostname: str = None, port: int = None): + self.zeebe_adapter = ZeebeAdapter(hostname=hostname, port=port) + + def run_workflow(self, bpmn_process_id: str, variables: Dict = None, version: int = -1) -> str: + return self.zeebe_adapter.create_workflow_instance(bpmn_process_id=bpmn_process_id, variables=variables or {}, + version=version) + + def run_workflow_with_result(self, bpmn_process_id: str, variables: Dict = None, version: int = -1, + timeout: int = 0, variables_to_fetch: List[str] = None) -> Dict: + return self.zeebe_adapter.create_workflow_instance_with_result(bpmn_process_id=bpmn_process_id, + variables=variables or {}, version=version, + timeout=timeout, + variables_to_fetch=variables_to_fetch or []) diff --git a/zeebepy/client/client_test.py b/zeebepy/client/client_test.py index e69de29b..4a50b2a1 100644 --- a/zeebepy/client/client_test.py +++ b/zeebepy/client/client_test.py @@ -0,0 +1,21 @@ +import pytest + +from zeebepy.client.client import ZeebeClient + +zeebe_client: ZeebeClient + + +@pytest.fixture(autouse=True) +def run_around_tests(): + global zeebe_client + zeebe_client = ZeebeClient() + yield + zeebe_client = ZeebeClient() + + +def test_start_workflow(): + assert isinstance(zeebe_client.run_workflow(), str) + + +def test_start_workflow_with_result(): + assert isinstance(zeebe_client.run_workflow_with_result(), dict) From 5cd097551bcd3b59e8ba845ceea323d8f6a29e74 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sun, 23 Aug 2020 18:45:18 +0300 Subject: [PATCH 04/57] Basic client functionality --- pyzeebe/client/__init__.py | 0 pyzeebe/client/client.py | 0 pyzeebe/client/client_test.py | 0 pyzeebe/common/gateway_mock.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 pyzeebe/client/__init__.py create mode 100644 pyzeebe/client/client.py create mode 100644 pyzeebe/client/client_test.py create mode 100644 pyzeebe/common/gateway_mock.py diff --git a/pyzeebe/client/__init__.py b/pyzeebe/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyzeebe/client/client.py b/pyzeebe/client/client.py new file mode 100644 index 00000000..e69de29b diff --git a/pyzeebe/client/client_test.py b/pyzeebe/client/client_test.py new file mode 100644 index 00000000..e69de29b diff --git a/pyzeebe/common/gateway_mock.py b/pyzeebe/common/gateway_mock.py new file mode 100644 index 00000000..e69de29b From dc4ae049c9e6d799c6cea2fc77bc9e81ed515549 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sun, 23 Aug 2020 18:45:47 +0300 Subject: [PATCH 05/57] Basic client functionality --- examples/worker.py | 7 +- pyzeebe/__init__.py | 1 + pyzeebe/client/client.py | 34 ++++++++++ pyzeebe/client/client_test.py | 56 ++++++++++++++++ pyzeebe/common/gateway_mock.py | 69 ++++++++++++++++++++ pyzeebe/grpc_internals/zeebe_adapter.py | 22 +++++-- pyzeebe/grpc_internals/zeebe_adapter_test.py | 52 +-------------- 7 files changed, 182 insertions(+), 59 deletions(-) diff --git a/examples/worker.py b/examples/worker.py index b4b48a3d..8ffac3bc 100644 --- a/examples/worker.py +++ b/examples/worker.py @@ -3,8 +3,8 @@ from pyzeebe import Task, TaskContext, TaskStatusController, ZeebeWorker -def example_task(input: str) -> Dict: - return {'output': f'Hello world, {input}!'} +def example_task() -> Dict: + return {'output': f'Hello world, test!'} def example_exception_handler(exc: Exception, context: TaskContext, controller: TaskStatusController) -> None: @@ -13,7 +13,8 @@ def example_exception_handler(exc: Exception, context: TaskContext, controller: controller.error(f'Failed to run task {context.type}. Reason: {exc}') -task = Task(task_type='example', task_handler=example_task, exception_handler=example_exception_handler) +task = Task(task_type='test', task_handler=example_task, exception_handler=example_exception_handler) +task_2 = Task(task_type='test2', task_handler=example_task, exception_handler=example_exception_handler) worker = ZeebeWorker() # Will use environment variable ZEEBE_ADDRESS or localhost:26500 diff --git a/pyzeebe/__init__.py b/pyzeebe/__init__.py index 409aa4eb..b359b060 100644 --- a/pyzeebe/__init__.py +++ b/pyzeebe/__init__.py @@ -1,3 +1,4 @@ +from pyzeebe.client.client import ZeebeClient from pyzeebe.task.task import Task from pyzeebe.task.task_context import TaskContext from pyzeebe.task.task_status_controller import TaskStatusController diff --git a/pyzeebe/client/client.py b/pyzeebe/client/client.py index e69de29b..ee4d2333 100644 --- a/pyzeebe/client/client.py +++ b/pyzeebe/client/client.py @@ -0,0 +1,34 @@ +from typing import Dict, List + +import grpc + +from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter + + +class ZeebeClient(object): + def __init__(self, hostname: str = None, port: int = None, channel: grpc.Channel = None): + self.zeebe_adapter = ZeebeAdapter(hostname=hostname, port=port, channel=channel) + + def run_workflow(self, bpmn_process_id: str, variables: Dict = None, version: int = -1) -> int: + return self.zeebe_adapter.create_workflow_instance(bpmn_process_id=bpmn_process_id, variables=variables or {}, + version=version) + + def run_workflow_with_result(self, bpmn_process_id: str, variables: Dict = None, version: int = -1, + timeout: int = 0, variables_to_fetch: List[str] = None) -> Dict: + return self.zeebe_adapter.create_workflow_instance_with_result(bpmn_process_id=bpmn_process_id, + variables=variables or {}, version=version, + timeout=timeout, + variables_to_fetch=variables_to_fetch or []) + + def cancel_workflow_instance(self, workflow_instance_key: int) -> int: + self.zeebe_adapter.cancel_workflow_instance(workflow_instance_key=workflow_instance_key) + return workflow_instance_key + + def deploy_workflow(self, workflow_file_path: str): + self.zeebe_adapter.deploy_workflow(workflow_file_path) + + def publish_message(self, name: str, correlation_key: str, variables: Dict = None, + time_to_live_in_milliseconds: int = 60000) -> None: + self.zeebe_adapter.publish_message(name=name, correlation_key=correlation_key, + time_to_live_in_milliseconds=time_to_live_in_milliseconds, + variables=variables or {}) diff --git a/pyzeebe/client/client_test.py b/pyzeebe/client/client_test.py index e69de29b..6196d906 100644 --- a/pyzeebe/client/client_test.py +++ b/pyzeebe/client/client_test.py @@ -0,0 +1,56 @@ +from random import randint +from uuid import uuid4 + +import pytest + +from pyzeebe.client.client import ZeebeClient +from pyzeebe.common.gateway_mock import GatewayMock +from pyzeebe.common.random_utils import RANDOM_RANGE + +zeebe_client: ZeebeClient + + +@pytest.fixture(scope='module') +def grpc_add_to_server(): + from pyzeebe.grpc_internals.zeebe_pb2_grpc import add_GatewayServicer_to_server + return add_GatewayServicer_to_server + + +@pytest.fixture(scope='module') +def grpc_servicer(): + return GatewayMock() + + +@pytest.fixture(scope='module') +def grpc_stub_cls(grpc_channel): + from pyzeebe.grpc_internals.zeebe_pb2_grpc import GatewayStub + return GatewayStub + + +@pytest.fixture(autouse=True) +def run_around_tests(grpc_channel): + global zeebe_client + zeebe_client = ZeebeClient(channel=grpc_channel) + yield + zeebe_client = ZeebeClient(channel=grpc_channel) + + +def test_run_workflow(): + assert isinstance(zeebe_client.run_workflow(bpmn_process_id=str(uuid4())), int) + + +def test_run_workflow_with_result(): + assert isinstance(zeebe_client.run_workflow_with_result(bpmn_process_id=str(uuid4())), dict) + + +def test_deploy_workflow(): + # TODO: Think about how to implement + pass + + +def test_cancel_workflow_instance(): + assert isinstance(zeebe_client.cancel_workflow_instance(workflow_instance_key=randint(0, RANDOM_RANGE)), int) + + +def test_publish_message(): + zeebe_client.publish_message(name=str(uuid4()), correlation_key=str(uuid4())) diff --git a/pyzeebe/common/gateway_mock.py b/pyzeebe/common/gateway_mock.py index e69de29b..d4305f45 100644 --- a/pyzeebe/common/gateway_mock.py +++ b/pyzeebe/common/gateway_mock.py @@ -0,0 +1,69 @@ +from random import randint +from typing import Dict +from unittest.mock import patch +from uuid import uuid4 + +import pytest + +from pyzeebe.common.random_utils import RANDOM_RANGE +from pyzeebe.grpc_internals.zeebe_pb2 import * +from pyzeebe.grpc_internals.zeebe_pb2_grpc import GatewayServicer + + +@patch('grpc.insecure_channel') +def mock_channel(): + pass + + +deployed_workflows: Dict + + +@pytest.fixture(autouse=True) +def run_around_tests(grpc_channel): + global deployed_workflows + deployed_workflows = {} + yield + deployed_workflows = {} + + +class GatewayMock(GatewayServicer): + # TODO: Mock behaviour of zeebe + + def __init__(self): + self.deployed_workflows = {} + + def CompleteJob(self, request, context): + return CompleteJobResponse() + + def FailJob(self, request, context): + return FailJobResponse() + + def ThrowError(self, request, context): + return ThrowErrorResponse() + + def CreateWorkflowInstance(self, request, context): + # context.set_code(grpc.StatusCode.NOT_FOUND) + return CreateWorkflowInstanceResponse(workflowKey=randint(0, RANDOM_RANGE), + bpmnProcessId=request.bpmnProcessId, + version=request.version, workflowInstanceKey=randint(0, RANDOM_RANGE)) + + def CreateWorkflowInstanceWithResult(self, request, context): + return CreateWorkflowInstanceWithResultResponse(workflowKey=request.request.workflowKey, + bpmnProcessId=request.request.bpmnProcessId, + version=randint(0, 10), variables=request.request.variables) + + def CancelWorkflowInstance(self, request, context): + return CancelWorkflowInstanceResponse() + + def DeployWorkflow(self, request, context): + workflows = [] + for workflow in request.workflows: + workflow_metadata = WorkflowMetadata(bpmnProcessId=str(uuid4()), version=randint(0, 10), + workflowKey=randint(0, RANDOM_RANGE), resourceName=workflow.name) + workflows.append(workflow_metadata) + deployed_workflows[workflow_metadata.bpmnProcessId] = workflow_metadata + + return DeployWorkflowResponse(key=randint(0, RANDOM_RANGE), workflows=workflows) + + def PublishMessage(self, request, context): + return PublishMessageResponse() diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index 68b045c7..1cfa508b 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -10,9 +10,14 @@ class ZeebeAdapter(object): - def __init__(self, hostname: str = None, port: int = None, channel: grpc.Channel = None, **kwargs): - self._connection_uri = f'{hostname}:{port}' or os.getenv('ZEEBE_ADDRESS') or 'localhost:26500' - self._channel = channel or grpc.insecure_channel(self._connection_uri) + def __init__(self, hostname: str = 'localhost', port: int = 26500, channel: grpc.Channel = None): + self._connection_uri = f'{hostname or "localhost"}:{port or 26500}' or os.getenv('ZEEBE_ADDRESS') + + if channel: + self._channel = channel + else: + self._channel = grpc.insecure_channel(self._connection_uri) + self.connected = False self.retrying_connection = True self._channel.subscribe(self._check_connectivity, try_to_connect=True) @@ -64,7 +69,7 @@ def throw_error(self, job_key: int, message: str) -> ThrowErrorResponse: return self.gateway_stub.ThrowError( ThrowErrorRequest(jobKey=job_key, errorMessage=message)) - def create_workflow_instance(self, bpmn_process_id: str, version: int, variables: Dict) -> str: + def create_workflow_instance(self, bpmn_process_id: str, version: int, variables: Dict) -> int: response = self.gateway_stub.CreateWorkflowInstance( CreateWorkflowInstanceRequest(bpmnProcessId=bpmn_process_id, version=version, variables=json.dumps(variables))) @@ -79,6 +84,10 @@ def create_workflow_instance_with_result(self, bpmn_process_id: str, version: in requestTimeout=timeout, fetchVariables=variables_to_fetch)) return json.loads(response.variables) + def cancel_workflow_instance(self, workflow_instance_key: int) -> CancelWorkflowInstanceResponse: + return self.gateway_stub.CancelWorkflowInstance( + CancelWorkflowInstanceRequest(workflowInstanceKey=workflow_instance_key)) + def publish_message(self, name: str, correlation_key: str, time_to_live_in_milliseconds: int, variables: Dict) -> PublishMessageResponse: return self.gateway_stub.PublishMessage( @@ -91,5 +100,6 @@ def deploy_workflow(self, *workflow_file_path: str) -> DeployWorkflowResponse: @staticmethod def _get_workflow_request_object(workflow_file_path: str) -> WorkflowRequestObject: - return WorkflowRequestObject(name=os.path.split(workflow_file_path)[-1], - definition=open(workflow_file_path).read()) + with open(workflow_file_path, mode='rb') as workflow_file: + return WorkflowRequestObject(name=os.path.split(workflow_file_path)[-1], + definition=workflow_file.read()) diff --git a/pyzeebe/grpc_internals/zeebe_adapter_test.py b/pyzeebe/grpc_internals/zeebe_adapter_test.py index 8550daa0..97256a1f 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_test.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_test.py @@ -1,65 +1,17 @@ from random import randint -from unittest.mock import patch from uuid import uuid4 import grpc import pytest +from pyzeebe.common.gateway_mock import GatewayMock from pyzeebe.common.random_utils import RANDOM_RANGE from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.grpc_internals.zeebe_pb2 import * -from pyzeebe.grpc_internals.zeebe_pb2_grpc import GatewayServicer zeebe_adapter: ZeebeAdapter -@patch('grpc.insecure_channel') -def mock_channel(): - pass - - -class TestGatewayServicer(GatewayServicer): - """ - def ActivateJobs(self, request, context): - return ActivateJobsResponse(jobs=[ActivatedJob()]) - """ - - def __init__(self): - self.workflows = {} - - def CompleteJob(self, request, context): - return CompleteJobResponse() - - def FailJob(self, request, context): - return FailJobResponse() - - def ThrowError(self, request, context): - return ThrowErrorResponse() - - def CreateWorkflowInstance(self, request, context): - return CreateWorkflowInstanceResponse(workflowKey=randint(0, RANDOM_RANGE), - bpmnProcessId=request.bpmnProcessId, - version=request.version, workflowInstanceKey=randint(0, RANDOM_RANGE)) - - def CreateWorkflowInstanceWithResult(self, request, context): - return CreateWorkflowInstanceWithResultResponse(workflowKey=request.request.workflowKey, - bpmnProcessId=request.request.bpmnProcessId, - version=randint(0, 10), variables=request.request.variables) - - def DeployWorkflow(self, request, context): - workflows = [] - for workflow in request.workflows: - workflow_metadata = WorkflowMetadata(bpmnProcessId=str(uuid4()), version=randint(0, 10), - workflowKey=randint(0, RANDOM_RANGE), resourceName=workflow.name) - workflows.append(workflow_metadata) - self.workflows[workflow_metadata.bpmnProcessId] = workflow_metadata - - return DeployWorkflowResponse(key=randint(0, RANDOM_RANGE), workflows=workflows) - - def PublishMessage(self, request, context): - return PublishMessageResponse() - - @pytest.fixture(scope='module') def grpc_add_to_server(): from pyzeebe.grpc_internals.zeebe_pb2_grpc import add_GatewayServicer_to_server @@ -68,7 +20,7 @@ def grpc_add_to_server(): @pytest.fixture(scope='module') def grpc_servicer(): - return TestGatewayServicer() + return GatewayMock() @pytest.fixture(scope='module') From 2e7f87a62997565afb58a645b42e1eae02ded8b9 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sun, 23 Aug 2020 22:25:52 +0300 Subject: [PATCH 06/57] [ADDED] e2e files --- tests/__init__.py | 0 tests/client.py | 0 tests/worker.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/client.py create mode 100644 tests/worker.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/client.py b/tests/client.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/worker.py b/tests/worker.py new file mode 100644 index 00000000..e69de29b From ab6db11deca24e2cd8868a86d57eb14e24da4f5a Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 25 Aug 2020 20:29:21 +0300 Subject: [PATCH 07/57] [ADDED] Tests for undeployed workflow --- pyzeebe/client/client_test.py | 33 +++++++++++++++---- pyzeebe/common/exceptions.py | 9 +++++- pyzeebe/common/gateway_mock.py | 43 +++++++++++++------------ pyzeebe/common/random_utils.py | 2 +- pyzeebe/grpc_internals/zeebe_adapter.py | 39 +++++++++++++++------- pyzeebe/worker/worker.py | 4 +-- pyzeebe/worker/worker_test.py | 6 ++-- 7 files changed, 90 insertions(+), 46 deletions(-) diff --git a/pyzeebe/client/client_test.py b/pyzeebe/client/client_test.py index 6196d906..6d2b8a49 100644 --- a/pyzeebe/client/client_test.py +++ b/pyzeebe/client/client_test.py @@ -4,6 +4,7 @@ import pytest from pyzeebe.client.client import ZeebeClient +from pyzeebe.common.exceptions import WorkflowDoesNotExist from pyzeebe.common.gateway_mock import GatewayMock from pyzeebe.common.random_utils import RANDOM_RANGE @@ -35,17 +36,35 @@ def run_around_tests(grpc_channel): zeebe_client = ZeebeClient(channel=grpc_channel) -def test_run_workflow(): - assert isinstance(zeebe_client.run_workflow(bpmn_process_id=str(uuid4())), int) +def test_run_workflow(grpc_servicer): + bpmn_process_id = str(uuid4()) + version = randint(0, 10) + grpc_servicer.mock_deploy_workflow(bpmn_process_id, version, []) + assert isinstance(zeebe_client.run_workflow(bpmn_process_id=bpmn_process_id, variables={}, version=version), int) -def test_run_workflow_with_result(): - assert isinstance(zeebe_client.run_workflow_with_result(bpmn_process_id=str(uuid4())), dict) +def test_run_workflow_with_result(grpc_servicer): + bpmn_process_id = str(uuid4()) + version = randint(0, 10) + grpc_servicer.mock_deploy_workflow(bpmn_process_id, version, []) + assert isinstance(zeebe_client.run_workflow(bpmn_process_id=bpmn_process_id, variables={}, version=version), int) -def test_deploy_workflow(): - # TODO: Think about how to implement - pass +def test_deploy_workflow(grpc_servicer): + bpmn_process_id = str(uuid4()) + version = randint(0, 10) + grpc_servicer.mock_deploy_workflow(bpmn_process_id, version, []) + assert bpmn_process_id in grpc_servicer.deployed_workflows.keys() + + +def test_run_non_existent_workflow(): + with pytest.raises(WorkflowDoesNotExist): + zeebe_client.run_workflow(bpmn_process_id=str(uuid4())) + + +def test_run_non_existent_workflow_with_result(): + with pytest.raises(WorkflowDoesNotExist): + zeebe_client.run_workflow_with_result(bpmn_process_id=str(uuid4())) def test_cancel_workflow_instance(): diff --git a/pyzeebe/common/exceptions.py b/pyzeebe/common/exceptions.py index 96b6896e..754b956f 100644 --- a/pyzeebe/common/exceptions.py +++ b/pyzeebe/common/exceptions.py @@ -1,2 +1,9 @@ -class TaskNotFoundException(Exception): +class TaskNotFound(Exception): pass + + +class WorkflowDoesNotExist(Exception): + def __init__(self, bpmn_process_id: str, version: int): + super().__init__(f'{bpmn_process_id} with {version} does not exist. Have you forgotten to deploy it?') + self.bpmn_process_id = bpmn_process_id + self.version = version diff --git a/pyzeebe/common/gateway_mock.py b/pyzeebe/common/gateway_mock.py index d4305f45..8a3f7727 100644 --- a/pyzeebe/common/gateway_mock.py +++ b/pyzeebe/common/gateway_mock.py @@ -1,9 +1,9 @@ from random import randint -from typing import Dict +from typing import List from unittest.mock import patch from uuid import uuid4 -import pytest +import grpc from pyzeebe.common.random_utils import RANDOM_RANGE from pyzeebe.grpc_internals.zeebe_pb2 import * @@ -15,22 +15,12 @@ def mock_channel(): pass -deployed_workflows: Dict - - -@pytest.fixture(autouse=True) -def run_around_tests(grpc_channel): - global deployed_workflows - deployed_workflows = {} - yield - deployed_workflows = {} - - class GatewayMock(GatewayServicer): # TODO: Mock behaviour of zeebe def __init__(self): self.deployed_workflows = {} + self.active_jobs = {} def CompleteJob(self, request, context): return CompleteJobResponse() @@ -39,18 +29,26 @@ def FailJob(self, request, context): return FailJobResponse() def ThrowError(self, request, context): + self.active_jobs[request.jobKey]['error'] = True return ThrowErrorResponse() def CreateWorkflowInstance(self, request, context): - # context.set_code(grpc.StatusCode.NOT_FOUND) - return CreateWorkflowInstanceResponse(workflowKey=randint(0, RANDOM_RANGE), - bpmnProcessId=request.bpmnProcessId, - version=request.version, workflowInstanceKey=randint(0, RANDOM_RANGE)) + if request.bpmnProcessId in self.deployed_workflows.keys(): + return CreateWorkflowInstanceResponse(workflowKey=randint(0, RANDOM_RANGE), + bpmnProcessId=request.bpmnProcessId, + version=request.version, workflowInstanceKey=randint(0, RANDOM_RANGE)) + else: + context.set_code(grpc.StatusCode.NOT_FOUND) + return CreateWorkflowInstanceResponse() def CreateWorkflowInstanceWithResult(self, request, context): - return CreateWorkflowInstanceWithResultResponse(workflowKey=request.request.workflowKey, - bpmnProcessId=request.request.bpmnProcessId, - version=randint(0, 10), variables=request.request.variables) + if request.request.bpmnProcessId in self.deployed_workflows.keys(): + return CreateWorkflowInstanceWithResultResponse(workflowKey=request.request.workflowKey, + bpmnProcessId=request.request.bpmnProcessId, + version=randint(0, 10), variables=request.request.variables) + else: + context.set_code(grpc.StatusCode.NOT_FOUND) + return CreateWorkflowInstanceWithResultResponse() def CancelWorkflowInstance(self, request, context): return CancelWorkflowInstanceResponse() @@ -61,9 +59,12 @@ def DeployWorkflow(self, request, context): workflow_metadata = WorkflowMetadata(bpmnProcessId=str(uuid4()), version=randint(0, 10), workflowKey=randint(0, RANDOM_RANGE), resourceName=workflow.name) workflows.append(workflow_metadata) - deployed_workflows[workflow_metadata.bpmnProcessId] = workflow_metadata return DeployWorkflowResponse(key=randint(0, RANDOM_RANGE), workflows=workflows) def PublishMessage(self, request, context): return PublishMessageResponse() + + def mock_deploy_workflow(self, bpmn_process_id: str, version: int, tasks: List[str]): + self.deployed_workflows[bpmn_process_id] = {'bpmn_process_id': bpmn_process_id, 'version': version, + 'tasks': tasks} diff --git a/pyzeebe/common/random_utils.py b/pyzeebe/common/random_utils.py index 7c48e4d3..f17c7e0f 100644 --- a/pyzeebe/common/random_utils.py +++ b/pyzeebe/common/random_utils.py @@ -4,7 +4,7 @@ from pyzeebe.task.task import Task from pyzeebe.task.task_context import TaskContext -RANDOM_RANGE = 100000 +RANDOM_RANGE = 1000000000 def random_task_context(task: Task = Task(task_type='test', task_handler=lambda x: {'x': x}, diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index 0b6f3a59..9d589f69 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -5,6 +5,7 @@ import grpc +from pyzeebe.common.exceptions import WorkflowDoesNotExist from pyzeebe.grpc_internals.zeebe_pb2 import * from pyzeebe.grpc_internals.zeebe_pb2_grpc import GatewayStub from pyzeebe.task.task_context import TaskContext @@ -79,20 +80,32 @@ def throw_error(self, job_key: int, message: str) -> ThrowErrorResponse: return self.gateway_stub.ThrowError( ThrowErrorRequest(jobKey=job_key, errorMessage=message)) - def create_workflow_instance(self, bpmn_process_id: str, version: int, variables: Dict) -> str: - response = self.gateway_stub.CreateWorkflowInstance( - CreateWorkflowInstanceRequest(bpmnProcessId=bpmn_process_id, version=version, - variables=json.dumps(variables))) - return response.workflowInstanceKey + def create_workflow_instance(self, bpmn_process_id: str, version: int, variables: Dict) -> int: + try: + response = self.gateway_stub.CreateWorkflowInstance( + CreateWorkflowInstanceRequest(bpmnProcessId=bpmn_process_id, version=version, + variables=json.dumps(variables))) + return response.workflowInstanceKey + except grpc.RpcError as rpc_error: + if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise WorkflowDoesNotExist(bpmn_process_id=bpmn_process_id, version=version) def create_workflow_instance_with_result(self, bpmn_process_id: str, version: int, variables: Dict, timeout: int, variables_to_fetch) -> Dict: - response = self.gateway_stub.CreateWorkflowInstanceWithResult( - CreateWorkflowInstanceWithResultRequest( - request=CreateWorkflowInstanceRequest(bpmnProcessId=bpmn_process_id, version=version, - variables=json.dumps(variables)), - requestTimeout=timeout, fetchVariables=variables_to_fetch)) - return json.loads(response.variables) + try: + response = self.gateway_stub.CreateWorkflowInstanceWithResult( + CreateWorkflowInstanceWithResultRequest( + request=CreateWorkflowInstanceRequest(bpmnProcessId=bpmn_process_id, version=version, + variables=json.dumps(variables)), + requestTimeout=timeout, fetchVariables=variables_to_fetch)) + return json.loads(response.variables) + except grpc.RpcError as rpc_error: + if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise WorkflowDoesNotExist(bpmn_process_id=bpmn_process_id, version=version) + + def cancel_workflow_instance(self, workflow_instance_key: int) -> None: + self.gateway_stub.CancelWorkflowInstance( + CancelWorkflowInstanceRequest(workflowInstanceKey=workflow_instance_key)) def publish_message(self, name: str, correlation_key: str, time_to_live_in_milliseconds: int, variables: Dict) -> PublishMessageResponse: @@ -108,3 +121,7 @@ def deploy_workflow(self, *workflow_file_path: str) -> DeployWorkflowResponse: def _get_workflow_request_object(workflow_file_path: str) -> WorkflowRequestObject: return WorkflowRequestObject(name=os.path.split(workflow_file_path)[-1], definition=open(workflow_file_path).read()) + + @staticmethod + def _is_error_of_status(rpc_error: grpc.RpcError, status_code: grpc.StatusCode): + return rpc_error.args[0].code == status_code diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 2d15c9b4..92c483d0 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -3,7 +3,7 @@ from concurrent.futures import ThreadPoolExecutor from typing import List, Callable, Generator, Tuple -from pyzeebe.common.exceptions import TaskNotFoundException +from pyzeebe.common.exceptions import TaskNotFound from pyzeebe.decorators.zeebe_decorator_base import ZeebeDecoratorBase from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.task.task import Task @@ -116,4 +116,4 @@ def _get_task_and_index(self, task_type: str) -> Tuple[Task, int]: for index, task in enumerate(self.tasks): if task.type == task_type: return task, index - raise TaskNotFoundException(f"Could not find task {task_type}") + raise TaskNotFound(f"Could not find task {task_type}") diff --git a/pyzeebe/worker/worker_test.py b/pyzeebe/worker/worker_test.py index 0ad19e44..295d28e6 100644 --- a/pyzeebe/worker/worker_test.py +++ b/pyzeebe/worker/worker_test.py @@ -4,7 +4,7 @@ import pytest -from pyzeebe.common.exceptions import TaskNotFoundException +from pyzeebe.common.exceptions import TaskNotFound from pyzeebe.common.random_utils import random_task_context from pyzeebe.task.task import Task from pyzeebe.task.task_context import TaskContext @@ -110,12 +110,12 @@ def test_remove_task_from_many(): def test_remove_fake_task(): - with pytest.raises(TaskNotFoundException): + with pytest.raises(TaskNotFound): zeebe_worker.remove_task(str(uuid4())) def test_get_fake_task(): - with pytest.raises(TaskNotFoundException): + with pytest.raises(TaskNotFound): zeebe_worker.get_task(str(uuid4())) From 6b9d88484e182f36928d7a097720b46eeb44a99f Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 25 Aug 2020 20:33:47 +0300 Subject: [PATCH 08/57] [FIXED] new mock tests --- pyzeebe/common/gateway_mock.py | 1 - pyzeebe/grpc_internals/zeebe_adapter.py | 4 ++++ pyzeebe/grpc_internals/zeebe_adapter_test.py | 18 +++++++++++------- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pyzeebe/common/gateway_mock.py b/pyzeebe/common/gateway_mock.py index 8a3f7727..18e2c366 100644 --- a/pyzeebe/common/gateway_mock.py +++ b/pyzeebe/common/gateway_mock.py @@ -29,7 +29,6 @@ def FailJob(self, request, context): return FailJobResponse() def ThrowError(self, request, context): - self.active_jobs[request.jobKey]['error'] = True return ThrowErrorResponse() def CreateWorkflowInstance(self, request, context): diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index 9d589f69..bc982c1f 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -89,6 +89,8 @@ def create_workflow_instance(self, bpmn_process_id: str, version: int, variables except grpc.RpcError as rpc_error: if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): raise WorkflowDoesNotExist(bpmn_process_id=bpmn_process_id, version=version) + else: + raise def create_workflow_instance_with_result(self, bpmn_process_id: str, version: int, variables: Dict, timeout: int, variables_to_fetch) -> Dict: @@ -102,6 +104,8 @@ def create_workflow_instance_with_result(self, bpmn_process_id: str, version: in except grpc.RpcError as rpc_error: if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): raise WorkflowDoesNotExist(bpmn_process_id=bpmn_process_id, version=version) + else: + raise def cancel_workflow_instance(self, workflow_instance_key: int) -> None: self.gateway_stub.CancelWorkflowInstance( diff --git a/pyzeebe/grpc_internals/zeebe_adapter_test.py b/pyzeebe/grpc_internals/zeebe_adapter_test.py index 97256a1f..6e979914 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_test.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_test.py @@ -81,16 +81,20 @@ def test_throw_error(): assert isinstance(response, ThrowErrorResponse) -def test_create_workflow_instance(): - response = zeebe_adapter.create_workflow_instance(bpmn_process_id=str(uuid4()), variables={}, - version=randint(0, 10)) +def test_create_workflow_instance(grpc_servicer): + bpmn_process_id = str(uuid4()) + version = randint(0, 10) + grpc_servicer.mock_deploy_workflow(bpmn_process_id, version, []) + response = zeebe_adapter.create_workflow_instance(bpmn_process_id=bpmn_process_id, variables={}, version=version) assert isinstance(response, int) -def test_create_workflow_instance_with_result(): - response = zeebe_adapter.create_workflow_instance_with_result(bpmn_process_id=str(uuid4()), variables={}, - version=randint(0, 10), timeout=0, - variables_to_fetch=[]) +def test_create_workflow_instance_with_result(grpc_servicer): + bpmn_process_id = str(uuid4()) + version = randint(0, 10) + grpc_servicer.mock_deploy_workflow(bpmn_process_id, version, []) + response = zeebe_adapter.create_workflow_instance_with_result(bpmn_process_id=bpmn_process_id, variables={}, + version=version, timeout=0, variables_to_fetch=[]) assert isinstance(response, dict) From 3587b1289934f9f74c97f773ebb6eb4bc148fa29 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 25 Aug 2020 20:40:19 +0300 Subject: [PATCH 09/57] [FIXED] deploy workflow --- pyzeebe/client/client.py | 4 ++-- pyzeebe/grpc_internals/zeebe_adapter.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyzeebe/client/client.py b/pyzeebe/client/client.py index ee4d2333..49adf6af 100644 --- a/pyzeebe/client/client.py +++ b/pyzeebe/client/client.py @@ -24,8 +24,8 @@ def cancel_workflow_instance(self, workflow_instance_key: int) -> int: self.zeebe_adapter.cancel_workflow_instance(workflow_instance_key=workflow_instance_key) return workflow_instance_key - def deploy_workflow(self, workflow_file_path: str): - self.zeebe_adapter.deploy_workflow(workflow_file_path) + def deploy_workflow(self, *workflow_file_path: str): + self.zeebe_adapter.deploy_workflow(*workflow_file_path) def publish_message(self, name: str, correlation_key: str, variables: Dict = None, time_to_live_in_milliseconds: int = 60000) -> None: diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index bc982c1f..d27de6da 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -123,8 +123,9 @@ def deploy_workflow(self, *workflow_file_path: str) -> DeployWorkflowResponse: @staticmethod def _get_workflow_request_object(workflow_file_path: str) -> WorkflowRequestObject: - return WorkflowRequestObject(name=os.path.split(workflow_file_path)[-1], - definition=open(workflow_file_path).read()) + with open(workflow_file_path, 'rb') as file: + return WorkflowRequestObject(name=os.path.split(workflow_file_path)[-1], + definition=file.read()) @staticmethod def _is_error_of_status(rpc_error: grpc.RpcError, status_code: grpc.StatusCode): From ff9938032f6c279412744376cdb57941a9fe7190 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 25 Aug 2020 21:11:02 +0300 Subject: [PATCH 10/57] [ENHANCED] WorkflowDoesNotExist exception --- pyzeebe/common/exceptions.py | 3 ++- pyzeebe/grpc_internals/zeebe_adapter.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyzeebe/common/exceptions.py b/pyzeebe/common/exceptions.py index 754b956f..9f618303 100644 --- a/pyzeebe/common/exceptions.py +++ b/pyzeebe/common/exceptions.py @@ -4,6 +4,7 @@ class TaskNotFound(Exception): class WorkflowDoesNotExist(Exception): def __init__(self, bpmn_process_id: str, version: int): - super().__init__(f'{bpmn_process_id} with {version} does not exist. Have you forgotten to deploy it?') + super().__init__( + f'Workflow definition: {bpmn_process_id} with {version} does not exist. Have you forgotten to deploy it?') self.bpmn_process_id = bpmn_process_id self.version = version diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index d27de6da..56489c84 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -104,6 +104,8 @@ def create_workflow_instance_with_result(self, bpmn_process_id: str, version: in except grpc.RpcError as rpc_error: if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): raise WorkflowDoesNotExist(bpmn_process_id=bpmn_process_id, version=version) + elif self._is_error_of_status(rpc_error, grpc.StatusCode.RESOURCE_EXHAUSTED): + pass else: raise From cf06afbacf5f2f003be61c1916e16d8473196d36 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 25 Aug 2020 21:12:08 +0300 Subject: [PATCH 11/57] [TEMP] Added all error codes for saving --- pyzeebe/common/exceptions.py | 147 +++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/pyzeebe/common/exceptions.py b/pyzeebe/common/exceptions.py index 9f618303..f66c3ca6 100644 --- a/pyzeebe/common/exceptions.py +++ b/pyzeebe/common/exceptions.py @@ -8,3 +8,150 @@ def __init__(self, bpmn_process_id: str, version: int): f'Workflow definition: {bpmn_process_id} with {version} does not exist. Have you forgotten to deploy it?') self.bpmn_process_id = bpmn_process_id self.version = version + + +class WorkflowInstanceDoesNotExist(Exception): + """ + GRPC_STATUS_NOT_FOUND + Returned if: + no workflow with the given key exists (if workflowKey was given) + no workflow with the given process ID exists (if bpmnProcessId was given but version was -1) + no workflow with the given process ID and version exists (if both bpmnProcessId and version were given) + + GRPC_STATUS_FAILED_PRECONDITION + Returned if: + the workflow definition does not contain a none start event; only workflows with none start event can be started manually. + + GRPC_STATUS_INVALID_ARGUMENT + Returned if: + the given variables argument is not a valid JSON document; it is expected to be a valid JSON document where the root node is an object. + + """ + + def __init__(self, workflow_instance_key: int): + super().__init__(f'Workflow instance key: {workflow_instance_key} does not exist') + self.workflow_instance_key = workflow_instance_key + + +class InvalidActivateJobs(Exception): + """ + GRPC_STATUS_INVALID_ARGUMENT + Returned if: + type is blank (empty string, null) + worker is blank (empty string, null) + timeout less than 1 (ms) + amount is less than 1 + """ + pass + + +class InvalidCompleteJob(Exception): + """ + GRPC_STATUS_NOT_FOUND + Returned if: + no job exists with the given job key. Note that since jobs are removed once completed, it could be that this job did exist at some point. + + GRPC_STATUS_FAILED_PRECONDITION + Returned if: + the job was marked as failed. In that case, the related incident must be resolved before the job can be activated again and completed. + """ + pass + + +class InvalidFailJob(Exception): + """ + GRPC_STATUS_NOT_FOUND + Returned if: + no job was found with the given key + + GRPC_STATUS_FAILED_PRECONDITION + Returned if: + the job was not activated + the job is already in a failed state, i.e. ran out of retries + """ + pass + + +class InvalidDeployWorkflow(Exception): + """ + GRPC_STATUS_INVALID_ARGUMENT + Returned if: + no resources given. + if at least one resource is invalid. A resource is considered invalid if: + it is not a BPMN or YAML file (currently detected through the file extension) + the resource data is not deserializable (e.g. detected as BPMN, but it's broken XML) + the workflow is invalid (e.g. an event-based gateway has an outgoing sequence flow to a task) + """ + pass + + +class InvalidPublishMessage(Exception): + """ + GRPC_STATUS_ALREADY_EXISTS + Returned if: + a message with the same ID was previously published (and is still alive) + """ + pass + + +class InvalidResolveIncident(Exception): + """ + GRPC_STATUS_NOT_FOUND + Returned if: + no incident with the given key exists + """ + pass + + +class InvalidSetVariables(Exception): + """ + GRPC_STATUS_NOT_FOUND + Returned if: + no element with the given elementInstanceKey was exists + + GRPC_STATUS_INVALID_ARGUMENT + Returned if: + the given payload is not a valid JSON document; all payloads are expected to be valid JSON documents where the root node is an object. + """ + pass + + +class InvalidThrowError(Exception): + """ + Errors + GRPC_STATUS_NOT_FOUND + Returned if: + no job was found with the given key + + GRPC_STATUS_FAILED_PRECONDITION + Returned if: + the job is already in a failed state, i.e. ran out of retries + """ + pass + + +class InvalidJobRetries(Exception): + """ + Errors + GRPC_STATUS_NOT_FOUND + Returned if: + no job exists with the given key + + GRPC_STATUS_INVALID_ARGUMENT + Returned if: + retries is not greater than 0 + """ + pass + + +class ZeebeBackPressure(Exception): + # TODO: Think about deploying some kind of retry strategy instead of raising this + pass + + +class ZeebeGatewayUnavailable(Exception): + pass + + +class ZeebeInternalError(Exception): + pass From c71db99b5bff7872e1dea8e1ff17eb1622c31bab Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 25 Aug 2020 21:49:40 +0300 Subject: [PATCH 12/57] Checkpoint --- pyzeebe/client/client_test.py | 6 +- pyzeebe/common/exceptions.py | 41 ++++++--- pyzeebe/grpc_internals/zeebe_adapter.py | 113 ++++++++++++++++++------ 3 files changed, 121 insertions(+), 39 deletions(-) diff --git a/pyzeebe/client/client_test.py b/pyzeebe/client/client_test.py index 6d2b8a49..f640ecd7 100644 --- a/pyzeebe/client/client_test.py +++ b/pyzeebe/client/client_test.py @@ -4,7 +4,7 @@ import pytest from pyzeebe.client.client import ZeebeClient -from pyzeebe.common.exceptions import WorkflowDoesNotExist +from pyzeebe.common.exceptions import WorkflowNotFound from pyzeebe.common.gateway_mock import GatewayMock from pyzeebe.common.random_utils import RANDOM_RANGE @@ -58,12 +58,12 @@ def test_deploy_workflow(grpc_servicer): def test_run_non_existent_workflow(): - with pytest.raises(WorkflowDoesNotExist): + with pytest.raises(WorkflowNotFound): zeebe_client.run_workflow(bpmn_process_id=str(uuid4())) def test_run_non_existent_workflow_with_result(): - with pytest.raises(WorkflowDoesNotExist): + with pytest.raises(WorkflowNotFound): zeebe_client.run_workflow_with_result(bpmn_process_id=str(uuid4())) diff --git a/pyzeebe/common/exceptions.py b/pyzeebe/common/exceptions.py index f66c3ca6..8b883639 100644 --- a/pyzeebe/common/exceptions.py +++ b/pyzeebe/common/exceptions.py @@ -2,15 +2,15 @@ class TaskNotFound(Exception): pass -class WorkflowDoesNotExist(Exception): +class WorkflowNotFound(Exception): def __init__(self, bpmn_process_id: str, version: int): super().__init__( - f'Workflow definition: {bpmn_process_id} with {version} does not exist. Have you forgotten to deploy it?') + f'Workflow definition: {bpmn_process_id} with {version} was not found') self.bpmn_process_id = bpmn_process_id self.version = version -class WorkflowInstanceDoesNotExist(Exception): +class WorkflowInstanceNotFound(Exception): """ GRPC_STATUS_NOT_FOUND Returned if: @@ -18,10 +18,6 @@ class WorkflowInstanceDoesNotExist(Exception): no workflow with the given process ID exists (if bpmnProcessId was given but version was -1) no workflow with the given process ID and version exists (if both bpmnProcessId and version were given) - GRPC_STATUS_FAILED_PRECONDITION - Returned if: - the workflow definition does not contain a none start event; only workflows with none start event can be started manually. - GRPC_STATUS_INVALID_ARGUMENT Returned if: the given variables argument is not a valid JSON document; it is expected to be a valid JSON document where the root node is an object. @@ -29,10 +25,19 @@ class WorkflowInstanceDoesNotExist(Exception): """ def __init__(self, workflow_instance_key: int): - super().__init__(f'Workflow instance key: {workflow_instance_key} does not exist') + super().__init__(f'Workflow instance key: {workflow_instance_key} was not found') self.workflow_instance_key = workflow_instance_key +class WorkflowHasNoStartEvent(Exception): + """ + GRPC_STATUS_FAILED_PRECONDITION + Returned if: + the workflow definition does not contain a none start event; only workflows with none start event can be started manually. + """ + pass + + class InvalidActivateJobs(Exception): """ GRPC_STATUS_INVALID_ARGUMENT @@ -58,6 +63,14 @@ class InvalidCompleteJob(Exception): pass +class JobAlreadyFailed(Exception): + pass + + +class JobNotFound(Exception): + pass + + class InvalidFailJob(Exception): """ GRPC_STATUS_NOT_FOUND @@ -85,7 +98,7 @@ class InvalidDeployWorkflow(Exception): pass -class InvalidPublishMessage(Exception): +class MessageAlreadyExists(Exception): """ GRPC_STATUS_ALREADY_EXISTS Returned if: @@ -94,7 +107,7 @@ class InvalidPublishMessage(Exception): pass -class InvalidResolveIncident(Exception): +class IncidentNotFound(Exception): """ GRPC_STATUS_NOT_FOUND Returned if: @@ -103,6 +116,14 @@ class InvalidResolveIncident(Exception): pass +class ElementNotFound(Exception): + pass + + +class InvalidJSON(Exception): + pass + + class InvalidSetVariables(Exception): """ GRPC_STATUS_NOT_FOUND diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index 56489c84..b133f37f 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -5,7 +5,7 @@ import grpc -from pyzeebe.common.exceptions import WorkflowDoesNotExist +from pyzeebe.common.exceptions import * from pyzeebe.grpc_internals.zeebe_pb2 import * from pyzeebe.grpc_internals.zeebe_pb2_grpc import GatewayStub from pyzeebe.task.task_context import TaskContext @@ -46,14 +46,20 @@ def _check_connectivity(self, value: grpc.ChannelConnectivity) -> None: def activate_jobs(self, task_type: str, worker: str, timeout: int, max_jobs_to_activate: int, variables_to_fetch: List[str], request_timeout: int) -> Generator[TaskContext, None, None]: - for response in self.gateway_stub.ActivateJobs( - ActivateJobsRequest(type=task_type, worker=worker, timeout=timeout, - maxJobsToActivate=max_jobs_to_activate, - fetchVariable=variables_to_fetch, requestTimeout=request_timeout)): - for job in response.jobs: - context = self._create_task_context_from_job(job) - logging.debug(f'Got job: {context} from zeebe') - yield context + try: + for response in self.gateway_stub.ActivateJobs( + ActivateJobsRequest(type=task_type, worker=worker, timeout=timeout, + maxJobsToActivate=max_jobs_to_activate, + fetchVariable=variables_to_fetch, requestTimeout=request_timeout)): + for job in response.jobs: + context = self._create_task_context_from_job(job) + logging.debug(f'Got job: {context} from zeebe') + yield context + except grpc.RpcError as rpc_error: + if self._is_error_of_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): + raise InvalidActivateJobs() # TODO make proper exception + else: + self._common_zeebe_grpc_errors(rpc_error) @staticmethod def _create_task_context_from_job(job) -> TaskContext: @@ -71,14 +77,38 @@ def _create_task_context_from_job(job) -> TaskContext: variables=json.loads(job.variables)) def complete_job(self, job_key: int, variables: Dict) -> CompleteJobResponse: - return self.gateway_stub.CompleteJob(CompleteJobRequest(jobKey=job_key, variables=json.dumps(variables))) + try: + return self.gateway_stub.CompleteJob(CompleteJobRequest(jobKey=job_key, variables=json.dumps(variables))) + except grpc.RpcError as rpc_error: + if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise JobNotFound() + elif self._is_error_of_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): + raise InvalidCompleteJob() # TODO proper exception + else: + self._common_zeebe_grpc_errors(rpc_error) def fail_job(self, job_key: int, message: str) -> FailJobResponse: - return self.gateway_stub.FailJob(FailJobRequest(jobKey=job_key, errorMessage=message)) + try: + return self.gateway_stub.FailJob(FailJobRequest(jobKey=job_key, errorMessage=message)) + except grpc.RpcError as rpc_error: + if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise JobNotFound() + elif self._is_error_of_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): + InvalidFailJob() # TODO make good exception + else: + self._common_zeebe_grpc_errors(rpc_error) def throw_error(self, job_key: int, message: str) -> ThrowErrorResponse: - return self.gateway_stub.ThrowError( - ThrowErrorRequest(jobKey=job_key, errorMessage=message)) + try: + return self.gateway_stub.ThrowError( + ThrowErrorRequest(jobKey=job_key, errorMessage=message)) + except grpc.RpcError as rpc_error: + if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise JobNotFound() + elif self._is_error_of_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): + InvalidThrowError() # TODO make good exception + else: + self._common_zeebe_grpc_errors(rpc_error) def create_workflow_instance(self, bpmn_process_id: str, version: int, variables: Dict) -> int: try: @@ -88,9 +118,11 @@ def create_workflow_instance(self, bpmn_process_id: str, version: int, variables return response.workflowInstanceKey except grpc.RpcError as rpc_error: if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise WorkflowDoesNotExist(bpmn_process_id=bpmn_process_id, version=version) + raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) + elif self._is_error_of_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): + raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) # TODO: Change exception else: - raise + self._common_zeebe_grpc_errors(rpc_error) def create_workflow_instance_with_result(self, bpmn_process_id: str, version: int, variables: Dict, timeout: int, variables_to_fetch) -> Dict: @@ -103,25 +135,54 @@ def create_workflow_instance_with_result(self, bpmn_process_id: str, version: in return json.loads(response.variables) except grpc.RpcError as rpc_error: if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise WorkflowDoesNotExist(bpmn_process_id=bpmn_process_id, version=version) - elif self._is_error_of_status(rpc_error, grpc.StatusCode.RESOURCE_EXHAUSTED): - pass + raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) + elif self._is_error_of_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): + raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) # TODO: Change exception else: - raise + self._common_zeebe_grpc_errors(rpc_error) def cancel_workflow_instance(self, workflow_instance_key: int) -> None: - self.gateway_stub.CancelWorkflowInstance( - CancelWorkflowInstanceRequest(workflowInstanceKey=workflow_instance_key)) + try: + self.gateway_stub.CancelWorkflowInstance( + CancelWorkflowInstanceRequest(workflowInstanceKey=workflow_instance_key)) + except grpc.RpcError as rpc_error: + if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise WorkflowInstanceNotFound(workflow_instance_key=workflow_instance_key) + else: + self._common_zeebe_grpc_errors(rpc_error) def publish_message(self, name: str, correlation_key: str, time_to_live_in_milliseconds: int, variables: Dict) -> PublishMessageResponse: - return self.gateway_stub.PublishMessage( - PublishMessageRequest(name=name, correlationKey=correlation_key, timeToLive=time_to_live_in_milliseconds, - variables=json.dumps(variables))) + try: + return self.gateway_stub.PublishMessage( + PublishMessageRequest(name=name, correlationKey=correlation_key, + timeToLive=time_to_live_in_milliseconds, + variables=json.dumps(variables))) + except grpc.RpcError as rpc_error: + if self._is_error_of_status(rpc_error, grpc.StatusCode.ALREADY_EXISTS): + raise MessageAlreadyExists() + else: + self._common_zeebe_grpc_errors(rpc_error) def deploy_workflow(self, *workflow_file_path: str) -> DeployWorkflowResponse: - return self.gateway_stub.DeployWorkflow( - DeployWorkflowRequest(workflows=map(self._get_workflow_request_object, workflow_file_path))) + try: + return self.gateway_stub.DeployWorkflow( + DeployWorkflowRequest(workflows=map(self._get_workflow_request_object, workflow_file_path))) + except grpc.RpcError as rpc_error: + if self._is_error_of_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): + raise InvalidDeployWorkflow() + else: + self._common_zeebe_grpc_errors(rpc_error) + + def _common_zeebe_grpc_errors(self, rpc_error: grpc.RpcError): + if self._is_error_of_status(rpc_error, grpc.StatusCode.RESOURCE_EXHAUSTED): + raise ZeebeBackPressure() + elif self._is_error_of_status(rpc_error, grpc.StatusCode.UNAVAILABLE): + raise ZeebeGatewayUnavailable() + elif self._is_error_of_status(rpc_error, grpc.StatusCode.INTERNAL): + raise ZeebeInternalError() + else: + raise rpc_error @staticmethod def _get_workflow_request_object(workflow_file_path: str) -> WorkflowRequestObject: From afeb3d8b80c895f089bc8fc5fa410ecc623209cd Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 25 Aug 2020 22:16:09 +0300 Subject: [PATCH 13/57] Using all exceptions --- pyzeebe/common/exceptions.py | 145 ++++-------------------- pyzeebe/grpc_internals/zeebe_adapter.py | 83 +++++++------- 2 files changed, 65 insertions(+), 163 deletions(-) diff --git a/pyzeebe/common/exceptions.py b/pyzeebe/common/exceptions.py index 8b883639..3785a764 100644 --- a/pyzeebe/common/exceptions.py +++ b/pyzeebe/common/exceptions.py @@ -11,108 +11,49 @@ def __init__(self, bpmn_process_id: str, version: int): class WorkflowInstanceNotFound(Exception): - """ - GRPC_STATUS_NOT_FOUND - Returned if: - no workflow with the given key exists (if workflowKey was given) - no workflow with the given process ID exists (if bpmnProcessId was given but version was -1) - no workflow with the given process ID and version exists (if both bpmnProcessId and version were given) - - GRPC_STATUS_INVALID_ARGUMENT - Returned if: - the given variables argument is not a valid JSON document; it is expected to be a valid JSON document where the root node is an object. - - """ - def __init__(self, workflow_instance_key: int): super().__init__(f'Workflow instance key: {workflow_instance_key} was not found') self.workflow_instance_key = workflow_instance_key class WorkflowHasNoStartEvent(Exception): - """ - GRPC_STATUS_FAILED_PRECONDITION - Returned if: - the workflow definition does not contain a none start event; only workflows with none start event can be started manually. - """ - pass - + def __init__(self, bpmn_process_id: str): + super().__init__(f"Workflow {bpmn_process_id} has no start event that can be called manually") + self.bpmn_process_id = bpmn_process_id -class InvalidActivateJobs(Exception): - """ - GRPC_STATUS_INVALID_ARGUMENT - Returned if: - type is blank (empty string, null) - worker is blank (empty string, null) - timeout less than 1 (ms) - amount is less than 1 - """ - pass +class ActivateJobsRequestInvalid(Exception): + def __init__(self, task_type: str, worker: str, timeout: int, max_jobs_to_activate: int): + msg = "Failed to activate jobs. Reasons:" + if task_type == "" or task_type is None: + msg = msg + "task_type is empty, " + if worker == "" or task_type is None: + msg = msg + "worker is empty, " + if timeout < 1: + msg = msg + "job timeout is smaller than 0ms, " + if max_jobs_to_activate < 1: + msg = msg + "max_jobs_to_activate is smaller than 0ms, " -class InvalidCompleteJob(Exception): - """ - GRPC_STATUS_NOT_FOUND - Returned if: - no job exists with the given job key. Note that since jobs are removed once completed, it could be that this job did exist at some point. - - GRPC_STATUS_FAILED_PRECONDITION - Returned if: - the job was marked as failed. In that case, the related incident must be resolved before the job can be activated again and completed. - """ - pass + super().__init__(msg) class JobAlreadyFailed(Exception): - pass + def __init__(self, job_key: int): + super().__init__(f"Job {job_key} already failed") + self.job_key = job_key class JobNotFound(Exception): - pass - - -class InvalidFailJob(Exception): - """ - GRPC_STATUS_NOT_FOUND - Returned if: - no job was found with the given key - - GRPC_STATUS_FAILED_PRECONDITION - Returned if: - the job was not activated - the job is already in a failed state, i.e. ran out of retries - """ - pass + def __init__(self, job_key: int): + super().__init__(f"Job {job_key} not found") + self.job_key = job_key -class InvalidDeployWorkflow(Exception): - """ - GRPC_STATUS_INVALID_ARGUMENT - Returned if: - no resources given. - if at least one resource is invalid. A resource is considered invalid if: - it is not a BPMN or YAML file (currently detected through the file extension) - the resource data is not deserializable (e.g. detected as BPMN, but it's broken XML) - the workflow is invalid (e.g. an event-based gateway has an outgoing sequence flow to a task) - """ +class WorkflowInvalid(Exception): pass class MessageAlreadyExists(Exception): - """ - GRPC_STATUS_ALREADY_EXISTS - Returned if: - a message with the same ID was previously published (and is still alive) - """ - pass - - -class IncidentNotFound(Exception): - """ - GRPC_STATUS_NOT_FOUND - Returned if: - no incident with the given key exists - """ pass @@ -124,49 +65,7 @@ class InvalidJSON(Exception): pass -class InvalidSetVariables(Exception): - """ - GRPC_STATUS_NOT_FOUND - Returned if: - no element with the given elementInstanceKey was exists - - GRPC_STATUS_INVALID_ARGUMENT - Returned if: - the given payload is not a valid JSON document; all payloads are expected to be valid JSON documents where the root node is an object. - """ - pass - - -class InvalidThrowError(Exception): - """ - Errors - GRPC_STATUS_NOT_FOUND - Returned if: - no job was found with the given key - - GRPC_STATUS_FAILED_PRECONDITION - Returned if: - the job is already in a failed state, i.e. ran out of retries - """ - pass - - -class InvalidJobRetries(Exception): - """ - Errors - GRPC_STATUS_NOT_FOUND - Returned if: - no job exists with the given key - - GRPC_STATUS_INVALID_ARGUMENT - Returned if: - retries is not greater than 0 - """ - pass - - class ZeebeBackPressure(Exception): - # TODO: Think about deploying some kind of retry strategy instead of raising this pass diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index b133f37f..6cee76d9 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -56,8 +56,8 @@ def activate_jobs(self, task_type: str, worker: str, timeout: int, max_jobs_to_a logging.debug(f'Got job: {context} from zeebe') yield context except grpc.RpcError as rpc_error: - if self._is_error_of_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): - raise InvalidActivateJobs() # TODO make proper exception + if self.is_error_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): + raise ActivateJobsRequestInvalid(task_type, worker, timeout, max_jobs_to_activate) else: self._common_zeebe_grpc_errors(rpc_error) @@ -80,10 +80,10 @@ def complete_job(self, job_key: int, variables: Dict) -> CompleteJobResponse: try: return self.gateway_stub.CompleteJob(CompleteJobRequest(jobKey=job_key, variables=json.dumps(variables))) except grpc.RpcError as rpc_error: - if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise JobNotFound() - elif self._is_error_of_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - raise InvalidCompleteJob() # TODO proper exception + if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise JobNotFound(job_key=job_key) + elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): + raise JobAlreadyFailed(job_key=job_key) else: self._common_zeebe_grpc_errors(rpc_error) @@ -91,10 +91,10 @@ def fail_job(self, job_key: int, message: str) -> FailJobResponse: try: return self.gateway_stub.FailJob(FailJobRequest(jobKey=job_key, errorMessage=message)) except grpc.RpcError as rpc_error: - if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise JobNotFound() - elif self._is_error_of_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - InvalidFailJob() # TODO make good exception + if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise JobNotFound(job_key=job_key) + elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): + raise JobAlreadyFailed(job_key=job_key) else: self._common_zeebe_grpc_errors(rpc_error) @@ -103,10 +103,10 @@ def throw_error(self, job_key: int, message: str) -> ThrowErrorResponse: return self.gateway_stub.ThrowError( ThrowErrorRequest(jobKey=job_key, errorMessage=message)) except grpc.RpcError as rpc_error: - if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise JobNotFound() - elif self._is_error_of_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - InvalidThrowError() # TODO make good exception + if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise JobNotFound(job_key=job_key) + elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): + raise JobAlreadyFailed(job_key=job_key) else: self._common_zeebe_grpc_errors(rpc_error) @@ -117,12 +117,7 @@ def create_workflow_instance(self, bpmn_process_id: str, version: int, variables variables=json.dumps(variables))) return response.workflowInstanceKey except grpc.RpcError as rpc_error: - if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) - elif self._is_error_of_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): - raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) # TODO: Change exception - else: - self._common_zeebe_grpc_errors(rpc_error) + self._create_workflow_errors(rpc_error, bpmn_process_id, version, variables) def create_workflow_instance_with_result(self, bpmn_process_id: str, version: int, variables: Dict, timeout: int, variables_to_fetch) -> Dict: @@ -134,19 +129,26 @@ def create_workflow_instance_with_result(self, bpmn_process_id: str, version: in requestTimeout=timeout, fetchVariables=variables_to_fetch)) return json.loads(response.variables) except grpc.RpcError as rpc_error: - if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) - elif self._is_error_of_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): - raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) # TODO: Change exception - else: - self._common_zeebe_grpc_errors(rpc_error) + self._create_workflow_errors(rpc_error, bpmn_process_id, version, variables) + + def _create_workflow_errors(self, rpc_error: grpc.RpcError, bpmn_process_id: str, version: int, + variables: Dict) -> None: + if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): + raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) + elif self.is_error_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): + raise InvalidJSON( + f"Cannot start workflow: {bpmn_process_id} with version {version}. Variables: {variables}") + elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): + raise WorkflowHasNoStartEvent(bpmn_process_id=bpmn_process_id) + else: + self._common_zeebe_grpc_errors(rpc_error) def cancel_workflow_instance(self, workflow_instance_key: int) -> None: try: self.gateway_stub.CancelWorkflowInstance( CancelWorkflowInstanceRequest(workflowInstanceKey=workflow_instance_key)) except grpc.RpcError as rpc_error: - if self._is_error_of_status(rpc_error, grpc.StatusCode.NOT_FOUND): + if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): raise WorkflowInstanceNotFound(workflow_instance_key=workflow_instance_key) else: self._common_zeebe_grpc_errors(rpc_error) @@ -159,37 +161,38 @@ def publish_message(self, name: str, correlation_key: str, time_to_live_in_milli timeToLive=time_to_live_in_milliseconds, variables=json.dumps(variables))) except grpc.RpcError as rpc_error: - if self._is_error_of_status(rpc_error, grpc.StatusCode.ALREADY_EXISTS): + if self.is_error_status(rpc_error, grpc.StatusCode.ALREADY_EXISTS): raise MessageAlreadyExists() else: self._common_zeebe_grpc_errors(rpc_error) def deploy_workflow(self, *workflow_file_path: str) -> DeployWorkflowResponse: + try: return self.gateway_stub.DeployWorkflow( DeployWorkflowRequest(workflows=map(self._get_workflow_request_object, workflow_file_path))) except grpc.RpcError as rpc_error: - if self._is_error_of_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): - raise InvalidDeployWorkflow() + if self.is_error_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): + raise WorkflowInvalid() else: self._common_zeebe_grpc_errors(rpc_error) + @staticmethod + def _get_workflow_request_object(workflow_file_path: str) -> WorkflowRequestObject: + with open(workflow_file_path, 'rb') as file: + return WorkflowRequestObject(name=os.path.split(workflow_file_path)[-1], + definition=file.read()) + def _common_zeebe_grpc_errors(self, rpc_error: grpc.RpcError): - if self._is_error_of_status(rpc_error, grpc.StatusCode.RESOURCE_EXHAUSTED): + if self.is_error_status(rpc_error, grpc.StatusCode.RESOURCE_EXHAUSTED): raise ZeebeBackPressure() - elif self._is_error_of_status(rpc_error, grpc.StatusCode.UNAVAILABLE): + elif self.is_error_status(rpc_error, grpc.StatusCode.UNAVAILABLE): raise ZeebeGatewayUnavailable() - elif self._is_error_of_status(rpc_error, grpc.StatusCode.INTERNAL): + elif self.is_error_status(rpc_error, grpc.StatusCode.INTERNAL): raise ZeebeInternalError() else: raise rpc_error @staticmethod - def _get_workflow_request_object(workflow_file_path: str) -> WorkflowRequestObject: - with open(workflow_file_path, 'rb') as file: - return WorkflowRequestObject(name=os.path.split(workflow_file_path)[-1], - definition=file.read()) - - @staticmethod - def _is_error_of_status(rpc_error: grpc.RpcError, status_code: grpc.StatusCode): + def is_error_status(rpc_error: grpc.RpcError, status_code: grpc.StatusCode): return rpc_error.args[0].code == status_code From eaacdc70481d75cff6b9a37d9f44553b3caa958d Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 25 Aug 2020 22:18:29 +0300 Subject: [PATCH 14/57] Adapter inherits from object --- pyzeebe/grpc_internals/zeebe_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index 6cee76d9..ac944b92 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -11,7 +11,7 @@ from pyzeebe.task.task_context import TaskContext -class ZeebeAdapter: +class ZeebeAdapter(object): def __init__(self, hostname: str = None, port: int = None, channel: grpc.Channel = None, **kwargs): self._connection_uri = f'{hostname}:{port}' or os.getenv('ZEEBE_ADDRESS') or 'localhost:26500' if channel: From 417a079bc31e9a91584d202dabf104e4cd4cb6a9 Mon Sep 17 00:00:00 2001 From: JonatanMartens <40060128+JonatanMartens@users.noreply.github.com> Date: Tue, 1 Sep 2020 19:09:22 +0300 Subject: [PATCH 15/57] [IN PROGRESS] Added some tests --- examples/worker.py | 2 ++ pyzeebe/common/gateway_mock.py | 34 ++++++++++++++++++-- pyzeebe/grpc_internals/zeebe_adapter_test.py | 19 +++++++++-- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/examples/worker.py b/examples/worker.py index 8ffac3bc..9a16ce75 100644 --- a/examples/worker.py +++ b/examples/worker.py @@ -1,7 +1,9 @@ +import logging from typing import Dict from pyzeebe import Task, TaskContext, TaskStatusController, ZeebeWorker +logging.basicConfig(level=logging.DEBUG) def example_task() -> Dict: return {'output': f'Hello world, test!'} diff --git a/pyzeebe/common/gateway_mock.py b/pyzeebe/common/gateway_mock.py index 18e2c366..82bddc00 100644 --- a/pyzeebe/common/gateway_mock.py +++ b/pyzeebe/common/gateway_mock.py @@ -1,13 +1,16 @@ +import json from random import randint -from typing import List +from typing import List, Dict from unittest.mock import patch from uuid import uuid4 import grpc from pyzeebe.common.random_utils import RANDOM_RANGE +from pyzeebe.common.random_utils import random_task_context from pyzeebe.grpc_internals.zeebe_pb2 import * from pyzeebe.grpc_internals.zeebe_pb2_grpc import GatewayServicer +from pyzeebe.task.task_context import TaskContext @patch('grpc.insecure_channel') @@ -20,19 +23,44 @@ class GatewayMock(GatewayServicer): def __init__(self): self.deployed_workflows = {} - self.active_jobs = {} + self.active_jobs: Dict[str, TaskContext] = {} + + def ActivateJobs(self, request, context): + return ActivateJobsResponse(jobs=self.get_active_job_generator(request.type)) + + def get_active_job_generator(self, task_type: str): + for active_job in self.active_jobs.values(): + if active_job.type == task_type: + yield ActivatedJob(key=active_job.key, type=active_job.type, + workflowInstanceKey=active_job.workflow_instance_key, + bpmnProcessId=active_job.bpmn_process_id, + workflowDefinitionVersion=active_job.workflow_definition_version, + workflowKey=active_job.workflow_key, + elementId=active_job.element_id, + elementInstanceKey=active_job.element_instance_key, + customHeaders=json.dumps(active_job.custom_headers), + worker=active_job.worker, retries=active_job.retries, + deadline=active_job.deadline, + variables=json.dumps(active_job.variables)) def CompleteJob(self, request, context): return CompleteJobResponse() def FailJob(self, request, context): - return FailJobResponse() + if request.jobKey in self.active_jobs.keys(): + return FailJobResponse() + else: + context.set_code(grpc.StatusCode.NOT_FOUND) + return FailJobResponse() def ThrowError(self, request, context): return ThrowErrorResponse() def CreateWorkflowInstance(self, request, context): if request.bpmnProcessId in self.deployed_workflows.keys(): + for task in self.deployed_workflows[request.bpmnProcessId]['tasks']: + task_context = random_task_context(task) + self.active_jobs[task_context.key] = task_context return CreateWorkflowInstanceResponse(workflowKey=randint(0, RANDOM_RANGE), bpmnProcessId=request.bpmnProcessId, version=request.version, workflowInstanceKey=randint(0, RANDOM_RANGE)) diff --git a/pyzeebe/grpc_internals/zeebe_adapter_test.py b/pyzeebe/grpc_internals/zeebe_adapter_test.py index 6e979914..59d4a441 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_test.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_test.py @@ -4,10 +4,12 @@ import grpc import pytest +from pyzeebe.common.exceptions import * from pyzeebe.common.gateway_mock import GatewayMock from pyzeebe.common.random_utils import RANDOM_RANGE from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.grpc_internals.zeebe_pb2 import * +from pyzeebe.task.task import Task zeebe_adapter: ZeebeAdapter @@ -71,11 +73,24 @@ def test_complete_job(): assert isinstance(response, CompleteJobResponse) -def test_fail_job(): - response = zeebe_adapter.fail_job(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) +def test_fail_job(grpc_servicer): + mock_workflow_id = str(uuid4()) + mock_workflow_version = randint(0, 10) + mock_task_type = str(uuid4()) + task = Task(task_type=mock_task_type, task_handler=lambda x: x, exception_handler=lambda x: x) + grpc_servicer.mock_deploy_workflow(mock_workflow_id, version=mock_workflow_version, tasks=[task]) + zeebe_adapter.create_workflow_instance(bpmn_process_id=mock_workflow_id, version=randint(0, 10), variables={}) + job = zeebe_adapter.activate_jobs(task_type=mock_task_type, max_jobs_to_activate=1, request_timeout=10, + timeout=100, variables_to_fetch=[], worker=str(uuid4())) + response = zeebe_adapter.fail_job(job_key=next(job).key, message=str(uuid4())) assert isinstance(response, FailJobResponse) +def test_fail_job_not_found(): + with pytest.raises(JobNotFound): + zeebe_adapter.fail_job(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) + + def test_throw_error(): response = zeebe_adapter.throw_error(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) assert isinstance(response, ThrowErrorResponse) From 7c66f4f0e56f1d0c57151e3d922d5832ff751150 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 1 Sep 2020 21:45:50 +0300 Subject: [PATCH 16/57] [FIXED] Gateway mock not implementing ActivateJobs correctly --- pyzeebe/common/gateway_mock.py | 35 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/pyzeebe/common/gateway_mock.py b/pyzeebe/common/gateway_mock.py index 82bddc00..cd5ac3fd 100644 --- a/pyzeebe/common/gateway_mock.py +++ b/pyzeebe/common/gateway_mock.py @@ -6,10 +6,10 @@ import grpc -from pyzeebe.common.random_utils import RANDOM_RANGE -from pyzeebe.common.random_utils import random_task_context +from pyzeebe.common.random_utils import RANDOM_RANGE, random_task_context from pyzeebe.grpc_internals.zeebe_pb2 import * from pyzeebe.grpc_internals.zeebe_pb2_grpc import GatewayServicer +from pyzeebe.task.task import Task from pyzeebe.task.task_context import TaskContext @@ -26,22 +26,21 @@ def __init__(self): self.active_jobs: Dict[str, TaskContext] = {} def ActivateJobs(self, request, context): - return ActivateJobsResponse(jobs=self.get_active_job_generator(request.type)) - - def get_active_job_generator(self, task_type: str): + jobs = [] for active_job in self.active_jobs.values(): - if active_job.type == task_type: - yield ActivatedJob(key=active_job.key, type=active_job.type, - workflowInstanceKey=active_job.workflow_instance_key, - bpmnProcessId=active_job.bpmn_process_id, - workflowDefinitionVersion=active_job.workflow_definition_version, - workflowKey=active_job.workflow_key, - elementId=active_job.element_id, - elementInstanceKey=active_job.element_instance_key, - customHeaders=json.dumps(active_job.custom_headers), - worker=active_job.worker, retries=active_job.retries, - deadline=active_job.deadline, - variables=json.dumps(active_job.variables)) + if active_job.type == request.type: + jobs.append(ActivatedJob(key=active_job.key, type=active_job.type, + workflowInstanceKey=active_job.workflow_instance_key, + bpmnProcessId=active_job.bpmn_process_id, + workflowDefinitionVersion=active_job.workflow_definition_version, + workflowKey=active_job.workflow_key, + elementId=active_job.element_id, + elementInstanceKey=active_job.element_instance_key, + customHeaders=json.dumps(active_job.custom_headers), + worker=active_job.worker, retries=active_job.retries, + deadline=active_job.deadline, + variables=json.dumps(active_job.variables))) + yield ActivateJobsResponse(jobs=jobs) def CompleteJob(self, request, context): return CompleteJobResponse() @@ -92,6 +91,6 @@ def DeployWorkflow(self, request, context): def PublishMessage(self, request, context): return PublishMessageResponse() - def mock_deploy_workflow(self, bpmn_process_id: str, version: int, tasks: List[str]): + def mock_deploy_workflow(self, bpmn_process_id: str, version: int, tasks: List[Task]): self.deployed_workflows[bpmn_process_id] = {'bpmn_process_id': bpmn_process_id, 'version': version, 'tasks': tasks} From d70d61d76452f2968cc2120bcda01ef8ab40a9ec Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 1 Sep 2020 22:10:03 +0300 Subject: [PATCH 17/57] [ADDED] TaskStatus --- pyzeebe/task/task_context.py | 6 +++++- pyzeebe/task/task_status.py | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 pyzeebe/task/task_status.py diff --git a/pyzeebe/task/task_context.py b/pyzeebe/task/task_context.py index 33a58fbb..4c88e972 100644 --- a/pyzeebe/task/task_context.py +++ b/pyzeebe/task/task_context.py @@ -1,10 +1,13 @@ from typing import Dict +from pyzeebe.task.task_status import TaskStatus + class TaskContext(object): def __init__(self, key: int, _type: str, workflow_instance_key: int, bpmn_process_id: str, workflow_definition_version: int, workflow_key: int, element_id: str, element_instance_key: int, - custom_headers: Dict, worker: str, retries: int, deadline: int, variables: Dict): + custom_headers: Dict, worker: str, retries: int, deadline: int, variables: Dict, + status: TaskStatus = TaskStatus.Running): self.key = key self.type = _type self.workflow_instance_key = workflow_instance_key @@ -18,6 +21,7 @@ def __init__(self, key: int, _type: str, workflow_instance_key: int, bpmn_proces self.retries = retries self.deadline = deadline self.variables = variables + self.status = status def __repr__(self): return str({'jobKey': self.key, 'taskType': self.type, 'workflowInstanceKey': self.workflow_instance_key, diff --git a/pyzeebe/task/task_status.py b/pyzeebe/task/task_status.py new file mode 100644 index 00000000..0655b701 --- /dev/null +++ b/pyzeebe/task/task_status.py @@ -0,0 +1,8 @@ +from enum import Enum + + +class TaskStatus(Enum): + Running = 'Running' + Completed = 'Completed' + Failed = 'Failed' + ErrorThrown = 'ErrorThrown' From aa6625274bdb3e8277ed19e236677ec292f12558 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 1 Sep 2020 22:10:54 +0300 Subject: [PATCH 18/57] [FIXED] GatewayMock FailJob --- pyzeebe/common/gateway_mock.py | 8 ++++- pyzeebe/grpc_internals/zeebe_adapter_test.py | 36 ++++++++++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/pyzeebe/common/gateway_mock.py b/pyzeebe/common/gateway_mock.py index cd5ac3fd..a66ac0fb 100644 --- a/pyzeebe/common/gateway_mock.py +++ b/pyzeebe/common/gateway_mock.py @@ -11,6 +11,7 @@ from pyzeebe.grpc_internals.zeebe_pb2_grpc import GatewayServicer from pyzeebe.task.task import Task from pyzeebe.task.task_context import TaskContext +from pyzeebe.task.task_status import TaskStatus @patch('grpc.insecure_channel') @@ -23,7 +24,7 @@ class GatewayMock(GatewayServicer): def __init__(self): self.deployed_workflows = {} - self.active_jobs: Dict[str, TaskContext] = {} + self.active_jobs: Dict[int, TaskContext] = {} def ActivateJobs(self, request, context): jobs = [] @@ -47,6 +48,11 @@ def CompleteJob(self, request, context): def FailJob(self, request, context): if request.jobKey in self.active_jobs.keys(): + active_job = self.active_jobs.get(request.jobKey) + if active_job.status != TaskStatus.Running: + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + else: + active_job.status = TaskStatus.Failed return FailJobResponse() else: context.set_code(grpc.StatusCode.NOT_FOUND) diff --git a/pyzeebe/grpc_internals/zeebe_adapter_test.py b/pyzeebe/grpc_internals/zeebe_adapter_test.py index 59d4a441..beda3e0f 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_test.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_test.py @@ -6,10 +6,11 @@ from pyzeebe.common.exceptions import * from pyzeebe.common.gateway_mock import GatewayMock -from pyzeebe.common.random_utils import RANDOM_RANGE +from pyzeebe.common.random_utils import RANDOM_RANGE, random_task_context from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.grpc_internals.zeebe_pb2 import * from pyzeebe.task.task import Task +from pyzeebe.task.task_context import TaskContext zeebe_adapter: ZeebeAdapter @@ -74,15 +75,9 @@ def test_complete_job(): def test_fail_job(grpc_servicer): - mock_workflow_id = str(uuid4()) - mock_workflow_version = randint(0, 10) - mock_task_type = str(uuid4()) - task = Task(task_type=mock_task_type, task_handler=lambda x: x, exception_handler=lambda x: x) - grpc_servicer.mock_deploy_workflow(mock_workflow_id, version=mock_workflow_version, tasks=[task]) - zeebe_adapter.create_workflow_instance(bpmn_process_id=mock_workflow_id, version=randint(0, 10), variables={}) - job = zeebe_adapter.activate_jobs(task_type=mock_task_type, max_jobs_to_activate=1, request_timeout=10, - timeout=100, variables_to_fetch=[], worker=str(uuid4())) - response = zeebe_adapter.fail_job(job_key=next(job).key, message=str(uuid4())) + mock_task_type = create_random_task_and_activate(grpc_servicer) + job = get_first_active_job(mock_task_type) + response = zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) assert isinstance(response, FailJobResponse) @@ -91,6 +86,14 @@ def test_fail_job_not_found(): zeebe_adapter.fail_job(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) +def test_fail_job_already_failed(grpc_servicer): + mock_task_type = create_random_task_and_activate(grpc_servicer) + job = get_first_active_job(mock_task_type) + zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) + with pytest.raises(JobAlreadyFailed): + zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) + + def test_throw_error(): response = zeebe_adapter.throw_error(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) assert isinstance(response, ThrowErrorResponse) @@ -117,3 +120,16 @@ def test_publish_message(): response = zeebe_adapter.publish_message(name=str(uuid4()), variables={}, correlation_key=str(uuid4()), time_to_live_in_milliseconds=randint(0, RANDOM_RANGE)) assert isinstance(response, PublishMessageResponse) + + +def create_random_task_and_activate(grpc_servicer) -> str: + mock_task_type = str(uuid4()) + task = Task(task_type=mock_task_type, task_handler=lambda x: x, exception_handler=lambda x: x) + task_context = random_task_context(task) + grpc_servicer.active_jobs[task_context.key] = task_context + return mock_task_type + + +def get_first_active_job(task_type) -> TaskContext: + return next(zeebe_adapter.activate_jobs(task_type=task_type, max_jobs_to_activate=1, request_timeout=10, + timeout=100, variables_to_fetch=[], worker=str(uuid4()))) From d196ecbfccc44c2b1cc535750e18892c1453775c Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 1 Sep 2020 22:25:08 +0300 Subject: [PATCH 19/57] [ADDED] Tests for throw_error, complete_job --- pyzeebe/common/exceptions.py | 4 +- pyzeebe/common/gateway_mock.py | 29 +++++++++--- pyzeebe/grpc_internals/zeebe_adapter.py | 6 +-- pyzeebe/grpc_internals/zeebe_adapter_test.py | 48 ++++++++++++++++---- 4 files changed, 67 insertions(+), 20 deletions(-) diff --git a/pyzeebe/common/exceptions.py b/pyzeebe/common/exceptions.py index 3785a764..9ed52b08 100644 --- a/pyzeebe/common/exceptions.py +++ b/pyzeebe/common/exceptions.py @@ -37,9 +37,9 @@ def __init__(self, task_type: str, worker: str, timeout: int, max_jobs_to_activa super().__init__(msg) -class JobAlreadyFailed(Exception): +class JobAlreadyDeactivated(Exception): def __init__(self, job_key: int): - super().__init__(f"Job {job_key} already failed") + super().__init__(f"Job {job_key} was already stopped (Completed/Failed/Error)") self.job_key = job_key diff --git a/pyzeebe/common/gateway_mock.py b/pyzeebe/common/gateway_mock.py index a66ac0fb..894eee3d 100644 --- a/pyzeebe/common/gateway_mock.py +++ b/pyzeebe/common/gateway_mock.py @@ -44,22 +44,39 @@ def ActivateJobs(self, request, context): yield ActivateJobsResponse(jobs=jobs) def CompleteJob(self, request, context): - return CompleteJobResponse() + if request.jobKey in self.active_jobs.keys(): + active_job = self.active_jobs.get(request.jobKey) + self.handle_job(active_job, TaskStatus.Completed, context) + return CompleteJobResponse() + else: + context.set_code(grpc.StatusCode.NOT_FOUND) + return CompleteJobResponse() def FailJob(self, request, context): if request.jobKey in self.active_jobs.keys(): active_job = self.active_jobs.get(request.jobKey) - if active_job.status != TaskStatus.Running: - context.set_code(grpc.StatusCode.FAILED_PRECONDITION) - else: - active_job.status = TaskStatus.Failed + self.handle_job(active_job, TaskStatus.Failed, context) return FailJobResponse() else: context.set_code(grpc.StatusCode.NOT_FOUND) return FailJobResponse() def ThrowError(self, request, context): - return ThrowErrorResponse() + if request.jobKey in self.active_jobs.keys(): + active_job = self.active_jobs.get(request.jobKey) + self.handle_job(active_job, TaskStatus.ErrorThrown, context) + return CompleteJobResponse() + else: + context.set_code(grpc.StatusCode.NOT_FOUND) + return CompleteJobResponse() + + @staticmethod + def handle_job(job: TaskContext, status_on_deactivate: TaskStatus, context): + if job.status != TaskStatus.Running: + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + else: + job.status = status_on_deactivate + return context def CreateWorkflowInstance(self, request, context): if request.bpmnProcessId in self.deployed_workflows.keys(): diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index ac944b92..6d52052d 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -83,7 +83,7 @@ def complete_job(self, job_key: int, variables: Dict) -> CompleteJobResponse: if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): raise JobNotFound(job_key=job_key) elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - raise JobAlreadyFailed(job_key=job_key) + raise JobAlreadyDeactivated(job_key=job_key) else: self._common_zeebe_grpc_errors(rpc_error) @@ -94,7 +94,7 @@ def fail_job(self, job_key: int, message: str) -> FailJobResponse: if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): raise JobNotFound(job_key=job_key) elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - raise JobAlreadyFailed(job_key=job_key) + raise JobAlreadyDeactivated(job_key=job_key) else: self._common_zeebe_grpc_errors(rpc_error) @@ -106,7 +106,7 @@ def throw_error(self, job_key: int, message: str) -> ThrowErrorResponse: if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): raise JobNotFound(job_key=job_key) elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - raise JobAlreadyFailed(job_key=job_key) + raise JobAlreadyDeactivated(job_key=job_key) else: self._common_zeebe_grpc_errors(rpc_error) diff --git a/pyzeebe/grpc_internals/zeebe_adapter_test.py b/pyzeebe/grpc_internals/zeebe_adapter_test.py index beda3e0f..38ad091e 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_test.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_test.py @@ -69,14 +69,29 @@ def test_connectivity_shutdown(): zeebe_adapter._check_connectivity(grpc.ChannelConnectivity.SHUTDOWN) -def test_complete_job(): - response = zeebe_adapter.complete_job(job_key=randint(0, RANDOM_RANGE), variables={}) +def test_complete_job(grpc_servicer): + task_type = create_random_task_and_activate(grpc_servicer) + job = get_first_active_job(task_type) + response = zeebe_adapter.complete_job(job_key=job.key, variables={}) assert isinstance(response, CompleteJobResponse) +def test_complete_job_not_found(grpc_servicer): + with pytest.raises(JobNotFound): + zeebe_adapter.complete_job(job_key=randint(0, RANDOM_RANGE), variables={}) + + +def test_complete_job_already_completed(grpc_servicer): + task_type = create_random_task_and_activate(grpc_servicer) + job = get_first_active_job(task_type) + zeebe_adapter.complete_job(job_key=job.key, variables={}) + with pytest.raises(JobAlreadyDeactivated): + zeebe_adapter.complete_job(job_key=job.key, variables={}) + + def test_fail_job(grpc_servicer): - mock_task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(mock_task_type) + task_type = create_random_task_and_activate(grpc_servicer) + job = get_first_active_job(task_type) response = zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) assert isinstance(response, FailJobResponse) @@ -87,18 +102,33 @@ def test_fail_job_not_found(): def test_fail_job_already_failed(grpc_servicer): - mock_task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(mock_task_type) + task_type = create_random_task_and_activate(grpc_servicer) + job = get_first_active_job(task_type) zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) - with pytest.raises(JobAlreadyFailed): + with pytest.raises(JobAlreadyDeactivated): zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) -def test_throw_error(): - response = zeebe_adapter.throw_error(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) +def test_throw_error(grpc_servicer): + task_type = create_random_task_and_activate(grpc_servicer) + job = get_first_active_job(task_type) + response = zeebe_adapter.throw_error(job_key=job.key, message=str(uuid4())) assert isinstance(response, ThrowErrorResponse) +def test_throw_error_job_not_found(): + with pytest.raises(JobNotFound): + zeebe_adapter.throw_error(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) + + +def test_throw_error_already_thrown(grpc_servicer): + task_type = create_random_task_and_activate(grpc_servicer) + job = get_first_active_job(task_type) + zeebe_adapter.throw_error(job_key=job.key, message=str(uuid4())) + with pytest.raises(JobAlreadyDeactivated): + zeebe_adapter.throw_error(job_key=job.key, message=str(uuid4())) + + def test_create_workflow_instance(grpc_servicer): bpmn_process_id = str(uuid4()) version = randint(0, 10) From fd206e267cd41efd96212b7a862e414a328002b7 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Tue, 1 Sep 2020 23:12:40 +0300 Subject: [PATCH 20/57] [ADDED] ActivateJobs behavior for mock --- pyzeebe/common/gateway_mock.py | 16 +++++++ pyzeebe/grpc_internals/zeebe_adapter.py | 2 +- pyzeebe/grpc_internals/zeebe_adapter_test.py | 46 +++++++++++++++++++- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/pyzeebe/common/gateway_mock.py b/pyzeebe/common/gateway_mock.py index 894eee3d..5e3fe079 100644 --- a/pyzeebe/common/gateway_mock.py +++ b/pyzeebe/common/gateway_mock.py @@ -27,6 +27,22 @@ def __init__(self): self.active_jobs: Dict[int, TaskContext] = {} def ActivateJobs(self, request, context): + if not request.type: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + return ActivateJobsResponse() + + if request.maxJobsToActivate <= 0: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + return ActivateJobsResponse() + + if request.timeout <= 0: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + return ActivateJobsResponse() + + if not request.worker: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + return ActivateJobsResponse() + jobs = [] for active_job in self.active_jobs.values(): if active_job.type == request.type: diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index 6d52052d..eef79ee6 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -195,4 +195,4 @@ def _common_zeebe_grpc_errors(self, rpc_error: grpc.RpcError): @staticmethod def is_error_status(rpc_error: grpc.RpcError, status_code: grpc.StatusCode): - return rpc_error.args[0].code == status_code + return rpc_error._state.code == status_code diff --git a/pyzeebe/grpc_internals/zeebe_adapter_test.py b/pyzeebe/grpc_internals/zeebe_adapter_test.py index 38ad091e..c6367cdf 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_test.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_test.py @@ -69,6 +69,45 @@ def test_connectivity_shutdown(): zeebe_adapter._check_connectivity(grpc.ChannelConnectivity.SHUTDOWN) +def test_activate_jobs(grpc_servicer): + task_type = create_random_task_and_activate(grpc_servicer) + active_jobs_count = randint(4, 100) + counter = 0 + for i in range(0, active_jobs_count): + create_random_task_and_activate(grpc_servicer, task_type) + + for job in zeebe_adapter.activate_jobs(task_type=task_type, worker=str(uuid4()), timeout=randint(10, 100), + request_timeout=100, max_jobs_to_activate=1, variables_to_fetch=[]): + counter += 1 + assert isinstance(job, TaskContext) + assert counter == active_jobs_count + 1 + + +def test_activate_jobs_invalid_worker(): + with pytest.raises(ActivateJobsRequestInvalid): + next(zeebe_adapter.activate_jobs(task_type=str(uuid4()), worker=None, timeout=randint(10, 100), + request_timeout=100, + max_jobs_to_activate=1, variables_to_fetch=[])) + + +def test_activate_jobs_invalid_job_timeout(): + with pytest.raises(ActivateJobsRequestInvalid): + next(zeebe_adapter.activate_jobs(task_type=str(uuid4()), worker=str(uuid4()), timeout=0, + request_timeout=100, max_jobs_to_activate=1, variables_to_fetch=[])) + + +def test_activate_jobs_invalid_task_type(): + with pytest.raises(ActivateJobsRequestInvalid): + next(zeebe_adapter.activate_jobs(task_type=None, worker=str(uuid4()), timeout=randint(10, 100), + request_timeout=100, max_jobs_to_activate=1, variables_to_fetch=[])) + + +def test_activate_jobs_invalid_max_jobs(): + with pytest.raises(ActivateJobsRequestInvalid): + next(zeebe_adapter.activate_jobs(task_type=str(uuid4()), worker=str(uuid4()), timeout=randint(10, 100), + request_timeout=100, max_jobs_to_activate=0, variables_to_fetch=[])) + + def test_complete_job(grpc_servicer): task_type = create_random_task_and_activate(grpc_servicer) job = get_first_active_job(task_type) @@ -152,8 +191,11 @@ def test_publish_message(): assert isinstance(response, PublishMessageResponse) -def create_random_task_and_activate(grpc_servicer) -> str: - mock_task_type = str(uuid4()) +def create_random_task_and_activate(grpc_servicer, task_type: str = None) -> str: + if task_type: + mock_task_type = task_type + else: + mock_task_type = str(uuid4()) task = Task(task_type=mock_task_type, task_handler=lambda x: x, exception_handler=lambda x: x) task_context = random_task_context(task) grpc_servicer.active_jobs[task_context.key] = task_context From 84ebe6ee94c9b09b263b6c80c300b182caa46e2f Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Wed, 2 Sep 2020 22:21:06 +0300 Subject: [PATCH 21/57] [FIXED] Documentation --- README.md | 27 +++++++++++++++++++++++++++ examples/client.py | 20 ++++++++++++++++++++ examples/worker.py | 1 - 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 examples/client.py diff --git a/README.md b/README.md index 62878135..e895eead 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,33 @@ worker.add_task(task) # Add task to zeebe worker worker.work() # Now every time that a task with type example is called example_task will be called ``` +### Client + +```python +from pyzeebe import ZeebeClient + +# Create a zeebe client +zeebe_client = ZeebeClient(hostname='localhost', port=26500) + +# Run a workflow +workflow_instance_key = zeebe_client.run_workflow(bpmn_process_id='My zeebe workflow', variables={}) + +# Run a workflow and receive the result +workflow_result = zeebe_client.run_workflow_with_result(bpmn_process_id='My zeebe workflow', + timeout=10000) # Will wait 10000 milliseconds (10 seconds) + +# Deploy a bpmn workflow definition +zeebe_client.deploy_workflow('workflow.bpmn') + +# Cancel a running workflow +zeebe_client.cancel_workflow_instance(workflow_instance_key=12345) + +# Publish message +zeebe_client.publish_message(name='message_name', correlation_key='some_id') + + +``` + ## Tests Use the package manager [pip](https://pip.pypa.io/en/stable/) to install pyzeebe diff --git a/examples/client.py b/examples/client.py new file mode 100644 index 00000000..ed4baaff --- /dev/null +++ b/examples/client.py @@ -0,0 +1,20 @@ +from pyzeebe import ZeebeClient + +# Create a zeebe client +zeebe_client = ZeebeClient(hostname='localhost', port=26500) + +# Run a workflow +workflow_instance_key = zeebe_client.run_workflow(bpmn_process_id='My zeebe workflow', variables={}) + +# Run a workflow and receive the result +workflow_result = zeebe_client.run_workflow_with_result(bpmn_process_id='My zeebe workflow', + timeout=10000) # Will wait 10000 milliseconds (10 seconds) + +# Deploy a bpmn workflow definition +zeebe_client.deploy_workflow('workflow.bpmn') + +# Cancel a running workflow +zeebe_client.cancel_workflow_instance(workflow_instance_key=12345) + +# Publish message +zeebe_client.publish_message(name='message_name', correlation_key='some_id') diff --git a/examples/worker.py b/examples/worker.py index 8ffac3bc..f301e21a 100644 --- a/examples/worker.py +++ b/examples/worker.py @@ -14,7 +14,6 @@ def example_exception_handler(exc: Exception, context: TaskContext, controller: task = Task(task_type='test', task_handler=example_task, exception_handler=example_exception_handler) -task_2 = Task(task_type='test2', task_handler=example_task, exception_handler=example_exception_handler) worker = ZeebeWorker() # Will use environment variable ZEEBE_ADDRESS or localhost:26500 From 22705484dc724c4fb28b5d863d8e6d021a3002bd Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Wed, 2 Sep 2020 22:27:16 +0300 Subject: [PATCH 22/57] [ADDED] Test for file reading --- pyzeebe/grpc_internals/zeebe_adapter_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pyzeebe/grpc_internals/zeebe_adapter_test.py b/pyzeebe/grpc_internals/zeebe_adapter_test.py index 6e979914..7ccf3367 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_test.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_test.py @@ -1,4 +1,6 @@ +from io import BytesIO from random import randint +from unittest.mock import patch from uuid import uuid4 import grpc @@ -102,3 +104,11 @@ def test_publish_message(): response = zeebe_adapter.publish_message(name=str(uuid4()), variables={}, correlation_key=str(uuid4()), time_to_live_in_milliseconds=randint(0, RANDOM_RANGE)) assert isinstance(response, PublishMessageResponse) + + +def test_get_workflow_request_object(): + with patch('builtins.open') as mock_open: + mock_open.return_value = BytesIO() + file_path = str(uuid4()) + zeebe_adapter._get_workflow_request_object(file_path) + mock_open.assert_called_with(file_path, 'rb') From ae6ef0dc26c1fa00ad7089115feab3bc4f86ec1d Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Wed, 2 Sep 2020 22:32:13 +0300 Subject: [PATCH 23/57] [REMOVED] logging in example --- examples/worker.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/worker.py b/examples/worker.py index 9859252c..f301e21a 100644 --- a/examples/worker.py +++ b/examples/worker.py @@ -1,9 +1,7 @@ -import logging from typing import Dict from pyzeebe import Task, TaskContext, TaskStatusController, ZeebeWorker -logging.basicConfig(level=logging.DEBUG) def example_task() -> Dict: return {'output': f'Hello world, test!'} From fce23a35fbd49c64fa1bf8ff6329567bd6d8186e Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Wed, 2 Sep 2020 22:39:56 +0300 Subject: [PATCH 24/57] [IMPROVED] Using dataclass --- pyzeebe/common/random_utils.py | 2 +- pyzeebe/grpc_internals/zeebe_adapter.py | 2 +- pyzeebe/task/task_context.py | 42 ++++++++++--------------- 3 files changed, 18 insertions(+), 28 deletions(-) diff --git a/pyzeebe/common/random_utils.py b/pyzeebe/common/random_utils.py index f17c7e0f..84faeebc 100644 --- a/pyzeebe/common/random_utils.py +++ b/pyzeebe/common/random_utils.py @@ -9,7 +9,7 @@ def random_task_context(task: Task = Task(task_type='test', task_handler=lambda x: {'x': x}, exception_handler=lambda x, y, z: x)) -> TaskContext: - return TaskContext(_type=task.type, key=randint(0, RANDOM_RANGE), worker=str(uuid4()), + return TaskContext(type=task.type, key=randint(0, RANDOM_RANGE), worker=str(uuid4()), retries=randint(0, 10), workflow_instance_key=randint(0, RANDOM_RANGE), bpmn_process_id=str(uuid4()), workflow_definition_version=randint(0, 100), workflow_key=randint(0, RANDOM_RANGE), element_id=str(uuid4()), diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index eef79ee6..6d2c46f8 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -63,7 +63,7 @@ def activate_jobs(self, task_type: str, worker: str, timeout: int, max_jobs_to_a @staticmethod def _create_task_context_from_job(job) -> TaskContext: - return TaskContext(key=job.key, _type=job.type, + return TaskContext(key=job.key, type=job.type, workflow_instance_key=job.workflowInstanceKey, bpmn_process_id=job.bpmnProcessId, workflow_definition_version=job.workflowDefinitionVersion, diff --git a/pyzeebe/task/task_context.py b/pyzeebe/task/task_context.py index 4c88e972..96bda9ac 100644 --- a/pyzeebe/task/task_context.py +++ b/pyzeebe/task/task_context.py @@ -1,32 +1,22 @@ +from dataclasses import dataclass from typing import Dict from pyzeebe.task.task_status import TaskStatus +@dataclass class TaskContext(object): - def __init__(self, key: int, _type: str, workflow_instance_key: int, bpmn_process_id: str, - workflow_definition_version: int, workflow_key: int, element_id: str, element_instance_key: int, - custom_headers: Dict, worker: str, retries: int, deadline: int, variables: Dict, - status: TaskStatus = TaskStatus.Running): - self.key = key - self.type = _type - self.workflow_instance_key = workflow_instance_key - self.bpmn_process_id = bpmn_process_id - self.workflow_definition_version = workflow_definition_version - self.workflow_key = workflow_key - self.element_id = element_id - self.element_instance_key = element_instance_key - self.custom_headers = custom_headers - self.worker = worker - self.retries = retries - self.deadline = deadline - self.variables = variables - self.status = status - - def __repr__(self): - return str({'jobKey': self.key, 'taskType': self.type, 'workflowInstanceKey': self.workflow_instance_key, - 'bpmnProcessId': self.bpmn_process_id, - 'workflowDefinitionVersion': self.workflow_definition_version, 'workflowKey': self.workflow_key, - 'elementId': self.element_id, 'elementInstanceKey': self.element_instance_key, - 'customHeaders': self.custom_headers, 'worker': self.worker, 'retries': self.retries, - 'deadline': self.deadline, 'variables': self.variables}) + key: int + type: str + workflow_instance_key: int + bpmn_process_id: str + workflow_definition_version: int + workflow_key: int + element_id: str + element_instance_key: int + custom_headers: Dict + worker: str + retries: int + deadline: int + variables: Dict + status: TaskStatus = TaskStatus.Running From e3fb0c914da84b06b1b77141cc25da2ea7458a88 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Wed, 2 Sep 2020 22:45:42 +0300 Subject: [PATCH 25/57] Revert "[IMPROVED] Using dataclass" for compatibility reasons --- pyzeebe/common/random_utils.py | 2 +- pyzeebe/grpc_internals/zeebe_adapter.py | 2 +- pyzeebe/task/task_context.py | 42 +++++++++++++++---------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/pyzeebe/common/random_utils.py b/pyzeebe/common/random_utils.py index 84faeebc..f17c7e0f 100644 --- a/pyzeebe/common/random_utils.py +++ b/pyzeebe/common/random_utils.py @@ -9,7 +9,7 @@ def random_task_context(task: Task = Task(task_type='test', task_handler=lambda x: {'x': x}, exception_handler=lambda x, y, z: x)) -> TaskContext: - return TaskContext(type=task.type, key=randint(0, RANDOM_RANGE), worker=str(uuid4()), + return TaskContext(_type=task.type, key=randint(0, RANDOM_RANGE), worker=str(uuid4()), retries=randint(0, 10), workflow_instance_key=randint(0, RANDOM_RANGE), bpmn_process_id=str(uuid4()), workflow_definition_version=randint(0, 100), workflow_key=randint(0, RANDOM_RANGE), element_id=str(uuid4()), diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index 6d2c46f8..eef79ee6 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -63,7 +63,7 @@ def activate_jobs(self, task_type: str, worker: str, timeout: int, max_jobs_to_a @staticmethod def _create_task_context_from_job(job) -> TaskContext: - return TaskContext(key=job.key, type=job.type, + return TaskContext(key=job.key, _type=job.type, workflow_instance_key=job.workflowInstanceKey, bpmn_process_id=job.bpmnProcessId, workflow_definition_version=job.workflowDefinitionVersion, diff --git a/pyzeebe/task/task_context.py b/pyzeebe/task/task_context.py index 96bda9ac..4c88e972 100644 --- a/pyzeebe/task/task_context.py +++ b/pyzeebe/task/task_context.py @@ -1,22 +1,32 @@ -from dataclasses import dataclass from typing import Dict from pyzeebe.task.task_status import TaskStatus -@dataclass class TaskContext(object): - key: int - type: str - workflow_instance_key: int - bpmn_process_id: str - workflow_definition_version: int - workflow_key: int - element_id: str - element_instance_key: int - custom_headers: Dict - worker: str - retries: int - deadline: int - variables: Dict - status: TaskStatus = TaskStatus.Running + def __init__(self, key: int, _type: str, workflow_instance_key: int, bpmn_process_id: str, + workflow_definition_version: int, workflow_key: int, element_id: str, element_instance_key: int, + custom_headers: Dict, worker: str, retries: int, deadline: int, variables: Dict, + status: TaskStatus = TaskStatus.Running): + self.key = key + self.type = _type + self.workflow_instance_key = workflow_instance_key + self.bpmn_process_id = bpmn_process_id + self.workflow_definition_version = workflow_definition_version + self.workflow_key = workflow_key + self.element_id = element_id + self.element_instance_key = element_instance_key + self.custom_headers = custom_headers + self.worker = worker + self.retries = retries + self.deadline = deadline + self.variables = variables + self.status = status + + def __repr__(self): + return str({'jobKey': self.key, 'taskType': self.type, 'workflowInstanceKey': self.workflow_instance_key, + 'bpmnProcessId': self.bpmn_process_id, + 'workflowDefinitionVersion': self.workflow_definition_version, 'workflowKey': self.workflow_key, + 'elementId': self.element_id, 'elementInstanceKey': self.element_instance_key, + 'customHeaders': self.custom_headers, 'worker': self.worker, 'retries': self.retries, + 'deadline': self.deadline, 'variables': self.variables}) From 33b755b1bbfaf2b331aab201337e5df255d81c36 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Wed, 2 Sep 2020 23:08:30 +0300 Subject: [PATCH 26/57] [TESTING] integration --- .github/workflows/test-zeebe-integration.yml | 46 ++++++++++++++++++++ tests/client.py | 0 tests/integration_test.py | 2 + tests/worker.py | 0 4 files changed, 48 insertions(+) create mode 100644 .github/workflows/test-zeebe-integration.yml delete mode 100644 tests/client.py create mode 100644 tests/integration_test.py delete mode 100644 tests/worker.py diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml new file mode 100644 index 00000000..a778aa43 --- /dev/null +++ b/.github/workflows/test-zeebe-integration.yml @@ -0,0 +1,46 @@ +name: Test pyzeebe + +on: + push: + branches: [ master, development, feature/*, bugfix/* ] + pull_request: + branches: [ master, development, feature/*, bugfix/* ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + zeebe-version: [ 0.20, 0.21, 0.22, 0.23, 0.24 ] + python-version: [ 3.5, 3.6, 3.7, 3.8 ] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install pipenv + uses: dschep/install-pipenv-action@v1 + + - name: Install dependencies + run: | + pip install pipenv + pipenv install --dev + + - name: Run integration tests + env: + ZEEBE_ADDRESS: zeebe:26500 + + with: + zeebe-version: ${{ matrix.zeebe-version }} + + services: + zeebe: + image: camunda/zeebe:${{zeebe-version}} + + ports: + - 26500/tcp + run: | + pipenv run pytest tests diff --git a/tests/client.py b/tests/client.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/integration_test.py b/tests/integration_test.py new file mode 100644 index 00000000..f1748238 --- /dev/null +++ b/tests/integration_test.py @@ -0,0 +1,2 @@ +def test(): + pass diff --git a/tests/worker.py b/tests/worker.py deleted file mode 100644 index e69de29b..00000000 From 650a9b20544ec4a67016ea2b897c83078b509058 Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Wed, 2 Sep 2020 23:13:18 +0300 Subject: [PATCH 27/57] Update test-zeebe-integration.yml --- .github/workflows/test-zeebe-integration.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index a778aa43..6e1d2e0f 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -7,9 +7,17 @@ on: branches: [ master, development, feature/*, bugfix/* ] jobs: - build: + test: runs-on: ubuntu-latest + + services: + zeebe: + image: camunda/zeebe:${{zeebe-version}} + ports: + - 26500/tcp + + strategy: matrix: zeebe-version: [ 0.20, 0.21, 0.22, 0.23, 0.24 ] @@ -36,11 +44,5 @@ jobs: with: zeebe-version: ${{ matrix.zeebe-version }} - services: - zeebe: - image: camunda/zeebe:${{zeebe-version}} - - ports: - - 26500/tcp run: | pipenv run pytest tests From badec685627fed78be4798bfeb84bdb7c55a178d Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Wed, 2 Sep 2020 23:13:59 +0300 Subject: [PATCH 28/57] Using matrix --- .github/workflows/test-zeebe-integration.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 6e1d2e0f..adb9d489 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -11,11 +11,6 @@ jobs: runs-on: ubuntu-latest - services: - zeebe: - image: camunda/zeebe:${{zeebe-version}} - ports: - - 26500/tcp strategy: @@ -23,6 +18,12 @@ jobs: zeebe-version: [ 0.20, 0.21, 0.22, 0.23, 0.24 ] python-version: [ 3.5, 3.6, 3.7, 3.8 ] + services: + zeebe: + image: camunda/zeebe:${{zeebe-version}} + ports: + - 26500/tcp + steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 06990d23ba970cbf3d3758097712cfce5a27b5e0 Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Wed, 2 Sep 2020 23:15:37 +0300 Subject: [PATCH 29/57] Maybe works --- .github/workflows/test-zeebe-integration.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index adb9d489..c5a2e122 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -20,7 +20,7 @@ jobs: services: zeebe: - image: camunda/zeebe:${{zeebe-version}} + image: camunda/zeebe:${{ matrix.zeebe-version }} ports: - 26500/tcp @@ -38,12 +38,14 @@ jobs: pip install pipenv pipenv install --dev + - name: Run on every zeebe version + with: + zeebe-version: ${{ matrix.zeebe-version }} + + - name: Run integration tests env: ZEEBE_ADDRESS: zeebe:26500 - with: - zeebe-version: ${{ matrix.zeebe-version }} - run: | pipenv run pytest tests From 50d3f422f92be77778e3c44e91b22b1762c08df3 Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Wed, 2 Sep 2020 23:17:45 +0300 Subject: [PATCH 30/57] Update test-zeebe-integration.yml --- .github/workflows/test-zeebe-integration.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index c5a2e122..2e1439ed 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -11,8 +11,6 @@ jobs: runs-on: ubuntu-latest - - strategy: matrix: zeebe-version: [ 0.20, 0.21, 0.22, 0.23, 0.24 ] @@ -30,6 +28,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + zeebe-version: ${{ matrix.zeebe-version }} - name: Install pipenv uses: dschep/install-pipenv-action@v1 @@ -38,11 +37,6 @@ jobs: pip install pipenv pipenv install --dev - - name: Run on every zeebe version - with: - zeebe-version: ${{ matrix.zeebe-version }} - - - name: Run integration tests env: ZEEBE_ADDRESS: zeebe:26500 From c9b000971e6eea0e4da77f8262f9e28b86ab4e8a Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Wed, 2 Sep 2020 23:20:08 +0300 Subject: [PATCH 31/57] Using correct zeebe versions --- .github/workflows/test-zeebe-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 2e1439ed..6af34601 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - zeebe-version: [ 0.20, 0.21, 0.22, 0.23, 0.24 ] + zeebe-version: [ "0.20.0", "0.21.0", "0.22.0", "0.23.0", "0.24.0" ] python-version: [ 3.5, 3.6, 3.7, 3.8 ] services: From 85764be4df0e703081043c7c320ca8874a6f0529 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Thu, 3 Sep 2020 20:43:31 +0300 Subject: [PATCH 32/57] [TESTING] Integration tests --- .github/workflows/test-python-package.yml | 2 +- tests/integration_test.py | 60 ++++++++++++++++++++++- tests/test.bpmn | 46 +++++++++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 tests/test.bpmn diff --git a/.github/workflows/test-python-package.yml b/.github/workflows/test-python-package.yml index 49cf1769..72357a62 100644 --- a/.github/workflows/test-python-package.yml +++ b/.github/workflows/test-python-package.yml @@ -29,7 +29,7 @@ jobs: pipenv install --dev - name: Test with pytest run: | - pipenv run coverage run --source=. -m py.test + pipenv run coverage run --source=. -m py.test pyzeebe - name: Upload to coveralls run: | pipenv run coveralls diff --git a/tests/integration_test.py b/tests/integration_test.py index f1748238..42b7d9a4 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -1,2 +1,60 @@ -def test(): +from threading import Thread +from typing import Dict +from unittest.mock import patch +from uuid import uuid4 + +from pyzeebe import Task, ZeebeWorker, ZeebeClient + + +def task_handler(should_throw: bool, input: str) -> Dict: + if should_throw: + raise Exception('Error thrown') + else: + return {'output': input + str(uuid4())} + + +def exception_handler(*args, **kwargs): pass + + +task = Task('test', task_handler, exception_handler) + +zeebe_client: ZeebeClient +thread: Thread + + +def setup_module(): + zeebe_worker = ZeebeWorker() + zeebe_worker.add_task(task) + + thread = Thread(target=zeebe_worker.work) + thread.start() + global zeebe_client + zeebe_client = ZeebeClient() + + +def teardown_module(): + thread.join() + + +def test_run_workflow(): + workflow_key = zeebe_client.run_workflow('test', {'input': str(uuid4()), 'should_throw': False}) + assert isinstance(workflow_key, int) + + +def test_run_workflow_with_result(): + input = str(uuid4()) + output = zeebe_client.run_workflow_with_result('test', {'input': input, 'should_throw': False}) + assert isinstance(output['output'], str) + assert output['output'].starts_with(input) + + +def test_cancel_workflow(): + workflow_key = zeebe_client.run_workflow('test', {'input': str(uuid4()), 'should_throw': False}) + zeebe_client.cancel_workflow_instance(workflow_key) + + +def test_run_error_workflow(): + with patch('tests.exception_handler') as exc_handler: + zeebe_client.run_workflow('test', {'input': str(uuid4()), 'should_throw': True}) + exc_handler.assert_called() diff --git a/tests/test.bpmn b/tests/test.bpmn new file mode 100644 index 00000000..aed7d1a6 --- /dev/null +++ b/tests/test.bpmn @@ -0,0 +1,46 @@ + + + + + Flow_0fpfmf3 + + + + + + + + + + + + Flow_0fpfmf3 + Flow_05ukp8d + + + + Flow_05ukp8d + + + + + + + + + + + + + + + + + + + + + + + + From 03b44e389f19f1be7ed11f59ecd2d5162a1c283f Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Thu, 3 Sep 2020 21:20:30 +0300 Subject: [PATCH 33/57] [FIXED] Integration tests now working --- pyzeebe/__init__.py | 1 + tests/integration_test.py | 45 ++++++++++++++++++++--------------- tests/test.bpmn | 50 +++++++++++++++++++-------------------- 3 files changed, 52 insertions(+), 44 deletions(-) diff --git a/pyzeebe/__init__.py b/pyzeebe/__init__.py index b359b060..96eb1f74 100644 --- a/pyzeebe/__init__.py +++ b/pyzeebe/__init__.py @@ -1,4 +1,5 @@ from pyzeebe.client.client import ZeebeClient +from pyzeebe.common import exceptions from pyzeebe.task.task import Task from pyzeebe.task.task_context import TaskContext from pyzeebe.task.task_status_controller import TaskStatusController diff --git a/tests/integration_test.py b/tests/integration_test.py index 42b7d9a4..5eba9ada 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -1,9 +1,10 @@ -from threading import Thread +from concurrent.futures import ThreadPoolExecutor from typing import Dict -from unittest.mock import patch from uuid import uuid4 -from pyzeebe import Task, ZeebeWorker, ZeebeClient +import pytest + +from pyzeebe import Task, ZeebeWorker, ZeebeClient, exceptions, TaskContext, TaskStatusController def task_handler(should_throw: bool, input: str) -> Dict: @@ -13,28 +14,35 @@ def task_handler(should_throw: bool, input: str) -> Dict: return {'output': input + str(uuid4())} -def exception_handler(*args, **kwargs): - pass +def exception_handler(exc: Exception, context: TaskContext, controller: TaskStatusController) -> None: + controller.error(f'Failed to run task {context.type}. Reason: {exc}') task = Task('test', task_handler, exception_handler) zeebe_client: ZeebeClient -thread: Thread +executor: ThreadPoolExecutor -def setup_module(): +def run_worker(): zeebe_worker = ZeebeWorker() zeebe_worker.add_task(task) + zeebe_worker.work() + + +@pytest.fixture(scope='module', autouse=True) +def setup(): + global p, zeebe_client, executor + + executor = ThreadPoolExecutor() + executor.submit(run_worker) - thread = Thread(target=zeebe_worker.work) - thread.start() - global zeebe_client zeebe_client = ZeebeClient() + zeebe_client.deploy_workflow('test.bpmn') + yield zeebe_client -def teardown_module(): - thread.join() + executor.shutdown(wait=False) def test_run_workflow(): @@ -42,19 +50,18 @@ def test_run_workflow(): assert isinstance(workflow_key, int) +def test_non_existent_workflow(): + with pytest.raises(exceptions.WorkflowNotFound): + zeebe_client.run_workflow(str(uuid4())) + + def test_run_workflow_with_result(): input = str(uuid4()) output = zeebe_client.run_workflow_with_result('test', {'input': input, 'should_throw': False}) assert isinstance(output['output'], str) - assert output['output'].starts_with(input) + assert output['output'].startswith(input) def test_cancel_workflow(): workflow_key = zeebe_client.run_workflow('test', {'input': str(uuid4()), 'should_throw': False}) zeebe_client.cancel_workflow_instance(workflow_key) - - -def test_run_error_workflow(): - with patch('tests.exception_handler') as exc_handler: - zeebe_client.run_workflow('test', {'input': str(uuid4()), 'should_throw': True}) - exc_handler.assert_called() diff --git a/tests/test.bpmn b/tests/test.bpmn index aed7d1a6..e6106603 100644 --- a/tests/test.bpmn +++ b/tests/test.bpmn @@ -1,45 +1,45 @@ - - - - Flow_0fpfmf3 + + + + Flow_1o209vx - - + + - - - + + + - Flow_0fpfmf3 - Flow_05ukp8d + Flow_1o209vx + Flow_00xikey - - - Flow_05ukp8d + + Flow_00xikey + - - - - - + - + + + + + - + - - + + - - + + From 7c2f80eee1b4eea91fdd379d9a6c98abc1dee401 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Thu, 3 Sep 2020 21:26:57 +0300 Subject: [PATCH 34/57] [FIXED] bpmn path --- tests/integration_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration_test.py b/tests/integration_test.py index 5eba9ada..2141edb1 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -1,3 +1,4 @@ +import os.path from concurrent.futures import ThreadPoolExecutor from typing import Dict from uuid import uuid4 @@ -38,7 +39,10 @@ def setup(): executor.submit(run_worker) zeebe_client = ZeebeClient() - zeebe_client.deploy_workflow('test.bpmn') + try: + zeebe_client.deploy_workflow(os.path.join('tests', 'test.bpmn')) + except FileNotFoundError: + zeebe_client.deploy_workflow('test.bpmn') yield zeebe_client From 0de7bff416efc0bc4ba8899a57ec2c2642ea8ca2 Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Thu, 3 Sep 2020 21:35:13 +0300 Subject: [PATCH 35/57] Using ubuntu latest --- .github/workflows/test-zeebe-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 6af34601..38844f11 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -8,7 +8,7 @@ on: jobs: test: - + container: ubuntu:latest runs-on: ubuntu-latest strategy: From 928cd496e5a1d5a41dd244fd66f44823443921dd Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Thu, 3 Sep 2020 21:36:41 +0300 Subject: [PATCH 36/57] No env --- .github/workflows/test-zeebe-integration.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 38844f11..8d003e05 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -8,7 +8,6 @@ on: jobs: test: - container: ubuntu:latest runs-on: ubuntu-latest strategy: @@ -38,8 +37,5 @@ jobs: pipenv install --dev - name: Run integration tests - env: - ZEEBE_ADDRESS: zeebe:26500 - run: | pipenv run pytest tests From 7584e8638893e782c9171664d9df2c8b081bba4a Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Thu, 3 Sep 2020 21:44:49 +0300 Subject: [PATCH 37/57] Using wait --- .github/workflows/test-zeebe-integration.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 8d003e05..12df0518 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -35,6 +35,10 @@ jobs: run: | pip install pipenv pipenv install --dev + - name: Wait / Sleep + uses: jakejarvis/wait-action@v0.1.0 + with: + time: 30s - name: Run integration tests run: | From 42ff4d70b7cf2902ad06cabc07bc2965d015b46e Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Fri, 4 Sep 2020 16:19:43 +0300 Subject: [PATCH 38/57] No python matrix --- .github/workflows/test-zeebe-integration.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 12df0518..d728fbcb 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -13,7 +13,6 @@ jobs: strategy: matrix: zeebe-version: [ "0.20.0", "0.21.0", "0.22.0", "0.23.0", "0.24.0" ] - python-version: [ 3.5, 3.6, 3.7, 3.8 ] services: zeebe: @@ -23,11 +22,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.6 uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} - zeebe-version: ${{ matrix.zeebe-version }} + python-version: 3.6 - name: Install pipenv uses: dschep/install-pipenv-action@v1 From 0c094a09c41f78f2c8682b6e0f0badc03dbfa3af Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 16:30:40 +0300 Subject: [PATCH 39/57] [TRYING] grpc.enable_http_proxy --- .github/workflows/test-zeebe-integration.yml | 6 ++++-- pyzeebe/grpc_internals/zeebe_adapter.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index d728fbcb..12df0518 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -13,6 +13,7 @@ jobs: strategy: matrix: zeebe-version: [ "0.20.0", "0.21.0", "0.22.0", "0.23.0", "0.24.0" ] + python-version: [ 3.5, 3.6, 3.7, 3.8 ] services: zeebe: @@ -22,10 +23,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: ${{ matrix.python-version }} + zeebe-version: ${{ matrix.zeebe-version }} - name: Install pipenv uses: dschep/install-pipenv-action@v1 diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index eef79ee6..57d73969 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -21,7 +21,7 @@ def __init__(self, hostname: str = None, port: int = None, channel: grpc.Channel self._connection_uri = f'{hostname or "localhost"}:{port or 26500}' else: self._connection_uri = os.getenv('ZEEBE_ADDRESS') or 'localhost:26500' - self._channel = grpc.insecure_channel(self._connection_uri) + self._channel = grpc.insecure_channel(self._connection_uri, options=(('grpc.enable_http_proxy', 0),)) self.connected = False self.retrying_connection = True From ee76e877dd176289742b883d4b4f128bcf05537e Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 16:37:52 +0300 Subject: [PATCH 40/57] [REMOVED] grpc.enable_http_proxy, [TRYING] run on container --- .github/workflows/test-zeebe-integration.yml | 9 +++------ pyzeebe/grpc_internals/zeebe_adapter.py | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 12df0518..93ab753f 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -9,12 +9,14 @@ on: jobs: test: runs-on: ubuntu-latest - strategy: matrix: zeebe-version: [ "0.20.0", "0.21.0", "0.22.0", "0.23.0", "0.24.0" ] python-version: [ 3.5, 3.6, 3.7, 3.8 ] + container: python:${{ matrix.python-version }} + + services: zeebe: image: camunda/zeebe:${{ matrix.zeebe-version }} @@ -23,11 +25,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - zeebe-version: ${{ matrix.zeebe-version }} - name: Install pipenv uses: dschep/install-pipenv-action@v1 diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index 57d73969..eef79ee6 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -21,7 +21,7 @@ def __init__(self, hostname: str = None, port: int = None, channel: grpc.Channel self._connection_uri = f'{hostname or "localhost"}:{port or 26500}' else: self._connection_uri = os.getenv('ZEEBE_ADDRESS') or 'localhost:26500' - self._channel = grpc.insecure_channel(self._connection_uri, options=(('grpc.enable_http_proxy', 0),)) + self._channel = grpc.insecure_channel(self._connection_uri) self.connected = False self.retrying_connection = True From c4f674e02c85b233ac738740624b4b7850bf162f Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 16:38:45 +0300 Subject: [PATCH 41/57] [FIXED] workflow name --- .github/workflows/test-zeebe-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 93ab753f..856fe6d5 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -1,4 +1,4 @@ -name: Test pyzeebe +name: Integration test pyzeebe on: push: From 3d2bb36207fe2fab1031815b02983da8f7314879 Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Fri, 4 Sep 2020 16:42:44 +0300 Subject: [PATCH 42/57] Not using pipenv install action --- .github/workflows/test-zeebe-integration.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 856fe6d5..7b480709 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -25,9 +25,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Install pipenv - uses: dschep/install-pipenv-action@v1 - - name: Install dependencies run: | pip install pipenv From 1a0d4fc3b36469f128593796d0ed40de2b15675b Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Fri, 4 Sep 2020 16:46:24 +0300 Subject: [PATCH 43/57] Address is correct --- .github/workflows/test-zeebe-integration.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 7b480709..315f4e53 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -8,6 +8,8 @@ on: jobs: test: + env: + ZEEBE_ADDRESS: "zeebe:26500" runs-on: ubuntu-latest strategy: matrix: From e5dafead8611190de02fe49f33780ad61e5c409c Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Fri, 4 Sep 2020 16:50:30 +0300 Subject: [PATCH 44/57] No python matrix --- .github/workflows/test-zeebe-integration.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 315f4e53..d85821a9 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -14,9 +14,8 @@ jobs: strategy: matrix: zeebe-version: [ "0.20.0", "0.21.0", "0.22.0", "0.23.0", "0.24.0" ] - python-version: [ 3.5, 3.6, 3.7, 3.8 ] - container: python:${{ matrix.python-version }} + container: python:3.6 services: From dab966d1deaef7387df468cc680bbb7add7f3e8c Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Fri, 4 Sep 2020 16:53:24 +0300 Subject: [PATCH 45/57] Using patched zeebe versions --- .github/workflows/test-zeebe-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index d85821a9..bf8fcc24 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - zeebe-version: [ "0.20.0", "0.21.0", "0.22.0", "0.23.0", "0.24.0" ] + zeebe-version: [ "0.20.3", "0.21.1", "0.22.5", "0.23.5", "0.24.2" ] container: python:3.6 From 2bbe6a91d0b0e108c8c3611d072214f485ddae1a Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 17:58:33 +0300 Subject: [PATCH 46/57] [FIXED] integration tests not stopping --- pyzeebe/worker/worker.py | 21 +++++++++++---------- tests/integration_test.py | 19 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 92c483d0..79c2bc70 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -1,6 +1,6 @@ import logging import socket -from concurrent.futures import ThreadPoolExecutor +from threading import Thread, Event from typing import List, Callable, Generator, Tuple from pyzeebe.common.exceptions import TaskNotFound @@ -31,15 +31,17 @@ def __init__(self, name: str = None, request_timeout: int = 0, hostname: str = N self.name = name or socket.gethostname() self.request_timeout = request_timeout self.tasks = [] + self._task_threads: List[Thread] = [] - def work(self): - with ThreadPoolExecutor(thread_name_prefix='zeebe-task') as executor: - for task in self.tasks: - executor.submit(self._handle_task, task) + def work(self, stop_event: Event = None): + for task in self.tasks: + task_thread = Thread(target=self._handle_task, args=(task, stop_event), daemon=True) + self._task_threads.append(task_thread) + task_thread.start() - def _handle_task(self, task: Task): + def _handle_task(self, task: Task, stop_event: Event): logging.debug(f'Handling task {task}') - while self.zeebe_adapter.connected or self.zeebe_adapter.retrying_connection: + while not stop_event.is_set() and self.zeebe_adapter.connected or self.zeebe_adapter.retrying_connection: if self.zeebe_adapter.retrying_connection: logging.debug(f'Retrying connection to {self.zeebe_adapter._connection_uri}') continue @@ -47,11 +49,10 @@ def _handle_task(self, task: Task): self._handle_task_contexts(task) def _handle_task_contexts(self, task: Task): - executor = ThreadPoolExecutor(thread_name_prefix=f'zeebe-job-{task.type}') for task_context in self._get_task_contexts(task): + thread = Thread(target=task.handler, args=(task_context,)) logging.debug(f'Running job: {task_context}') - executor.submit(task.handler, task_context) - executor.shutdown(wait=False) + thread.start() def _get_task_contexts(self, task: Task) -> Generator[TaskContext, None, None]: logging.debug(f'Activating jobs for task: {task}') diff --git a/tests/integration_test.py b/tests/integration_test.py index 2141edb1..9c64f17e 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -1,5 +1,5 @@ -import os.path -from concurrent.futures import ThreadPoolExecutor +import os +from threading import Thread, Event from typing import Dict from uuid import uuid4 @@ -22,21 +22,21 @@ def exception_handler(exc: Exception, context: TaskContext, controller: TaskStat task = Task('test', task_handler, exception_handler) zeebe_client: ZeebeClient -executor: ThreadPoolExecutor -def run_worker(): +def run_worker(stop_event): zeebe_worker = ZeebeWorker() zeebe_worker.add_task(task) - zeebe_worker.work() + zeebe_worker.work(stop_event) @pytest.fixture(scope='module', autouse=True) def setup(): - global p, zeebe_client, executor + global zeebe_client - executor = ThreadPoolExecutor() - executor.submit(run_worker) + stop_event = Event() + t = Thread(target=run_worker, args=(stop_event,)) + t.start() zeebe_client = ZeebeClient() try: @@ -45,8 +45,7 @@ def setup(): zeebe_client.deploy_workflow('test.bpmn') yield zeebe_client - - executor.shutdown(wait=False) + stop_event.set() def test_run_workflow(): From c5468ae0acce67bc0d3a377c621e6b7728d1a06f Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 18:02:24 +0300 Subject: [PATCH 47/57] [REMOVED] zeebe 0.20 integration --- .github/workflows/test-zeebe-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index bf8fcc24..fcec9aea 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - zeebe-version: [ "0.20.3", "0.21.1", "0.22.5", "0.23.5", "0.24.2" ] + zeebe-version: [ "0.21.1", "0.22.5", "0.23.5", "0.24.2" ] container: python:3.6 From d4d0686801e8dd2deda0e602171e41b6f66e8ec1 Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Fri, 4 Sep 2020 18:11:44 +0300 Subject: [PATCH 48/57] No wait + no 0.21 --- .github/workflows/test-zeebe-integration.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index fcec9aea..2fc8ba63 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -13,11 +13,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - zeebe-version: [ "0.21.1", "0.22.5", "0.23.5", "0.24.2" ] + zeebe-version: [ "0.22.5", "0.23.5", "0.24.2" ] container: python:3.6 - services: zeebe: image: camunda/zeebe:${{ matrix.zeebe-version }} @@ -30,10 +29,10 @@ jobs: run: | pip install pipenv pipenv install --dev - - name: Wait / Sleep - uses: jakejarvis/wait-action@v0.1.0 - with: - time: 30s + #- name: Wait / Sleep + # uses: jakejarvis/wait-action@v0.1.0 + # with: + # time: 30s - name: Run integration tests run: | From a0df190d6e7027014129c7cbe81fb1a6e8bb1d16 Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Fri, 4 Sep 2020 18:31:01 +0300 Subject: [PATCH 49/57] Removed 0.22.5 --- .github/workflows/test-zeebe-integration.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 2fc8ba63..8a4fa3bf 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - zeebe-version: [ "0.22.5", "0.23.5", "0.24.2" ] + zeebe-version: [ "0.23.5", "0.24.2" ] container: python:3.6 @@ -29,10 +29,6 @@ jobs: run: | pip install pipenv pipenv install --dev - #- name: Wait / Sleep - # uses: jakejarvis/wait-action@v0.1.0 - # with: - # time: 30s - name: Run integration tests run: | From 3b8e51ee8d22e232e7b4302623174bf79ad292e8 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 18:34:09 +0300 Subject: [PATCH 50/57] [FIXED] Zeebe versions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e895eead..0ee8378e 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Zeebe version support: | Pyzeebe version | Tested Zeebe versions | |:---------------:|----------------| -| 1.0.1 | 0.24.2 | +| 1.0.1 | 0.23, 0.24 | ## Getting Started To install: From 1fa58d4d5d6d16dd5d7b5c034fcfecce68d5ed42 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 18:36:09 +0300 Subject: [PATCH 51/57] [FIXED] pyzeebe version --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0ee8378e..17ad9868 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Zeebe version support: | Pyzeebe version | Tested Zeebe versions | |:---------------:|----------------| -| 1.0.1 | 0.23, 0.24 | +| 1.1.0 | 0.23, 0.24 | ## Getting Started To install: diff --git a/setup.py b/setup.py index 68a99f1c..36076fb9 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="pyzeebe", - version="1.0.1", + version="1.1.0", author="Jonatan Martens", author_email="jonatanmartenstav@gmail.com", description="Zeebe client api", From b98cd8ca3666d06f07e279b09db87cca0a556e20 Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Fri, 4 Sep 2020 18:51:05 +0300 Subject: [PATCH 52/57] Using python matrix --- .github/workflows/test-zeebe-integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 8a4fa3bf..0ddd0d0a 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -14,8 +14,9 @@ jobs: strategy: matrix: zeebe-version: [ "0.23.5", "0.24.2" ] + python-version: [ 3.5, 3.6, 3.7, 3.8 ] - container: python:3.6 + container: python:${{ matrix.python-version }} services: zeebe: From 21479584d5293ea712d3980125e614c7cd08dcb7 Mon Sep 17 00:00:00 2001 From: Jonatan Martens <40060128+JonatanMartens@users.noreply.github.com> Date: Fri, 4 Sep 2020 18:51:57 +0300 Subject: [PATCH 53/57] Using python container and removed steps --- .github/workflows/test-python-package.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-python-package.yml b/.github/workflows/test-python-package.yml index 72357a62..575f3acf 100644 --- a/.github/workflows/test-python-package.yml +++ b/.github/workflows/test-python-package.yml @@ -13,16 +13,10 @@ jobs: strategy: matrix: python-version: [3.5, 3.6, 3.7, 3.8] - + + container: python:${{ matrix.python-version }} steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install pipenv - uses: dschep/install-pipenv-action@v1 - - name: Install dependencies run: | pip install pipenv From e5b5e41729c6fd55ff4e658178472c36bea1fbc7 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 19:22:47 +0300 Subject: [PATCH 54/57] [CHANGED] ZeebAdapter connection_uri + tests --- pyzeebe/grpc_internals/zeebe_adapter.py | 10 +++++----- pyzeebe/grpc_internals/zeebe_adapter_test.py | 19 +++++++++++++++++++ pyzeebe/worker/worker.py | 2 +- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/pyzeebe/grpc_internals/zeebe_adapter.py b/pyzeebe/grpc_internals/zeebe_adapter.py index eef79ee6..4f55cee5 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_adapter.py @@ -13,15 +13,15 @@ class ZeebeAdapter(object): def __init__(self, hostname: str = None, port: int = None, channel: grpc.Channel = None, **kwargs): - self._connection_uri = f'{hostname}:{port}' or os.getenv('ZEEBE_ADDRESS') or 'localhost:26500' if channel: self._channel = channel + self.connection_uri = None else: if hostname or port: - self._connection_uri = f'{hostname or "localhost"}:{port or 26500}' + self.connection_uri = f'{hostname or "localhost"}:{port or 26500}' else: - self._connection_uri = os.getenv('ZEEBE_ADDRESS') or 'localhost:26500' - self._channel = grpc.insecure_channel(self._connection_uri) + self.connection_uri = os.getenv('ZEEBE_ADDRESS', 'localhost:26500') + self._channel = grpc.insecure_channel(self.connection_uri) self.connected = False self.retrying_connection = True @@ -42,7 +42,7 @@ def _check_connectivity(self, value: grpc.ChannelConnectivity) -> None: logging.error('Failed to establish connection to Zeebe. Non recoverable') self.connected = False self.retrying_connection = False - raise ConnectionAbortedError(f'Lost connection to {self._connection_uri}') + raise ConnectionAbortedError(f'Lost connection to {self.connection_uri}') def activate_jobs(self, task_type: str, worker: str, timeout: int, max_jobs_to_activate: int, variables_to_fetch: List[str], request_timeout: int) -> Generator[TaskContext, None, None]: diff --git a/pyzeebe/grpc_internals/zeebe_adapter_test.py b/pyzeebe/grpc_internals/zeebe_adapter_test.py index d07888df..37e465b7 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_test.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_test.py @@ -71,6 +71,25 @@ def test_connectivity_shutdown(): zeebe_adapter._check_connectivity(grpc.ChannelConnectivity.SHUTDOWN) +def test_only_port(): + port = randint(0, 10000) + zeebe_adapter = ZeebeAdapter(port=port) + assert zeebe_adapter.connection_uri == f'localhost:{port}' + + +def test_only_host(): + hostname = str(uuid4()) + zeebe_adapter = ZeebeAdapter(hostname=hostname) + assert zeebe_adapter.connection_uri == f'{hostname}:26500' + + +def test_host_and_port(): + hostname = str(uuid4()) + port = randint(0, 10000) + zeebe_adapter = ZeebeAdapter(hostname=hostname, port=port) + assert zeebe_adapter.connection_uri == f'{hostname}:{port}' + + def test_activate_jobs(grpc_servicer): task_type = create_random_task_and_activate(grpc_servicer) active_jobs_count = randint(4, 100) diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 79c2bc70..50b9fcff 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -43,7 +43,7 @@ def _handle_task(self, task: Task, stop_event: Event): logging.debug(f'Handling task {task}') while not stop_event.is_set() and self.zeebe_adapter.connected or self.zeebe_adapter.retrying_connection: if self.zeebe_adapter.retrying_connection: - logging.debug(f'Retrying connection to {self.zeebe_adapter._connection_uri}') + logging.debug(f'Retrying connection to {self.zeebe_adapter.connection_uri}') continue self._handle_task_contexts(task) From ee2442ee789b7bcb1103c7c738c9bfcf91938624 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 20:23:41 +0300 Subject: [PATCH 55/57] [ADDED] stop worker test --- pyzeebe/grpc_internals/zeebe_adapter_test.py | 1 + pyzeebe/worker/worker_test.py | 25 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/pyzeebe/grpc_internals/zeebe_adapter_test.py b/pyzeebe/grpc_internals/zeebe_adapter_test.py index 37e465b7..7c20ef41 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_test.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_test.py @@ -227,6 +227,7 @@ def get_first_active_job(task_type) -> TaskContext: return next(zeebe_adapter.activate_jobs(task_type=task_type, max_jobs_to_activate=1, request_timeout=10, timeout=100, variables_to_fetch=[], worker=str(uuid4()))) + def test_get_workflow_request_object(): with patch('builtins.open') as mock_open: mock_open.return_value = BytesIO() diff --git a/pyzeebe/worker/worker_test.py b/pyzeebe/worker/worker_test.py index 295d28e6..9de282d4 100644 --- a/pyzeebe/worker/worker_test.py +++ b/pyzeebe/worker/worker_test.py @@ -1,10 +1,12 @@ from random import randint +from threading import Event from unittest.mock import patch from uuid import uuid4 import pytest from pyzeebe.common.exceptions import TaskNotFound +from pyzeebe.common.gateway_mock import GatewayMock from pyzeebe.common.random_utils import random_task_context from pyzeebe.task.task import Task from pyzeebe.task.task_context import TaskContext @@ -18,6 +20,23 @@ def decorator(context: TaskContext) -> TaskContext: return context +@pytest.fixture(scope='module') +def grpc_add_to_server(): + from pyzeebe.grpc_internals.zeebe_pb2_grpc import add_GatewayServicer_to_server + return add_GatewayServicer_to_server + + +@pytest.fixture(scope='module') +def grpc_servicer(): + return GatewayMock() + + +@pytest.fixture(scope='module') +def grpc_stub_cls(grpc_channel): + from pyzeebe.grpc_internals.zeebe_pb2_grpc import GatewayStub + return GatewayStub + + @pytest.fixture(autouse=True) def run_around_tests(): global zeebe_worker, task @@ -206,3 +225,9 @@ def test_handle_many_jobs(): task_handler_mock.return_value = {'x': str(uuid4())} zeebe_worker._handle_task_contexts(task) task_handler_mock.assert_called_with(context) + + +def test_stop_worker(): + stop_event = Event() + zeebe_worker.work(stop_event=stop_event) + stop_event.set() From d3f6f162a8ebf49739484a59102a5d889b7238d7 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 20:43:34 +0300 Subject: [PATCH 56/57] [FIXED] Thread daemon --- pyzeebe/worker/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 50b9fcff..70a89017 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -35,7 +35,7 @@ def __init__(self, name: str = None, request_timeout: int = 0, hostname: str = N def work(self, stop_event: Event = None): for task in self.tasks: - task_thread = Thread(target=self._handle_task, args=(task, stop_event), daemon=True) + task_thread = Thread(target=self._handle_task, args=(task, stop_event)) self._task_threads.append(task_thread) task_thread.start() From a8feeda4cfda43f3a82422e7d28a81b22a654778 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Fri, 4 Sep 2020 21:08:43 +0300 Subject: [PATCH 57/57] [ADDED] stop worker example, small threading explanation --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 17ad9868..dac9ea47 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ To install: ### Worker +The `ZeebeWorker` class uses threading to get and run jobs. + ```python from pyzeebe import ZeebeWorker, Task, TaskStatusController, TaskContext @@ -48,6 +50,16 @@ worker.add_task(task) # Add task to zeebe worker worker.work() # Now every time that a task with type example is called example_task will be called ``` +Stop a worker: +```python +from threading import Event + + +stop_event = Event() +zeebe_worker.work(stop_event=stop_event) # Worker will begin working +stop_event.set() # Stops worker and all running jobs +``` + ### Client ```python