From 58e827f007309a376296864ddabf76e3c477cd17 Mon Sep 17 00:00:00 2001 From: liivw <164842155+liivw@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:33:53 +0800 Subject: [PATCH] Worker doc update --- .../applications.md | 5 + .../rbac-overview.md | 1 + docs/access-control-and-security/tags.md | 5 + .../users-and-groups.md | 5 + docs/developer-guides/using-workers.md | 446 +++++++++++++----- sidebars.js | 2 +- 6 files changed, 355 insertions(+), 109 deletions(-) diff --git a/docs/access-control-and-security/applications.md b/docs/access-control-and-security/applications.md index a4eee0d3..f5d13f7d 100644 --- a/docs/access-control-and-security/applications.md +++ b/docs/access-control-and-security/applications.md @@ -1,3 +1,8 @@ +--- +slug: "/access-control-and-security/applications" +description: "Use applications to manage secure access for all connections to your Conductor cluster." +--- + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/access-control-and-security/rbac-overview.md b/docs/access-control-and-security/rbac-overview.md index d33356a7..cb37e4ab 100644 --- a/docs/access-control-and-security/rbac-overview.md +++ b/docs/access-control-and-security/rbac-overview.md @@ -1,5 +1,6 @@ --- slug: /category/access-control-and-security +description: "This guide focuses on how to configure Conductor features for access control and security, including applications, users, groups, and tags." --- import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/access-control-and-security/tags.md b/docs/access-control-and-security/tags.md index 5700a489..6c6ac959 100644 --- a/docs/access-control-and-security/tags.md +++ b/docs/access-control-and-security/tags.md @@ -1,3 +1,8 @@ +--- +slug: "/access-control-and-security/tags" +description: "Use tags to group tasks, workflows, schedulers, or secrets and easily share these tags with a group of users with the correct permissions." +--- + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/access-control-and-security/users-and-groups.md b/docs/access-control-and-security/users-and-groups.md index 8f3b2452..311e726b 100644 --- a/docs/access-control-and-security/users-and-groups.md +++ b/docs/access-control-and-security/users-and-groups.md @@ -1,3 +1,8 @@ +--- +slug: "/access-control-and-security/users-and-groups" +description: "Manage the users and groups on your Conductor cluster to provide the appropriate level of access and permissions. " +--- + # Managing Users and Groups :::info diff --git a/docs/developer-guides/using-workers.md b/docs/developer-guides/using-workers.md index 99400950..b3bfb816 100644 --- a/docs/developer-guides/using-workers.md +++ b/docs/developer-guides/using-workers.md @@ -1,14 +1,24 @@ +--- +slug: "/developer-guides/using-workers" +description: "Learn how to set up workers that can be deployed anywhere to service custom tasks in Conductor." +--- + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; # Using Workers -A worker is a piece of code responsible for executing a task, which is custom logic that can be deployed anywhere. In Conductor, workers can be implemented in any language. +A worker is a piece of code responsible for executing a task. In Conductor, workers can be implemented in any language and deployed anywhere. Here is the overview for using custom workers: +1. Create a worker project in your preferred language. +2. Set up the worker. +3. Use the worker task in a workflow. + -To create a worker task, you can use Conductor’s polyglot SDKs. These SDKs contain features that make it easier to create workers, such as polling threads, metrics, and server communication. Conductor maintains a registry of worker tasks. A task must be registered before being used in a workflow. This can be done by creating a Task Definition. +## Creating a worker +To create a worker, you can use Conductor’s polyglot SDKs. These SDKs contain features that make it easier to create workers, such as polling threads, metrics, and server communication. -## Best practices for writing workers +### Best practices for writing workers In Conductor, each worker should embody the microservice design pattern and follow these basic principles: * **Defined I/O**—Each worker executes a specific task and produces a well-defined output, given specific inputs. @@ -17,176 +27,365 @@ In Conductor, each worker should embody the microservice design pattern and foll * **Decoupled implementation**—Workers do not implement the logic to handle retries, timeouts, and other implementation details, which are handled by the Conductor server. -## Using workers in Conductor +### Step 1: Set up your worker project -In Conductor, you can use workers in two ways: the UI approach or the SDK-only approach. This guide will explore the UI approach. +Begin by creating a project to run workers for your custom task. -**To use a worker in Conductor (UI approach):** -1. Create a worker to run a task. -2. Define the task in Conductor. -3. Add the task to a workflow. -4. Add worker to an application account and grant permissions for execution. -5. Launch the worker to start. +For Python projects, it is a good practice to use a virtual environment. + + -### Step 1: Create a worker task +``` shell +// Using venv +python -m venv myProject +source myProject/bin/activate -Begin by creating a worker and defining the logic for your task. It is possible to use one worker to service multiple tasks. +// Using virtualenv +virtualenv myProject +source myProject/bin/activate +``` -**Prerequisites** + -Ensure that you have installed the [Conductor SDK](/category/sdks) for your chosen language. + - - +``` shell +// Using venv +python -m venv myProject +myProject\Scripts\activate -``` bash -pip3 install conductor-python +// Using virtualenv +virtualenv myProject +myProject\Scripts\activate ``` + - -``` bash +For Go projects, you should initialize your Go module. + + + + +``` go +go mod init +``` + + + + +### Step 2: Get Conductor SDK + +Get [Conductor SDK](../category/sdks) in your preferred language. + + + + +``` python +python3 -m pip install conductor-python +``` + + + + + +**For Gradle:** +``` java +implementation 'org.conductoross:conductor-client:4.0.0' +implementation 'io.orkes:orkes-conductor-client:4.0.0' +``` + +**For Maven:** +``` xml + + org.conductoross + conductor-client + 4.0.0 + + + io.orkes + orkes-conductor-client + 4.0.0 + +``` + + + + + +**npm:** +``` shell npm i @io-orkes/conductor-javascript ``` + +**yarn:** +``` bash +yarn add @io-orkes/conductor-javascript +``` + + + + + +``` csharp +dotnet add package conductor-csharp +``` + + + + + +``` go +go get github.com/conductor-sdk/conductor-go +``` + + + + + +Get the Conductor Clojure package from [clojars](https://clojars.org/io.orkes/conductor-clojure). + +``` clojure +:deps {org.clojure/clojure {:mvn/version "1.11.0"} + io.orkes/conductor-clojure {:mvn/version "0.3.0"}} +``` + -**Example worker task** +### Step 3: Write your worker logic -Here is an example of a task that generates a random product inventory number. In this example, a domain is used to route the task to the correct worker. +In general, workers can be instantiated from a WorkerInterface class, defined as a function, or annotated using a WorkerTask decorator, depending on the language. The worker file in your project may look like this: - - + + ``` python -@worker_task(task_definition_name='get-inventory', domain='john') -def get_inventory(product_id: str) -> dict: - return dict(product_id = product_id, inventory = randrange(10)) +from conductor.client.worker.worker_task import worker_task + +@worker_task(task_definition_name='myTask') +def greet(name: str) -> str: + return f'Hello {name}' ``` + - + ``` java -@Service -public class Worker { - @WorkerTask(value = "get-inventory", domain = "john") - public Map getInventory(Map input) { - return Map.of( - "product_id", input.get("product_id"), - "inventory", ThreadLocalRandom.current().nextInt(1, 11) - ); +import com.netflix.conductor.sdk.workflow.task.InputParam; +import com.netflix.conductor.sdk.workflow.task.WorkerTask; + +public class Workers { + @WorkerTask("myTask") + public String greeting(@InputParam("name") String name) { + return ("Hello " + name); } } ``` + - + ``` javascript -const { - orkesConductorClient, TaskManager, -} = require("@io-orkes/conductor-javascript"); - -const getInventory = { - taskDefName: 'get-inventory', - domain: 'john', - execute: async ({ inputData }) => ({ - outputData: { - product_id: inputData.product_id, - inventory: 1 + Math.floor(Math.random() * 10) - }, - status: "COMPLETED", - }), +import { ConductorWorker, Task } from "@io-orkes/conductor-javascript"; + +const worker: ConductorWorker = { + taskDefName: "myTask", + execute: async ( + task: Task + ): Promise> => {}, }; +``` + + + + + +*Coming soon.* + + + + + +``` go +import ( + "fmt" + + "github.com/conductor-sdk/conductor-go/sdk/model" +) + +func myTask(task *model.Task) (interface{}, error) { + return map[string]interface{}{ + "greetings": "Hello, " + fmt.Sprintf("%v", task.InputData["name"]), + }, nil +} +``` + + + + + +*Coming soon.* + + + + + +### Step 4: Run your worker + +Start the workers using the SDK-provided interface. The interface polls the server for work, executes worker code, and updates the results back to the server. + + + + +Run the Python worker by calling a TaskHandler. + +``` python +from conductor.client.automator.task_handler import TaskHandler +from conductor.client.configuration.configuration import Configuration + +task_handler = TaskHandler(configuration=Configuration()) +task_handler.start_processes() +task_handler.join_processes() +``` + +Full worker example: [https://github.com/conductor-oss/python-worker-container/tree/main](https://github.com/conductor-oss/python-worker-container/tree/main) + + + + + +Run the annotated Java worker using WorkflowExecutor. + +``` java +WorkflowExecutor executor = new WorkflowExecutor("http://server/api/"); +//List of packages (comma separated) to scan for annotated workers. +// Please note, the worker method MUST be public and the class in which they are defined +//MUST have a no-args constructor +executor.initWorkers("com.company.package1,com.company.package2"); +``` + + + + + +Import TaskRunner to run the JavaScript worker. + +``` javascript +import { + OrkesApiConfig, + orkesConductorClient, + TaskRunner, +} from "@io-orkes/conductor-javascript"; orkesConductorClient({ keyId: process.env.CONDUCTOR_AUTH_KEY, keySecret: process.env.CONDUCTOR_AUTH_SECRET, serverUrl: process.env.CONDUCTOR_SERVER_URL, }).then((client) => - new TaskManager(client, [ getInventory ]).startPolling() + new TaskManager(client, [ myWorker ]).startPolling() ); ``` - + -For more information on creating workers in your preferred language, refer to the [SDK guides](/category/sdk). +*Coming soon.* + -### Step 2: Define the task in Conductor + -All worker tasks must be defined in Conductor before they can be used in a workflow. Tasks can be defined on Orkes Platform or using an API. +Run the Go worker using TaskRunner. -**To define a worker task:** +``` go +import ( + "github.com/conductor-sdk/conductor-go/sdk/client" + "github.com/conductor-sdk/conductor-go/sdk/model" + "github.com/conductor-sdk/conductor-go/sdk/settings" - - + "github.com/conductor-sdk/conductor-go/sdk/worker" +) -1. Go to your Conductor cluster on Orkes Platform. -2. In the left navigation menu, go to **Definitions** > **Task**. -3. Select **(+) Define task**. -4. Enter the task details, such as the rate limits, retry settings, timeout settings, and expected inputs and outputs. The **Name** must match the task name defined previously in your code. -

Define task in Orkes Platform

-5. Select **Save** > **Confirm Save**. +apiClient := client.NewAPIClient( + settings.NewAuthenticationSettings( + KEY, + SECRET, + ), + settings.NewHttpSettings( + "https://play.orkes.io/api", +)) + +taskRunner = worker.NewTaskRunnerWithApiClient(apiClient) +taskRunner.StartWorker("simple_task", examples.SimpleWorker, 1, time.Second*1) +``` - + + +*Coming soon.* -Refer to the [Create Task Definition API](https://orkes.io/content/reference-docs/api/metadata/creating-task-definitions). -### Step 3: Add the task to a workflow +For more information on creating workers in your preferred language, refer to the [SDK guides](../category/sdks). -Once the task is defined, it can be added to a workflow definition. -**To add a worker task to a workflow:** +## Setting up the worker - - +To use the worker in a workflow, you must register the worker task to the Conductor server, set up authorized access, and launch your worker project. -1. Go to your Conductor cluster on Orkes Platform. -2. In the left navigation menu, go to **Definitions** > **Workflow**. -3. Select or create a workflow. -4. In the visual workflow editor, select the **(+)** icon to add a new task. There are two ways to add a worker task: -* Search for your task using its task name and select to add it to the workflow. -* Add a **Worker Task (Simple)** and enter the task name in **Task Definition**. -5. Configure the task, such as its inputs, caching, and optionality. -6. On the top right, select **Save** > **Confirm**. +The worker task cannot begin execution until the worker is connected to the Conductor server. If the workflow is run, the task will remain in the Scheduled status until the worker comes online to service the task. + +### Register worker task + +All worker tasks must be registered to the Conductor server, which is done by creating a task definition. Tasks can be defined in UI, using API, or SDK. + +**To define a worker task:** + + + + +1. Go to your Orkes Conductor cluster. +2. In the left navigation menu, go to **Definitions** > **Task**. +3. Select **(+) Define task**. +4. Enter the task details, such as the rate limits, retry settings, timeout settings, and expected inputs and outputs.
The **Name** must match the task name defined previously in your code. +

Define task in Orkes Platform

+5. Select **Save** > **Confirm Save**. -

Add task in Orkes Platform

 - + + +Refer to the [Create Task Definition API](../reference-docs/api/metadata/creating-task-definitions). -Refer to the [Create Workflow Definition](https://orkes.io/content/reference-docs/api/metadata/creating-workflow-definition) or [Update Workflow Definition](https://orkes.io/content/reference-docs/api/metadata/update-workflow-definitions) APIs. -The task cannot begin execution until the worker is connected to the Conductor server. If the workflow is run, the custom task will remain in the Scheduled state until the worker comes online to service the task. This is achieved by creating the appropriate application accounts and permissions for the worker. +### Set up authorized access -### Step 4: Add worker to an application account +In Orkes Conductor, an [application account](/category/access-control-and-security#applications) with a Worker role type enables workers to authenticate and authorize against the Conductor server. To set up authorized access, you need to add the worker to an application and grant it Execute permission. -In Orkes, an application account with a Worker role type enables workers to authenticate and authorize against the Conductor server. To start a worker, you need to add the worker to an application account and grant Execute permission to the application. +:::note +For well-defined access controls, your worker application should be kept separate from your workflow client application. Learn more about [proper application separation](https://orkes.io/content/access-control-and-security/applications#example-application-setup). +::: -**To add worker to application account:** - -1. Go to the application account. - 1. Go to your Conductor cluster on Orkes Platform. +**To set up authorized access:** +1. Configure an application account. + 1. Go to your Orkes Conductor cluster. 2. In the left navigation menu, go to **Access Control** > **Applications**. - 3. Select an application that you will be adding your worker to. Otherwise, create an application. Ensure that the application role has **Worker** enabled. + 3. Create a new application or select an application to which you will be adding your worker.
Ensure that the application role has **Worker** enabled.

Add worker to application account in Orkes Platform

-2. Get the application access key for your worker task. +2. Get the application access key for your worker project. 1. Under Access Keys, select **Create access key** and store your credentials securely. - 2. Set the Key ID and Key Secret in your environment variables. + 2. Set the Key ID and Key Secret in your project environment variables. ``` bash export CONDUCTOR_SERVER_URL= // eg: https://play.orkes.io/api export CONDUCTOR_AUTH_KEY= @@ -196,23 +395,20 @@ In Orkes, an application account with a Worker role type enables workers to auth 1. Under Permissions, select **Add permission**. 2. Select the **Task** tab and then your worker task. 3. Enable the **Execute** toggle. - 4. (If Task to Domain is used) In Domain, enter the domain name used in your worker code. + 4. (If [task-to-domain](task-to-domain) is used) In Domain, enter the domain name used in your worker code. 5. Select **Add Permissions**. The application account can now execute the worker task.

Add permissions to application account in Orkes Platform

-For more information on application accounts, refer to [Enabling Security via Applications](https://orkes.io/content/access-control-and-security/applications). - -### Step 5: Launch the worker -You must launch the worker project before it can start executing the custom task. +### Launch the worker -**To launch the worker:** -Start your project using the respective method of your language and editor. +Launch the worker to begin polling the Conductor server. The specific method depends on your language and project configuration. +**Example** @@ -234,17 +430,51 @@ mvn spring-boot:run // using Maven -``` +``` bash node index.js ``` -Run the workflow to ensure that it starts up successfully. If not, return to the previous steps and verify that all details have been entered correctly, including: +## Using the worker task + +All custom worker tasks are denoted as [Simple tasks](/reference-docs/worker-task) in Conductor. To use a worker task, add it to a workflow definition. + + +### Add to workflow + +**To add a worker task to a workflow:** + + + + +1. Go to your Orkes Conductor cluster. +2. In the left navigation menu, go to **Definitions** > **Workflow**. +3. Select or create a workflow. +4. In the visual workflow editor, select the **(+)** icon to add a new task. There are two ways to add a worker task: +* Search for your task using its task name and select to add it to the workflow. +* Add a **Worker Task (Simple)** and enter the task name in **Task Definition**. +5. Configure the task, such as its inputs, caching, and optionality. +6. On the top right, select **Save** > **Confirm**. + +

Add task in Orkes Platform

+ + + + +Refer to the [Create Workflow Definition](../reference-docs/api/metadata/creating-workflow-definition) or [Update Workflow Definition](../reference-docs/api/metadata/update-workflow-definitions) APIs. + + + +### Run workflow + +Run the workflow to ensure that your worker has started successfully. If not, return to the previous steps and verify that all details have been entered correctly, such as: * **Server URL, Key ID, and Key Secret**— Added to your environment variables. * **Execute permissions**—Added for the worker task in your application account. -* **(if applicable) Domain**—The domain in your code matches the domain in the run mapping and application permissions. - ``` - // run mapping during workflow execution - { "get-inventory": "john" } - ``` \ No newline at end of file +* (if applicable) **Domain**—The domain in your code matches the domain used during workflow execution and in the application permissions. + + +## Advanced topics +* Monitor task queues: [Monitoring Task Queues](https://orkes.io/content/developer-guides/monitoring-task-queues) +* Scale up your worker instances: [Metrics for Scaling Workers](https://orkes.io/content/developer-guides/scaling-workers) +* Implement domains for your workers to split the traffic: [Routing Tasks](https://orkes.io/content/developer-guides/task-to-domain) diff --git a/sidebars.js b/sidebars.js index f39e125a..0bc8e087 100644 --- a/sidebars.js +++ b/sidebars.js @@ -113,9 +113,9 @@ const sidebars = { }, items: [ 'developer-guides/using-workers', - 'developer-guides/task-to-domain', 'developer-guides/monitoring-task-queues', 'developer-guides/scaling-workers', + 'developer-guides/task-to-domain', ], collapsible: false, },