Merge pull request #688 from writer/feat-async-jobs

feat: Async jobs

pyproject.toml

@@ -49,7 +49,6 @@ websockets = ">= 12, < 13"
 writer-sdk = ">= 1.5.0, < 2"
 python-multipart = ">=0.0.7, < 1"
 
-
 [tool.poetry.group.build]
 optional = true
 
@@ -68,13 +67,20 @@ pytest-asyncio = ">= 0.23.4, < 1"
 ruff = "^0.3.4"
 types-requests = "^2.31.0.20240406"
 
+[tool.poetry.group.redis]
+optional = true
+
+[tool.poetry.group.redis.dependencies]
+redis = "^5.2.1"
 
 [tool.poetry.group.dev.dependencies]
 types-python-dateutil = "^2.9.0.20240316"
 
 [tool.poetry.scripts]
 writer = 'writer.command_line:main'
 
+[tool.poetry.extras]
+redis = ["redis"]
 
 [tool.ruff]
 exclude = [

src/ui/src/builder/BuilderEmbeddedCodeEditor.vue

@@ -59,6 +59,7 @@ onMounted(() => {
 	editor = monaco.editor.create(editorContainerEl.value, {
 		value: modelValue.value,
 		language: props.language,
+		readOnly: props.disabled,
 		...VARIANTS_SETTINGS[props.variant],
 	});
 	editor.getModel().onDidChangeContent(() => {
@@ -84,11 +85,11 @@ onUnmounted(() => {
 .BuilderEmbeddedCodeEditor {
 	height: 100%;
 	width: 100%;
-	min-height: 200px;
+	min-height: 100px;
 }
 
 .editorContainer {
-	min-height: 200px;
+	min-height: 100px;
 	width: 100%;
 	height: 100%;
 	overflow: hidden;

src/ui/src/builder/BuilderTooltip.vue

@@ -37,7 +37,9 @@ const position = ref<{
 }>({ top: 0, left: 0 });
 
 async function setUpAndShowTooltip() {
+	if (!trackedElement) return;
 	tooltipText.value = trackedElement.dataset.writerTooltip;
+	if (!tooltipText.value) return;
 	const gapPx = trackedElement.dataset.writerTooltipGap
 		? parseInt(trackedElement.dataset.writerTooltipGap)
 		: DEFAULT_GAP_PX;

src/ui/src/builder/settings/BuilderSettingsAPICode.vue

@@ -0,0 +1,132 @@
+<template>
+	<div v-if="isWorkflow" class="BuilderSettingsAPICode">
+		<BuilderModal
+			v-if="isModalVisible"
+			:close-action="modalCloseAction"
+			icon="code"
+			modal-title="API Code"
+		>
+			<div class="modalContents">
+				<template v-if="isHashAvailable">
+					The following call will create the job and provide you with
+					a job ID and a job token.
+					<div class="codeContainer">
+						<BuilderEmbeddedCodeEditor
+							v-model="code"
+							class="editor"
+							variant="minimal"
+							language="shell"
+						></BuilderEmbeddedCodeEditor>
+					</div>
+					Using the job ID and token obtained in the previous call,
+					check the status of the job. You can use the code below,
+					after replacing JOB_ID and JOB_TOKEN for the right values.
+					<div class="codeContainer">
+						<BuilderEmbeddedCodeEditor
+							v-model="codePost"
+							class="editor"
+							variant="minimal"
+							language="shell"
+						></BuilderEmbeddedCodeEditor>
+					</div>
+					<strong
+						>Note: For API calls to work, the --enable-jobs-api flag
+						must be active.</strong
+					>
+				</template>
+				<template v-else>
+					API code cannot be generated. Please make sure the
+					environment variable WRITER_SECRET_KEY has been set up.
+				</template>
+			</div>
+		</BuilderModal>
+		<template v-if="workflowKey">
+			<WdsButton variant="tertiary" size="small" @click="showCode">
+				<i class="material-symbols-outlined"> code </i> Call via
+				API</WdsButton
+			>
+		</template>
+		<template v-else>
+			<WdsButton
+				:disabled="true"
+				variant="tertiary"
+				size="small"
+				data-writer-tooltip="You need to specify a workflow key before this workflow can be
+				used in the UI or called via API."
+			>
+				<i class="material-symbols-outlined"> code </i> Call via
+				API</WdsButton
+			>
+		</template>
+	</div>
+</template>
+
+<script setup lang="ts">
+import { computed, inject, ref } from "vue";
+import injectionKeys from "../../injectionKeys";
+import BuilderModal, { ModalAction } from "../BuilderModal.vue";
+import BuilderEmbeddedCodeEditor from "../BuilderEmbeddedCodeEditor.vue";
+import WdsButton from "@/wds/WdsButton.vue";
+
+const wf = inject(injectionKeys.core);
+const wfbm = inject(injectionKeys.builderManager);
+
+const component = computed(() => wf.getComponentById(wfbm.getSelectedId()));
+
+const isWorkflow = computed(
+	() => component.value?.type === "workflows_workflow",
+);
+
+const workflowKey = computed(() => {
+	if (!isWorkflow.value) return;
+	return component.value.content?.["key"];
+});
+
+const isModalVisible = ref(false);
+
+const code = ref("");
+const codePost = ref("");
+const isHashAvailable = ref(false);
+
+const modalCloseAction: ModalAction = {
+	desc: "Close",
+	fn: () => {
+		isModalVisible.value = false;
+	},
+};
+
+function showCode() {
+	generateCode();
+	isModalVisible.value = true;
+}
+
+async function generateCode() {
+	const bearerToken = await wf.hashMessage(`create_job_${workflowKey.value}`);
+	isHashAvailable.value = Boolean(bearerToken);
+	if (!isHashAvailable.value) return;
+
+	const baseURL = window.location.origin + window.location.pathname;
+	code.value = `
+curl --location --request POST '${baseURL}api/job/workflow/${workflowKey.value}' \\
+--header 'Content-Type: application/json' \\
+--header 'Authorization: Bearer ${bearerToken}' \\
+--data '{
+    "my_var": 1
+}'`.trim();
+	codePost.value = `
+curl --location '${baseURL}api/job/JOB_ID' \\
+--header 'Authorization: Bearer JOB_TOKEN'`.trim();
+}
+</script>
+
+<style scoped>
+.modalContents {
+	display: flex;
+	gap: 16px;
+	flex-direction: column;
+}
+
+.codeContainer {
+	height: 100px;
+}
+</style>

src/ui/src/builder/settings/BuilderSettingsMain.vue

@@ -22,6 +22,7 @@
 				<BuilderSettingsHandlers></BuilderSettingsHandlers>
 				<BuilderSettingsVisibility></BuilderSettingsVisibility>
 			</template>
+			<BuilderSettingsAPICode>Execute via API</BuilderSettingsAPICode>
 		</div>
 
 		<div class="sections debug">
@@ -42,6 +43,7 @@ import BuilderSettingsBinding from "./BuilderSettingsBinding.vue";
 import BuilderSettingsVisibility from "./BuilderSettingsVisibility.vue";
 import BuilderCopyText from "../BuilderCopyText.vue";
 import BuilderAsyncLoader from "../BuilderAsyncLoader.vue";
+import BuilderSettingsAPICode from "./BuilderSettingsAPICode.vue";
 
 const BuilderSettingsHandlers = defineAsyncComponent({
 	loader: () => import("./BuilderSettingsHandlers.vue"),

src/ui/src/builder/sidebar/BuilderSidebarComponentTreeBranch.vue

@@ -141,7 +141,7 @@ const name = computed(() => {
 		return component.value.content?.["element"];
 	}
 	if (type == "workflows_workflow") {
-		return component.value.content?.["key"] ?? "Workflow";
+		return component.value.content?.["key"] || "Workflow";
 	}
 	return def.value?.name ?? `Unknown (${component.value.type})`;
 });

src/ui/src/core/index.ts

@@ -333,6 +333,35 @@ export function generateCore() {
 		sendFrontendMessage("event", messagePayload, callback, true);
 	}
 
+	/**
+	 * Sends a message to be hashed in the backend using the relevant keys.
+	 * Due to security reasons, it works only in edit mode.
+	 * 
+	 * @param message Messaged to be hashed
+	 * @returns The hashed message
+	 */
+	async function hashMessage(message: string):Promise<string> {
+		return new Promise((resolve, reject) => {
+			const messageCallback = (r: {
+				ok: boolean;
+				payload?: Record<string, any>;
+			}) => {
+				if (!r.ok) {
+					reject("Couldn't connect to the server.");
+					return;
+				}
+				resolve(r.payload?.message);
+			};
+
+			sendFrontendMessage(
+				"hashRequest",
+				{ message },
+				messageCallback,
+			);
+		});
+
+	}
+
 	async function sendCodeSaveRequest(newCode: string): Promise<void> {
 		const messageData = {
 			code: newCode,
@@ -572,6 +601,7 @@ export function generateCore() {
 		addMailSubscription,
 		init,
 		forwardEvent,
+		hashMessage,
 		runCode: readonly(runCode),
 		sendCodeSaveRequest,
 		sendCodeUpdate,

src/writer/app_runner.py

@@ -17,7 +17,7 @@
 from pydantic import ValidationError
 from watchdog.observers.polling import PollingObserver
 
-from writer import VERSION, audit_and_fix, core_ui, wf_project
+from writer import VERSION, audit_and_fix, core_ui, crypto, wf_project
 from writer.core import (
     Config,
     EventHandlerRegistry,
@@ -36,6 +36,9 @@
     ComponentUpdateRequestPayload,
     EventRequest,
     EventResponsePayload,
+    HashRequest,
+    HashRequestPayload,
+    HashRequestResponsePayload,
     InitSessionRequest,
     InitSessionRequestPayload,
     InitSessionResponsePayload,
@@ -199,7 +202,7 @@ def _handle_event(self, session: WriterSession, event: WriterEvent) -> EventResp
         session.session_state.clear_mail()
 
         return res_payload
-    
+
     def _handle_state_enquiry(self, session: WriterSession) -> StateEnquiryResponsePayload:
         import traceback as tb
 
@@ -236,7 +239,13 @@ def _handle_state_content(self, session: WriterSession) -> StateContentResponseP
                                                 tb.format_exc())
 
         return StateContentResponsePayload(state=serialized_state)
-
+
+    def _handle_hash_request(self, req_payload: HashRequestPayload) -> HashRequestResponsePayload:
+        res_payload = HashRequestResponsePayload(
+            message=crypto.get_hash(req_payload.message)
+        )
+        return res_payload
+
     def _handle_component_update(self, session: WriterSession, payload: ComponentUpdateRequestPayload) -> None:
         import writer
         ingest_bmc_component_tree(writer.base_component_tree, payload.components)
@@ -303,6 +312,14 @@ def _handle_message(self, session_id: str, request: AppProcessServerRequest) ->
                     payload=None
                 )
 
+            if self.mode == "edit" and type == "hashRequest":
+                hash_request_payload = HashRequestPayload.model_validate(request.payload)
+                return AppProcessServerResponse(
+                    status="ok",
+                    status_message=None,
+                    payload=self._handle_hash_request(hash_request_payload)
+                )
+
             if self.mode == "edit" and type == "componentUpdate":
                 cu_req_payload = ComponentUpdateRequestPayload.parse_obj(
                     request.payload)
@@ -745,6 +762,12 @@ async def handle_event(self, session_id: str, event: WriterEvent) -> AppProcessS
             payload=event
         ))
 
+    async def handle_hash_request(self, session_id: str, payload: HashRequestPayload) -> AppProcessServerResponse:
+        return await self.dispatch_message(session_id, HashRequest(
+            type="hashRequest",
+            payload=payload
+        ))
+
     async def handle_state_enquiry(self, session_id: str) -> AppProcessServerResponse:
         return await self.dispatch_message(session_id, StateEnquiryRequest(
             type="stateEnquiry"

src/writer/blocks/foreach.py

@@ -22,19 +22,12 @@ def register(cls, type: str):
                     },
                     "items": {
                         "name": "Items",
-                        "desc": "The item value will be passed in the execution environment and will be available at @{item}, its id at @{itemId}.",
+                        "desc": "The item value will be passed in the execution environment and will be available at @{payload.item}, its id at @{payload.itemId}.",
                         "default": "{}",
                         "init": '{ "fr": "France", "pl": "Poland" }',
                         "type": "Object",
                         "control": "Textarea"
-                    },
-                    "executionEnv": {
-                        "name": "Execution environment",
-                        "desc": "You can add other values to the execution environment.",
-                        "default": "{}",
-                        "type": "Object",
-                        "control": "Textarea"
-                    },
+                    }
                 },
                 "outs": {
                     "success": {
@@ -52,14 +45,20 @@ def register(cls, type: str):
         ))
 
     def _run_workflow_for_item(self, workflow_key, base_execution_environment, item_id, item):
-        expanded_execution_environment = base_execution_environment | { "itemId": item_id, "item": item }
+        expanded_execution_environment = base_execution_environment | { 
+            "payload": {
+                "itemId": item_id,
+                "item": item
+            }
+        }
         return self.runner.run_workflow_by_key(workflow_key, expanded_execution_environment)
 
     def run(self):
         try:
             workflow_key = self._get_field("workflowKey")
             items = self._get_field("items", as_json=True)
-            base_execution_environment = self._get_field("executionEnv", as_json=True)
+            base_execution_environment = self.execution_environment
+
             std_items = items
             result = None
             if isinstance(items, list):