From 4e122e62295d1c98aa1f27c0b29ea0cd682ca4df Mon Sep 17 00:00:00 2001 From: mmikita95 Date: Thu, 5 Dec 2024 12:31:10 +0300 Subject: [PATCH 01/60] chore: add a TypedDict to parameters definition --- src/writer/ai.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/writer/ai.py b/src/writer/ai.py index bbdbbfbb8..71af28870 100644 --- a/src/writer/ai.py +++ b/src/writer/ai.py @@ -90,17 +90,31 @@ class GraphTool(Tool): subqueries: bool +class FunctionToolParameterMeta(TypedDict): + type: Union[ + Literal["string"], + Literal["number"], + Literal["integer"], + Literal["float"], + Literal["boolean"], + Literal["array"], + Literal["object"], + Literal["null"] + ] + description: str + + class FunctionTool(Tool): callable: Callable name: str description: Optional[str] - parameters: Dict[str, Dict[str, str]] + parameters: Dict[str, FunctionToolParameterMeta] def create_function_tool( callable: Callable, name: str, - parameters: Optional[Dict[str, Dict[str, str]]] = None, + parameters: Optional[Dict[str, FunctionToolParameterMeta]] = None, description: Optional[str] = None ) -> FunctionTool: parameters = parameters or {} From 00244b1a720c10cc2012f67690ca36dabba269f8 Mon Sep 17 00:00:00 2001 From: mmikita95 Date: Thu, 5 Dec 2024 14:32:35 +0300 Subject: [PATCH 02/60] fix: typing --- src/writer/ai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/writer/ai.py b/src/writer/ai.py index 71af28870..557f6003d 100644 --- a/src/writer/ai.py +++ b/src/writer/ai.py @@ -1210,7 +1210,7 @@ def _register_callable( self, callable_to_register: Callable, name: str, - parameters: Dict[str, Dict[str, str]] + parameters: Dict[str, FunctionToolParameterMeta] ): """ Internal helper function to store a provided callable @@ -1280,7 +1280,9 @@ def _prepare_tool( Internal helper function to process a tool instance into the required format. """ - def validate_parameters(parameters: Dict[str, Dict[str, str]]) -> bool: + def validate_parameters( + parameters: Dict[str, FunctionToolParameterMeta] + ) -> bool: """ Validates the `parameters` dictionary to ensure that each key is a parameter name, and each value is a dictionary containing From e879f349f6c4969281aaa0b3b1e5a7d2acfa9c10 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:39:47 +0000 Subject: [PATCH 03/60] feat: Async jobs --- src/writer/core.py | 4 +- src/writer/crypto.py | 25 ++++++++ src/writer/serve.py | 132 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 src/writer/crypto.py diff --git a/src/writer/core.py b/src/writer/core.py index 9c0074340..d2bcc25d1 100644 --- a/src/writer/core.py +++ b/src/writer/core.py @@ -1605,9 +1605,9 @@ def fn(payload, context, session): "session": session } if workflow_key: - self.workflow_runner.run_workflow_by_key(workflow_key, execution_environment) + return self.workflow_runner.run_workflow_by_key(workflow_key, execution_environment) elif workflow_id: - self.workflow_runner.run_workflow(workflow_id, execution_environment, "Workflow execution triggered on demand") + return self.workflow_runner.run_workflow(workflow_id, execution_environment, "Workflow execution triggered on demand") return fn def _get_handler_callable(self, handler: str) -> Optional[Callable]: diff --git a/src/writer/crypto.py b/src/writer/crypto.py new file mode 100644 index 000000000..c95c700ce --- /dev/null +++ b/src/writer/crypto.py @@ -0,0 +1,25 @@ + +import hashlib +import os + +from fastapi import HTTPException, Request + +HASH_SALT = "a9zHYfIeL0" + +def get_hash(message: str): + base_hash = os.getenv("WRITER_BASE_HASH") + if not base_hash: + raise ValueError("Environment variable WRITER_BASE_HASH needs to be set up in" + \ + "order to enable operations which require hash generation, such as creating async jobs.") + assert HASH_SALT + combined = base_hash + HASH_SALT + message + return hashlib.sha256(combined.encode()).hexdigest() + +def verify_hash_in_request(message: str, request: Request): + auth_header = request.headers.get("Authorization") + if auth_header != f"Bearer {get_hash(message)}": + raise HTTPException(status_code=403) + + +print("hashi") +print(get_hash("create_job_car_story")) diff --git a/src/writer/serve.py b/src/writer/serve.py index 459069d1b..6524f1a55 100644 --- a/src/writer/serve.py +++ b/src/writer/serve.py @@ -2,13 +2,16 @@ import html import importlib.util import io +import json import logging import mimetypes +import redis import os import os.path import pathlib import socket import textwrap +import time import typing from contextlib import asynccontextmanager from importlib.machinery import ModuleSpec @@ -17,13 +20,13 @@ import uvicorn from fastapi import FastAPI, HTTPException, Request, Response -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, JSONResponse from fastapi.routing import Mount from fastapi.staticfiles import StaticFiles from pydantic import ValidationError from starlette.websockets import WebSocket, WebSocketDisconnect, WebSocketState -from writer import VERSION, abstract +from writer import VERSION, abstract, crypto from writer.app_runner import AppRunner from writer.ss_types import ( AppProcessServerResponse, @@ -56,7 +59,7 @@ def __init__(self): def generate_job_id(self): self.counter += 1 - return self.counter + return str(self.counter) def set(self, job_id: str, value: Any): self.vault[job_id] = value @@ -64,6 +67,39 @@ def set(self, job_id: str, value: Any): def get(self, job_id: str): return self.vault.get(job_id) + @classmethod + def create_vault(cls): + redis_connection_string = os.getenv("WRITER_CONNECTION_STRING_REDIS") + if redis_connection_string: + return RedisJobVault() + else: + return cls() + + +class RedisJobVault(JobVault): + + def __init__(self): + super().__init__() + redis_connection_string = os.getenv("WRITER_CONNECTION_STRING_REDIS") + self.redis_client = redis.from_url(redis_connection_string, decode_responses=True) + self.counter_key = "job_counter" + if not self.redis_client.exists(self.counter_key): + self.redis_client.set(self.counter_key, 0) + + def generate_job_id(self): + job_id = self.redis_client.incr(self.counter_key) + return str(job_id) + + def set(self, job_id: str, value: Any): + json_str = json.dumps(value) + self.redis_client.set(f"job:{job_id}", json_str) + + def get(self, job_id: str): + json_str = self.redis_client.get(f"job:{job_id}") + return json.loads(json_str) + + + class WriterState(typing.Protocol): app_runner: AppRunner writer_app: bool @@ -139,7 +175,7 @@ async def lifespan(asgi_app: FastAPI): """ app.state.writer_app = True app.state.app_runner = app_runner - app.state.job_vault = JobVault() + app.state.job_vault = JobVault.create_vault() def _get_extension_paths() -> List[str]: extensions_path = pathlib.Path(user_app_path) / "extensions" @@ -242,6 +278,94 @@ async def init(initBody: InitRequestBody, request: Request, response: Response) if serve_mode == "edit": return _get_edit_starter_pack(app_response.payload) + # Jobs + + @app.post("/api/job/workflow/{workflow_key}") + async def create_workflow_job(workflow_key: str, request: Request, response: Response): + crypto.verify_hash_in_request(f"create_job_{workflow_key}", request) + + def serialize_result(data): + if isinstance(data, list): + return [serialize_result(item) for item in data] + if isinstance(data, dict): + return {k : serialize_result(v) for k, v in data.items()} + if isinstance(data, (str, int, float, bool, type(None))): + return data + try: + return json.loads(json.dumps(data)) + except (TypeError, OverflowError): + return f"Can't be displayed. Value of type: {str(type(data))}." + + def update_job(job_id: str, job_info: dict): + current_job_info = app.state.job_vault.get(job_id) + if not current_job_info: + raise RuntimeError("Job not found.") + merged_info = current_job_info | { "finished_at": int(time.time()) } | job_info + app.state.job_vault.set(job_id, merged_info) + + def job_done_callback(task, job_id: str): + try: + apsr: Optional[AppProcessServerResponse] = None + apsr = task.result() + if apsr.status != "ok": + update_job(job_id, {"status": "error"}) + return + result = apsr.payload.result.get("result") + update_job(job_id, { + "status": "complete", + "result": serialize_result(result) + }) + except Exception as e: + update_job(job_id, {"status": "error"}) + raise e + + app_response = await app_runner.init_session(InitSessionRequestPayload( + cookies=dict(request.cookies), + headers=dict(request.headers), + proposedSessionId=None + )) + + session_id = app_response.payload.sessionId + is_session_ok = await app_runner.check_session(session_id) + if not is_session_ok: + return + + loop = asyncio.get_running_loop() + task = loop.create_task(app_runner.handle_event( + session_id, WriterEvent( + type="wf-builtin-run", + isSafe=True, + handler=f"$runWorkflow_{workflow_key}" + ))) + + job_id = app.state.job_vault.generate_job_id() + app.state.job_vault.set(job_id, { + "id": job_id, + "status": "in progress", + "created_at": int(time.time()) + }) + task.add_done_callback(lambda t: job_done_callback(t, job_id)) + return { + "id": job_id, + "token": crypto.get_hash(f"get_job_{job_id}") + } + + @app.get("/api/job/{job_id}") + async def get_workflow_job(job_id: str, request: Request, response: Response): + crypto.verify_hash_in_request(f"get_job_{job_id}", request) + job = app.state.job_vault.get(job_id) + + if not job: + return JSONResponse(status_code=404, content={ + "id": job_id, + "status": "not found" + }) + + status_code = 200 + if job.get("status") == "error": + status_code = 400 + + return JSONResponse(status_code=status_code, content=job) # Streaming From d9468be42002c4eaeecf4366a2e8c367b00c2f2e Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Sat, 7 Dec 2024 22:03:13 +0100 Subject: [PATCH 04/60] feat: Async jobs --- .../src/builder/BuilderEmbeddedCodeEditor.vue | 5 +- src/ui/src/builder/BuilderTooltip.vue | 2 + .../settings/BuilderSettingsAPICode.vue | 111 ++++++++++++++++++ .../builder/settings/BuilderSettingsMain.vue | 2 + src/ui/src/core/index.ts | 30 +++++ src/writer/app_runner.py | 28 ++++- src/writer/crypto.py | 11 +- src/writer/serve.py | 46 ++++++-- src/writer/ss_types.py | 14 +++ 9 files changed, 230 insertions(+), 19 deletions(-) create mode 100644 src/ui/src/builder/settings/BuilderSettingsAPICode.vue diff --git a/src/ui/src/builder/BuilderEmbeddedCodeEditor.vue b/src/ui/src/builder/BuilderEmbeddedCodeEditor.vue index 4a2e3c923..7c348f1ac 100644 --- a/src/ui/src/builder/BuilderEmbeddedCodeEditor.vue +++ b/src/ui/src/builder/BuilderEmbeddedCodeEditor.vue @@ -59,6 +59,7 @@ onMounted(() => { editor = monaco.editor.create(editorContainerEl.value, { value: modelValue.value, language: props.language, + readOnly: props.disabled, ...VARIANTS_SETTINGS[props.variant], }); editor.getModel().onDidChangeContent(() => { @@ -84,11 +85,11 @@ onUnmounted(() => { .BuilderEmbeddedCodeEditor { height: 100%; width: 100%; - min-height: 200px; + min-height: 100px; } .editorContainer { - min-height: 200px; + min-height: 100px; width: 100%; height: 100%; overflow: hidden; diff --git a/src/ui/src/builder/BuilderTooltip.vue b/src/ui/src/builder/BuilderTooltip.vue index 889ec1e87..cbca66472 100644 --- a/src/ui/src/builder/BuilderTooltip.vue +++ b/src/ui/src/builder/BuilderTooltip.vue @@ -37,7 +37,9 @@ const position = ref<{ }>({ top: 0, left: 0 }); async function setUpAndShowTooltip() { + if (!trackedElement) return; tooltipText.value = trackedElement.dataset.writerTooltip; + if (!tooltipText.value) return; const gapPx = trackedElement.dataset.writerTooltipGap ? parseInt(trackedElement.dataset.writerTooltipGap) : DEFAULT_GAP_PX; diff --git a/src/ui/src/builder/settings/BuilderSettingsAPICode.vue b/src/ui/src/builder/settings/BuilderSettingsAPICode.vue new file mode 100644 index 000000000..ee3612b24 --- /dev/null +++ b/src/ui/src/builder/settings/BuilderSettingsAPICode.vue @@ -0,0 +1,111 @@ + + + + + diff --git a/src/ui/src/builder/settings/BuilderSettingsMain.vue b/src/ui/src/builder/settings/BuilderSettingsMain.vue index 68ddd2564..fd98b7f47 100644 --- a/src/ui/src/builder/settings/BuilderSettingsMain.vue +++ b/src/ui/src/builder/settings/BuilderSettingsMain.vue @@ -22,6 +22,7 @@ + Execute via API
@@ -42,6 +43,7 @@ import BuilderSettingsBinding from "./BuilderSettingsBinding.vue"; import BuilderSettingsVisibility from "./BuilderSettingsVisibility.vue"; import BuilderCopyText from "../BuilderCopyText.vue"; import BuilderAsyncLoader from "../BuilderAsyncLoader.vue"; +import BuilderSettingsAPICode from "./BuilderSettingsAPICode.vue"; const BuilderSettingsHandlers = defineAsyncComponent({ loader: () => import("./BuilderSettingsHandlers.vue"), diff --git a/src/ui/src/core/index.ts b/src/ui/src/core/index.ts index 8368d99ea..7a1e33f70 100644 --- a/src/ui/src/core/index.ts +++ b/src/ui/src/core/index.ts @@ -333,6 +333,35 @@ export function generateCore() { sendFrontendMessage("event", messagePayload, callback, true); } + /** + * Sends a message to be hashed in the backend using the relevant keys. + * Due to security reasons, it works only in edit mode. + * + * @param message Messaged to be hashed + * @returns The hashed message + */ + async function hashMessage(message: string):Promise { + return new Promise((resolve, reject) => { + const messageCallback = (r: { + ok: boolean; + payload?: Record; + }) => { + if (!r.ok) { + reject("Couldn't connect to the server."); + return; + } + resolve(r.payload?.message); + }; + + sendFrontendMessage( + "hashRequest", + { message }, + messageCallback, + ); + }); + + } + async function sendCodeSaveRequest(newCode: string): Promise { const messageData = { code: newCode, @@ -572,6 +601,7 @@ export function generateCore() { addMailSubscription, init, forwardEvent, + hashMessage, runCode: readonly(runCode), sendCodeSaveRequest, sendCodeUpdate, diff --git a/src/writer/app_runner.py b/src/writer/app_runner.py index 616093a6e..5286b12c4 100644 --- a/src/writer/app_runner.py +++ b/src/writer/app_runner.py @@ -17,7 +17,7 @@ from pydantic import ValidationError from watchdog.observers.polling import PollingObserver -from writer import VERSION, audit_and_fix, core_ui, wf_project +from writer import VERSION, audit_and_fix, core_ui, crypto, wf_project from writer.core import ( Config, EventHandlerRegistry, @@ -36,6 +36,9 @@ ComponentUpdateRequestPayload, EventRequest, EventResponsePayload, + HashRequest, + HashRequestPayload, + HashRequestResponsePayload, InitSessionRequest, InitSessionRequestPayload, InitSessionResponsePayload, @@ -199,7 +202,7 @@ def _handle_event(self, session: WriterSession, event: WriterEvent) -> EventResp session.session_state.clear_mail() return res_payload - + def _handle_state_enquiry(self, session: WriterSession) -> StateEnquiryResponsePayload: import traceback as tb @@ -236,7 +239,13 @@ def _handle_state_content(self, session: WriterSession) -> StateContentResponseP tb.format_exc()) return StateContentResponsePayload(state=serialized_state) - + + def _handle_hash_request(self, req_payload: HashRequestPayload) -> HashRequestResponsePayload: + res_payload = HashRequestResponsePayload( + message=crypto.get_hash(req_payload.message) + ) + return res_payload + def _handle_component_update(self, session: WriterSession, payload: ComponentUpdateRequestPayload) -> None: import writer ingest_bmc_component_tree(writer.base_component_tree, payload.components) @@ -303,6 +312,13 @@ def _handle_message(self, session_id: str, request: AppProcessServerRequest) -> payload=None ) + if self.mode == "edit" and type == "hashRequest": + return AppProcessServerResponse( + status="ok", + status_message=None, + payload=self._handle_hash_request(request.payload) + ) + if self.mode == "edit" and type == "componentUpdate": cu_req_payload = ComponentUpdateRequestPayload.parse_obj( request.payload) @@ -745,6 +761,12 @@ async def handle_event(self, session_id: str, event: WriterEvent) -> AppProcessS payload=event )) + async def handle_hash_request(self, session_id: str, payload: HashRequestPayload) -> AppProcessServerResponse: + return await self.dispatch_message(session_id, HashRequest( + type="hashRequest", + payload=payload + )) + async def handle_state_enquiry(self, session_id: str) -> AppProcessServerResponse: return await self.dispatch_message(session_id, StateEnquiryRequest( type="stateEnquiry" diff --git a/src/writer/crypto.py b/src/writer/crypto.py index c95c700ce..25be0f447 100644 --- a/src/writer/crypto.py +++ b/src/writer/crypto.py @@ -6,8 +6,7 @@ HASH_SALT = "a9zHYfIeL0" -def get_hash(message: str): - base_hash = os.getenv("WRITER_BASE_HASH") +def get_hash(message: str, base_hash = os.getenv("WRITER_BASE_HASH")): if not base_hash: raise ValueError("Environment variable WRITER_BASE_HASH needs to be set up in" + \ "order to enable operations which require hash generation, such as creating async jobs.") @@ -17,9 +16,7 @@ def get_hash(message: str): def verify_hash_in_request(message: str, request: Request): auth_header = request.headers.get("Authorization") + if not auth_header: + raise HTTPException(status_code=401, detail=f"Unauthorized. Token not specified.") if auth_header != f"Bearer {get_hash(message)}": - raise HTTPException(status_code=403) - - -print("hashi") -print(get_hash("create_job_car_story")) + raise HTTPException(status_code=403, detail=f"Forbidden. Incorrect token.") \ No newline at end of file diff --git a/src/writer/serve.py b/src/writer/serve.py index 6524f1a55..eee0b9fd0 100644 --- a/src/writer/serve.py +++ b/src/writer/serve.py @@ -32,6 +32,8 @@ AppProcessServerResponse, ComponentUpdateRequestPayload, EventResponsePayload, + HashRequestPayload, + HashRequestResponsePayload, InitRequestBody, InitResponseBodyEdit, InitResponseBodyRun, @@ -69,19 +71,25 @@ def get(self, job_id: str): @classmethod def create_vault(cls): - redis_connection_string = os.getenv("WRITER_CONNECTION_STRING_REDIS") - if redis_connection_string: - return RedisJobVault() - else: + redis_connection_string = os.getenv("WRITER_REDIS") + if not redis_connection_string: + return cls() + try: + redis_vault = RedisJobVault() + return redis_vault + except Exception as e: + logging.error(f"There was an error connecting to Redis. Falling back to in-memory JobVault. {repr(e)}") return cls() class RedisJobVault(JobVault): + DEFAULT_TTL = 86400 + def __init__(self): super().__init__() - redis_connection_string = os.getenv("WRITER_CONNECTION_STRING_REDIS") - self.redis_client = redis.from_url(redis_connection_string, decode_responses=True) + redis_connection_string = os.getenv("WRITER_REDIS") + self.redis_client = redis.from_url(redis_connection_string, decode_responses=True, socket_timeout=30) self.counter_key = "job_counter" if not self.redis_client.exists(self.counter_key): self.redis_client.set(self.counter_key, 0) @@ -91,11 +99,14 @@ def generate_job_id(self): return str(job_id) def set(self, job_id: str, value: Any): + ttl = int(os.getenv("WRITER_REDIS_TTL")) if os.getenv("WRITER_REDIS_TTL") else RedisJobVault.DEFAULT_TTL json_str = json.dumps(value) - self.redis_client.set(f"job:{job_id}", json_str) + self.redis_client.set(f"job:{job_id}", json_str, ex=ttl) def get(self, job_id: str): json_str = self.redis_client.get(f"job:{job_id}") + if not json_str: + return None return json.loads(json_str) @@ -425,6 +436,9 @@ async def _stream_incoming_requests(websocket: WebSocket, session_id: str): elif req_message.type == "stateEnquiry": new_task = asyncio.create_task( _handle_state_enquiry_message(websocket, session_id, req_message)) + elif serve_mode == "edit" and req_message.type == "hashRequest": + new_task = asyncio.create_task( + _handle_hash_request(websocket, session_id, req_message)) elif serve_mode == "edit": new_task = asyncio.create_task( _handle_incoming_edit_message(websocket, session_id, req_message)) @@ -516,6 +530,24 @@ async def _handle_state_enquiry_message(websocket: WebSocket, session_id: str, r response.payload = res_payload await websocket.send_json(response.model_dump()) + async def _handle_hash_request(websocket: WebSocket, session_id: str, req_message: WriterWebsocketIncoming): + response = WriterWebsocketOutgoing( + messageType=f"{req_message.type}Response", + trackingId=req_message.trackingId, + payload=None + ) + res_payload: str = None + apsr: Optional[AppProcessServerResponse] = None + apsr = await app_runner.handle_hash_request(session_id, HashRequestPayload( + message=req_message.payload.get("message", "") + )) + if apsr is not None and apsr.payload is not None: + res_payload = typing.cast( + HashRequestResponsePayload, apsr.payload).model_dump() + if res_payload is not None: + response.payload = res_payload + await websocket.send_json(response.model_dump()) + async def _stream_outgoing_announcements(websocket: WebSocket): """ diff --git a/src/writer/ss_types.py b/src/writer/ss_types.py index 3bd05ba76..f453e386c 100644 --- a/src/writer/ss_types.py +++ b/src/writer/ss_types.py @@ -120,6 +120,13 @@ class StateContentRequest(AppProcessServerRequest): type: Literal["stateContent"] +class HashRequestPayload(BaseModel): + message: str + +class HashRequest(AppProcessServerRequest): + type: Literal["hashRequest"] + payload: HashRequestPayload + AppProcessServerRequestPacket = Tuple[int, Optional[str], AppProcessServerRequest] @@ -171,6 +178,13 @@ class StateEnquiryResponse(AppProcessServerResponse): payload: Optional[StateEnquiryResponsePayload] +class HashRequestResponsePayload(BaseModel): + message: str + +class HashRequestResponse(AppProcessServerRequest): + type: Literal["hashRequest"] + payload: HashRequestResponsePayload + AppProcessServerResponsePacket = Tuple[int, Optional[str], AppProcessServerResponse] From 629ccb205d6ea92855a2960c909965dd603df54d Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:04:45 +0100 Subject: [PATCH 05/60] chore: Added Redis --- poetry.lock | 33 +++++++++++++++++++++++++++++++-- pyproject.toml | 1 + 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 95edbbaef..f076fad47 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "alfred-cli" @@ -74,6 +74,17 @@ doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] trio = ["trio (>=0.26.1)"] +[[package]] +name = "async-timeout" +version = "5.0.1" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, + {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, +] + [[package]] name = "attrs" version = "24.2.0" @@ -1235,6 +1246,24 @@ files = [ {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, ] +[[package]] +name = "redis" +version = "5.2.1" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.8" +files = [ + {file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"}, + {file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} + +[package.extras] +hiredis = ["hiredis (>=3.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] + [[package]] name = "referencing" version = "0.35.1" @@ -1761,4 +1790,4 @@ typing-extensions = ">=4.7,<5" [metadata] lock-version = "2.0" python-versions = ">=3.9.2, <4.0" -content-hash = "ddd67e80eb5e5990db930710506e771990d96c0af8647ec2c0c9e15cefdfd7a5" +content-hash = "7aee6e63f83e44ef7c4c2fd5d37534df2a084952a17d6043f7c28668fe03476a" diff --git a/pyproject.toml b/pyproject.toml index dff683345..d0bc9c19f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ watchdog = ">= 3.0.0, < 4" websockets = ">= 12, < 13" writer-sdk = ">= 1.5.0, < 2" python-multipart = ">=0.0.7, < 1" +redis = "^5.2.1" [tool.poetry.group.build] From 4be1ae98451d3634e0262204a8774d5cbffebce3 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:02:47 +0100 Subject: [PATCH 06/60] fix: JSON strict false for more resilient parsing --- src/writer/blocks/httprequest.py | 2 +- src/writer/evaluator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/writer/blocks/httprequest.py b/src/writer/blocks/httprequest.py index 6d403734c..874fc3f26 100644 --- a/src/writer/blocks/httprequest.py +++ b/src/writer/blocks/httprequest.py @@ -80,7 +80,7 @@ def run(self): self.result = { "headers": dict(req.headers), "status_code": req.status_code, - "body": req.json() if is_json else req.text + "body": req.json(strict=False) if is_json else req.text } if req.ok: self.outcome = "success" diff --git a/src/writer/evaluator.py b/src/writer/evaluator.py index 1d8ccf685..c4100ac27 100644 --- a/src/writer/evaluator.py +++ b/src/writer/evaluator.py @@ -32,7 +32,7 @@ def __init__(self, state: "WriterState", component_tree: "ComponentTree"): def evaluate_field(self, instance_path: InstancePath, field_key: str, as_json=False, default_field_value="", base_context={}) -> Any: def decode_json(text): try: - return json.loads(text) + return json.loads(text, strict=False) except json.JSONDecodeError as exception: raise WriterConfigurationError("Error decoding JSON. " + str(exception)) from exception From e99b2c65e2a38a89bf38f9efb2fcb3ed1982da95 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:07:03 +0100 Subject: [PATCH 07/60] chore: Clearer UX for API call --- .../src/builder/settings/BuilderSettingsAPICode.vue | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ui/src/builder/settings/BuilderSettingsAPICode.vue b/src/ui/src/builder/settings/BuilderSettingsAPICode.vue index ee3612b24..f2303812d 100644 --- a/src/ui/src/builder/settings/BuilderSettingsAPICode.vue +++ b/src/ui/src/builder/settings/BuilderSettingsAPICode.vue @@ -37,8 +37,16 @@ >
From 9f296f81bbfb61a9f9768a6721f107b21bc81586 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:07:26 +0100 Subject: [PATCH 08/60] fix: Show workflow when key is null, but also empty --- .../src/builder/sidebar/BuilderSidebarComponentTreeBranch.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ui/src/builder/sidebar/BuilderSidebarComponentTreeBranch.vue b/src/ui/src/builder/sidebar/BuilderSidebarComponentTreeBranch.vue index a3e168119..75808ddfb 100644 --- a/src/ui/src/builder/sidebar/BuilderSidebarComponentTreeBranch.vue +++ b/src/ui/src/builder/sidebar/BuilderSidebarComponentTreeBranch.vue @@ -141,7 +141,7 @@ const name = computed(() => { return component.value.content?.["element"]; } if (type == "workflows_workflow") { - return component.value.content?.["key"] ?? "Workflow"; + return component.value.content?.["key"] || "Workflow"; } return def.value?.name ?? `Unknown (${component.value.type})`; }); From 0a6697b72b81f9c05249373eac5b61220f862679 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:27:55 +0100 Subject: [PATCH 09/60] chore: Allow arbitrary payload for safe events --- src/writer/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/writer/core.py b/src/writer/core.py index d2bcc25d1..74f40cd99 100644 --- a/src/writer/core.py +++ b/src/writer/core.py @@ -1240,6 +1240,8 @@ def transform(self, ev: WriterEvent) -> None: custom_event_name = ev.type[3:] func_name = "_transform_" + custom_event_name.replace("-", "_") if not hasattr(self, func_name): + if ev.isSafe: + return ev.payload = {} raise ValueError( "No payload transformer available for custom event type.") From e5bd32861522332327b323eedf6037d71db7bc4c Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:28:23 +0100 Subject: [PATCH 10/60] chore: Payload parsing --- src/writer/serve.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/writer/serve.py b/src/writer/serve.py index eee0b9fd0..792cd3e5a 100644 --- a/src/writer/serve.py +++ b/src/writer/serve.py @@ -291,6 +291,17 @@ async def init(initBody: InitRequestBody, request: Request, response: Response) # Jobs + async def _get_payload_as_json(request: Request): + payload = None + body = await request.body() + if not body: + return None + try: + payload = await request.json() + except json.JSONDecodeError: + raise HTTPException(status_code=400, detail="Cannot parse the payload.") + return payload + @app.post("/api/job/workflow/{workflow_key}") async def create_workflow_job(workflow_key: str, request: Request, response: Response): crypto.verify_hash_in_request(f"create_job_{workflow_key}", request) @@ -346,7 +357,8 @@ def job_done_callback(task, job_id: str): session_id, WriterEvent( type="wf-builtin-run", isSafe=True, - handler=f"$runWorkflow_{workflow_key}" + handler=f"$runWorkflow_{workflow_key}", + payload=await _get_payload_as_json(request) ))) job_id = app.state.job_vault.generate_job_id() From fd6c6d6211a2faf04137188d5c2fdb95f1893cfc Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:29:00 +0100 Subject: [PATCH 11/60] chore: Prevent execution environment pollution and use payload instead for passing values --- src/writer/blocks/foreach.py | 21 ++++++++++----------- src/writer/blocks/httprequest.py | 12 +++++++++++- src/writer/blocks/runworkflow.py | 12 ++++++------ tests/backend/blocks/conftest.py | 9 +++++---- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/writer/blocks/foreach.py b/src/writer/blocks/foreach.py index b5268d625..b8f56a82c 100644 --- a/src/writer/blocks/foreach.py +++ b/src/writer/blocks/foreach.py @@ -22,19 +22,12 @@ def register(cls, type: str): }, "items": { "name": "Items", - "desc": "The item value will be passed in the execution environment and will be available at @{item}, its id at @{itemId}.", + "desc": "The item value will be passed in the execution environment and will be available at @{payload.item}, its id at @{payload.itemId}.", "default": "{}", "init": '{ "fr": "France", "pl": "Poland" }', "type": "Object", "control": "Textarea" - }, - "executionEnv": { - "name": "Execution environment", - "desc": "You can add other values to the execution environment.", - "default": "{}", - "type": "Object", - "control": "Textarea" - }, + } }, "outs": { "success": { @@ -52,14 +45,20 @@ def register(cls, type: str): )) def _run_workflow_for_item(self, workflow_key, base_execution_environment, item_id, item): - expanded_execution_environment = base_execution_environment | { "itemId": item_id, "item": item } + expanded_execution_environment = base_execution_environment | { + "payload": { + "itemId": item_id, + "item": item + } + } return self.runner.run_workflow_by_key(workflow_key, expanded_execution_environment) def run(self): try: workflow_key = self._get_field("workflowKey") items = self._get_field("items", as_json=True) - base_execution_environment = self._get_field("executionEnv", as_json=True) + base_execution_environment = self.execution_environment + std_items = items result = None if isinstance(items, list): diff --git a/src/writer/blocks/httprequest.py b/src/writer/blocks/httprequest.py index 874fc3f26..c5a229543 100644 --- a/src/writer/blocks/httprequest.py +++ b/src/writer/blocks/httprequest.py @@ -1,3 +1,4 @@ +import re import requests from writer.abstract import register_abstract_template @@ -7,6 +8,8 @@ class HTTPRequest(WorkflowBlock): + CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") + @classmethod def register(cls, type: str): super(HTTPRequest, cls).register(type) @@ -64,6 +67,12 @@ def register(cls, type: str): } )) + def _clean_json_string(self, s: str) -> str: + + """ Remove control characters, which aren't tolerated by JSON loads() strict mode, from string.""" + + return HTTPRequest.CONTROL_CHARS.sub("", s) + def run(self): import json @@ -72,7 +81,8 @@ def run(self): url = self._get_field("url") headers = self._get_field("headers", True) body = self._get_field("body") - req = requests.request(method, url, headers=headers, data=body) + clean_body = self._clean_json_string(body) + req = requests.request(method, url, headers=headers, data=clean_body) content_type = req.headers.get("Content-Type") is_json = content_type and "application/json" in content_type diff --git a/src/writer/blocks/runworkflow.py b/src/writer/blocks/runworkflow.py index e9042cc42..94645a326 100644 --- a/src/writer/blocks/runworkflow.py +++ b/src/writer/blocks/runworkflow.py @@ -19,9 +19,9 @@ def register(cls, type: str): "name": "Workflow key", "type": "Text", }, - "executionEnv": { - "name": "Execution environment", - "desc": "Values passed will be available using the template syntax i.e. @{my_var}", + "payload": { + "name": "Payload", + "desc": "The value specified will be available using the template syntax i.e. @{payload}", "default": "{}", "type": "Object", "control": "Textarea" @@ -45,9 +45,9 @@ def register(cls, type: str): def run(self): try: workflow_key = self._get_field("workflowKey") - execution_environment = self._get_field("executionEnv", as_json=True) - - return_value = self.runner.run_workflow_by_key(workflow_key, execution_environment) + payload = self._get_field("payload", as_json=True) + expanded_execution_environment = self.execution_environment | { "payload": payload } + return_value = self.runner.run_workflow_by_key(workflow_key, expanded_execution_environment) self.result = return_value self.outcome = "success" except BaseException as e: diff --git a/tests/backend/blocks/conftest.py b/tests/backend/blocks/conftest.py index d41f61e6f..fd84d077d 100644 --- a/tests/backend/blocks/conftest.py +++ b/tests/backend/blocks/conftest.py @@ -27,16 +27,17 @@ def run_branch(self, component_id: str, base_outcome_id: str, execution_environm return f"Branch run {component_id} {base_outcome_id}" def run_workflow_by_key(self, workflow_key: str, execution_environment: Dict): - if "env_injection_test" in execution_environment: - return execution_environment.get("env_injection_test") + payload = execution_environment.get("payload") + if "env_injection_test" in payload: + return payload.get("env_injection_test") if workflow_key == "workflow1": return 1 if workflow_key == "workflowDict": return { "a": "b" } if workflow_key == "duplicator": - return execution_environment.get("item") * 2 + return payload.get("item") * 2 if workflow_key == "showId": - return execution_environment.get("itemId") + return payload.get("itemId") if workflow_key == "boom": return 1/0 raise ValueError("Workflow not found.") From 388cd38b541eac0cc750eaebb3cd8c1f7ba4c5a0 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:00:21 +0100 Subject: [PATCH 12/60] chore: Remove control characters --- src/writer/blocks/httprequest.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/writer/blocks/httprequest.py b/src/writer/blocks/httprequest.py index c5a229543..aedb86de7 100644 --- a/src/writer/blocks/httprequest.py +++ b/src/writer/blocks/httprequest.py @@ -80,9 +80,8 @@ def run(self): method = self._get_field("method", False, "GET") url = self._get_field("url") headers = self._get_field("headers", True) - body = self._get_field("body") - clean_body = self._clean_json_string(body) - req = requests.request(method, url, headers=headers, data=clean_body) + body = self._clean_json_string(self._get_field("body")) + req = requests.request(method, url, headers=headers, data=body) content_type = req.headers.get("Content-Type") is_json = content_type and "application/json" in content_type @@ -90,7 +89,7 @@ def run(self): self.result = { "headers": dict(req.headers), "status_code": req.status_code, - "body": req.json(strict=False) if is_json else req.text + "body": req.json() if is_json else req.text } if req.ok: self.outcome = "success" From c53dea9ad02a6ae231075fe6a65696ad257548d2 Mon Sep 17 00:00:00 2001 From: Ramiro Medina <64783088+ramedina86@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:00:46 +0100 Subject: [PATCH 13/60] test: Fix monkeypatching, change to match new payload focused workflow running --- tests/backend/blocks/test_httprequest.py | 16 ++++++++-------- tests/backend/blocks/test_runworkflow.py | 2 +- tests/backend/blocks/test_writernocodeapp.py | 1 - 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/backend/blocks/test_httprequest.py b/tests/backend/blocks/test_httprequest.py index 3d3388ff0..9aab07316 100644 --- a/tests/backend/blocks/test_httprequest.py +++ b/tests/backend/blocks/test_httprequest.py @@ -71,8 +71,8 @@ def test_actual_request_with_bad_path(session, runner): assert block.outcome == "responseError" -def test_patched_request(session, runner): - requests.request = fake_request +def test_patched_request(session, runner, monkeypatch): + monkeypatch.setattr("requests.request", fake_request) session.add_fake_component({ "url": "https://www.duck.com" }) @@ -82,8 +82,8 @@ def test_patched_request(session, runner): assert block.result.get("body") == "Ducks are birds." -def test_patched_request_to_nowhere(session, runner): - requests.request = fake_request +def test_patched_request_to_nowhere(session, runner, monkeypatch): + monkeypatch.setattr("requests.request", fake_request) session.add_fake_component({ "url": "https://www.cat.com" }) @@ -93,8 +93,8 @@ def test_patched_request_to_nowhere(session, runner): assert block.outcome == "connectionError" -def test_patched_request_with_json(session, runner): - requests.request = fake_request +def test_patched_request_with_json(session, runner, monkeypatch): + monkeypatch.setattr("requests.request", fake_request) session.add_fake_component({ "url": "https://www.elephant.com", "method": "POST", @@ -107,8 +107,8 @@ def test_patched_request_with_json(session, runner): assert block.result.get("body").get("request_body") == "Posting the elephant." -def test_patched_request_with_json_and_bad_path(session, runner): - requests.request = fake_request +def test_patched_request_with_json_and_bad_path(session, runner, monkeypatch): + monkeypatch.setattr("requests.request", fake_request) session.add_fake_component({ "url": "https://www.elephant.com/history", "method": "POST", diff --git a/tests/backend/blocks/test_runworkflow.py b/tests/backend/blocks/test_runworkflow.py index 2e3a0d90d..dbfeaee2b 100644 --- a/tests/backend/blocks/test_runworkflow.py +++ b/tests/backend/blocks/test_runworkflow.py @@ -14,7 +14,7 @@ def test_workflow_that_does_not_exist(session, runner): def test_duplicator(session, runner): session.add_fake_component({ "workflowKey": "duplicator", - "executionEnv": '{"item": 23}' + "payload": '{"item": 23}' }) block = RunWorkflow("fake_id", runner, {}) block.run() diff --git a/tests/backend/blocks/test_writernocodeapp.py b/tests/backend/blocks/test_writernocodeapp.py index 9c8f576cd..65c4cd2e0 100644 --- a/tests/backend/blocks/test_writernocodeapp.py +++ b/tests/backend/blocks/test_writernocodeapp.py @@ -15,7 +15,6 @@ def fake_generate_content(application_id, app_inputs): def test_call_nocode_app(monkeypatch, session, runner): monkeypatch.setattr("writer.ai.apps.generate_content", fake_generate_content) - writer.ai.apps.generate_content = fake_generate_content session.add_fake_component({ "appId": "123", "appInputs": json.dumps({ From e1322cda6728babd9552730f12c72dd3024b4be7 Mon Sep 17 00:00:00 2001 From: Alexandre Rousseau Date: Thu, 28 Nov 2024 23:12:25 +0100 Subject: [PATCH 14/60] feat(ui): use design system in builder - WF-127 --- src/ui/src/builder/BuilderModal.vue | 33 +++- src/ui/src/builder/BuilderSelect.vue | 44 ++--- .../builder/settings/BuilderFieldsAlign.vue | 5 +- .../builder/settings/BuilderFieldsColor.vue | 14 +- .../settings/BuilderFieldsKeyValue.vue | 12 +- .../builder/settings/BuilderFieldsObject.vue | 1 - .../builder/settings/BuilderFieldsPadding.vue | 130 +++++++------ .../builder/settings/BuilderFieldsShadow.vue | 22 ++- .../builder/settings/BuilderFieldsText.vue | 1 - .../builder/settings/BuilderFieldsWidth.vue | 28 ++- .../builder/settings/BuilderSectionTitle.vue | 23 +++ .../settings/BuilderSettingsBinding.vue | 20 +- .../settings/BuilderSettingsHandlers.vue | 178 +++++++----------- .../settings/BuilderSettingsProperties.vue | 63 +++---- .../settings/BuilderSettingsVisibility.vue | 28 +-- .../builder/settings/BuilderTemplateInput.vue | 38 ++-- src/ui/src/builder/sharedStyles.css | 70 ------- src/ui/src/wds/WdsFieldWrapper.vue | 71 ++++--- src/ui/src/wds/WdsTextInput.vue | 43 ++++- src/ui/src/wds/WdsTextareaInput.vue | 32 +++- tests/e2e/tests/state.spec.ts | 2 +- tests/e2e/tests/stateAutocompletion.spec.ts | 22 +-- 22 files changed, 429 insertions(+), 451 deletions(-) create mode 100644 src/ui/src/builder/settings/BuilderSectionTitle.vue diff --git a/src/ui/src/builder/BuilderModal.vue b/src/ui/src/builder/BuilderModal.vue index 62cbf4a4d..12ab381a7 100644 --- a/src/ui/src/builder/BuilderModal.vue +++ b/src/ui/src/builder/BuilderModal.vue @@ -1,7 +1,12 @@