diff --git a/apps/shinkai-desktop/src-tauri/Cargo.lock b/apps/shinkai-desktop/src-tauri/Cargo.lock index de6fd97a7..569d1eaba 100644 --- a/apps/shinkai-desktop/src-tauri/Cargo.lock +++ b/apps/shinkai-desktop/src-tauri/Cargo.lock @@ -372,6 +372,61 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "axum" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", + "hyper-util", + "itoa 1.0.11", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "tokio", + "tower 0.5.1", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.1", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -2401,6 +2456,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa 1.0.11", "pin-project-lite", "smallvec", @@ -2454,7 +2510,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", + "tower 0.4.13", "tower-service", "tracing", ] @@ -2910,6 +2966,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "memchr" version = "2.7.1" @@ -4440,6 +4502,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + [[package]] name = "ryu" version = "1.0.17" @@ -4614,6 +4682,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa 1.0.11", + "serde", +] + [[package]] name = "serde_repr" version = "0.1.18" @@ -4745,6 +4823,7 @@ name = "shinkai-desktop" version = "0.0.0" dependencies = [ "anyhow", + "axum", "base64 0.22.1", "blake3", "chrono", @@ -5831,6 +5910,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -5839,9 +5934,9 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -5849,6 +5944,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", diff --git a/apps/shinkai-desktop/src-tauri/Cargo.toml b/apps/shinkai-desktop/src-tauri/Cargo.toml index 4f1197a7a..5af805567 100644 --- a/apps/shinkai-desktop/src-tauri/Cargo.toml +++ b/apps/shinkai-desktop/src-tauri/Cargo.toml @@ -49,6 +49,7 @@ tauri-plugin-fs="2.0.1" tauri-plugin-os = "2.0.1" tauri-plugin-process = "2.0.1" tauri-plugin-log = "2.0.1" +axum = "0.7.7" [features] # this feature is used for production builds or when `devPath` points to the filesystem diff --git a/apps/shinkai-desktop/src-tauri/capabilities/python-code-runner.json b/apps/shinkai-desktop/src-tauri/capabilities/python-code-runner.json new file mode 100644 index 000000000..eb2333a87 --- /dev/null +++ b/apps/shinkai-desktop/src-tauri/capabilities/python-code-runner.json @@ -0,0 +1,13 @@ +{ + "identifier": "python-code-runner", + "description": "permissions for python-code-runner window", + "local": true, + "windows": ["python-code-runner"], + "permissions": [ + "core:event:allow-listen", + "core:event:allow-unlisten", + "core:event:allow-emit", + "log:default" + ], + "commands.allow": [] +} diff --git a/apps/shinkai-desktop/src-tauri/src/external_api/mod.rs b/apps/shinkai-desktop/src-tauri/src/external_api/mod.rs new file mode 100644 index 000000000..758770f3f --- /dev/null +++ b/apps/shinkai-desktop/src-tauri/src/external_api/mod.rs @@ -0,0 +1,158 @@ +use std::sync::Arc; + +use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::post, Json, Router}; +use futures_util::{FutureExt, TryFutureExt}; +use log::debug; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use tauri::{AppHandle, Emitter, EventTarget, Listener}; + +use crate::windows::Window; + +#[derive(Deserialize, Debug)] +struct RunCodeRequest { + parameters: Value, + configurations: Value, + code: String, +} + +// the output to our `create_user` handler +#[derive(Serialize)] +struct RunCodeResponse { + result: Value, +} + +#[derive(Serialize, Clone)] +struct RunPythonCodeRequestEventPayload { + id: String, + parameters: Value, + configurations: Value, + code: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +struct RunPythonCodeResponseEventPayload { + result: Value, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +struct RunPythonCodeResponseErrorEventPayload { + message: String, +} + +async fn run_code( + State(app_handle): State, + Json(payload): Json, +) -> Result, StatusCode> { + // (StatusCode::OK, "asda") + log::info!("api /run-code {:?}", payload); + let id = uuid::Uuid::new_v4().to_string(); + let _ = app_handle.emit_to( + EventTarget::webview_window(Window::PythonCodeRunner.as_str()), + "run-python-code-request", + RunPythonCodeRequestEventPayload { + id: id.clone(), + parameters: payload.parameters, + configurations: payload.configurations, + code: payload.code, + }, + ); + let run_python_code_response_event_str = format!("run-python-code-response-{}", id); + let run_python_code_response_error_event_str = format!("run-python-code-response-error-{}", id); + let (tx, mut rx) = tokio::sync::mpsc::channel::>(1); + + let tx_clone = tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; + log::info!("execution timeout 30s"); + let _ = tx_clone.send(None).await; + }); + + let app_handle_clone = app_handle.clone(); + let tx_clone = tx.clone(); + tokio::spawn(async move { + debug!( + "liteninig code response {}", + run_python_code_response_event_str.clone() + ); + app_handle_clone.once( + run_python_code_response_event_str.clone(), + move |event: tauri::Event| { + log::info!("{} {:?}", run_python_code_response_event_str.clone(), event); + let event: RunPythonCodeResponseEventPayload = + serde_json::from_str(event.payload()) + .expect("failed to deserialize event payload"); + log::info!("{:?}", event); + let _ = tx_clone.blocking_send(Some( + serde_json::to_value(event).expect("failed to serialize event"), + )); + }, + ); + }); + + let app_handle_clone = app_handle.clone(); + let tx_clone = tx.clone(); + tokio::spawn(async move { + app_handle_clone.once( + run_python_code_response_error_event_str, + move |event: tauri::Event| { + let event: RunPythonCodeResponseErrorEventPayload = + serde_json::from_str(&event.payload()) + .expect("failed to deserialize error event payload"); + log::info!("{:?}", event); + let _ = tx_clone.blocking_send(Some( + serde_json::to_value(event).expect("failed to serialize event"), + )); + }, + ) + }); + + let result = rx.recv().await; + if let Some(Some(event)) = result { + match event { + serde_json::Value::Object(event) => { + if let Some(result) = event.get("result") { + // Handle RunPythonCodeResponseEventPayload + log::info!("python code execution successful: {:?}", result); + Ok(Json(RunCodeResponse { + result: result.clone(), + })) + } else if let Some(message) = event.get("message") { + // Handle RunPythonCodeResponseErrorEventPayload + log::error!("python code execution failed: {:?}", message); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } else { + log::error!("invalid event format received"); + return Err(StatusCode::BAD_REQUEST); + } + } + _ => { + log::error!("invalid event format received"); + Err(StatusCode::BAD_REQUEST) + } + } + } else { + log::info!("Timeout reached before receiving a response"); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } +} + +pub async fn init(app_handle: AppHandle, port: usize) { + let address = format!("0.0.0.0:{}", port); + log::info!("initializing external api {}", address.clone()); + let app = Router::new().route("/run-code", post(run_code).with_state(app_handle.clone())); + + log::info!("binding address {}", address.clone()); + let listener = match tokio::net::TcpListener::bind(address.clone()).await { + Ok(listener) => listener, + Err(e) => { + log::error!("failed to bind to {}: {}", address.clone(), e); + return; + } + }; + log::info!("port bound successfully {}", address.clone()); + if let Err(e) = axum::serve(listener, app).await { + log::error!("failed to serve external API: {}", e); + } + log::info!("external api initialized"); +} diff --git a/apps/shinkai-desktop/src-tauri/src/main.rs b/apps/shinkai-desktop/src-tauri/src/main.rs index f339e7cee..5fc7ad970 100644 --- a/apps/shinkai-desktop/src-tauri/src/main.rs +++ b/apps/shinkai-desktop/src-tauri/src/main.rs @@ -14,6 +14,7 @@ use crate::commands::shinkai_node_manager_commands::{ }; use commands::spotlight_commands::hide_spotlight_window_app; +use external_api::init; use global_shortcuts::global_shortcut_handler; use globals::SHINKAI_NODE_MANAGER_INSTANCE; use local_shinkai_node::shinkai_node_manager::ShinkaiNodeManager; @@ -25,6 +26,7 @@ use windows::Window; mod audio; mod commands; +mod external_api; mod galxe; mod global_shortcuts; mod globals; @@ -116,9 +118,11 @@ fn main() { } } }); - create_tray(app.handle())?; - + let app_handle_clone = app.handle().clone(); + tauri::async_runtime::spawn(async move { + init(app_handle_clone, 11436).await; + }); Ok(()) }) .build(tauri::generate_context!()) diff --git a/apps/shinkai-desktop/src-tauri/src/windows/mod.rs b/apps/shinkai-desktop/src-tauri/src/windows/mod.rs index 78efa7aaf..651d83635 100644 --- a/apps/shinkai-desktop/src-tauri/src/windows/mod.rs +++ b/apps/shinkai-desktop/src-tauri/src/windows/mod.rs @@ -5,6 +5,8 @@ pub enum Window { Main, ShinkaiNodeManager, Spotlight, + Coordinator, + PythonCodeRunner, } impl Window { @@ -13,6 +15,8 @@ impl Window { Window::Main => "main", Window::ShinkaiNodeManager => "shinkai-node-manager", Window::Spotlight => "spotlight", + Window::Coordinator => "coordinator", + Window::PythonCodeRunner => "python-code-runner", } } } @@ -52,3 +56,4 @@ pub fn hide_spotlight_window(app_handle: AppHandle) { window.hide().unwrap(); } } + diff --git a/apps/shinkai-desktop/src-tauri/tauri.conf.json b/apps/shinkai-desktop/src-tauri/tauri.conf.json index 8d279461a..d98105246 100644 --- a/apps/shinkai-desktop/src-tauri/tauri.conf.json +++ b/apps/shinkai-desktop/src-tauri/tauri.conf.json @@ -97,6 +97,28 @@ "parent": null, "create": true }, + { + "fullscreen": false, + "label": "python-code-runner", + "transparent": false, + "title": "Python Code Runner", + "width": 720, + "height": 470, + "resizable": true, + "dragDropEnabled": true, + "titleBarStyle": "Overlay", + "hiddenTitle": false, + "url": "src/windows/python-code-runner/index.html", + "center": true, + "skipTaskbar": false, + "shadow": false, + "decorations": false, + "visible": true, + "closable": false, + "parent": null, + "create": true, + + }, { "label": "shinkai-node-manager", "url": "src/windows/shinkai-node-manager/index.html", diff --git a/apps/shinkai-desktop/src/windows/python-code-runner/index.html b/apps/shinkai-desktop/src/windows/python-code-runner/index.html new file mode 100644 index 000000000..d1ce11fb1 --- /dev/null +++ b/apps/shinkai-desktop/src/windows/python-code-runner/index.html @@ -0,0 +1,14 @@ + + + + + + + Python Code Runner + + + +
+ + + diff --git a/apps/shinkai-desktop/src/windows/python-code-runner/main.tsx b/apps/shinkai-desktop/src/windows/python-code-runner/main.tsx new file mode 100644 index 000000000..fe379006e --- /dev/null +++ b/apps/shinkai-desktop/src/windows/python-code-runner/main.tsx @@ -0,0 +1,99 @@ +import { emit, listen, UnlistenFn } from '@tauri-apps/api/event'; +import { debug, info } from '@tauri-apps/plugin-log'; +import React from 'react'; +import { useEffect } from 'react'; +import ReactDOM from 'react-dom/client'; + +import { + PythonCodeRunnerWebWorkerMessage, + RunResult, +} from './python-code-runner-web-worker'; +import PythonRunnerWorker from './python-code-runner-web-worker?worker'; + +type RunPythonCodeRequest = { + id: string; + parameters: object; + configurations: object; + code: string; +}; + +type RunPythonCodeResponse = { + result: RunResult; +}; + +type RunPythonCodeResponseError = { + message: string; +}; + +const useRunPythonCodeEventListener = () => { + useEffect(() => { + let unlisten: UnlistenFn; + + const setupListener = async () => { + unlisten = await listen( + 'run-python-code-request', + async (event) => { + debug(`received run-python-code request: ${event.payload}`); + const worker = new PythonRunnerWorker(); + worker.postMessage({ + type: 'run', + payload: { code: event.payload.code }, + }); + let result: RunResult; + try { + result = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error('execution timed out')); + }, 30000); + worker.onmessage = (event: { + data: PythonCodeRunnerWebWorkerMessage; + }) => { + if (event.data.type === 'run-done') { + clearTimeout(timeout); + console.log('worker event', event); + resolve(event.data.payload); + } + }; + worker.onerror = (error: { message: string }) => { + clearTimeout(timeout); + info(`python code runner worker error ${String(error)}`); + reject(new Error(`worker error: ${error.message}`)); + }; + }).finally(() => { + worker.terminate(); + }); + } catch (e) { + await emit(`run-python-code-response-error-${event.payload.id}`, { + message: String(e), + } as RunPythonCodeResponseError); + return; + } + info(`emiting result for run-python-code-response-${event.payload.id} ${JSON.stringify(result)}`); + await emit(`run-python-code-response-${event.payload.id}`, { + result, + } as RunPythonCodeResponse); + }, + ); + }; + + setupListener(); + + return () => { + if (unlisten) { + unlisten(); + } + }; + }, []); +}; + +const App = () => { + info('initializing window python-code-runner'); + useRunPythonCodeEventListener(); + return null; +}; + +ReactDOM.createRoot(document.querySelector('#root') as HTMLElement).render( + + + , +); diff --git a/apps/shinkai-desktop/src/windows/python-code-runner/python-code-runner-web-worker.ts b/apps/shinkai-desktop/src/windows/python-code-runner/python-code-runner-web-worker.ts new file mode 100644 index 000000000..f5e16fbd0 --- /dev/null +++ b/apps/shinkai-desktop/src/windows/python-code-runner/python-code-runner-web-worker.ts @@ -0,0 +1,206 @@ +/* eslint-disable no-restricted-globals */ +import { loadPyodide, PyodideInterface } from 'pyodide'; + +export type CodeOutput = { + rawOutput: string; + figures: { type: 'plotly' | 'html'; data: string }[]; +}; + +export type RunResult = + | { + state: 'success'; + stdout: string[]; + stderr: string[]; + result: { + rawOutput: string; + figures: { type: 'plotly' | 'html'; data: string }[]; + }; + } + | { + state: 'error'; + stdout: string[]; + stderr: string[]; + message: string; + }; + +export type PythonCodeRunnerWebWorkerMessage = { + type: 'run-done'; + payload: RunResult; +}; + +const INDEX_URL = 'https://cdn.jsdelivr.net/pyodide/v0.26.2/full/'; + +let pyodide: PyodideInterface; +const stdout: string[] = []; +const stderr: string[] = []; + +const wrapCode = (code: string): string => { + const wrappedCode = ` +import sys +import pandas as pd +import plotly.graph_objects as go +import plotly.express as px +import plotly.io as pio +import json +import array + +import pyodide_http +pyodide_http.patch_all() + +import matplotlib +matplotlib.use("AGG") + +# Function to capture DataFrame display as HTML +def capture_df_display(df): + return df.to_html() + +output = None +outputError = None +figures = [] + +def execute_user_code(): +${code + .split('\n') + .map((line) => ` ${line}`) + .join('\n')} + return locals() + +try: + user_code_result = execute_user_code() + # Capture the last variable in the scope + last_var = list(user_code_result.values())[-1] if user_code_result else None + if isinstance(last_var, pd.DataFrame): + output = capture_df_display(last_var) + elif isinstance(last_var, (str, int, float, list, dict)): + output = json.dumps(last_var) + else: + output = '' + + for var_name, var_value in user_code_result.items(): + if isinstance(var_value, go.Figure): + figures.append({ 'type': 'plotly', 'data': pio.to_json(var_value) }) + if isinstance(var_value, pd.DataFrame): + figures.append({ 'type': 'html', 'data': capture_df_display(var_value) }) + +except Exception as e: + outputError = str(e) + +figures = json.dumps(figures) +(output, outputError, figures) + `; + console.log('Running code:', wrappedCode); + return wrappedCode; +}; + +const findImportsFromCodeString = async (code: string): Promise => { + const wrappedCode = ` +from pyodide.code import find_imports +import json +code = """${code.replace(/"""/g, '\\"\\"\\"')}""" +imports = find_imports(code) +json.dumps(imports) +`; + const jsonResult = await pyodide.runPythonAsync(wrappedCode); + const result = JSON.parse(jsonResult); + return result; +}; + +/** + * Attempts to install dependencies for the given Python code using micropip. + * + * This method does its best effort to install all dependencies using micropip, + * but it's important to note that not all packages may be available or compatible + * with the Pyodide environment. Even after this method completes, some dependencies + * might still not be found or properly installed due to various constraints of + * the web-based Python runtime. + * + * This just do the best effort to install micropip dependencies + * Native pyodide dependencies are install uner the hood when call runPythonAsync + * + * The function performs the following steps: + * 1. Loads the 'micropip' package. + * 2. Finds imports from both the wrapped code template and the user's code. + * 3. Attempts to install each found dependency using micropip. + * + * @param code - The Python code string to analyze for dependencies. + * @returns A Promise that resolves when the installation attempts are complete. + */ +const installDependencies = async (code: string): Promise => { + console.time('install micropip dependencies'); + await pyodide.loadPackage(['micropip']); + const micropip = pyodide.pyimport('micropip'); + const codeDependencies = [ + // Our code wrapper constains dependencies so we need to install them + ...(await findImportsFromCodeString(wrapCode(''))), + ...(await findImportsFromCodeString(code)), + ]; + console.log('trying to install the following dependencies', codeDependencies); + const installPromises = codeDependencies.map((dependency) => + micropip.install(dependency), + ); + await Promise.allSettled(installPromises); + console.timeEnd('install micropip dependencies'); +}; + +const run = async (code: string) => { + console.time('run code'); + const wrappedCode = wrapCode(code); + const [output, outputError, figures] = + await pyodide.runPythonAsync(wrappedCode); + if (outputError) { + throw new Error(outputError); + } + console.timeEnd('run code'); + return { rawOutput: output, figures: JSON.parse(figures) }; +}; + +const initialize = async () => { + console.time('initialize'); + pyodide = await loadPyodide({ + indexURL: INDEX_URL, + stdout: (message) => { + console.log('python stdout', message); + stdout.push(message); + }, + stderr: (message) => { + console.log('python stderr', message); + stderr.push(message); + }, + fullStdLib: false, + }); + console.timeEnd('initialize'); +}; + +self.onmessage = async (event) => { + switch (event.data?.type) { + case 'run': + console.time('total run time'); + try { + await initialize(); + await installDependencies(event.data.payload.code); + const runResult = await run(event.data.payload.code); + self.postMessage({ + type: 'run-done', + payload: { + state: 'success', + stdout, + stderr, + result: runResult, + } as RunResult, + }); + } catch (e) { + self.postMessage({ + type: 'run-done', + payload: { + state: 'error', + stdout, + stderr, + message: String(e), + } as RunResult, + }); + } finally { + console.timeEnd('total run time'); + } + break; + } +}; diff --git a/apps/shinkai-desktop/vite.config.ts b/apps/shinkai-desktop/vite.config.ts index d8948825a..09e359a9c 100644 --- a/apps/shinkai-desktop/vite.config.ts +++ b/apps/shinkai-desktop/vite.config.ts @@ -65,10 +65,14 @@ export default defineConfig(() => ({ ), spotlight: resolve(__dirname, 'src/windows/spotlight/index.html'), coordinator: resolve(__dirname, 'src/windows/coordinator/index.html'), + 'python-code-runner': resolve( + __dirname, + 'src/windows/python-code-runner/index.html', + ), }, }, }, worker: { - format: 'es' as const + format: 'es' as const, }, }));